diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index c55c3a9c9a9..a66cab4b0a0 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -160,6 +160,15 @@ jobs: HARNESS_RUNS: ${{ runner.temp }}/moq-test-runs HARNESS_RESULTS: ${{ runner.temp }}/moq-test-runs/results.jsonl + - name: VR catalog fields + id: vr + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + run: nix develop --command just test vr + shell: bash -leo pipefail {0} + env: + HARNESS_RUNS: ${{ runner.temp }}/moq-test-runs + HARNESS_RESULTS: ${{ runner.temp }}/moq-test-runs/results.jsonl + - name: Interop negative control id: negative if: ${{ !cancelled() && steps.setup.outcome == 'success' }} @@ -236,6 +245,7 @@ jobs: "Interop (moq-transport)": "${{ steps.interop-ietf.outcome }}", "Released wire compatibility": "${{ steps.wire-compat.outcome }}", "Portrait transcode": "${{ steps.portrait.outcome }}", + "VR catalog fields": "${{ steps.vr.outcome }}", "Interop negative control": "${{ steps.negative.outcome }}", "Media output and lifecycle": "${{ steps.media.outcome }}", "Media track identity changes": "${{ steps.media-rename.outcome }}" diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 9324e5f9190..9ae28ad624e 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -125,6 +125,8 @@ jobs: - "test/interop/**" - "test/max-age/**" - "test/portrait/**" + - "test/vr/**" + - "demo/vr/**" - "rs/moq-tokio/tests/integration/max_age.rs" - "test/datagram-tracks/**" - "rs/moq-tokio/tests/integration/datagram_tracks.rs" diff --git a/bun.lock b/bun.lock index 2f51f8c5b49..51d9ab390de 100644 --- a/bun.lock +++ b/bun.lock @@ -72,6 +72,19 @@ "vite": "^8.3.6", }, }, + "demo/vr": { + "name": "@moq/demo-vr", + "dependencies": { + "@moq/watch": "workspace:^", + "three": "^0.186.1", + }, + "devDependencies": { + "@types/three": "^0.186.0", + "rolldown": "^1.2.16", + "typescript": "7.0.2", + "vite": "^8.3.5", + }, + }, "demo/web": { "name": "@moq/demo", "version": "0.1.1", @@ -685,6 +698,8 @@ "@cspotcode/source-map-support": ["@cspotcode/source-map-support@0.8.1", "", { "dependencies": { "@jridgewell/trace-mapping": "0.3.9" } }, "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw=="], + "@dimforge/rapier3d-compat": ["@dimforge/rapier3d-compat@0.12.0", "", {}, "sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow=="], + "@discoveryjs/json-ext": ["@discoveryjs/json-ext@1.1.0", "", {}, "sha512-Xc3VhU02wqZ1HvHRJUwL09HkZSTvidqY5Ya0NXBSYOxAp+Ln9dcJr9fySI+CkONzP3PekQo9WdzCv0PGER/mOA=="], "@docsearch/css": ["@docsearch/css@3.8.2", "", {}, "sha512-y05ayQFyUmCXze79+56v/4HpycYF3uFqB78pLPrSV5ZKAlDuIAAJNhaRi8tTdRNXh05yxX/TyNnzD6LwSM89vQ=="], @@ -847,6 +862,8 @@ "@moq/demo-carla": ["@moq/demo-carla@workspace:demo/carla/web"], + "@moq/demo-vr": ["@moq/demo-vr@workspace:demo/vr"], + "@moq/drain-test": ["@moq/drain-test@workspace:test/drain"], "@moq/hang": ["@moq/hang@workspace:js/hang"], @@ -1069,6 +1086,8 @@ "@tybys/wasm-util": ["@tybys/wasm-util@0.10.3", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg=="], + "@tweenjs/tween.js": ["@tweenjs/tween.js@23.1.3", "", {}, "sha512-vJmvvwFxYuGnF2axRtPYocag6Clbb5YS7kLL+SO/TeVFzHqDIWrNKYtcsPMibjDx9O+bu+psAy9NKfWklassUA=="], + "@types/audioworklet": ["@types/audioworklet@0.0.100", "", {}, "sha512-mWt4Z5CWi0CupDnSYWJgx6dw61XUk9JSRGrM9F/YNrpOoY/0XG+sxP+8rAjQq80H4febOe8PrzGmmN/X3w8LsQ=="], "@types/babel__core": ["@types/babel__core@7.20.5", "", { "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", "@types/babel__generator": "*", "@types/babel__template": "*", "@types/babel__traverse": "*" } }, "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA=="], @@ -1115,14 +1134,20 @@ "@types/react-dom": ["@types/react-dom@19.3.0", "", { "peerDependencies": { "@types/react": "^19.3.0" } }, "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q=="], + "@types/stats.js": ["@types/stats.js@0.17.4", "", {}, "sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA=="], + "@types/supports-color": ["@types/supports-color@8.1.3", "", {}, "sha512-Hy6UMpxhE3j1tLpl27exp1XqHD7n8chAiNPzWfz16LPZoMMoSc4dzLl6w9qijkEb/r5O1ozdu1CWGA2L83ZeZg=="], "@types/text-table": ["@types/text-table@0.2.5", "", {}, "sha512-hcZhlNvMkQG/k1vcZ6yHOl6WAYftQ2MLfTHcYRZ2xYZFD8tGVnE3qFV0lj1smQeDSR7/yY0PyuUalauf33bJeA=="], + "@types/three": ["@types/three@0.186.0", "", { "dependencies": { "@dimforge/rapier3d-compat": "~0.12.0", "@tweenjs/tween.js": "~23.1.3", "@types/stats.js": "*", "@types/webxr": ">=0.5.17", "fflate": "~0.8.3", "meshoptimizer": "~1.1.1" } }, "sha512-mxYSBpDC+D0pLfSP6sW4WZTcT+nrtmZcimMqnVmy36Hte3XpeYSrvgg4TRdaM1GemGog1AWzI5qL2VoIfMXbJQ=="], + "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], "@types/web-bluetooth": ["@types/web-bluetooth@0.0.21", "", {}, "sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA=="], + "@types/webxr": ["@types/webxr@0.5.24", "", {}, "sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg=="], + "@types/whatwg-mimetype": ["@types/whatwg-mimetype@3.0.2", "", {}, "sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA=="], "@types/ws": ["@types/ws@8.18.2", "", { "dependencies": { "@types/node": "*" } }, "sha512-67MQl+fpWKVTT1NYdnmo3U4sc/xPo/zQBncVnI74qmQa0z/b+1g6iYqNmGCPbxO+zz2aklb08a0oHfegiVd0/w=="], @@ -1393,6 +1418,8 @@ "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], + "fflate": ["fflate@0.8.3", "", {}, "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA=="], + "flat": ["flat@5.0.2", "", { "bin": { "flat": "cli.js" } }, "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ=="], "focus-trap": ["focus-trap@7.8.0", "", { "dependencies": { "tabbable": "^6.4.0" } }, "sha512-/yNdlIkpWbM0ptxno3ONTuf+2g318kh2ez3KSeZN5dZ8YC6AAmgeWz+GasYYiBJPFaYcSAPeu4GfhUaChzIJXA=="], @@ -1577,6 +1604,8 @@ "merge-stream": ["merge-stream@2.0.0", "", {}, "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="], + "meshoptimizer": ["meshoptimizer@1.1.1", "", {}, "sha512-oRFNWJRDA/WTrVj7NWvqa5HqE1t9MYDj2VaWirQCzCCrAd2GHrqR/sQezCxiWATPNlKTcRaPRHPJwIRoPBAp5g=="], + "micromark": ["micromark@4.0.2", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="], "micromark-core-commonmark": ["micromark-core-commonmark@2.0.3", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg=="], @@ -1895,6 +1924,8 @@ "text-table": ["text-table@0.2.0", "", {}, "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw=="], + "three": ["three@0.186.1", "", {}, "sha512-blFeqb49wRCSGUGj7gtpfnSGHy2lwDk94RhUmS1c/hTby70kvChbWpkJ4Pm1390LqzzvTmzgXKHPEafJwCb8jA=="], + "tinyexec": ["tinyexec@1.3.0", "", {}, "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ=="], "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="], diff --git a/demo/vr/justfile b/demo/vr/justfile new file mode 100644 index 00000000000..9a56ec58af9 --- /dev/null +++ b/demo/vr/justfile @@ -0,0 +1,9 @@ +set shell := ["bash", "-euo", "pipefail", "-c"] + +# Serve the VR demo against a relay; the page takes ?url= and ?name= to override. +serve url="http://localhost:4443": install + VITE_RELAY_URL="{{ url }}" bun --bun vite + +# Install the workspace deps that vite and the demo's imports need. +install: + uv run --no-project python ../../sh/js/install.py update diff --git a/demo/vr/package.json b/demo/vr/package.json new file mode 100644 index 00000000000..69858b6ebac --- /dev/null +++ b/demo/vr/package.json @@ -0,0 +1,22 @@ +{ + "name": "@moq/demo-vr", + "private": true, + "type": "module", + "description": "Media over QUIC - VR demo", + "license": "(MIT OR Apache-2.0)", + "scripts": { + "dev": "vite", + "build": "vite build", + "check": "tsc --noEmit" + }, + "dependencies": { + "@moq/watch": "workspace:^", + "three": "^0.186.1" + }, + "devDependencies": { + "@types/three": "^0.186.0", + "rolldown": "^1.2.16", + "typescript": "7.0.2", + "vite": "^8.3.5" + } +} diff --git a/demo/vr/src/audio.ts b/demo/vr/src/audio.ts new file mode 100644 index 00000000000..45581f11e18 --- /dev/null +++ b/demo/vr/src/audio.ts @@ -0,0 +1,134 @@ +import * as THREE from "three"; + +export type Placement = { kind: "ambisonic" } | { kind: "point"; position: THREE.Vector3 } | { kind: "head" }; + +// Virtual speakers at the corners of a cube in ambisonic axes (W, Y, Z, X) +const SPEAKERS = [-1, 1].flatMap((x) => + [-1, 1].flatMap((y) => [-1, 1].map((z) => [x, y, z].map((v) => v / Math.sqrt(3)))), +); + +const MAX_RE = 1 / Math.sqrt(3); + +// Scratch for pose(), which runs every XR frame. +const POSE = { + position: new THREE.Vector3(), + rotation: new THREE.Quaternion(), + scale: new THREE.Vector3(), + forward: new THREE.Vector3(), + up: new THREE.Vector3(), +}; + +// Renders audio graph around a viewer's head +export class Spatial { + #context: BaseAudioContext; + #nodes: AudioNode[] = []; + #tracksPosition: boolean; + // Every placement mixes here, so muting is one param. + #out: GainNode; + + constructor(root: AudioNode, placement: Placement) { + this.#context = root.context; + this.#out = new GainNode(this.#context); + this.#out.connect(this.#context.destination); + this.#nodes.push(this.#out); + const destination = this.#out; + + switch (placement.kind) { + case "head": + root.connect(destination); + this.#tracksPosition = false; + break; + case "point": { + const panner = this.#panner(placement.position, 1); + root.connect(panner).connect(destination); + this.#tracksPosition = true; + break; + } + case "ambisonic": + this.#ambisonic(root, destination); + // The field is infinitely far, so only the head's rotation moves it. + this.#tracksPosition = false; + break; + } + this.#nodes.push(root); + } + + // Decodes out to fixed speakers around the scene. A higher order plays at first order: the + // splitter's discrete downmix keeps ACN channels 0 to 3 and drops the rest. + #ambisonic(root: AudioNode, destination: AudioNode) { + if (root.channelCount < 4) throw new Error(`ambisonics needs at least 4 channels, got ${root.channelCount}`); + + const split = new ChannelSplitterNode(this.#context, { numberOfOutputs: 4 }); + root.connect(split); + this.#nodes.push(split); + + for (const [x, y, z] of SPEAKERS) { + // ACN order is W, Y, Z, X. + const weights = [1, 3 * MAX_RE * y, 3 * MAX_RE * z, 3 * MAX_RE * x].map((w) => w / SPEAKERS.length); + // Ambisonic axes to WebAudio + const panner = this.#panner(new THREE.Vector3(-y, z, -x), 0); + panner.connect(destination); + + weights.forEach((weight, channel) => { + const gain = new GainNode(this.#context, { gain: weight }); + split.connect(gain, channel).connect(panner); + this.#nodes.push(gain); + }); + } + } + + #panner(position: THREE.Vector3, rolloffFactor: number): PannerNode { + const panner = new PannerNode(this.#context, { + panningModel: "HRTF", + distanceModel: "inverse", + refDistance: 1, + rolloffFactor, + positionX: position.x, + positionY: position.y, + positionZ: position.z, + }); + this.#nodes.push(panner); + return panner; + } + + // The mix every placement feeds, for meters and the harness. + get output(): AudioNode { + return this.#out; + } + + set muted(muted: boolean) { + this.#out.gain.setTargetAtTime(muted ? 0 : 1, this.#context.currentTime, 0.01); + } + + // Move the listener to the viewer's head, in the same space as the placement. + pose(head: THREE.Matrix4) { + const { position, rotation, scale, forward, up } = POSE; + head.decompose(position, rotation, scale); + forward.set(0, 0, -1).applyQuaternion(rotation); + up.set(0, 1, 0).applyQuaternion(rotation); + + const listener = this.#context.listener; + if (!this.#tracksPosition) position.set(0, 0, 0); + // Firefox has no listener AudioParams, only the older setters. + if (!listener.positionX) { + listener.setPosition(position.x, position.y, position.z); + listener.setOrientation(forward.x, forward.y, forward.z, up.x, up.y, up.z); + return; + } + const at = this.#context.currentTime; + const set = (param: AudioParam, value: number) => param.setTargetAtTime(value, at, 0.01); + set(listener.positionX, position.x); + set(listener.positionY, position.y); + set(listener.positionZ, position.z); + set(listener.forwardX, forward.x); + set(listener.forwardY, forward.y); + set(listener.forwardZ, forward.z); + set(listener.upX, up.x); + set(listener.upY, up.y); + set(listener.upZ, up.z); + } + + close() { + for (const node of this.#nodes) node.disconnect(); + } +} diff --git a/demo/vr/src/index.html b/demo/vr/src/index.html new file mode 100644 index 00000000000..9ac55d45d39 --- /dev/null +++ b/demo/vr/src/index.html @@ -0,0 +1,29 @@ + + + + + + MoQ VR + + + + +
drag to look · R recentre · M mute
+ + + diff --git a/demo/vr/src/index.ts b/demo/vr/src/index.ts new file mode 100644 index 00000000000..db538ff05d0 --- /dev/null +++ b/demo/vr/src/index.ts @@ -0,0 +1,78 @@ +import { Audio, Broadcast, Net, Signals, Sync, Video } from "@moq/watch"; +import { type Placement, Spatial } from "./audio"; +import { Scene } from "./scene"; + +const params = new URLSearchParams(location.search); +const url = new URL(params.get("url") ?? import.meta.env.VITE_RELAY_URL ?? "http://localhost:4443"); +const name = Net.Path.from(params.get("name") ?? "vr.hang"); + +const connection = new Net.Connection({ url }); +const broadcast = new Broadcast({ origin: connection.origin, name, enabled: true }); +const audioSrc = new Audio.Source({ broadcast, supported: Audio.Decoder.supported }); +const videoSrc = new Video.Source({ broadcast, supported: Video.Decoder.supported }); +const sync = new Sync({}); +const audio = new Audio.Decoder({ source: audioSrc, sync, enabled: true }); +const video = new Video.Decoder({ source: videoSrc, sync, enabled: true }); +const scene = new Scene(document.body); +const effect = new Signals.Effect(); + +// The console and test/vr read the catalog and the scene through this. +Object.assign(window, { vr: { audio: audioSrc, video: videoSrc, scene } }); + +// clone the frame once decoder gets it out +effect.run((e) => { + const frame = e.get(video.out.frame); + if (frame) scene.frame(frame.clone()); +}); + +effect.run((e) => { + const error = e.get(video.out.error) ?? e.get(audio.out.error); + const muted = e.get(scene.muted) ? " · muted" : ""; + const config = e.get(videoSrc.out.config); + const display = e.get(video.out.display); + const delay = e.get(sync.out.delay); + const fps = e.get(scene.fps); + // Stats change every frame, so sample them on the once-a-second fps tick instead. + const stats = video.out.stats.peek(); + + const projection = config?.projection; + const shape = projection?.kind === "equirectangular" ? `equirect ${projection.fov ?? 360}°` : "flat"; + const lines = [ + error + ? `error: ${error.message}` + : `${e.get(connection.status)} · ${name} ${e.get(broadcast.out.status)}${muted}`, + config + ? `${display?.width ?? "?"}×${display?.height ?? "?"} ${config.codec} · ${shape} · ${config.stereo ?? "mono"}` + : "no video", + `render ${fps.render} fps · video ${fps.video} fps`, + `buffer ${Math.round(delay.asMillis())} ms · late ${stats?.late ?? 0} · stalls ${stats?.stalls ?? 0}`, + ]; + scene.status(lines); +}); + +effect.run((e) => { + const config = e.get(videoSrc.out.config); + if (config) scene.layout(config.projection, config.stereo); +}); +effect.run((e) => { + const root = e.get(audio.out.root); + const config = e.get(audioSrc.out.config); + if (!root || !config) return; + const projection = e.get(videoSrc.out.config)?.projection?.kind; + const placement: Placement = + config.layout === "ambisonic" + ? { kind: "ambisonic" } + : projection === "equirectangular" + ? { kind: "head" } + : { kind: "point", position: scene.speaker }; + const spatial = new Spatial(root, placement); + scene.listener = spatial; + e.run((inner) => { + spatial.muted = inner.get(scene.muted); + }); + + e.cleanup(() => { + spatial.close(); + scene.listener = undefined; + }); +}); diff --git a/demo/vr/src/scene.ts b/demo/vr/src/scene.ts new file mode 100644 index 00000000000..5ef567e2c82 --- /dev/null +++ b/demo/vr/src/scene.ts @@ -0,0 +1,295 @@ +import { type Hang, Signals } from "@moq/watch"; +import * as THREE from "three"; +import { VRButton } from "three/addons/webxr/VRButton.js"; +import type { Spatial } from "./audio"; + +type Projection = Hang.Catalog.Projection; + +const EYE = 1.6; +const SCREEN_HEIGHT = 1.8; +const SPHERE_RADIUS = 10; +const HUD_WIDTH = 0.9; +const HUD_LINES = 4; +const HUD_LINE = 56; +// Where the panel sits relative to the head: below the line of sight, at arm's length. +const HUD_OFFSET = new THREE.Vector3(0, -0.55, -1.2); +// How far the head turns from the panel before it swings back in front. +const HUD_FOLLOW = THREE.MathUtils.degToRad(35); +const UP = new THREE.Vector3(0, 1, 0); + +// Each eye's [u, v, width, height] in the frame. +const PACKING: Record = { + mono: [ + [0, 0, 1, 1], + [0, 0, 1, 1], + ], + "left-right": [ + [0, 0, 0.5, 1], + [0.5, 0, 0.5, 1], + ], + "top-bottom": [ + [0, 0.5, 1, 0.5], + [0, 0, 1, 0.5], + ], +}; + +// Paints decoded frames onto a screen or sphere according to projection +export class Scene { + // Screen centre in content space, shared as the point-audio source. + readonly speaker = new THREE.Vector3(0, EYE, -2); + + // Toggled by a controller squeeze or the M key. + readonly muted = new Signals.Signal(false); + + // Rendered and decoded frames per second, updated once a second. + readonly fps = new Signals.Signal({ render: 0, video: 0 }); + + // Posed in content space each frame, so audio turns with the picture on a recentre. + listener?: Spatial; + + #renderer: THREE.WebGLRenderer; + #scene = new THREE.Scene(); + #content = new THREE.Group(); + #camera: THREE.PerspectiveCamera; + #texture = new THREE.VideoFrameTexture(); + #frame?: VideoFrame; + #eyes: THREE.Mesh[]; + #hud: THREE.Mesh; + #hudCanvas = document.createElement("canvas"); + // The panel's current and target yaw; unset snaps it in front on the next frame. + #hudYaw?: number; + #hudTarget = 0; + #headPosition = new THREE.Vector3(); + #headDirection = new THREE.Vector3(); + #relative = new THREE.Matrix4(); + #renders = 0; + #frames = 0; + #since = performance.now(); + + #projection?: Projection; + #stereo = "mono"; + #aspect = 16 / 9; + + constructor(parent: HTMLElement) { + this.#renderer = new THREE.WebGLRenderer({ antialias: true }); + this.#renderer.setPixelRatio(window.devicePixelRatio); + this.#renderer.setSize(window.innerWidth, window.innerHeight); + this.#renderer.xr.enabled = true; + this.#renderer.xr.setReferenceSpaceType("local-floor"); + parent.append(this.#renderer.domElement, VRButton.createButton(this.#renderer)); + + this.#camera = new THREE.PerspectiveCamera(70, window.innerWidth / window.innerHeight, 0.01, 100); + this.#camera.position.set(0, EYE, 0); + this.#camera.layers.enable(1); + + this.#scene.background = new THREE.Color(0x111111); + this.#scene.add(this.#content); + this.#texture.colorSpace = THREE.SRGBColorSpace; + + this.#eyes = [1, 2].map((layer) => { + const map = this.#texture.clone(); + const mesh = new THREE.Mesh( + new THREE.BufferGeometry(), + new THREE.MeshBasicMaterial({ map, toneMapped: false }), + ); + mesh.layers.set(layer); + mesh.visible = false; + this.#content.add(mesh); + return mesh; + }); + this.#build(); + + this.#hudCanvas.width = 1024; + this.#hudCanvas.height = HUD_LINES * HUD_LINE + 32; + const hudMap = new THREE.CanvasTexture(this.#hudCanvas); + hudMap.colorSpace = THREE.SRGBColorSpace; + this.#hud = new THREE.Mesh( + new THREE.PlaneGeometry(HUD_WIDTH, (HUD_WIDTH * this.#hudCanvas.height) / this.#hudCanvas.width), + new THREE.MeshBasicMaterial({ map: hudMap, transparent: true, toneMapped: false }), + ); + // Outside the content, since it follows the head rather than the picture. + this.#scene.add(this.#hud); + this.#renderer.xr.addEventListener("sessionstart", () => { + this.#hudYaw = undefined; + }); + + // Trigger or pinch recentres; grip toggles mute. + for (const index of [0, 1]) { + const controller = this.#renderer.xr.getController(index); + controller.addEventListener("select", this.#recentre); + controller.addEventListener("squeeze", this.#toggleMute); + } + + window.addEventListener("resize", this.#resize); + window.addEventListener("keydown", this.#key); + this.#look(this.#renderer.domElement); + + this.#renderer.setAnimationLoop(() => { + this.#follow(); + this.#renderer.render(this.#scene, this.#camera); + this.#relative.copy(this.#content.matrixWorld).invert().multiply(this.#head().matrixWorld); + this.listener?.pose(this.#relative); + this.#renders++; + this.#count(); + }); + } + + frame(frame: VideoFrame) { + this.#frames++; + this.#texture.setFrame(frame); + for (const mesh of this.#eyes) { + if (mesh.material.map) mesh.material.map.needsUpdate = true; + mesh.visible = true; + } + this.#frame?.close(); + this.#frame = frame; + + const aspect = frame.displayWidth / frame.displayHeight; + if (aspect !== this.#aspect) { + this.#aspect = aspect; + this.#build(); + } + } + + // Layout for the picture according to projection and stereo + layout(projection?: Projection, stereo?: string) { + this.#projection = projection; + this.#stereo = stereo && stereo in PACKING ? stereo : "mono"; + this.#build(); + } + + // Up to HUD_LINES lines on the in-headset panel. + status(lines: string[]) { + const context = this.#hudCanvas.getContext("2d"); + if (!context) return; + const { width, height } = this.#hudCanvas; + context.clearRect(0, 0, width, height); + context.fillStyle = "rgba(0, 0, 0, 0.6)"; + context.beginPath(); + context.roundRect(0, 0, width, height, 24); + context.fill(); + context.fillStyle = "#fff"; + context.font = "40px monospace"; + context.textBaseline = "middle"; + for (const [index, line] of lines.slice(0, HUD_LINES).entries()) { + context.fillText(line, 32, 16 + HUD_LINE * (index + 0.5), width - 64); + } + if (this.#hud.material.map) this.#hud.material.map.needsUpdate = true; + } + + // Hold the panel still through a glance, then glide it back in front once the head turns away. + #follow() { + const head = this.#head(); + head.getWorldPosition(this.#headPosition); + head.getWorldDirection(this.#headDirection); + const yaw = Math.atan2(-this.#headDirection.x, -this.#headDirection.z); + const wrap = (angle: number) => Math.atan2(Math.sin(angle), Math.cos(angle)); + + if (this.#hudYaw === undefined) this.#hudYaw = this.#hudTarget = yaw; + if (Math.abs(wrap(yaw - this.#hudTarget)) > HUD_FOLLOW) this.#hudTarget = yaw; + this.#hudYaw += wrap(this.#hudTarget - this.#hudYaw) * 0.1; + + this.#hud.position.copy(HUD_OFFSET).applyAxisAngle(UP, this.#hudYaw).add(this.#headPosition); + // Turned to face the head, then tilted up toward the eyes. + this.#hud.rotation.set(-0.4, this.#hudYaw, 0, "YXZ"); + } + + #count() { + const now = performance.now(); + const elapsed = now - this.#since; + if (elapsed < 1000) return; + const rate = (count: number) => Math.round((count * 1000) / elapsed); + this.fps.set({ render: rate(this.#renders), video: rate(this.#frames) }); + this.#renders = 0; + this.#frames = 0; + this.#since = now; + } + + #head(): THREE.Camera { + return this.#renderer.xr.isPresenting ? this.#renderer.xr.getCamera() : this.#camera; + } + + // Turn the content to face wherever the viewer is looking. + #recentre = () => { + const head = this.#head(); + head.getWorldPosition(this.#headPosition); + head.getWorldDirection(this.#headDirection); + const yaw = Math.atan2(-this.#headDirection.x, -this.#headDirection.z); + this.#content.rotation.set(0, yaw, 0); + this.#content.position.set(this.#headPosition.x, 0, this.#headPosition.z); + this.#content.updateMatrixWorld(); + }; + + #toggleMute = () => { + this.muted.set(!this.muted.peek()); + }; + + #key = (event: KeyboardEvent) => { + if (event.key === "r") this.#recentre(); + if (event.key === "m") this.#toggleMute(); + }; + + #build() { + const packing = PACKING[this.#stereo]; + const projection = this.#projection?.kind === "equirectangular" ? this.#projection : undefined; + + for (const [eye, mesh] of this.#eyes.entries()) { + const [u, v, width, height] = packing[eye]; + mesh.material.map?.offset.set(u, v); + mesh.material.map?.repeat.set(width, height); + + mesh.geometry.dispose(); + if (projection) { + const fov = THREE.MathUtils.degToRad(projection.fov ?? 360); + // Centred on the seam opposite phi = 0, then mirrored and turned so the picture's centre + // faces -Z and its left half wraps to the viewer's left. + mesh.geometry = new THREE.SphereGeometry(SPHERE_RADIUS, 64, 32, Math.PI - fov / 2, fov).scale(-1, 1, 1); + mesh.position.set(0, EYE, 0); + mesh.rotation.y = -Math.PI / 2; + } else { + // The packed frame's aspect, unpacked to one eye's. + const aspect = (this.#aspect * width) / height; + mesh.geometry = new THREE.PlaneGeometry(SCREEN_HEIGHT * aspect, SCREEN_HEIGHT); + mesh.position.copy(this.speaker); + mesh.rotation.y = 0; + } + } + } + + #resize = () => { + this.#camera.aspect = window.innerWidth / window.innerHeight; + this.#camera.updateProjectionMatrix(); + this.#renderer.setSize(window.innerWidth, window.innerHeight); + }; + + // Looking around is dragging on desktop; a headset tracks it. + #look(element: HTMLElement) { + const euler = new THREE.Euler(0, 0, 0, "YXZ"); + element.addEventListener("pointermove", (event) => { + if (event.buttons !== 1) return; + euler.setFromQuaternion(this.#camera.quaternion); + euler.y += event.movementX * 0.005; + euler.x = THREE.MathUtils.clamp(euler.x + event.movementY * 0.005, -Math.PI / 2, Math.PI / 2); + this.#camera.quaternion.setFromEuler(euler); + }); + } + + close() { + this.#renderer.setAnimationLoop(null); + window.removeEventListener("resize", this.#resize); + window.removeEventListener("keydown", this.#key); + for (const index of [0, 1]) { + const controller = this.#renderer.xr.getController(index); + controller.removeEventListener("select", this.#recentre); + controller.removeEventListener("squeeze", this.#toggleMute); + } + for (const mesh of [...this.#eyes, this.#hud]) { + mesh.geometry.dispose(); + mesh.material.map?.dispose(); + mesh.material.dispose(); + } + this.#texture.dispose(); + this.#renderer.dispose(); + this.#frame?.close(); + } +} diff --git a/demo/vr/src/vite-env.d.ts b/demo/vr/src/vite-env.d.ts new file mode 100644 index 00000000000..4ef6324115a --- /dev/null +++ b/demo/vr/src/vite-env.d.ts @@ -0,0 +1,9 @@ +/// + +interface ImportMetaEnv { + readonly VITE_RELAY_URL?: string; +} + +interface ImportMeta { + readonly env: ImportMetaEnv; +} diff --git a/demo/vr/tsconfig.json b/demo/vr/tsconfig.json new file mode 100644 index 00000000000..616947250ab --- /dev/null +++ b/demo/vr/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../js/tsconfig.json", + "compilerOptions": { "module": "esnext", "moduleResolution": "bundler", "outDir": "dist" }, + "include": ["src", "../../js/common/worklet.d.ts"] +} diff --git a/demo/vr/vite.config.ts b/demo/vr/vite.config.ts new file mode 100644 index 00000000000..1560c3f72bf --- /dev/null +++ b/demo/vr/vite.config.ts @@ -0,0 +1,11 @@ +import { resolve } from "path"; +import { defineConfig } from "vite"; +import { worklet } from "../../js/common/vite-plugin-worklet.ts"; + +export default defineConfig({ + root: "src", + envDir: resolve(import.meta.dirname), + plugins: [worklet()], + build: { target: "esnext" }, + server: { hmr: false }, +}); diff --git a/doc/bin/demo.md b/doc/bin/demo.md index 2d779d90fc9..e893d8c451e 100644 --- a/doc/bin/demo.md +++ b/doc/bin/demo.md @@ -1,6 +1,6 @@ --- title: Demos -description: The web demo, MoQ Boy, and CARLA +description: The web demo, MoQ Boy, CARLA, and VR --- # Demos @@ -69,3 +69,28 @@ What it demonstrates: - **Measured feedback.** Each truck's `status` track carries what the gate accepted and what the vehicle did, so the player draws requested against measured steering. Code: [`demo/carla`](https://github.com/moq-dev/moq/tree/main/demo/carla). + +## VR + +A WebXR player for 360 and VR180 video with spatial audio, in a headset or a +desktop tab. It lays a broadcast out by the catalog's +[spatial fields](/concept/hang#catalog): an equirectangular `projection` wraps +the picture on a sphere spanning its field of view, `stereo` crops each eye +from a packed frame, and an `ambisonic` audio layout decodes the sound field +around the viewer's head. Without them it shows a flat screen with the sound +coming from it. + +```bash +just relay # a local relay +ffmpeg -re -i vr180.mp4 -c copy -strict unofficial \ + -movflags cmaf+frag_keyframe+empty_moov+default_base_moof -f mp4 - | + moq --connect http://localhost:4443 --broadcast vr.hang import fmp4 +just vr serve # the player: Enter VR in a headset, or drag to look around +``` + +The importer reads the projection and stereo packing from an MP4's Spherical +Video V2 boxes, which ffmpeg writes only past `-strict unofficial`, and the +ambisonic layout from Opus channel mapping family 2. In the player, a trigger +or `R` recentres the picture where you look, and a grip or `M` mutes. + +Code: [`demo/vr`](https://github.com/moq-dev/moq/tree/main/demo/vr). diff --git a/doc/concept/hang.md b/doc/concept/hang.md index d4c0269ded6..fe1b0afc9ba 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -69,7 +69,7 @@ A few things the catalog can express beyond decoder config: - **Delay.** A rendition can also say how far its frames' timestamps trail those another rendition already wrote, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. A rendition alone measures none. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. - **Reorder depth.** A video rendition may declare `reorder`, the most frames that can precede one in decode order and follow it in presentation, as H.264 `max_num_reorder_frames` counts them; `0` declares none, which is not the same as absent. The Rust H.264 and H.265 importers copy it from the SPS when the SPS declares one (progressive H.264 only), and `@moq/publish` declares `0`. The MPEG-TS, Matroska, and FLV exporters then derive each group's decode times from that group's frames alone, so exports that joined at different groups agree, and they end the export on a frame reordered deeper than declared or a depth the codec level cannot reach. Without it they infer decode times from the timestamps they have read, as before. A publisher never changes it once declared. - **Colour.** A video rendition may carry `color`: the H.273 primaries, transfer, matrix, and range, plus the HDR10 mastering display and content light levels, so a renderer can set up HDR without parsing the bitstream. The Rust importers fill it from the SPS VUI, AV1 sequence header, and VP9 header, then fill gaps from a container's `colr` box or Matroska `Colour` element; the stream wins a disagreement. The HDR10 values come from H.264 and H.265 SEI or AV1 metadata OBUs in MPEG-TS and raw elementary streams, an AV1 track's av1C, or Matroska, but not yet from H.264 or H.265 in fMP4 or FLV. Nothing in `@moq/watch` reads it yet. -- **Spatial media.** A video rendition may carry `projection` (flat `rectilinear`, or `equirectangular` with the degrees of longitude it spans, 180 for VR180) and `stereo` (`mono`, `left-right`, or `top-bottom` frame packing), and an audio rendition may carry `layout: "ambisonic"` for AmbiX channels. Readers treat a value they don't recognize as absent. Nothing in `@moq/watch` renders them yet. +- **Spatial media.** A video rendition may carry `projection` (flat `rectilinear`, or `equirectangular` with the degrees of longitude it spans, 180 for VR180) and `stereo` (`mono`, `left-right`, or `top-bottom` frame packing), and an audio rendition may carry `layout: "ambisonic"` for AmbiX channels. Readers treat a value they don't recognize as absent. The Rust importers fill them from MP4 `sv3d` and `st3d`, Matroska, and Opus channel mapping family 2. `@moq/watch` plays the picture and channels as they are; the [VR demo](/bin/demo#vr) lays them out in a headset. - **Disabled renditions.** An audio or video rendition with `enabled: false` has no new frames coming, and a player must not select it. A subscriber may still get audio retained from before the disable, as when it subscribed a moment earlier. Disabling a rendition in `@moq/publish`, as hiding video does, keeps it in the catalog this way rather than removing it, so it is a one-field update; a disabled video rendition encodes one black keyframe first, so a player that predates the field shows black instead of a frozen picture. The field is written only when false. The older video `stalled` field is no longer written, and readers ignore it. A muted `@moq/publish` microphone is not disabled: it stays enabled and sends nothing, ending each utterance with a discontinuity marker, so listeners stay subscribed and play the gap as intentional silence. - **Preference.** A video rendition may carry a signed `preference`, default `0`. A player keeps only the highest preference it can decode, then picks by size and bitrate within it, as do the single-rendition egresses (RTMP play, WHEP, FLV, the transcoder's source). A publisher that adds its own H.264 copy of an H.265 source marks the copy `-1`, so only a viewer that can't decode H.265 pulls it. Picking a rendition by name still overrides it. The field is written only when nonzero. - **Join interval.** The root `media.join.interval` promises that every audio and video track starts a group at least that often: the longest GOP any of them publishes. It is fixed for the broadcast, and the [HLS gateway](/bin/hls) numbers every rendition's segments on the grid it fixes, segments the smallest whole multiple of it at least a second long, from each group's first timestamp, so renditions share segment numbers. A publisher that knows its GOP declares it; Rust publishers (`moq-mux` and everything built on it) without one measure each track over its first second or group, whichever is longer, and `@moq/publish` declares its encoders' `join`. A broadcast without it gets no HLS playlists. diff --git a/doc/setup/examples.md b/doc/setup/examples.md index d8de784ac25..698c73ce6e7 100644 --- a/doc/setup/examples.md +++ b/doc/setup/examples.md @@ -18,6 +18,7 @@ migration guides. | Build a meeting with a roster and chat | [Rooms](/lib/js/room) | [`meet.ts`](https://github.com/moq-dev/moq/blob/main/demo/web/src/meet.ts) | | Send input back to a live application | [MoQ Boy](/bin/demo#moq-boy) | [`demo/boy`](https://github.com/moq-dev/moq/tree/main/demo/boy) | | Stream a simulator and send controls | [CARLA](/bin/demo#carla) | [`demo/carla`](https://github.com/moq-dev/moq/tree/main/demo/carla) | +| Play 360 or VR180 video with spatial audio in a headset | [VR](/bin/demo#vr) | [`demo/vr`](https://github.com/moq-dev/moq/tree/main/demo/vr) | | Bring an existing media tool | [OBS](/bin/obs), [GStreamer](/bin/gstreamer), or [gateways](/bin/#gateways) | [`demo/pub`](https://github.com/moq-dev/moq/tree/main/demo/pub), [`demo/sub`](https://github.com/moq-dev/moq/tree/main/demo/sub) | Run checkout examples inside the [Nix dev shell](/setup/dev). Each demo's diff --git a/js/watch/src/audio/config.test.ts b/js/watch/src/audio/config.test.ts index fa9a5de101e..d7c3cb24b51 100644 --- a/js/watch/src/audio/config.test.ts +++ b/js/watch/src/audio/config.test.ts @@ -2,7 +2,7 @@ import { expect, test } from "bun:test"; import { Duration, yieldNow } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; import { Effect, Signal } from "@moq/signals"; -import { decoderConfig, frameDuration, packetDuration, playbackIdentity } from "./config"; +import { decoderConfig, decoderDescription, frameDuration, packetDuration, playbackIdentity } from "./config"; const flush = () => yieldNow(); @@ -90,3 +90,42 @@ test("an implicit CMAF duration falls through to the Opus TOC", () => { expect(packetDuration("opus", frame)).toEqual(Duration.millis(20)); expect(packetDuration("mp4a.40.2", frame)).toBeUndefined(); }); + +test("an Opus description reaches WebCodecs with its mapping and no pre-skip", () => { + // OpusHead, version 1, 4 channels, pre-skip 312, 48 kHz, gain 0, family 2 with 4 streams. + const head = new Uint8Array([ + ...new TextEncoder().encode("OpusHead"), + 1, + 4, + 0x38, + 0x01, + 0x80, + 0xbb, + 0, + 0, + 0, + 0, + 2, + 4, + 0, + 0, + 1, + 2, + 3, + ]); + const description = decoderDescription("opus", head); + expect(description?.slice(10, 12)).toEqual(new Uint8Array([0, 0])); + expect([...(description ?? [])].filter((_, index) => index < 10 || index > 11)).toEqual( + [...head].filter((_, index) => index < 10 || index > 11), + ); + // The catalog's bytes are left alone. + expect(head[10]).toBe(0x38); + + // WebCodecs reads only a signed OpusHead, so a dOps stays dropped, as a truncated header fails. + const dops = new Uint8Array([0, 2, 0x01, 0x38, 0, 0, 0xbb, 0x80, 0, 0, 0]); + expect(decoderDescription("opus", dops)).toBeUndefined(); + expect(() => decoderDescription("opus", head.slice(0, 12))).toThrow(); + expect(decoderDescription("opus", undefined)).toBeUndefined(); + const asc = new Uint8Array([0x12, 0x10]); + expect(decoderDescription("mp4a.40.2", asc)).toBe(asc); +}); diff --git a/js/watch/src/audio/config.ts b/js/watch/src/audio/config.ts index 531dd4b5da9..9e52ee077c2 100644 --- a/js/watch/src/audio/config.ts +++ b/js/watch/src/audio/config.ts @@ -51,6 +51,27 @@ export function playbackIdentity(config: Catalog.AudioConfig): PlaybackIdentity }; } +const OPUS_HEAD = new TextEncoder().encode("OpusHead"); + +/** + * The description to configure WebCodecs with: the catalog's, or the init segment's when it has none. + * + * An Opus stream past stereo cannot decode without its OpusHead's channel mapping, but Chromium then + * trims the header's pre-skip itself, again after every flush. The decoder's Terminal owns that trim, + * gap resumes included, so WebCodecs gets a copy with the pre-skip zeroed. WebCodecs reads only a + * signed OpusHead, so a dOps or an unsigned header stays dropped, as before. + */ +export function decoderDescription(codec: string, description: Uint8Array | undefined): Uint8Array | undefined { + if (!description || codec !== "opus") return description; + if (!OPUS_HEAD.every((byte, index) => description[index] === byte)) return undefined; + // Validates the header's length. + Util.Opus.preSkip(description); + const copy = description.slice(); + // The two little-endian bytes after the version and channel count. + copy.fill(0, OPUS_HEAD.length + 2, OPUS_HEAD.length + 4); + return copy; +} + /** * A frame's duration as a constant over the catalog's sample rate, for codecs that fix it. * diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 07fcd4ef723..f38fa967a86 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -20,7 +20,14 @@ import { arrivedAt, type Counter, type Key, type Received, Tally } from "../feed import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; import { DELAY_MAX, type Sync, type Track } from "../sync"; import { AudioBuffer } from "./buffer"; -import { type DecoderConfig, frameDuration, type PlaybackIdentity, packetDuration, playbackIdentity } from "./config"; +import { + type DecoderConfig, + decoderDescription, + frameDuration, + type PlaybackIdentity, + packetDuration, + playbackIdentity, +} from "./config"; import { Handover } from "./handover"; import { bound, maxDelay, ringHold, target } from "./latency"; import { Order } from "./order"; @@ -687,13 +694,10 @@ export class Decoder { this.#ring.peek()?.dropped(); }); - // Opus in CMAF uses raw packets; dOps is not a valid OGG Identification Header. - const description = - config.codec === "opus" - ? undefined - : config.description - ? Util.Hex.toBytes(config.description) - : undefined; + const description = decoderDescription( + config.codec, + config.description ? Util.Hex.toBytes(config.description) : undefined, + ); const decoderConfig: AudioDecoderConfig = { codec: config.codec, sampleRate: config.sampleRate, @@ -759,14 +763,11 @@ export class Decoder { .min(DELAY_MAX) .asMicros(), ); - // Opus in CMAF uses raw packets (not OGG-wrapped), so description must be omitted. - // The dOps box from the init segment is not a valid OGG Identification Header. - const description = - config.codec === "opus" - ? undefined - : config.description - ? Util.Hex.toBytes(config.description) - : init.description; + // The init segment never yields Opus's dOps, which is not the OpusHead WebCodecs takes. + const description = decoderDescription( + config.codec, + config.description ? Util.Hex.toBytes(config.description) : init.description, + ); const { consumer, read } = this.#consume(effect, sub, new Cmaf.Format(init), { config, path }); diff --git a/js/watch/src/audio/supported.ts b/js/watch/src/audio/supported.ts index 81cd8cb0a80..2e59517ba10 100644 --- a/js/watch/src/audio/supported.ts +++ b/js/watch/src/audio/supported.ts @@ -3,7 +3,7 @@ import * as Util from "@moq/hang/util"; import { hosted } from "../assets"; import { base64ToBytes } from "../base64"; import { supportCacheKey } from "../support-probe"; -import { decoderConfigKey } from "./config"; +import { decoderConfigKey, decoderDescription } from "./config"; /** Whether the WebCodecs audio decoder can play this config. */ export async function supported(config: Catalog.AudioConfig): Promise { @@ -21,23 +21,21 @@ export async function supported(config: Catalog.AudioConfig): Promise { console.warn(`audio: opus advertised at ${config.sampleRate}Hz, which some browsers cannot decode`); } - // Opus in CMAF uses raw packets; dOps is not a valid OGG Identification Header. + // For Opus this is the OpusHead; the init segment never yields Opus's dOps in its place. let description: Uint8Array | undefined; - if (config.codec !== "opus") { - if (config.description) { - description = Util.Hex.toBytes(config.description); - } else if (config.container.kind === "cmaf") { - try { - const Cmaf = await import("@moq/hang/cmaf"); - description = Cmaf.decodeInitSegment(base64ToBytes(config.container.init)).description; - } catch (err) { - // A malformed init segment means we can't extract the codec - // description, so we can't probe support reliably. Reject the - // track rather than letting isConfigSupported pass on a - // description-less config and then having decode() fail later. - console.warn(`audio: malformed CMAF init segment for codec ${config.codec}`, err); - return false; - } + if (config.description) { + description = Util.Hex.toBytes(config.description); + } else if (config.container.kind === "cmaf") { + try { + const Cmaf = await import("@moq/hang/cmaf"); + description = Cmaf.decodeInitSegment(base64ToBytes(config.container.init)).description; + } catch (err) { + // A malformed init segment means we can't extract the codec + // description, so we can't probe support reliably. Reject the + // track rather than letting isConfigSupported pass on a + // description-less config and then having decode() fail later. + console.warn(`audio: malformed CMAF init segment for codec ${config.codec}`, err); + return false; } } // Install the WebCodecs polyfill before probing, since playback would decode through it where @@ -45,7 +43,7 @@ export async function supported(config: Catalog.AudioConfig): Promise { await Util.Libav.polyfill(hosted.url); const res = await AudioDecoder.isConfigSupported({ ...config, - description, + description: decoderDescription(config.codec, description), }); return res.supported ?? false; } diff --git a/justfile b/justfile index 59e7d4528ec..d8905f89bda 100644 --- a/justfile +++ b/justfile @@ -34,6 +34,7 @@ mod carla 'demo/carla' mod pub 'demo/pub' mod relay 'demo/relay' mod sub 'demo/sub' +mod vr 'demo/vr' mod web 'demo/web' # Run the demo by default. diff --git a/package.json b/package.json index 60a83df9346..a09cfccd755 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "demo/boy", "demo/web", "demo/carla/web", + "demo/vr", "test/audio-quality/clients/js", "test/interop/clients/js", "test/interop/clients/js-native", diff --git a/quest/m1/vr-demo-try.md b/quest/m1/vr-demo-try.md new file mode 100644 index 00000000000..5c65cef0d0e --- /dev/null +++ b/quest/m1/vr-demo-try.md @@ -0,0 +1,24 @@ +# [XS] Try the VR demo in a headset, then host it at moq.dev + +## Goal + +The maintainer has played `demo/vr` in a WebXR headset, and once it plays +well there, it is hosted beside the web demo at moq.dev with a looping VR180 +broadcast to watch. + +## Plan + +Decided 2026-11-14 (maintainer, pick round for +[moq-dev/moq#8781](https://github.com/moq-dev/moq/pull/8781)): ✅ host it at +moq.dev with a looping VR180 broadcast once it plays well in a headset +(rejected: keep it local-only as a reference consumer). + +The harness (`just test vr`) proves the layout in headless Chromium on a +desktop view only; a headset's per-eye render, recentre, and HRTF audio have +had no human check. Run it with `just relay`, the `moq import fmp4` command in +`doc/bin/demo.md#vr`, and `just vr serve`, then open the page on the headset +(it needs a secure context, so serve it over HTTPS or tunnel to localhost). + +## Waiting + +- Maintainer: try it in a headset (an agent has none) and say whether it plays well enough to host. Check by this quest's Plan recording the result. diff --git a/quest/m1/vr-demo.md b/quest/m1/vr-demo.md deleted file mode 100644 index 8b69a945cce..00000000000 --- a/quest/m1/vr-demo.md +++ /dev/null @@ -1,54 +0,0 @@ -# [M] A VR demo reads the catalog's spatial fields, tested end to end - -## Goal - -`demo/vr` plays a broadcast in a WebXR headset (or a desktop tab) and picks -its layout from the hang catalog: video `projection` and its `fov` choose a -flat screen or an equirectangular sphere, `stereo` crops each eye from a -left-right or top-bottom frame, and an audio `layout` of `ambisonic` decodes -the sound field around the head. A harness imports a small VR180 left-right -fixture with first-order ambisonic Opus, publishes it through a relay, and -asserts the demo sees and applies the right projection, fov, stereo, and -layout. - -This gives the fields that -[moq-dev/moq#5543](https://github.com/moq-dev/moq/pull/5543) added and -[moq-dev/moq#7308](https://github.com/moq-dev/moq/pull/7308) fills on import -their first reader, so they ship with the cut instead of being deleted. - -## Plan - -Decided 2026-11-14 (maintainer): keep the fields and finish -[moq-dev/moq#6373](https://github.com/moq-dev/moq/pull/6373) so they are -exercised (rejected: deleting them before the cut, as the feature review's -`catalog-vr-fields` quest in -[moq-dev/moq#8764](https://github.com/moq-dev/moq/pull/8764) recommended). -✅ m1, since a demo and a harness break nothing (rejected: m2, which holds -breaking work). - -- Build on @nuts-rice's commits from `nuts-rice/moq:webxr_vr_demo`, keeping - their authorship; the PR credits them and links #6373. Maintainer edits - are off on their fork, so the work lands from a moq-dev branch. -- Drop #6373's catalog plumbing that #7308 made redundant: - `moq_mux::catalog::Spatial`, its producer and track hunks, the `import - --projection/--stereo/--audio-layout` flags, and the `pub` on the hang - `check` helpers. The importers fill the fields from MP4 `st3d`/`sv3d`, - Matroska, and Opus family 2. Adding CLI or catalog API needs the - maintainer first. -- `@moq/watch` drops every Opus `description`, so Chromium refuses Opus past - stereo ("description is required for multi-channel Opus"). Passing the - catalog's OpusHead to both the support probe and the decoder, as the Rust - importers intend, is part of this quest, since the ambisonic decode needs - it. Chromium then trims the header's pre-skip itself, again after every - flush (measured: 648 of 960 frames), so WebCodecs gets a copy with the - pre-skip zeroed and the decoder's `Terminal` stays its one owner, gap - resumes included. -- The harness publishes the fixture through `moq import fmp4`, so it proves - the importer's side too, and runs nightly in `interop.yml` beside the - portrait harness, outside the merge gate. - -Public API: none (a behavior fix in `@moq/watch`). Wire: none. - -## Related - -- [fMP4 import fills the ambisonic layout](/quest/m5/mux-vr-ambisonic-fmp4.md) - the harness's fixture exercises that importer path diff --git a/test/README.md b/test/README.md index 9f54a250896..ab9dc98c219 100644 --- a/test/README.md +++ b/test/README.md @@ -20,6 +20,7 @@ questions to a harness. For builds and CI selection, see the | [drain](drain/README.md) | `just test drain` | a JS viewer migrates off a draining relay without a dropped group | | [stream-credit](stream-credit/README.md) | `just test stream-credit` | a browser tab reads 300 live tracks past its ~100-stream WebTransport cap, the highest priority first, and nothing hangs | | qmux | `just test qmux` | `@moq/qmux`, as client and as server, interoperates with the Rust qmux 0.6.3 release on every QMux draft | +| vr | `just test vr` | a VR180 left-right MP4 with ambisonic Opus keeps its catalog projection, stereo, and layout through import and a relay, and `demo/vr` lays out each eye and turns the sound field with the head | | video | `just test video` | the `@moq/video` renderer paints through WebGPU and falls back to Canvas2D after losing the GPU | | [json-properties](json-properties.md) | `just test json-properties` | both Snapshot codecs reconstruct, and both Window codecs emit the exact events for, random JSON sequences with replayable seeds | diff --git a/test/interop/clients/js/vr.ts b/test/interop/clients/js/vr.ts new file mode 100644 index 00000000000..6ad2166fa75 --- /dev/null +++ b/test/interop/clients/js/vr.ts @@ -0,0 +1,189 @@ +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import type { Page } from "playwright"; +import { check, finishTraces, launch, open, throwPageErrors } from "./harness"; + +const { values } = parseArgs({ options: { url: { type: "string" } } }); +if (!values.url) throw new Error("vr.ts requires --url"); +const run = process.env.HARNESS_RUN_DIR; +if (!run) throw new Error("vr.ts runs under test/vr/run.sh, which builds the page into the run directory"); + +const root = join(run, "vr-dist"); +const server = Bun.serve({ + port: 0, + async fetch(req) { + let path = new URL(req.url).pathname; + if (path === "/") path = "/index.html"; + const file = Bun.file(join(root, path)); + if (!(await file.exists())) return new Response("not found", { status: 404 }); + return new Response(file); + }, +}); +const origin = `http://localhost:${server.port}`; + +const VIEWPORT = { width: 640, height: 360 }; +// Radians the demo turns the camera per dragged pixel. +const DRAG = 0.005; +type Colour = "red" | "lime" | "blue" | "magenta" | "dark" | "other"; + +// What demo/vr puts on `window.vr`, as far as this reads it. +type Demo = { + video: { out: { config: { peek(): unknown } } }; + audio: { out: { config: { peek(): unknown } } }; + scene: { listener?: { output: AudioNode } }; +}; + +// The colour at the centre of the page, read from a screenshot: WebGL discards its drawing buffer after +// each frame, so the canvas itself reads back blank. +async function centre(page: Page): Promise<{ colour: Colour; rgb: number[] }> { + const png = (await page.screenshot()).toString("base64"); + const rgb = await page.evaluate( + async ({ png, x, y }) => { + const image = await createImageBitmap(await (await fetch(`data:image/png;base64,${png}`)).blob()); + const canvas = new OffscreenCanvas(image.width, image.height); + const context = canvas.getContext("2d"); + if (!context) throw new Error("no 2d context"); + context.drawImage(image, 0, 0); + return Array.from(context.getImageData(x, y, 1, 1).data.slice(0, 3)); + }, + { png, x: VIEWPORT.width / 2, y: VIEWPORT.height / 2 }, + ); + const [r, g, b] = rgb; + const hi = (v: number) => v > 150; + const lo = (v: number) => v < 100; + const colour: Colour = + r < 40 && g < 40 && b < 40 + ? "dark" + : hi(r) && lo(g) && lo(b) + ? "red" + : lo(r) && hi(g) && lo(b) + ? "lime" + : lo(r) && lo(g) && hi(b) + ? "blue" + : hi(r) && lo(g) && hi(b) + ? "magenta" + : "other"; + return { colour, rgb }; +} + +async function waitForCentre(page: Page, want: Colour, timeout: number): Promise { + const deadline = Date.now() + timeout; + let last = await centre(page); + while (last.colour !== want && Date.now() < deadline) { + await page.waitForTimeout(250); + last = await centre(page); + } + check(last.colour === want, `centre is ${want}`, () => `got ${last.colour} rgb(${last.rgb.join(", ")})`); +} + +// Turn the view left (right when negative) by dragging, as a desktop viewer looks around. The drag +// starts at the far edge so it stays on the page, where every pointer move lands. +async function turnLeft(page: Page, radians: number): Promise { + const { width, height } = VIEWPORT; + const distance = radians / DRAG; + check(Math.abs(distance) < width - 2, "the turn fits one drag", () => `${distance} px`); + const start = distance > 0 ? 1 : width - 1; + await page.mouse.move(start, height / 2); + await page.mouse.down(); + await page.mouse.move(start + distance, height / 2, { steps: 20 }); + await page.mouse.up(); +} + +// Each ear's RMS level at the demo's spatial mix, averaged over half a second of live playback. +async function ears(page: Page): Promise<{ L: number; R: number }> { + return page.evaluate(async () => { + const output = (window as unknown as { vr: Demo }).vr.scene.listener?.output; + if (!output) throw new Error("the demo has no spatial audio"); + const context = output.context; + const split = new ChannelSplitterNode(context, { numberOfOutputs: 2 }); + output.connect(split); + const meters = [0, 1].map((channel) => { + const analyser = new AnalyserNode(context, { fftSize: 2048 }); + split.connect(analyser, channel); + return analyser; + }); + const sums = [0, 0]; + const data = new Float32Array(2048); + for (let i = 0; i < 10; i++) { + await new Promise((resolve) => setTimeout(resolve, 50)); + for (const [channel, meter] of meters.entries()) { + meter.getFloatTimeDomainData(data); + sums[channel] += data.reduce((sum, v) => sum + v * v, 0) / data.length; + } + } + output.disconnect(split); + const [L, R] = sums.map((sum) => Math.sqrt(sum / 10)); + return { L, R }; + }); +} + +function louder(level: { L: number; R: number }, ear: "L" | "neither", assertion: string) { + const ratio = level.L / level.R; + const ok = level.L + level.R > 0.01 && (ear === "L" ? ratio > 2 : ratio > 0.8 && ratio < 1.25); + check(ok, assertion, () => `L ${level.L.toFixed(3)} R ${level.R.toFixed(3)}`); +} + +const browser = await launch(["--use-gl=angle", "--use-angle=swiftshader", "--enable-unsafe-swiftshader"]); +let failed = true; +try { + const query = new URLSearchParams({ url: values.url, name: "vr.hang" }); + const [page, errors] = await open(browser, `${origin}/?${query}`, "vr", true, { viewport: VIEWPORT }); + + // The catalog the demo read carries what the importer found in the MP4's sv3d, st3d, and dOps. + await page.waitForFunction( + () => { + const { vr } = window as unknown as { vr: Demo }; + return vr.video.out.config.peek() !== undefined && vr.audio.out.config.peek() !== undefined; + }, + undefined, + { timeout: 30_000 }, + ); + const catalog = await page.evaluate(() => { + const { vr } = window as unknown as { vr: Demo }; + const video = vr.video.out.config.peek() as { projection?: unknown; stereo?: unknown }; + const audio = vr.audio.out.config.peek() as { layout?: unknown; numberOfChannels?: unknown }; + return { + projection: video.projection, + stereo: video.stereo, + layout: audio.layout, + channels: audio.numberOfChannels, + }; + }); + const want = { + projection: { kind: "equirectangular", fov: 180 }, + stereo: "left-right", + layout: "ambisonic", + channels: 4, + }; + check(Bun.deepEquals(catalog, want), "the catalog carries the fixture's VR fields", () => JSON.stringify(catalog)); + + // Straight ahead is the left eye's centre: the stereo packing cropped the frame and the + // equirectangular projection faced its centre forward. + await waitForCentre(page, "red", 30_000); + + // The left eye's left quarter lies 45 degrees left over a 180 degree field. + await turnLeft(page, Math.PI / 4); + await waitForCentre(page, "lime", 5_000); + + // 120 degrees left is past the field's edge: empty scene. A full sphere would show the frame's blue. + await turnLeft(page, (Math.PI * 75) / 180); + await waitForCentre(page, "dark", 5_000); + + // Face forward again. The drags were gestures, so the audio graph is running. The fixture's sound + // is a plane wave from the left: the left ear hears it, and turning to face it centres it. + await turnLeft(page, (-Math.PI * 120) / 180); + const ahead = await ears(page); + louder(ahead, "L", "a source on the left is louder in the left ear"); + await turnLeft(page, Math.PI / 2); + const facing = await ears(page); + louder(facing, "neither", "turning left to face that source centres it"); + + throwPageErrors(errors); + const level = ({ L, R }: { L: number; R: number }) => `L ${L.toFixed(3)} R ${R.toFixed(3)}`; + console.log(`vr: ${JSON.stringify(catalog)} laid out; ahead ${level(ahead)}, facing ${level(facing)}`); + failed = false; +} finally { + await finishTraces(failed); + await browser.close(); + server.stop(true); +} diff --git a/test/justfile b/test/justfile index 070c538f528..7023141a463 100644 --- a/test/justfile +++ b/test/justfile @@ -115,6 +115,10 @@ smoke *args: portrait: ./portrait/run.sh +# Import a VR180 MP4 with ambisonic Opus and check demo/vr lays it out by the catalog's VR fields. +vr: + ./vr/run.sh + # Negative control: no publisher, every subscriber must time out (proves the # harness can actually report failure). diff --git a/test/vr/relay.toml b/test/vr/relay.toml new file mode 100644 index 00000000000..e9a52bad3ca --- /dev/null +++ b/test/vr/relay.toml @@ -0,0 +1,15 @@ +# Private loopback relay for the VR projection harness. +[log] +level = "info" + +[listen] +bind = "127.0.0.1:4443" +tls.generate = ["localhost", "127.0.0.1"] +# The CLI publisher negotiates lite-07; Chromium offers lite-06. +version = ["moq-lite-07-wip", "moq-lite-06"] + +[web.http] +listen = "127.0.0.1:4443" + +[auth] +public = "**" diff --git a/test/vr/run.sh b/test/vr/run.sh new file mode 100755 index 00000000000..62f6ce6ccbc --- /dev/null +++ b/test/vr/run.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Import a VR180 left-right MP4 with ambisonic Opus, publish it through a relay, and check demo/vr +# reads the catalog's projection, stereo, and layout and lays out the picture and sound by them. +set -euo pipefail + +if [[ "${GITHUB_ACTIONS:-}" != true && "${LOCAL_SUPERVISED:-}" != 1 ]]; then + exec bash "$(dirname "$0")/../../sh/local.sh" bash "$0" "$@" +fi +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$DIR/../.." && pwd) +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$DIR/../lib/harness.sh" +harness_begin vr 'just test vr' +cd "$WORKSPACE" +cargo build --locked -p moq-relay -p moq-cli +uv run --no-project python sh/js/install.py frozen +(cd demo/vr && bunx vite build --outDir "$HARNESS_RUN/vr-dist" --emptyOutDir) + +# Each eye is 480x480 over 180 degrees. Left eye: blue, red straight ahead (u 0.5), lime 45 degrees +# left (u 0.25). Right eye: magenta, so an uncropped frame puts magenta ahead. The sound is a plane +# wave from the left in first-order AmbiX (ACN W, Y, Z, X), so W and Y carry the same tone. +ffmpeg -hide_banner -loglevel error -f lavfi \ + -i "color=c=blue:s=960x480:r=15:d=2,drawbox=x=210:y=210:w=60:h=60:c=red:t=fill,drawbox=x=90:y=210:w=60:h=60:c=lime:t=fill,drawbox=x=480:y=0:w=480:h=480:c=magenta:t=fill" \ + -f lavfi -i "aevalsrc=exprs='0.5*sin(2*PI*1000*t)|0.5*sin(2*PI*1000*t)|0|0':s=48000:d=2" \ + -c:v libx264 -preset ultrafast -profile:v baseline -pix_fmt yuv420p -g 15 \ + -c:a libopus -mapping_family 2 -b:a 128k "$HARNESS_RUN/flat.mp4" +uv run --no-project python -I "$DIR/spherical.py" "$HARNESS_RUN/flat.mp4" "$HARNESS_RUN/vr180.mp4" + +harness_port relay +URL="http://127.0.0.1:$HARNESS_PORT" +sed "s/4443/$HARNESS_PORT/g" "$DIR/relay.toml" >"$HARNESS_RUN/relay.toml" +harness_spawn relay "$HARNESS_RUN/relay.log" "$WORKSPACE/target/debug/moq-relay" "$HARNESS_RUN/relay.toml" +harness_ready "$URL/certificate.sha256" 30 "$HARNESS_PID" +# ffmpeg carries st3d and sv3d into each fragmented init segment only past -strict unofficial. +# shellcheck disable=SC2329 # invoked by harness_spawn +publish_vr() { + ffmpeg -hide_banner -loglevel error -re -stream_loop -1 -i "$HARNESS_RUN/vr180.mp4" -c copy -strict unofficial \ + -movflags cmaf+frag_keyframe+empty_moov+default_base_moof -f mp4 - | + "$WORKSPACE/target/debug/moq" --connect "$URL" --connect-version moq-lite-07-wip --broadcast vr.hang import fmp4 +} +harness_spawn publisher "$HARNESS_RUN/publisher.log" publish_vr +cd test/interop/clients/js +# SwiftShader stands in for a GPU. ANGLE asks its Vulkan for a Wayland surface, which it lacks, +# whenever a Wayland session is visible, so a desktop run would have no WebGL. +env -u WAYLAND_DISPLAY bun vr.ts --url "$URL" diff --git a/test/vr/spherical.py b/test/vr/spherical.py new file mode 100644 index 00000000000..cba22dedb8e --- /dev/null +++ b/test/vr/spherical.py @@ -0,0 +1,76 @@ +"""Mark an MP4's video as VR180 left-right with Spherical Video V2 boxes. + +ffmpeg reads and remuxes `st3d` and `sv3d` but cannot create them, so this appends both to the +`avc1` sample entry and grows every enclosing box. The `moov` must follow `mdat`, as ffmpeg writes +it without `+faststart`, so no chunk offset moves. + +Usage: python -I spherical.py in.mp4 out.mp4 +""" + +import struct +import sys + +CONTAINERS = {b"moov", b"trak", b"mdia", b"minf", b"stbl"} + + +def box(kind: bytes, body: bytes) -> bytes: + return struct.pack(">I", 8 + len(body)) + kind + body + + +def full(kind: bytes, body: bytes) -> bytes: + return box(kind, b"\0\0\0\0" + body) + + +# Each equirectangular bound is the 0.32 fixed-point fraction of the sphere left uncovered: a +# quarter off each side leaves 180 degrees of longitude. +QUARTER = 1 << 30 +POSE = full(b"prhd", struct.pack(">iii", 0, 0, 0)) +EQUI = full(b"equi", struct.pack(">IIII", 0, 0, QUARTER, QUARTER)) +# Stereo mode 2 is left-right. +SPATIAL = full(b"st3d", bytes([2])) + box(b"sv3d", full(b"svhd", b"moq\0") + box(b"proj", POSE + EQUI)) + + +def patch(data: bytearray, start: int, end: int) -> int: + """Append SPATIAL to the avc1 entry within [start, end), returning how much the range grew.""" + at = start + while at < end: + size, kind = struct.unpack(">I4s", data[at : at + 8]) + if size < 8: + raise ValueError(f"unsupported box size {size} for {kind!r}") + grown = 0 + if kind in CONTAINERS: + grown = patch(data, at + 8, at + size) + elif kind == b"stsd": + # A full box and an entry count precede the sample entries. + grown = patch(data, at + 16, at + size) + elif kind == b"avc1": + data[at + size : at + size] = SPATIAL + grown = len(SPATIAL) + if grown: + struct.pack_into(">I", data, at, size + grown) + return grown + at += size + return 0 + + +def main() -> None: + source, target = sys.argv[1:] + with open(source, "rb") as file: + data = bytearray(file.read()) + kinds, at = [], 0 + while at < len(data): + size, kind = struct.unpack(">I4s", data[at : at + 8]) + if size < 8: + raise ValueError(f"unsupported box size {size} for {kind!r}") + kinds.append(kind) + at += size + # The mdat must precede the moov, so growing the moov moves no sample. + if kinds.index(b"moov") < kinds.index(b"mdat"): + raise ValueError("moov precedes mdat; write the source without +faststart") + if not patch(data, 0, len(data)): + raise ValueError("no avc1 sample entry") + with open(target, "wb") as file: + file.write(data) + + +main()