Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions .github/workflows/interop.yml
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,15 @@ jobs:
HARNESS_RUNS: ${{ runner.temp }}/moq-test-runs
HARNESS_RESULTS: ${{ runner.temp }}/moq-test-runs/results.jsonl

- name: VR catalog fields
id: vr
if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
run: nix develop --command just test vr
shell: bash -leo pipefail {0}
env:
HARNESS_RUNS: ${{ runner.temp }}/moq-test-runs
HARNESS_RESULTS: ${{ runner.temp }}/moq-test-runs/results.jsonl

- name: Interop negative control
id: negative
if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
Expand Down Expand Up @@ -236,6 +245,7 @@ jobs:
"Interop (moq-transport)": "${{ steps.interop-ietf.outcome }}",
"Released wire compatibility": "${{ steps.wire-compat.outcome }}",
"Portrait transcode": "${{ steps.portrait.outcome }}",
"VR catalog fields": "${{ steps.vr.outcome }}",
"Interop negative control": "${{ steps.negative.outcome }}",
"Media output and lifecycle": "${{ steps.media.outcome }}",
"Media track identity changes": "${{ steps.media-rename.outcome }}"
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/release-gate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ jobs:
- "test/interop/**"
- "test/max-age/**"
- "test/portrait/**"
- "test/vr/**"
- "demo/vr/**"
- "rs/moq-tokio/tests/integration/max_age.rs"
- "test/datagram-tracks/**"
- "rs/moq-tokio/tests/integration/datagram_tracks.rs"
Expand Down
31 changes: 31 additions & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions demo/vr/justfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
set shell := ["bash", "-euo", "pipefail", "-c"]

# Serve the VR demo against a relay; the page takes ?url= and ?name= to override.
serve url="http://localhost:4443": install
VITE_RELAY_URL="{{ url }}" bun --bun vite

# Install the workspace deps that vite and the demo's imports need.
install:
uv run --no-project python ../../sh/js/install.py update
22 changes: 22 additions & 0 deletions demo/vr/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
{
"name": "@moq/demo-vr",
"private": true,
"type": "module",
"description": "Media over QUIC - VR demo",
"license": "(MIT OR Apache-2.0)",
"scripts": {
"dev": "vite",
"build": "vite build",
"check": "tsc --noEmit"
},
"dependencies": {
"@moq/watch": "workspace:^",
"three": "^0.186.1"
},
"devDependencies": {
"@types/three": "^0.186.0",
"rolldown": "^1.2.16",
"typescript": "7.0.2",
"vite": "^8.3.5"
}
}
134 changes: 134 additions & 0 deletions demo/vr/src/audio.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
import * as THREE from "three";

export type Placement = { kind: "ambisonic" } | { kind: "point"; position: THREE.Vector3 } | { kind: "head" };

// Virtual speakers at the corners of a cube in ambisonic axes (W, Y, Z, X)
const SPEAKERS = [-1, 1].flatMap((x) =>
[-1, 1].flatMap((y) => [-1, 1].map((z) => [x, y, z].map((v) => v / Math.sqrt(3)))),
);

const MAX_RE = 1 / Math.sqrt(3);

// Scratch for pose(), which runs every XR frame.
const POSE = {
position: new THREE.Vector3(),
rotation: new THREE.Quaternion(),
scale: new THREE.Vector3(),
forward: new THREE.Vector3(),
up: new THREE.Vector3(),
};

// Renders audio graph around a viewer's head
export class Spatial {
#context: BaseAudioContext;
#nodes: AudioNode[] = [];
#tracksPosition: boolean;
// Every placement mixes here, so muting is one param.
#out: GainNode;

constructor(root: AudioNode, placement: Placement) {
this.#context = root.context;
this.#out = new GainNode(this.#context);
this.#out.connect(this.#context.destination);
this.#nodes.push(this.#out);
const destination = this.#out;

switch (placement.kind) {
case "head":
root.connect(destination);
this.#tracksPosition = false;
break;
case "point": {
const panner = this.#panner(placement.position, 1);
root.connect(panner).connect(destination);
this.#tracksPosition = true;
break;
}
case "ambisonic":
this.#ambisonic(root, destination);
// The field is infinitely far, so only the head's rotation moves it.
this.#tracksPosition = false;
break;
}
this.#nodes.push(root);
}

// Decodes out to fixed speakers around the scene. A higher order plays at first order: the
// splitter's discrete downmix keeps ACN channels 0 to 3 and drops the rest.
#ambisonic(root: AudioNode, destination: AudioNode) {
if (root.channelCount < 4) throw new Error(`ambisonics needs at least 4 channels, got ${root.channelCount}`);

const split = new ChannelSplitterNode(this.#context, { numberOfOutputs: 4 });
root.connect(split);
this.#nodes.push(split);

for (const [x, y, z] of SPEAKERS) {
// ACN order is W, Y, Z, X.
const weights = [1, 3 * MAX_RE * y, 3 * MAX_RE * z, 3 * MAX_RE * x].map((w) => w / SPEAKERS.length);
// Ambisonic axes to WebAudio
const panner = this.#panner(new THREE.Vector3(-y, z, -x), 0);
panner.connect(destination);

weights.forEach((weight, channel) => {
const gain = new GainNode(this.#context, { gain: weight });
split.connect(gain, channel).connect(panner);
this.#nodes.push(gain);
});
}
}

#panner(position: THREE.Vector3, rolloffFactor: number): PannerNode {
const panner = new PannerNode(this.#context, {
panningModel: "HRTF",
distanceModel: "inverse",
refDistance: 1,
rolloffFactor,
positionX: position.x,
positionY: position.y,
positionZ: position.z,
});
this.#nodes.push(panner);
return panner;
}

// The mix every placement feeds, for meters and the harness.
get output(): AudioNode {
return this.#out;
}

set muted(muted: boolean) {
this.#out.gain.setTargetAtTime(muted ? 0 : 1, this.#context.currentTime, 0.01);
}

// Move the listener to the viewer's head, in the same space as the placement.
pose(head: THREE.Matrix4) {
const { position, rotation, scale, forward, up } = POSE;
head.decompose(position, rotation, scale);
forward.set(0, 0, -1).applyQuaternion(rotation);
up.set(0, 1, 0).applyQuaternion(rotation);

const listener = this.#context.listener;
if (!this.#tracksPosition) position.set(0, 0, 0);
// Firefox has no listener AudioParams, only the older setters.
if (!listener.positionX) {
listener.setPosition(position.x, position.y, position.z);
listener.setOrientation(forward.x, forward.y, forward.z, up.x, up.y, up.z);
return;
}
const at = this.#context.currentTime;
const set = (param: AudioParam, value: number) => param.setTargetAtTime(value, at, 0.01);
set(listener.positionX, position.x);
set(listener.positionY, position.y);
set(listener.positionZ, position.z);
set(listener.forwardX, forward.x);
set(listener.forwardY, forward.y);
set(listener.forwardZ, forward.z);
set(listener.upX, up.x);
set(listener.upY, up.y);
set(listener.upZ, up.z);
Comment thread
kixelated marked this conversation as resolved.
Resolved
}

close() {
for (const node of this.#nodes) node.disconnect();
}
}
29 changes: 29 additions & 0 deletions demo/vr/src/index.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>MoQ VR</title>
<link rel="icon" href="data:," />
<style>
body {
margin: 0;
overflow: hidden;
background: #111;
color: #ddd;
font: 14px monospace;
}
#help {
position: fixed;
left: 8px;
bottom: 8px;
opacity: 0.6;
pointer-events: none;
}
</style>
</head>
<body>
<div id="help">drag to look · R recentre · M mute</div>
<script type="module" src="./index.ts"></script>
</body>
</html>
Loading