Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 31 additions & 2 deletions .github/workflows/interop.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,30 @@ on:
# Nightly: catch an interop regression in the current tree before a user does.
schedule:
- cron: "0 8 * * *"
# release-gate.yml, on release PRs that touch the harness.
workflow_call:
# On release PRs that touch the harness itself, so a backported change to the
# interop test is validated before it merges. Main PRs skip it; the schedule
# above covers main. `edited` catches a main PR retargeted to release.
pull_request:
branches: [release]
types: [opened, synchronize, reopened, edited]
paths:
- "test/lib/**"
- "test/p2p/**"
- "js/p2p/**"
- "test/interop/**"
- "test/max-age/**"
- "test/portrait/**"
- "test/vr/**"
- "demo/vr/**"
- "rs/moq-tokio/tests/integration/max_age.rs"
# The interop job also depends on the recipe that invokes it and the
# workspace membership that links the JS clients to source.
- "test/justfile"
- "package.json"
# The Go client builds against the modules this script stages.
- "sh/go/**"
- "cpp/moq/**"
- ".github/workflows/interop.yml"

# Keyed by event too, so a manual run on main cannot cancel the scheduled one that
# publishes the night's report.
Expand Down Expand Up @@ -131,6 +153,13 @@ jobs:
MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs
MOQ_TEST_RESULTS: ${{ runner.temp }}/moq-test-runs/results.jsonl

- name: VR projection
if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
run: nix develop --command just test vr
shell: bash -leo pipefail {0}
env:
MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs

- name: Interop negative control
id: negative
if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
Expand Down
4 changes: 4 additions & 0 deletions demo/vr/justfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
set shell := ["bash", "-euo", "pipefail", "-c"]
serve url = "http://localhost:4443":
bun install
VITE_RELAY_URL="{{url}}" bun --bun vite
21 changes: 21 additions & 0 deletions demo/vr/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"name": "@moq/demo-vr",
"private": true,
"type": "module",
"description": "Media over QUIC - VR demo",
"license": "(MIT OR Apache-2.0)",
"scripts": {
"dev": "vite",
"build": "vite build",
"check": "tsc --noEmit"
},
"dependencies": {
"@moq/watch": "workspace:^",
"three": "^0.186.1"
},
"devDependencies": {
"@types/three": "^0.186.0",
"typescript": "7.0.2",
"vite": "^8.3.4"
}
}
110 changes: 110 additions & 0 deletions demo/vr/src/audio.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
import * as THREE from "three";

export type Placement =
| { kind: "ambisonic" }
| { kind: "point"; position: THREE.Vector3 }
| { kind: "head" };

// Virtual speakers at the corners of a cube in ambisonic axes (W, Y, Z, X)
const SPEAKERS = [-1, 1].flatMap((x) =>
[-1, 1].flatMap((y) => [-1, 1].map((z) => [x, y, z].map((v) => v / Math.sqrt(3)))),
);

const MAX_RE = 1 / Math.sqrt(3);

// Renders audio graph around a viewer's head
export class Spatial {
#context: BaseAudioContext;
#nodes: AudioNode[] = [];
#tracksPosition: boolean;

constructor(root: AudioNode, placement: Placement) {
this.#context = root.context;
const destination = this.#context.destination;

switch (placement.kind) {
case "head":
root.connect(destination);
this.#tracksPosition = false;
break;
case "point": {
const panner = this.#panner(placement.position, 1);
root.connect(panner).connect(destination);
this.#tracksPosition = true;
break;
}
case "ambisonic":
this.#ambisonic(root, destination);
// The field is infinitely far, so only the head's rotation moves it.
this.#tracksPosition = false;
break;
}
this.#nodes.push(root);
}

// Decodes out to fixed speakers around the scene
#ambisonic(root: AudioNode, destination: AudioNode) {
if (root.channelCount !== 4)
throw new Error(`first-order ambisonics needs 4 channels, got ${root.channelCount}`);

const split = new ChannelSplitterNode(this.#context, { numberOfOutputs: 4 });
root.connect(split);
this.#nodes.push(split);

for (const [x, y, z] of SPEAKERS) {
// ACN order is W, Y, Z, X.
const weights = [1, 3 * MAX_RE * y, 3 * MAX_RE * z, 3 * MAX_RE * x].map((w) => w / SPEAKERS.length);
// Ambisonic axes to WebAudio
const panner = this.#panner(new THREE.Vector3(-y, z, -x), 0);
panner.connect(destination);

weights.forEach((weight, channel) => {
const gain = new GainNode(this.#context, { gain: weight });
split.connect(gain, channel).connect(panner);
this.#nodes.push(gain);
});
}
}

#panner(position: THREE.Vector3, rolloffFactor: number): PannerNode {
const panner = new PannerNode(this.#context, {
panningModel: "HRTF",
distanceModel: "inverse",
refDistance: 1,
rolloffFactor,
positionX: position.x,
positionY: position.y,
positionZ: position.z,
});
this.#nodes.push(panner);
return panner;
}


// Move the audio with the transform of viewer's head
pose(head: THREE.Matrix4) {
const position = new THREE.Vector3();
const rotation = new THREE.Quaternion();
head.decompose(position, rotation, new THREE.Vector3());
const forward = new THREE.Vector3(0, 0, -1).applyQuaternion(rotation);
const up = new THREE.Vector3(0, 1, 0).applyQuaternion(rotation);

const listener = this.#context.listener;
const at = this.#context.currentTime;
const set = (param: AudioParam, value: number) => param.setTargetAtTime(value, at, 0.01);
if (!this.#tracksPosition) position.set(0, 0, 0);
set(listener.positionX, position.x);
set(listener.positionY, position.y);
set(listener.positionZ, position.z);
set(listener.forwardX, forward.x);
set(listener.forwardY, forward.y);
set(listener.forwardZ, forward.z);
set(listener.upX, up.x);
set(listener.upY, up.y);
set(listener.upZ, up.z);
}

close() {
for (const node of this.#nodes) node.disconnect();
}
}
28 changes: 28 additions & 0 deletions demo/vr/src/index.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>MoQ VR</title>
<link rel="icon" href="data:," />
<style>
body {
margin: 0;
overflow: hidden;
background: #111;
color: #ddd;
font: 14px monospace;
}
#status {
position: fixed;
top: 8px;
left: 8px;
pointer-events: none;
}
</style>
</head>
<body>
<div id="status">connecting…</div>
<script type="module" src="./index.ts"></script>
</body>
</html>
56 changes: 56 additions & 0 deletions demo/vr/src/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import { Audio, Broadcast, Net, Signals, Sync, Video } from "@moq/watch";
import { type Placement, Spatial } from "./audio";
import { Scene } from "./scene";

const params = new URLSearchParams(location.search);
const url = new URL(params.get("url") ?? import.meta.env.VITE_RELAY_URL ?? "http://localhost:4443");
const name = Net.Path.from(params.get("name") ?? "vr.hang");

const connection = new Net.Connection({ url });
const broadcast = new Broadcast({ origin: connection.origin, name, enabled: true, announced: true });
const audioSrc = new Audio.Source({ broadcast, supported: Audio.Decoder.supported });
const videoSrc = new Video.Source({ broadcast, supported: Video.Decoder.supported, probe: connection.probe });
const sync = new Sync({});
const audio = new Audio.Decoder({ source: audioSrc, sync, enabled: true });
const video = new Video.Decoder({ source: videoSrc, sync, enabled: true });
const scene = new Scene(document.body);
const effect = new Signals.Effect();

// clone the frame once decoder gets it out
effect.run((e) => {
const frame = e.get(video.out.frame);
if (frame) scene.frame(frame.clone());
});

const status = document.getElementById("status");
effect.run((e) => {
if (!status) return;
const error = e.get(video.out.error) ?? e.get(audio.out.error);
status.textContent = error
? `error: ${error.message}`
: `${e.get(connection.status)} · ${name} ${e.get(broadcast.out.status)}`;
});

effect.run((e) => {
const config = e.get(videoSrc.out.config);
if (config) scene.layout(config.projection, config.stereo);
});
effect.run((e) => {
const root = e.get(audio.out.root);
const config = e.get(audioSrc.out.config);
if (!root || !config) return;
const projection = e.get(videoSrc.out.config)?.projection?.kind;
const placement: Placement =
config.layout === "ambisonic"
? { kind: "ambisonic" }
: projection === "equirectangular"
? { kind: "head" }
: { kind: "point", position: scene.speaker };
const spatial = new Spatial(root, placement);
scene.listener = spatial;

e.cleanup(() => {
spatial.close();
scene.listener = undefined;
});
});
Loading