Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions drafts/draft-lcurley-moq-hang.md
Original file line number Diff line number Diff line change
Expand Up @@ -944,6 +944,7 @@ Audio publishers do not append per-group duration markers because the codec defi
Data tracks retain empty payloads as data, without endpoint semantics.

For example, h.264 with no `description` field would be annex.b encoded, while h.264 with a `description` field would be AVCC encoded.
For AAC (`mp4a.40.*`), each nonempty codec payload MUST be one complete ADTS frame when `description` is absent; with an AudioSpecificConfig in `description`, it MUST be one raw AAC access unit without an ADTS header.
For a text track, the remainder is the cue in the track's declared `format` (for example a `WEBVTT` segment).

## cmaf
Expand All @@ -961,6 +962,7 @@ A consumer MUST feed `init` to the decoder before the first frame.

## loc
Each frame is a Low Overhead Container frame {{!I-D.ietf-moq-loc}}: a property block, carrying the timestamp among other properties, followed by the codec payload.
The codec payload uses the same framing as {{legacy}}.

For audio and video, consumers accept empty codec payload metadata with the same video-duration and audio-terminal-trimming meanings as the legacy container.
Data tracks retain empty payloads as data.
Expand Down Expand Up @@ -1162,6 +1164,7 @@ This document has no IANA actions.
## moq-hang-04
{:numbered="false"}

- Defined AAC payloads without a `description` as ADTS, and payloads with an AudioSpecificConfig as raw AAC.
- A `broadcast` reference and an echo `path` are spelled like a path: one with an empty segment or a lone `\` refuses the catalog instead of collapsing.
- Removed `catalog.json.z` and the data track `compression` field: compression is a track property the transport carries ({{compression}}), so each compressed track is one name, and the catalog, `stats` and `echo` snapshots, HLS record tracks, and JSON data tracks SHOULD be compressed.
- Replaced the root `archive` section and its per-track timelines with an optional, per-rendition `hls` field on video and audio ({{hls}}). Each record is a segment's first `group` and `pts` in milliseconds, with an optional `duration` where zero marks a stop; frame positions, `end`, `keyframe`, `timescale`, `replay`, `replayEpoch`, `store`, and `version` are gone. Each rendition numbers its own segments, and an edge maps a rendition's record `n` to its segment `n`. The record track is named `<track>.hls` and is compressed ({{compression}}).
Expand Down
82 changes: 42 additions & 40 deletions js/hang/src/util/libav.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ const SAMPLE_RATE = 48000;
// -filter_complex amerge -c:a aac -b:a 64k -f adts libav.test.aac
const FIXTURE = new URL("./libav.test.aac", import.meta.url);

// Strip the ADTS headers: a hang catalog carries the AudioSpecificConfig and frames are raw.
function rawFrames(adts: Uint8Array): Uint8Array[] {
// A description selects raw blocks; without one, each packet keeps its ADTS header.
function aacFrames(adts: Uint8Array, raw: boolean): Uint8Array[] {
const frames: Uint8Array[] = [];
for (let i = 0; i < adts.length; ) {
const length = ((adts[i + 3] & 0x03) << 11) | (adts[i + 4] << 3) | (adts[i + 5] >> 5);
const header = adts[i + 1] & 0x01 ? 7 : 9;
frames.push(adts.subarray(i + header, i + length));
frames.push(adts.subarray(i + (raw ? header : 0), i + length));
i += length;
}
return frames;
Expand Down Expand Up @@ -51,45 +51,47 @@ test("the fallback decodes AAC without a native AudioDecoder", async () => {
expect((await AudioDecoder.isConfigSupported({ ...config, codec })).supported).toBe(true);
}

const planes: Float32Array[][] = [[], []];
let failure: Error | undefined;
const decoder = new AudioDecoder({
output: (data) => {
for (const [index, plane] of planes.entries()) {
const samples = new Float32Array(data.numberOfFrames);
data.copyTo(samples, { planeIndex: index, format: "f32-planar" });
plane.push(samples);
}
data.close();
},
error: (error) => {
failure = error;
},
});
decoder.configure(config);

const frames = rawFrames(new Uint8Array(await Bun.file(FIXTURE).arrayBuffer()));
for (const [index, data] of frames.entries()) {
decoder.decode(
new EncodedAudioChunk({ type: "key", timestamp: (index * 1024 * 1_000_000) / SAMPLE_RATE, data }),
);
}
await decoder.flush();
decoder.close();
expect(failure).toBeUndefined();
for (const raw of [true, false]) {
const planes: Float32Array[][] = [[], []];
let failure: Error | undefined;
const decoder = new AudioDecoder({
output: (data) => {
for (const [index, plane] of planes.entries()) {
const samples = new Float32Array(data.numberOfFrames);
data.copyTo(samples, { planeIndex: index, format: "f32-planar" });
plane.push(samples);
}
data.close();
},
error: (error) => {
failure = error;
},
});
decoder.configure({ ...config, description: raw ? config.description : undefined });

const [left, right] = planes.map((chunks) => {
const out = new Float32Array(chunks.reduce((sum, chunk) => sum + chunk.length, 0));
let offset = 0;
for (const chunk of chunks) {
out.set(chunk, offset);
offset += chunk.length;
const frames = aacFrames(new Uint8Array(await Bun.file(FIXTURE).arrayBuffer()), raw);
for (const [index, data] of frames.entries()) {
decoder.decode(
new EncodedAudioChunk({ type: "key", timestamp: (index * 1024 * 1_000_000) / SAMPLE_RATE, data }),
);
}
return out;
});
expect(left.length).toBe(frames.length * 1024);
expect(power(left, 440)).toBeGreaterThan(100 * power(left, 880));
expect(power(right, 880)).toBeGreaterThan(100 * power(right, 440));
await decoder.flush();
decoder.close();
expect(failure).toBeUndefined();

const [left, right] = planes.map((chunks) => {
const out = new Float32Array(chunks.reduce((sum, chunk) => sum + chunk.length, 0));
let offset = 0;
for (const chunk of chunks) {
out.set(chunk, offset);
offset += chunk.length;
}
return out;
});
expect(left.length).toBe(frames.length * 1024);
expect(power(left, 440)).toBeGreaterThan(100 * power(left, 880));
expect(power(right, 880)).toBeGreaterThan(100 * power(right, 440));
}
});

// Runs on the polyfill's AudioData, which libopus decodes into. After the AAC test, which needs Bun's
Expand Down
32 changes: 31 additions & 1 deletion js/watch/src/audio/decoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,8 @@ let decodeThrows: Error | undefined;
let deferAudioOutput = false;
let pendingAudio: (() => void)[] = [];
let audioInputs: number[] = [];
let audioConfigs: AudioDecoderConfig[] = [];
let audioChunks: EncodedAudioChunkInit[] = [];

beforeEach(() => {
frameTimestamp = 0;
Expand All @@ -166,6 +168,8 @@ beforeEach(() => {
deferAudioOutput = false;
pendingAudio = [];
audioInputs = [];
audioConfigs = [];
audioChunks = [];
contexts = [];
worklets = [];
moduleLoaded = Promise.resolve();
Expand All @@ -180,7 +184,8 @@ beforeEach(() => {
codecErrors.push(init.error);
codecs++;
}
configure() {
configure(config: AudioDecoderConfig) {
audioConfigs.push(config);
this.needsKey = true;
}
reset() {
Expand All @@ -204,6 +209,7 @@ beforeEach(() => {
readonly timestamp: number;
readonly type: EncodedAudioChunkType;
constructor(init: EncodedAudioChunkInit) {
audioChunks.push(init);
this.timestamp = init.timestamp;
this.type = init.type;
}
Expand Down Expand Up @@ -392,6 +398,30 @@ async function play(initial: Delay, buffered = false, cmaf = false) {
};
}

for (const description of [undefined, "1190"]) {
it(`passes AAC to WebCodecs as ${description ? "raw" : "ADTS"}`, async () => {
const playback = await play(Milli(100));
try {
playback.restore({ codec: "mp4a.40.2", sampleRate: 48_000, numberOfChannels: 2, description });
await microtasks();
const config = audioConfigs.at(-1);
expect(config?.codec).toBe("mp4a.40.2");
expect(config?.description).toEqual(description ? new Uint8Array([0x11, 0x90]) : undefined);
const packet = new Uint8Array(
description ? [0x01, 0x52, 0xf2] : [0xff, 0xf1, 0x4c, 0x80, 0x01, 0x5f, 0xfc, 0x01, 0x52, 0xf2],
);
const read = frame();
read.frame.payload = packet;
playback.push(read);
await microtasks();
expect(audioChunks.at(-1)?.data).toEqual(packet);
} finally {
playback.close();
await microtasks();
}
});
}

for (const cmaf of [false, true]) {
it(`a ${cmaf ? "CMAF" : "legacy"} gap releases accounting for codec outputs it discarded`, async () => {
const clock = new Manual();
Expand Down
1 change: 0 additions & 1 deletion quest/m1/cut-audit/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ leaving JS-only redesigns of `js/net` to [rs2ts](/quest/m2/rs2ts-js.md).
## Required

- [A reserved group survives a route returning](/quest/m1/cut-audit/reserved-route-return.md) - a route flip A to B to A no longer panics a peeked reader
- [A description-less AAC rendition means one thing](/quest/m1/cut-audit/aac-description-less.md) - every player and exporter agrees on the frame format
- [Stream-accept loops yield on a per-poll budget](/quest/m1/cut-audit/accept-budget.md) - a peer opening many streams cannot monopolize a session driver
- [Rust and JS skip a backlog by the same measure](/quest/m1/cut-audit/catalog-parity.md) - a backlog join starts at the same group in every player
- [Text and data sections keep a prototype-named ID](/quest/m1/cut-audit/catalog-proto-ids.md) - text and data entries keep the ID JS's `z.record` drops
Expand Down
30 changes: 0 additions & 30 deletions quest/m1/cut-audit/aac-description-less.md

This file was deleted.

6 changes: 3 additions & 3 deletions rs/moq-audio/src/aac.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ fn validate(config: &moq_mux::codec::aac::Config) -> Result<(), Error> {
/// The AudioSpecificConfig to open a decoder with, validated as AAC-LC.
///
/// Prefers the catalog `description` verbatim, since re-encoding the parsed
/// fields would drop any extension it carries. A catalog without one (an MSF
/// track, or a producer that only filled in the plain fields) gets one
/// synthesized from the declared profile, rate, and channel count.
/// fields would drop any extension it carries. Without one, the decoder's config is
/// synthesized from the declared profile, rate, and channels. Legacy and LOC packets
/// then carry ADTS headers that must agree with it; CMAF samples remain raw.
///
/// The profile is read back out of the resulting config rather than taken from
/// the catalog codec string: the description is what the decoder acts on, and
Expand Down
24 changes: 21 additions & 3 deletions rs/moq-audio/src/decode/backend/symphonia.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ pub(super) struct Symphonia {
inner: symphonia_codec_aac::AacDecoder,
sample_rate: u32,
layout: Layout,
adts: bool,
}

impl Symphonia {
Expand Down Expand Up @@ -58,15 +59,32 @@ impl Symphonia {
inner,
sample_rate,
layout: Layout::from_channels(channel_count as u32)?,
adts: catalog.description.is_none() && !matches!(catalog.container, hang::catalog::Container::Cmaf { .. }),
}))
}
}

impl Backend for Symphonia {
fn decode(&mut self, packet: &[u8]) -> Result<Decoded, Error> {
// The packet is a raw AAC frame, not ADTS, so there is nothing to timestamp
// it with here: the container carries the timestamp and the decoder only
// reads the payload.
let packet = if self.adts {
let header =
moq_mux::codec::aac::adts::Header::parse(packet).map_err(|err| Error::Decode(format!("aac: {err}")))?;
if header.object_type != 2
|| header.sample_rate != self.sample_rate
|| u32::from(header.channel_config) != self.layout.channels()
|| header.blocks != 1
{
return Err(Error::Decode(
"ADTS header disagrees with the AAC-LC catalog or carries multiple blocks".into(),
));
}
header
.payload(packet)
.map_err(|err| Error::Decode(format!("aac: {err}")))?
} else {
packet
};
// The container carries the timestamp; Symphonia reads only the raw AAC block.
let packet = symphonia_core::packet::PacketRef::new(
0,
symphonia_core::units::Timestamp::ZERO,
Expand Down
63 changes: 60 additions & 3 deletions rs/moq-audio/src/decode/decoder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ impl Decoder {
/// `description` declares, refusing a malformed head; without a description
/// it takes the catalog's channel count and applies neither. PCM uses the
/// catalog fields directly and requires an absent `description`. AAC reads
/// its AudioSpecificConfig, synthesizing one from the catalog when absent.
/// its AudioSpecificConfig; without one, legacy and LOC packets carry ADTS headers,
/// while demuxed CMAF samples remain raw.
pub fn new(catalog: &hang::catalog::AudioConfig, config: &Config) -> Result<Self, Error> {
let backend = backend::open(catalog, config)?;
let delay = backend.delay();
Expand Down Expand Up @@ -198,13 +199,69 @@ pub(crate) mod tests {

#[cfg(feature = "aac")]
#[test]
fn aac_synthesizes_a_missing_description() {
// An MSF catalog carries the shape in its own fields instead.
fn aac_without_a_description_refuses_raw_packets() {
let mut catalog = aac_catalog();
catalog.description = None;

let mut decoder = Decoder::new(&catalog, &Config::default()).unwrap();
assert_eq!(decoder.sample_rate(), 44_100);
assert!(matches!(decoder.decode(AAC_FRAMES[0]), Err(Error::Decode(_))));
}

#[cfg(feature = "aac")]
#[test]
fn aac_without_a_description_decodes_adts() {
let mut catalog = aac_catalog();
catalog.description = None;
for crc in [false, true] {
let mut adts = Decoder::new(&catalog, &Config::default()).unwrap();
let mut raw = Decoder::new(&aac_catalog(), &Config::default()).unwrap();
for frame in AAC_FRAMES {
let len = frame.len() + if crc { 9 } else { 7 };
let mut packet = vec![
0xff,
if crc { 0xf0 } else { 0xf1 },
0x50,
0x40 | (len >> 11) as u8,
(len >> 3) as u8,
((len & 7) << 5) as u8 | 0x1f,
0xfc,
];
if crc {
packet.extend_from_slice(&[0, 0]);
}
packet.extend_from_slice(frame);
let got = adts.decode(&packet).unwrap().samples;
let expected = raw.decode(frame).unwrap().samples;
assert!(got == expected, "ADTS framing changed decoded PCM");
for altered in [2, 3, 6] {
let mut invalid = packet.clone();
invalid[altered] ^= if altered == 2 {
4
} else if altered == 3 {
0xc0
} else {
1
};
assert!(matches!(adts.decode(&invalid), Err(Error::Decode(_))));
}
assert!(matches!(
adts.decode(&packet[..packet.len() - 1]),
Err(Error::Decode(_))
));
}
}
}

#[cfg(feature = "aac")]
#[test]
fn cmaf_aac_samples_remain_raw_without_a_catalog_description() {
let mut catalog = aac_catalog();
catalog.description = None;
catalog.container = hang::catalog::Container::Cmaf {
init: bytes::Bytes::new(),
};
let mut decoder = Decoder::new(&catalog, &Config::default()).unwrap();
assert_eq!(decoder.decode(AAC_FRAMES[0]).unwrap().samples.len(), 1024);
}

Expand Down
Loading