From bd7c1a1449d8d6aa6f4f6765c01eee7a39621972 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 9 Oct 2026 07:27:42 +1300 Subject: [PATCH 1/4] Normalize Annex B start codes in E2EE unencrypted header Some H.264/H.265 encoders (e.g. Intel Quick Sync on Windows) emit 3-byte start codes (00 00 01). WebRTC's depacketizer always reconstructs NALUs with 4-byte start codes, so the unencrypted header authenticated as AES-GCM additionalData on the sender no longer matched on the receiver and every frame failed to decrypt. The sender now rewrites 3-byte start codes to 4-byte in the unencrypted header before encrypting and sends that header. Frames that already use 4-byte start codes are byte-identical on the wire, so receivers need no change. Fixes #2140 Co-Authored-By: Claude Opus 5.5 (1M context) --- .changeset/e2ee-annexb-start-codes.md | 5 ++ src/e2ee/worker/FrameCryptor.test.ts | 75 +++++++++++++++++++++++++++ src/e2ee/worker/FrameCryptor.ts | 15 ++++-- src/e2ee/worker/naluUtils.test.ts | 56 ++++++++++++++++++++ src/e2ee/worker/naluUtils.ts | 33 ++++++++++++ 5 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 .changeset/e2ee-annexb-start-codes.md create mode 100644 src/e2ee/worker/naluUtils.test.ts diff --git a/.changeset/e2ee-annexb-start-codes.md b/.changeset/e2ee-annexb-start-codes.md new file mode 100644 index 0000000000..d721691f30 --- /dev/null +++ b/.changeset/e2ee-annexb-start-codes.md @@ -0,0 +1,5 @@ +--- +'livekit-client': patch +--- + +Fix E2EE H.264/H.265 decryption failing for encoders that emit 3-byte Annex B start codes (e.g. Intel Quick Sync on Windows). The sender now rewrites them to 4-byte start codes in the unencrypted header, matching what the receiver's depacketizer reconstructs. diff --git a/src/e2ee/worker/FrameCryptor.test.ts b/src/e2ee/worker/FrameCryptor.test.ts index 366887960b..72a71a5193 100644 --- a/src/e2ee/worker/FrameCryptor.test.ts +++ b/src/e2ee/worker/FrameCryptor.test.ts @@ -233,6 +233,81 @@ describe('FrameCryptor', () => { } }); + describe('H.264 start codes (#2140)', () => { + const sps = [0x67, 0x64, 0x00, 0x1f, 0xac, 0xd9]; + const pps = [0x68, 0xee, 0x3c, 0xb0]; + const idr = [0x65, 0x88, 0x84, 0x21, 0x43, 0x55, 0x66, 0x77]; + + async function encryptAndDecrypt(plainTextData: NonSharedUint8Array) { + const encoder = prepareParticipantTestEncoder(participantIdentity, {}); + await encoder.keys.setKey(await createKeyMaterialFromString('key1'), 1); + const encrypted = mockRTCEncodedVideoFrame(plainTextData); + encoder.input.write(encrypted); + await vitest.waitFor(() => expect(encoder.output.chunks).toHaveLength(1)); + const encryptedData = new Uint8Array(encrypted.data.slice(0)); + + const decoder = prepareParticipantTestDecoder(participantIdentity, {}); + await decoder.keys.setKey(await createKeyMaterialFromString('key1'), 1); + const decrypted = mockRTCEncodedVideoFrame(new Uint8Array(encryptedData)); + decoder.input.write(decrypted); + await vitest.waitFor(() => expect(decoder.output.chunks).toHaveLength(1)); + + return { encryptedData, decryptedData: new Uint8Array(decrypted.data) }; + } + + it('rewrites 3-byte start codes in the unencrypted header to 4-byte start codes', async () => { + const { encryptedData, decryptedData } = await encryptAndDecrypt( + new Uint8Array([0, 0, 1, ...sps, 0, 0, 1, ...pps, 0, 0, 1, ...idr]), + ); + + // The unencrypted header runs through the first 2 bytes of the slice NALU. + const expectedHeader = [0, 0, 0, 1, ...sps, 0, 0, 0, 1, ...pps, 0, 0, 0, 1, 0x65, 0x88]; + expect(Array.from(encryptedData.subarray(0, expectedHeader.length))).toEqual( + expectedHeader, + ); + expect(Array.from(decryptedData)).toEqual([ + 0, + 0, + 0, + 1, + ...sps, + 0, + 0, + 0, + 1, + ...pps, + 0, + 0, + 0, + 1, + ...idr, + ]); + }); + + it('rewrites a 3-byte start code on a delta frame', async () => { + const pSlice = [0x41, 0x9a, 0x21, 0x43, 0x55, 0x66]; + const { encryptedData, decryptedData } = await encryptAndDecrypt( + new Uint8Array([0, 0, 1, ...pSlice]), + ); + + expect(Array.from(encryptedData.subarray(0, 6))).toEqual([0, 0, 0, 1, 0x41, 0x9a]); + expect(Array.from(decryptedData)).toEqual([0, 0, 0, 1, ...pSlice]); + }); + + it('leaves 4-byte start codes in the unencrypted header unchanged', async () => { + const plainTextData = [0, 0, 0, 1, ...sps, 0, 0, 0, 1, ...pps, 0, 0, 0, 1, ...idr]; + const { encryptedData, decryptedData } = await encryptAndDecrypt( + new Uint8Array(plainTextData), + ); + + const expectedHeader = [0, 0, 0, 1, ...sps, 0, 0, 0, 1, ...pps, 0, 0, 0, 1, 0x65, 0x88]; + expect(Array.from(encryptedData.subarray(0, expectedHeader.length))).toEqual( + expectedHeader, + ); + expect(Array.from(decryptedData)).toEqual(plainTextData); + }); + }); + it('appends packet trailer after encryption', async () => { vitest.useFakeTimers(); const now = new Date('2025-04-10T12:00:00.123Z'); diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index bc0b771a61..de5a5c8642 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -25,7 +25,7 @@ import type { import { deriveKeys, isVideoFrame, needsRbspUnescaping, parseRbsp, writeRbsp } from '../utils'; import { ErrorRateLimiter } from './ErrorRateLimiter'; import type { ParticipantKeyHandler } from './ParticipantKeyHandler'; -import { processNALUsForEncryption } from './naluUtils'; +import { normalizeAnnexBStartCodes, processNALUsForEncryption } from './naluUtils'; import { identifySifPayload } from './sifPayload'; export const encryptionEnabledMap: Map = new Map(); @@ -490,7 +490,16 @@ export class FrameCryptor extends BaseFrameCryptor { let frameInfo = this.getUnencryptedBytes(encodedFrame); // Thіs is not encrypted and contains the VP8 payload descriptor or the Opus TOC byte. - const frameHeader = new Uint8Array(encodedFrame.data, 0, frameInfo.unencryptedBytes); + let frameHeader: NonSharedUint8Array = new Uint8Array( + encodedFrame.data, + 0, + frameInfo.unencryptedBytes, + ); + if (frameInfo.requiresNALUProcessing) { + // The receiver's depacketizer rebuilds NALUs with 4-byte start codes, so authenticate (and + // send) the header in that form or decryption fails for encoders emitting 3-byte ones. + frameHeader = normalizeAnnexBStartCodes(frameHeader); + } // Frame trailer contains the R|IV_LENGTH and key index const frameTrailer = new Uint8Array(2); @@ -510,7 +519,7 @@ export class FrameCryptor extends BaseFrameCryptor { { name: ENCRYPTION_ALGORITHM, iv, - additionalData: new Uint8Array(encodedFrame.data, 0, frameHeader.byteLength), + additionalData: frameHeader, }, encryptionKey, new Uint8Array(encodedFrame.data, frameInfo.unencryptedBytes), diff --git a/src/e2ee/worker/naluUtils.test.ts b/src/e2ee/worker/naluUtils.test.ts new file mode 100644 index 0000000000..383bf09469 --- /dev/null +++ b/src/e2ee/worker/naluUtils.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest'; +import { normalizeAnnexBStartCodes } from './naluUtils'; + +describe('normalizeAnnexBStartCodes', () => { + it('rewrites 3-byte start codes to 4-byte start codes', () => { + // prettier-ignore + const input = new Uint8Array([ + 0x00, 0x00, 0x01, 0x67, 0x64, // SPS + 0x00, 0x00, 0x01, 0x68, 0xee, // PPS + 0x00, 0x00, 0x01, 0x65, 0x88, // IDR slice + ]); + // prettier-ignore + const expected = new Uint8Array([ + 0x00, 0x00, 0x00, 0x01, 0x67, 0x64, + 0x00, 0x00, 0x00, 0x01, 0x68, 0xee, + 0x00, 0x00, 0x00, 0x01, 0x65, 0x88, + ]); + + expect(normalizeAnnexBStartCodes(input)).toEqual(expected); + }); + + it('rewrites only the 3-byte start codes when mixed with 4-byte ones', () => { + // prettier-ignore + const input = new Uint8Array([ + 0x00, 0x00, 0x00, 0x01, 0x67, 0x64, + 0x00, 0x00, 0x01, 0x68, 0xee, + 0x00, 0x00, 0x00, 0x01, 0x06, 0x05, + 0x00, 0x00, 0x01, 0x65, 0x88, + ]); + // prettier-ignore + const expected = new Uint8Array([ + 0x00, 0x00, 0x00, 0x01, 0x67, 0x64, + 0x00, 0x00, 0x00, 0x01, 0x68, 0xee, + 0x00, 0x00, 0x00, 0x01, 0x06, 0x05, + 0x00, 0x00, 0x00, 0x01, 0x65, 0x88, + ]); + + expect(normalizeAnnexBStartCodes(input)).toEqual(expected); + }); + + it('returns the same instance when all start codes are already 4-byte', () => { + // prettier-ignore + const input = new Uint8Array([ + 0x00, 0x00, 0x00, 0x01, 0x67, 0x64, + 0x00, 0x00, 0x00, 0x01, 0x65, 0x88, + ]); + + expect(normalizeAnnexBStartCodes(input)).toBe(input); + }); + + it('does not add a zero when a start code already has extra leading zeros', () => { + const input = new Uint8Array([0x00, 0x00, 0x00, 0x00, 0x01, 0x65, 0x88]); + + expect(normalizeAnnexBStartCodes(input)).toBe(input); + }); +}); diff --git a/src/e2ee/worker/naluUtils.ts b/src/e2ee/worker/naluUtils.ts index c067ac87df..f99d5ed1ae 100644 --- a/src/e2ee/worker/naluUtils.ts +++ b/src/e2ee/worker/naluUtils.ts @@ -327,3 +327,36 @@ export function processNALUsForEncryption( return { unencryptedBytes, detectedCodec, requiresNALUProcessing: true }; } + +/** + * Rewrite every 3-byte Annex B start code (00 00 01) in `data` to the 4-byte form (00 00 00 01). + * + * WebRTC's H.264/H.265 depacketizer always reconstructs NALUs with 4-byte start codes, regardless + * of what the encoder emitted. Some encoders (e.g. Intel Quick Sync) emit 3-byte start codes, so + * the unencrypted header a receiver sees would differ from what the sender authenticated as + * AES-GCM additionalData. Normalizing on the sender makes both sides agree. + * + * Returns `data` unchanged (same instance) when it contains no 3-byte start codes. + */ +export function normalizeAnnexBStartCodes(data: NonSharedUint8Array): NonSharedUint8Array { + const isShortStartCode = (i: number) => + data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 1 && (i === 0 || data[i - 1] !== 0); + + let shortStartCodes = 0; + for (let i = 0; i + 2 < data.length; i++) { + if (isShortStartCode(i)) shortStartCodes++; + } + if (shortStartCodes === 0) { + return data; + } + + const result = new Uint8Array(data.length + shortStartCodes); + let out = 0; + for (let i = 0; i < data.length; i++) { + if (i + 2 < data.length && isShortStartCode(i)) { + result[out++] = 0; + } + result[out++] = data[i]; + } + return result; +} From 65e1f0e9c401991d1aafd9a144a93b0fda613b39 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 9 Oct 2026 07:33:50 +1300 Subject: [PATCH 2/4] Use explicit increments in normalizeAnnexBStartCodes Co-Authored-By: Claude Opus 5.5 (1M context) --- src/e2ee/worker/naluUtils.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/e2ee/worker/naluUtils.ts b/src/e2ee/worker/naluUtils.ts index f99d5ed1ae..54f1c7c553 100644 --- a/src/e2ee/worker/naluUtils.ts +++ b/src/e2ee/worker/naluUtils.ts @@ -344,7 +344,9 @@ export function normalizeAnnexBStartCodes(data: NonSharedUint8Array): NonSharedU let shortStartCodes = 0; for (let i = 0; i + 2 < data.length; i++) { - if (isShortStartCode(i)) shortStartCodes++; + if (isShortStartCode(i)) { + shortStartCodes += 1; + } } if (shortStartCodes === 0) { return data; @@ -354,9 +356,11 @@ export function normalizeAnnexBStartCodes(data: NonSharedUint8Array): NonSharedU let out = 0; for (let i = 0; i < data.length; i++) { if (i + 2 < data.length && isShortStartCode(i)) { - result[out++] = 0; + result[out] = 0; + out += 1; } - result[out++] = data[i]; + result[out] = data[i]; + out += 1; } return result; } From 8fa21cc8da4adf9c63a8128beb8eb51eb11f355e Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Mon, 12 Oct 2026 09:56:33 +1300 Subject: [PATCH 3/4] Update src/e2ee/worker/naluUtils.ts Co-authored-by: lukasIO --- src/e2ee/worker/naluUtils.ts | 41 ++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/src/e2ee/worker/naluUtils.ts b/src/e2ee/worker/naluUtils.ts index 54f1c7c553..cf36198fdc 100644 --- a/src/e2ee/worker/naluUtils.ts +++ b/src/e2ee/worker/naluUtils.ts @@ -339,28 +339,23 @@ export function processNALUsForEncryption( * Returns `data` unchanged (same instance) when it contains no 3-byte start codes. */ export function normalizeAnnexBStartCodes(data: NonSharedUint8Array): NonSharedUint8Array { - const isShortStartCode = (i: number) => - data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 1 && (i === 0 || data[i - 1] !== 0); - - let shortStartCodes = 0; - for (let i = 0; i + 2 < data.length; i++) { - if (isShortStartCode(i)) { - shortStartCodes += 1; - } - } - if (shortStartCodes === 0) { - return data; - } - - const result = new Uint8Array(data.length + shortStartCodes); - let out = 0; - for (let i = 0; i < data.length; i++) { - if (i + 2 < data.length && isShortStartCode(i)) { - result[out] = 0; - out += 1; - } - result[out] = data[i]; - out += 1; +const shortStartCodes: number[] = []; +for (let i = 0; i + 2 < data.length; ) { + if (data[i + 2] === 0) { i += 1; continue; } + if (data[i + 2] === 1 && data[i] === 0 && data[i + 1] === 0 && (i === 0 || data[i - 1] !== 0)) { + shortStartCodes.push(i); } - return result; + // data[i + 2] is non-zero, so no start code can begin at i + 1 or i + 2 either. + i += 3; +} +if (shortStartCodes.length === 0) return data; + +const result = new Uint8Array(data.length + shortStartCodes.length); +let from = 0; +shortStartCodes.forEach((to, inserted) => { + result.set(data.subarray(from, to), from + inserted); + from = to; +}); +result.set(data.subarray(from), from + shortStartCodes.length); +return result; } From 1c9819ec0bfc0c689cd8981f3696c448a8b18022 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Mon, 12 Oct 2026 09:57:43 +1300 Subject: [PATCH 4/4] Format normalizeAnnexBStartCodes and clarify block copy Indent and prettier-format the applied review suggestion, use an indexed for loop instead of forEach to match the rest of naluUtils.ts, and note that the zero-filled result array supplies each inserted leading 00. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/e2ee/worker/naluUtils.ts | 44 +++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/e2ee/worker/naluUtils.ts b/src/e2ee/worker/naluUtils.ts index cf36198fdc..922e1bf645 100644 --- a/src/e2ee/worker/naluUtils.ts +++ b/src/e2ee/worker/naluUtils.ts @@ -339,23 +339,31 @@ export function processNALUsForEncryption( * Returns `data` unchanged (same instance) when it contains no 3-byte start codes. */ export function normalizeAnnexBStartCodes(data: NonSharedUint8Array): NonSharedUint8Array { -const shortStartCodes: number[] = []; -for (let i = 0; i + 2 < data.length; ) { - if (data[i + 2] === 0) { i += 1; continue; } - if (data[i + 2] === 1 && data[i] === 0 && data[i + 1] === 0 && (i === 0 || data[i - 1] !== 0)) { - shortStartCodes.push(i); + const shortStartCodes: number[] = []; + for (let i = 0; i + 2 < data.length;) { + if (data[i + 2] === 0) { + i += 1; + continue; + } + if (data[i + 2] === 1 && data[i] === 0 && data[i + 1] === 0 && (i === 0 || data[i - 1] !== 0)) { + shortStartCodes.push(i); + } + // data[i + 2] is non-zero, so no start code can begin at i + 1 or i + 2 either. + i += 3; } - // data[i + 2] is non-zero, so no start code can begin at i + 1 or i + 2 either. - i += 3; -} -if (shortStartCodes.length === 0) return data; - -const result = new Uint8Array(data.length + shortStartCodes.length); -let from = 0; -shortStartCodes.forEach((to, inserted) => { - result.set(data.subarray(from, to), from + inserted); - from = to; -}); -result.set(data.subarray(from), from + shortStartCodes.length); -return result; + if (shortStartCodes.length === 0) { + return data; + } + + // The new array is zero-filled, so the one-byte gap left before each short start code becomes + // the inserted leading 00. + const result = new Uint8Array(data.length + shortStartCodes.length); + let from = 0; + for (let inserted = 0; inserted < shortStartCodes.length; inserted++) { + const to = shortStartCodes[inserted]; + result.set(data.subarray(from, to), from + inserted); + from = to; + } + result.set(data.subarray(from), from + shortStartCodes.length); + return result; }