Fix E2EE decryption for H.264/H.265 frames with 3-byte start codes

.changeset/e2ee-annexb-start-codes.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Fix E2EE H.264/H.265 decryption failing for encoders that emit 3-byte Annex B start codes (e.g. Intel Quick Sync on Windows). The sender now rewrites them to 4-byte start codes in the unencrypted header, matching what the receiver's depacketizer reconstructs.

src/e2ee/worker/FrameCryptor.test.ts

@@ -233,6 +233,81 @@ describe('FrameCryptor', () => {
       }
     });
 
+    describe('H.264 start codes (#2140)', () => {
+      const sps = [0x67, 0x64, 0x00, 0x1f, 0xac, 0xd9];
+      const pps = [0x68, 0xee, 0x3c, 0xb0];
+      const idr = [0x65, 0x88, 0x84, 0x21, 0x43, 0x55, 0x66, 0x77];
+
+      async function encryptAndDecrypt(plainTextData: NonSharedUint8Array) {
+        const encoder = prepareParticipantTestEncoder(participantIdentity, {});
+        await encoder.keys.setKey(await createKeyMaterialFromString('key1'), 1);
+        const encrypted = mockRTCEncodedVideoFrame(plainTextData);
+        encoder.input.write(encrypted);
+        await vitest.waitFor(() => expect(encoder.output.chunks).toHaveLength(1));
+        const encryptedData = new Uint8Array(encrypted.data.slice(0));
+
+        const decoder = prepareParticipantTestDecoder(participantIdentity, {});
+        await decoder.keys.setKey(await createKeyMaterialFromString('key1'), 1);
+        const decrypted = mockRTCEncodedVideoFrame(new Uint8Array(encryptedData));
+        decoder.input.write(decrypted);
+        await vitest.waitFor(() => expect(decoder.output.chunks).toHaveLength(1));
+
+        return { encryptedData, decryptedData: new Uint8Array(decrypted.data) };
+      }
+
+      it('rewrites 3-byte start codes in the unencrypted header to 4-byte start codes', async () => {
+        const { encryptedData, decryptedData } = await encryptAndDecrypt(
+          new Uint8Array([0, 0, 1, ...sps, 0, 0, 1, ...pps, 0, 0, 1, ...idr]),
+        );
+
+        // The unencrypted header runs through the first 2 bytes of the slice NALU.
+        const expectedHeader = [0, 0, 0, 1, ...sps, 0, 0, 0, 1, ...pps, 0, 0, 0, 1, 0x65, 0x88];
+        expect(Array.from(encryptedData.subarray(0, expectedHeader.length))).toEqual(
+          expectedHeader,
+        );
+        expect(Array.from(decryptedData)).toEqual([
+          0,
+          0,
+          0,
+          1,
+          ...sps,
+          0,
+          0,
+          0,
+          1,
+          ...pps,
+          0,
+          0,
+          0,
+          1,
+          ...idr,
+        ]);
+      });
+
+      it('rewrites a 3-byte start code on a delta frame', async () => {
+        const pSlice = [0x41, 0x9a, 0x21, 0x43, 0x55, 0x66];
+        const { encryptedData, decryptedData } = await encryptAndDecrypt(
+          new Uint8Array([0, 0, 1, ...pSlice]),
+        );
+
+        expect(Array.from(encryptedData.subarray(0, 6))).toEqual([0, 0, 0, 1, 0x41, 0x9a]);
+        expect(Array.from(decryptedData)).toEqual([0, 0, 0, 1, ...pSlice]);
+      });
+
+      it('leaves 4-byte start codes in the unencrypted header unchanged', async () => {
+        const plainTextData = [0, 0, 0, 1, ...sps, 0, 0, 0, 1, ...pps, 0, 0, 0, 1, ...idr];
+        const { encryptedData, decryptedData } = await encryptAndDecrypt(
+          new Uint8Array(plainTextData),
+        );
+
+        const expectedHeader = [0, 0, 0, 1, ...sps, 0, 0, 0, 1, ...pps, 0, 0, 0, 1, 0x65, 0x88];
+        expect(Array.from(encryptedData.subarray(0, expectedHeader.length))).toEqual(
+          expectedHeader,
+        );
+        expect(Array.from(decryptedData)).toEqual(plainTextData);
+      });
+    });
+
     it('appends packet trailer after encryption', async () => {
       vitest.useFakeTimers();
       const now = new Date('2025-04-10T12:00:00.123Z');

src/e2ee/worker/FrameCryptor.ts

@@ -25,7 +25,7 @@ import type {
 import { deriveKeys, isVideoFrame, needsRbspUnescaping, parseRbsp, writeRbsp } from '../utils';
 import { ErrorRateLimiter } from './ErrorRateLimiter';
 import type { ParticipantKeyHandler } from './ParticipantKeyHandler';
-import { processNALUsForEncryption } from './naluUtils';
+import { normalizeAnnexBStartCodes, processNALUsForEncryption } from './naluUtils';
 import { identifySifPayload } from './sifPayload';
 
 export const encryptionEnabledMap: Map<string, boolean> = new Map();
@@ -490,7 +490,16 @@ export class FrameCryptor extends BaseFrameCryptor {
       let frameInfo = this.getUnencryptedBytes(encodedFrame);
 
       // Thіs is not encrypted and contains the VP8 payload descriptor or the Opus TOC byte.
-      const frameHeader = new Uint8Array(encodedFrame.data, 0, frameInfo.unencryptedBytes);
+      let frameHeader: NonSharedUint8Array = new Uint8Array(
+        encodedFrame.data,
+        0,
+        frameInfo.unencryptedBytes,
+      );
+      if (frameInfo.requiresNALUProcessing) {
+        // The receiver's depacketizer rebuilds NALUs with 4-byte start codes, so authenticate (and
+        // send) the header in that form or decryption fails for encoders emitting 3-byte ones.
+        frameHeader = normalizeAnnexBStartCodes(frameHeader);
+      }
 
       // Frame trailer contains the R|IV_LENGTH and key index
       const frameTrailer = new Uint8Array(2);
@@ -510,7 +519,7 @@ export class FrameCryptor extends BaseFrameCryptor {
           {
             name: ENCRYPTION_ALGORITHM,
             iv,
-            additionalData: new Uint8Array(encodedFrame.data, 0, frameHeader.byteLength),
+            additionalData: frameHeader,
           },
           encryptionKey,
           new Uint8Array(encodedFrame.data, frameInfo.unencryptedBytes),

src/e2ee/worker/naluUtils.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest';
+import { normalizeAnnexBStartCodes } from './naluUtils';
+
+describe('normalizeAnnexBStartCodes', () => {
+  it('rewrites 3-byte start codes to 4-byte start codes', () => {
+    // prettier-ignore
+    const input = new Uint8Array([
+      0x00, 0x00, 0x01, 0x67, 0x64, // SPS
+      0x00, 0x00, 0x01, 0x68, 0xee, // PPS
+      0x00, 0x00, 0x01, 0x65, 0x88, // IDR slice
+    ]);
+    // prettier-ignore
+    const expected = new Uint8Array([
+      0x00, 0x00, 0x00, 0x01, 0x67, 0x64,
+      0x00, 0x00, 0x00, 0x01, 0x68, 0xee,
+      0x00, 0x00, 0x00, 0x01, 0x65, 0x88,
+    ]);
+
+    expect(normalizeAnnexBStartCodes(input)).toEqual(expected);
+  });
+
+  it('rewrites only the 3-byte start codes when mixed with 4-byte ones', () => {
+    // prettier-ignore
+    const input = new Uint8Array([
+      0x00, 0x00, 0x00, 0x01, 0x67, 0x64,
+      0x00, 0x00, 0x01, 0x68, 0xee,
+      0x00, 0x00, 0x00, 0x01, 0x06, 0x05,
+      0x00, 0x00, 0x01, 0x65, 0x88,
+    ]);
+    // prettier-ignore
+    const expected = new Uint8Array([
+      0x00, 0x00, 0x00, 0x01, 0x67, 0x64,
+      0x00, 0x00, 0x00, 0x01, 0x68, 0xee,
+      0x00, 0x00, 0x00, 0x01, 0x06, 0x05,
+      0x00, 0x00, 0x00, 0x01, 0x65, 0x88,
+    ]);
+
+    expect(normalizeAnnexBStartCodes(input)).toEqual(expected);
+  });
+
+  it('returns the same instance when all start codes are already 4-byte', () => {
+    // prettier-ignore
+    const input = new Uint8Array([
+      0x00, 0x00, 0x00, 0x01, 0x67, 0x64,
+      0x00, 0x00, 0x00, 0x01, 0x65, 0x88,
+    ]);
+
+    expect(normalizeAnnexBStartCodes(input)).toBe(input);
+  });
+
+  it('does not add a zero when a start code already has extra leading zeros', () => {
+    const input = new Uint8Array([0x00, 0x00, 0x00, 0x00, 0x01, 0x65, 0x88]);
+
+    expect(normalizeAnnexBStartCodes(input)).toBe(input);
+  });
+});

src/e2ee/worker/naluUtils.ts

@@ -327,3 +327,43 @@ export function processNALUsForEncryption(
 
   return { unencryptedBytes, detectedCodec, requiresNALUProcessing: true };
 }
+
+/**
+ * Rewrite every 3-byte Annex B start code (00 00 01) in `data` to the 4-byte form (00 00 00 01).
+ *
+ * WebRTC's H.264/H.265 depacketizer always reconstructs NALUs with 4-byte start codes, regardless
+ * of what the encoder emitted. Some encoders (e.g. Intel Quick Sync) emit 3-byte start codes, so
+ * the unencrypted header a receiver sees would differ from what the sender authenticated as
+ * AES-GCM additionalData. Normalizing on the sender makes both sides agree.
+ *
+ * Returns `data` unchanged (same instance) when it contains no 3-byte start codes.
+ */
+export function normalizeAnnexBStartCodes(data: NonSharedUint8Array): NonSharedUint8Array {
+  const shortStartCodes: number[] = [];
+  for (let i = 0; i + 2 < data.length;) {
+    if (data[i + 2] === 0) {
+      i += 1;
+      continue;
+    }
+    if (data[i + 2] === 1 && data[i] === 0 && data[i + 1] === 0 && (i === 0 || data[i - 1] !== 0)) {
+      shortStartCodes.push(i);
+    }
+    // data[i + 2] is non-zero, so no start code can begin at i + 1 or i + 2 either.
+    i += 3;
+  }
+  if (shortStartCodes.length === 0) {
+    return data;
+  }
+
+  // The new array is zero-filled, so the one-byte gap left before each short start code becomes
+  // the inserted leading 00.
+  const result = new Uint8Array(data.length + shortStartCodes.length);
+  let from = 0;
+  for (let inserted = 0; inserted < shortStartCodes.length; inserted++) {
+    const to = shortStartCodes[inserted];
+    result.set(data.subarray(from, to), from + inserted);
+    from = to;
+  }
+  result.set(data.subarray(from), from + shortStartCodes.length);
+  return result;
+}