From a78655dfa5e1d82e0ed3790978c8fa5ce7f8c35b Mon Sep 17 00:00:00 2001 From: Sen Chang Date: Thu, 8 Oct 2026 09:07:28 -0700 Subject: [PATCH] Scale the video start bitrate hint by connection setup time Port of livekit/rust-sdks#1467 and #1505 (CLT-3380). The camera x-google-start-bitrate cap is 1 Mbps for connections that set up within 1.5 s and ramps linearly down to 300 kbps at 3.5 s or slower. Screen share stays uncapped. RTCEngine times each join attempt and hands the setup time to the publisher when the primary transport first connects. A video published during connect is seeded from the time elapsed so far rather than delaying its offer. --- ...tart-bitrate-from-connection-setup-time.md | 5 + src/room/PCTransport.test.ts | 93 +++++++++++++- src/room/PCTransport.ts | 115 ++++++++++++++++-- src/room/RTCEngine.ts | 26 ++++ 4 files changed, 228 insertions(+), 11 deletions(-) create mode 100644 .changeset/adaptive-start-bitrate-from-connection-setup-time.md diff --git a/.changeset/adaptive-start-bitrate-from-connection-setup-time.md b/.changeset/adaptive-start-bitrate-from-connection-setup-time.md new file mode 100644 index 0000000000..604309c8c2 --- /dev/null +++ b/.changeset/adaptive-start-bitrate-from-connection-setup-time.md @@ -0,0 +1,5 @@ +--- +'livekit-client': patch +--- + +Scale the `x-google-start-bitrate` hint by connection setup time: the 1 Mbps camera cap now applies to connections that set up within 1.5 s and ramps linearly down to 300 kbps at 3.5 s or slower; screen share stays uncapped. diff --git a/src/room/PCTransport.test.ts b/src/room/PCTransport.test.ts index 32bc541f12..bb4c0c2b30 100644 --- a/src/room/PCTransport.test.ts +++ b/src/room/PCTransport.test.ts @@ -1,6 +1,6 @@ import { type MediaDescription, parse } from 'sdp-transform'; -import { describe, expect, it } from 'vitest'; -import { +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import PCTransport, { applyVideoStartBitrate, computeConnectionStartBitrate, computeTrackStartBitrate, @@ -156,6 +156,95 @@ describe('video start bitrate', () => { expect(computeTrackStartBitrate({ cid: 'c', codec: 'VP8', maxbr: 300 })).toBe(270); }); + it('ramps the cap down with connection setup time', () => { + // A 3 Mbps camera target, so only the cap moves. + const camera = (ms: number) => + computeTrackStartBitrate({ cid: 'c', codec: 'VP8', maxbr: 3_000 }, ms); + + expect(camera(0)).toBe(1_000); + expect(camera(471)).toBe(1_000); // unshaped baseline + expect(camera(1_273)).toBe(1_000); // 1 Mbps link median + expect(camera(1_500)).toBe(1_000); // the fast anchor + expect(camera(1_724)).toBe(922); // 1 Mbps link slow attempt + expect(camera(2_334)).toBe(708); // 500 kbps link median + expect(camera(2_500)).toBe(650); // midpoint of the ramp + expect(camera(3_093)).toBe(442); // 300 kbps link fastest attempt + expect(camera(3_500)).toBe(300); // the slow anchor + expect(camera(18_131)).toBe(300); // anything slower stays at the floor + }); + + it('keeps the 1 Mbps ceiling without a setup time', () => { + expect(computeTrackStartBitrate({ cid: 'c', codec: 'VP8', maxbr: 3_000 }, undefined)).toBe( + 1_000, + ); + }); + + it('still applies 90% of the target under the ramp, and hints at the floor', () => { + expect(computeTrackStartBitrate({ cid: 'c', codec: 'VP8', maxbr: 500 }, 2_500)).toBe(450); + expect(computeTrackStartBitrate({ cid: 'c', codec: 'VP8', maxbr: 300 }, 3_500)).toBe(270); + }); + + it('leaves screen share uncapped however slow the setup', () => { + const screenShare = (ms: number) => + computeTrackStartBitrate({ cid: 's', codec: 'VP8', maxbr: 3_000, isScreenShare: true }, ms); + + expect(screenShare(1_273)).toBe(2_700); + expect(screenShare(2_500)).toBe(2_700); + expect(screenShare(4_061)).toBe(2_700); + }); + + it('applies the setup time to the connection-level value', () => { + const { media } = parse(TWO_VIDEO_SECTIONS); + const camera = { cid: 'camera-cid', codec: 'VP8', maxbr: 3_000 }; + const screenShare = { cid: 'other-track', codec: 'VP8', maxbr: 3_000, isScreenShare: true }; + + expect(computeConnectionStartBitrate(media, [camera], 1_000)).toBe(1_000); + expect(computeConnectionStartBitrate(media, [camera], 2_500)).toBe(650); + // The uncapped screen share still wins the connection-level max on a slow setup. + expect(computeConnectionStartBitrate(media, [camera, screenShare], 2_500)).toBe(2_700); + }); + + describe('connection setup time while still connecting', () => { + beforeEach(() => { + // The constructor creates its peer connection eagerly; only its event handlers are set. + vi.stubGlobal('RTCPeerConnection', class {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + const estimate = (transport: PCTransport) => + ( + transport as unknown as { connectionSetupTimeEstimateMs(): number | undefined } + ).connectionSetupTimeEstimateMs(); + + it('uses the time elapsed since the join started before the setup time is measured', () => { + const transport = new PCTransport(); + vi.spyOn(performance, 'now').mockReturnValue(10_000); + transport.setConnectionStartedAt(7_500); + + expect(estimate(transport)).toBe(2_500); + expect( + computeTrackStartBitrate({ cid: 'c', codec: 'VP8', maxbr: 3_000 }, estimate(transport)), + ).toBe(650); + }); + + it('prefers the measured setup time once known', () => { + const transport = new PCTransport(); + vi.spyOn(performance, 'now').mockReturnValue(10_000); + transport.setConnectionStartedAt(7_500); + transport.setConnectionSetupTime(3_600); + + expect(estimate(transport)).toBe(3_600); + }); + + it('has no estimate without any timing', () => { + expect(estimate(new PCTransport())).toBeUndefined(); + }); + }); + it('uses one connection-level value: the largest hint across video sections', () => { const { media } = parse(TWO_VIDEO_SECTIONS); diff --git a/src/room/PCTransport.ts b/src/room/PCTransport.ts index a470a19a04..8709e95315 100644 --- a/src/room/PCTransport.ts +++ b/src/room/PCTransport.ts @@ -28,6 +28,8 @@ interface TrackBitrateInfo { * Why same for all codecs: Target bitrate already accounts for codec efficiency * (e.g., users set lower targets for VP9/AV1 knowing they're more efficient). * Why cap at 1 Mbps: Prevents BWE from starting too aggressively on high bitrate tracks. + * The cap is lowered further for connections that were slow to set up, see + * `computeStartBitrateCap`. */ const startBitrateMultiplier = 0.9; @@ -35,11 +37,24 @@ const startBitrateMultiplier = 0.9; const maxStartBitrateKbps = 1000; /** - * Minimum target bitrate in kbps for the start bitrate hint. Below this, seeding above the - * real capacity costs more than the ramp it saves, so libwebrtc's default is left in place. + * Minimum x-google-start-bitrate in kbps: libwebrtc's own default starting estimate. A target + * below this gets no hint, since seeding above the real capacity costs more than the ramp it + * saves, and a slow connection is seeded no higher than this. */ const minTargetBitrateKbps = 300; +/** + * Connection setup times (ms) bounding the ramp in `computeStartBitrateCap`: at or below the + * first the hint is capped at `maxStartBitrateKbps`, at or above the second at + * `minTargetBitrateKbps`. Measured on shaped links (CLT-3380): healthy links set up in under + * 860 ms, a 1 Mbps link with 150 ms RTT took 1.1–1.7 s, a 500 kbps link ~2.3 s at the median, + * and a 300 kbps link never under 3 s. The first sits above the 1 Mbps link, which a lower seed + * only slows down, and the second above the 500 kbps link's median, so only links that cannot + * carry more land at the floor. + */ +const setupTimeForMaxStartBitrateMs = 1500; +const setupTimeForMinStartBitrateMs = 3500; + const debounceInterval = 20; /** @@ -71,17 +86,51 @@ export function findTrackCodecPayload( * high-bitrate track; screen share is exempt, because its content needs the bitrate * immediately to stay legible. * - * TODO: adjust dynamically from network conditions (e.g. a previous BWE estimate) rather - * than a fixed cap. + * The cap is lowered for a connection that was slow to set up (`computeStartBitrateCap`). + * Screen share is exempt from that ramp as well, however slow the setup. * * @internal */ -export function computeTrackStartBitrate(trackbr: TrackBitrateInfo): number | undefined { +export function computeTrackStartBitrate( + trackbr: TrackBitrateInfo, + connectionSetupTimeMs?: number, +): number | undefined { if (trackbr.maxbr < minTargetBitrateKbps) { return undefined; } - const calculated = Math.round(trackbr.maxbr * startBitrateMultiplier); - return trackbr.isScreenShare ? calculated : Math.min(calculated, maxStartBitrateKbps); + const calculated = Math.min(Math.round(trackbr.maxbr * startBitrateMultiplier), trackbr.maxbr); + const cap = computeStartBitrateCap(connectionSetupTimeMs); + if (trackbr.isScreenShare) { + return calculated; + } + return Math.min(calculated, cap); +} + +/** + * Cap on the start bitrate hint, in kbps, from how long the connection took to set up. + * + * Connection setup time (signaling join plus ICE/DTLS) is the only network signal there is + * before the first video offer, since libwebrtc cannot probe the path until a video sender + * exists. It grows with round-trip time and loss, which also mark the links where a 1 Mbps + * seed overshoots, so the cap ramps linearly from `maxStartBitrateKbps` at + * `setupTimeForMaxStartBitrateMs` down to `minTargetBitrateKbps` at + * `setupTimeForMinStartBitrateMs`. A video offer built while the connection is still being set + * up (publishing during connect) passes the time elapsed so far, a lower bound on the setup time. + * Without any timing the cap stays at the 1 Mbps ceiling. + */ +function computeStartBitrateCap(connectionSetupTimeMs?: number): number { + if (connectionSetupTimeMs === undefined) { + return maxStartBitrateKbps; + } + const ramp = Math.min( + Math.max( + (connectionSetupTimeMs - setupTimeForMaxStartBitrateMs) / + (setupTimeForMinStartBitrateMs - setupTimeForMaxStartBitrateMs), + 0, + ), + 1, + ); + return Math.round(maxStartBitrateKbps - ramp * (maxStartBitrateKbps - minTargetBitrateKbps)); } /** @@ -114,6 +163,7 @@ export function computeTrackStartBitrate(trackbr: TrackBitrateInfo): number | un export function computeConnectionStartBitrate( media: MediaDescription[], trackBitrates: TrackBitrateInfo[], + connectionSetupTimeMs?: number, ): number | undefined { let connectionStartBitrate: number | undefined; for (const m of media) { @@ -128,7 +178,8 @@ export function computeConnectionStartBitrate( if (codecPayload === undefined) { continue; } - const startBitrate = codecPayload > 0 ? computeTrackStartBitrate(trackbr) : undefined; + const startBitrate = + codecPayload > 0 ? computeTrackStartBitrate(trackbr, connectionSetupTimeMs) : undefined; if ( startBitrate !== undefined && (connectionStartBitrate === undefined || startBitrate > connectionStartBitrate) @@ -234,6 +285,20 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter */ private hasAppliedVideoStartBitrate = false; + /** + * How long this peer connection took to set up, in ms, recorded once the initial connect + * succeeds. Lowers the start bitrate hint for a slow connection; see `computeStartBitrateCap`. + */ + private connectionSetupTimeMs?: number; + + /** + * When the connection attempt that built this peer connection began (`performance.now()`). + * Until `connectionSetupTimeMs` is known, the time elapsed since then is used as a lower bound + * on the setup time, so a video published during connect is still seeded from it rather than + * waiting for the connection, which would delay the offer. + */ + private connectionStartedAt?: number; + remoteStereoMids: string[] = []; remoteNackMids: string[] = []; @@ -584,7 +649,11 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter // published (data channel or audio only) find no target and leave the latch unset. const connectionStartBitrate = this.hasAppliedVideoStartBitrate ? undefined - : computeConnectionStartBitrate(sdpParsed.media, this.trackBitrates); + : computeConnectionStartBitrate( + sdpParsed.media, + this.trackBitrates, + this.connectionSetupTimeEstimateMs(), + ); let appliedVideoStartBitrate = false; sdpParsed.media.forEach((media) => { ensureIPAddrMatchVersion(media); @@ -641,6 +710,10 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter // one-shot hint once the SDP carrying it has been accepted locally. if (appliedVideoStartBitrate && offer.sdp === mungedSdp) { this.hasAppliedVideoStartBitrate = true; + this.log.info('applied x-google-start-bitrate', { + startBitrateKbps: connectionStartBitrate, + connectionSetupTimeMs: this.connectionSetupTimeEstimateMs(), + }); } this.onOffer(offer, this.latestOfferId); } finally { @@ -695,6 +768,30 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter this.trackBitrates.push(info); } + /** + * Record how long this peer connection took to set up. Called once, when the initial connect + * succeeds; resumes and ICE restarts keep the estimator they have and never call this. + */ + setConnectionSetupTime(ms: number) { + this.connectionSetupTimeMs = ms; + } + + /** Record when the connection attempt that built this peer connection began. */ + setConnectionStartedAt(startedAt: number) { + this.connectionStartedAt = startedAt; + } + + /** The measured setup time, or while still connecting, the time elapsed so far. */ + private connectionSetupTimeEstimateMs(): number | undefined { + if (this.connectionSetupTimeMs !== undefined) { + return this.connectionSetupTimeMs; + } + if (this.connectionStartedAt === undefined) { + return undefined; + } + return Math.round(performance.now() - this.connectionStartedAt); + } + setConfiguration(rtcConfig: RTCConfiguration) { if (!this._pc) { throw new UnexpectedConnectionState('PC closed, cannot configure'); diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index 332c97ddb7..156b38f436 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -236,6 +236,14 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit /** keeps track of how often an initial join connection has been tried */ private joinAttempts: number = 0; + /** + * When the current join attempt began (`performance.now()`), taken at the top of `join` before + * the signal connection or the peer connections exist. Every attempt (a join retry, or the join + * inside a full restart) times itself from scratch. Consumed when the primary transport first + * connects, to hand the publisher its connection setup time; resumes never set it. + */ + private joinStartedAt?: number; + /** specifies how often an initial join connection is allowed to retry */ private maxJoinAttempts: number = 1; @@ -352,6 +360,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit this.maxJoinAttempts = opts.maxRetries; try { this.joinAttempts += 1; + this.joinStartedAt = performance.now(); this.setupSignalClientCallbacks(); // Whether the initial publisher offer is bundled with the join request. Computed once and @@ -420,6 +429,12 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } } + if (this.joinStartedAt !== undefined) { + // Lets a video published before the transports connect be seeded from the time elapsed + // so far, rather than delaying its offer until the setup time is measured. + this.pcManager?.publisher.setConnectionStartedAt(this.joinStartedAt); + } + this.registerOnLineListener(); this.clientConfiguration = joinResponse.clientConfiguration; this.emit(EngineEvent.SignalConnected, joinResponse); @@ -471,6 +486,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit try { this._isClosed = true; this.joinAttempts = 0; + this.joinStartedAt = undefined; this.emit(EngineEvent.Closing); this.removeAllListeners(); this.deregisterOnLineListener(); @@ -629,6 +645,16 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit this.publisherConnectionPromise = undefined; } if (connectionState === PCTransportState.CONNECTED) { + if (this.joinStartedAt !== undefined) { + // Connection setup time is the only network signal there is before the first video + // offer. Set synchronously, so it is in place before `room.connect()` resolves and the + // app can publish. A full restart joins again and builds new transports, so it is + // measured again; a resume keeps its estimator and never restarts the timer. + const setupTimeMs = Math.round(performance.now() - this.joinStartedAt); + this.joinStartedAt = undefined; + this.log.info(`connection setup took ${setupTimeMs} ms`); + this.pcManager?.publisher.setConnectionSetupTime(setupTimeMs); + } const shouldEmit = this.pcState === PCState.New; this.pcState = PCState.Connected; if (shouldEmit) {