Skip to content
Merged
182 changes: 152 additions & 30 deletions apps/sandbox/templates/moq-publisher/fake-media.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ const FAKE_AUDIO_BITRATE = 1.28e5;
* `internal/design/spf/features/publisher-multi-source-capture.md`). That
* extra independence isn't this fake's job to demonstrate; it only needs to
* satisfy the additive capability contract for the sandbox's `?fake` mode.
* `micActive` runs a third, audio-only pipeline for the explicit mic seam,
* so a mic-only publish is drivable here too.
*/
export class FakePublishMedia
extends HTMLVideoElementHost
Expand All @@ -48,15 +50,22 @@ export class FakePublishMedia

#cameraActive = false;
#screenShareActive = false;
#micActive = false;
#cameraState: MediaCaptureState = 'idle';
#screenShareState: MediaCaptureState = 'idle';
#cameraStream: MediaStream | null = null;
#screenShareStream: MediaStream | null = null;
/** Owned by the explicit mic pipeline; never exposed or previewed. */
#micStream: MediaStream | null = null;
/** Bumped per kind to cancel in-flight acquisitions when it's re-toggled. */
#cameraAcquireToken = 0;
#screenAcquireToken = 0;
#micAcquireToken = 0;
/** Owned: losing the merged audio track doesn't end its stream, so `micState` has nothing live to read 'ended' off. */
#micEnded = false;
/** Owned: the explicit pipeline's in-flight/denied phases have no stream to derive from either. */
#micAcquiring = false;
#micDenied = false;

#captureDevices: MediaCaptureDeviceInfo[] = [];
#videoInputDeviceId = '';
Expand Down Expand Up @@ -101,12 +110,14 @@ export class FakePublishMedia
this.#disposed.abort();
this.#cameraAcquireToken++;
this.#screenAcquireToken++;
this.#micAcquireToken++;
this.#clearPublishTimer();
this.#stopStats();
this.#pendingPublish?.reject(new Error('The media host was destroyed.'));
this.#pendingPublish = null;
this.#releaseStream('camera');
this.#releaseStream('screen');
this.#releaseMicStream();
super.destroy();
}

Expand All @@ -127,9 +138,7 @@ export class FakePublishMedia
this.#cameraAcquireToken++;
this.#releaseStream('camera');
this.#setCaptureState('camera', 'idle');
if (!this.#screenShareActive && (this.#publishState === 'connecting' || this.#publishState === 'live')) {
this.unpublish();
}
this.#unpublishWithoutSources();
}
}

Expand All @@ -148,9 +157,30 @@ export class FakePublishMedia
this.#screenAcquireToken++;
this.#releaseStream('screen');
this.#setCaptureState('screen', 'idle');
if (!this.#cameraActive && (this.#publishState === 'connecting' || this.#publishState === 'live')) {
this.unpublish();
}
this.#unpublishWithoutSources();
}
}

get micActive(): boolean {
return this.#micActive;
}

set micActive(value: boolean) {
if (value === this.#micActive) return;
this.#micActive = value;
this.dispatchEvent(new Event('capturesourcechange'));

if (value) {
void this.#acquireMic();
} else {
this.#micAcquireToken++;
this.#updateMicState(() => {
this.#releaseMicStream();
this.#micAcquiring = false;
this.#micDenied = false;
this.#micEnded = false;
});
this.#unpublishWithoutSources();
}
}

Expand All @@ -163,16 +193,20 @@ export class FakePublishMedia
}

/**
* Mostly derived: this fake merges mic audio into each capture stream
* (see the header note) instead of running an independent mic pipeline,
* so 'active' reads straight off the live streams. 'ended' needs the
* owned `#micEnded` flag — losing the audio track removes it from its
* stream without ending the stream itself, so there's nothing live to
* derive 'ended' from.
* Mostly derived: this fake merges mic audio into each video capture
* stream (see the header note), and the explicit `micActive` pipeline
* owns `#micStream`, so 'active' reads straight off the live streams.
* The remaining phases need owned flags — losing a merged audio track
* removes it from its stream without ending the stream itself, and the
* explicit pipeline's acquiring/denied phases have no stream at all.
*/
get micState(): MediaCaptureState {
const hasAudio = (stream: MediaStream | null) => (stream?.getAudioTracks().length ?? 0) > 0;
if (hasAudio(this.#cameraStream) || hasAudio(this.#screenShareStream)) return 'active';
if (hasAudio(this.#micStream) || hasAudio(this.#cameraStream) || hasAudio(this.#screenShareStream)) {
return 'active';
}
if (this.#micAcquiring) return 'acquiring';
if (this.#micDenied) return 'denied';
return this.#micEnded ? 'ended' : 'idle';
}

Expand Down Expand Up @@ -208,6 +242,7 @@ export class FakePublishMedia
if (value === this.#audioInputDeviceId) return;
this.#audioInputDeviceId = value;
this.#reacquireCamera();
this.#reacquireMic();
}

// ── MediaCaptureToggleCapability ────────────────────────────────────────
Expand All @@ -232,6 +267,7 @@ export class FakePublishMedia
this.#micMuted = value;
for (const track of this.#cameraStream?.getAudioTracks() ?? []) track.enabled = !value;
for (const track of this.#screenShareStream?.getAudioTracks() ?? []) track.enabled = !value;
for (const track of this.#micStream?.getAudioTracks() ?? []) track.enabled = !value;
this.dispatchEvent(new Event('capturetogglechange'));
}

Expand All @@ -253,7 +289,11 @@ export class FakePublishMedia
if (this.#publishState === 'connecting' || this.#publishState === 'live' || this.#publishState === 'stopping') {
return Promise.reject(new Error(`publish() is not allowed while the session is ${this.#publishState}.`));
}
if (this.#cameraState !== 'active' && this.#screenShareState !== 'active') {
if (
this.#cameraState !== 'active' &&
this.#screenShareState !== 'active' &&
!(this.#micActive && this.micState === 'active')
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
) {
this.#publishError = { code: 0, message: 'publish() requires an active capture source.' };
this.#setPublishState('error');
return Promise.reject(new Error(this.#publishError.message));
Expand Down Expand Up @@ -337,22 +377,93 @@ export class FakePublishMedia
* click. Sets the backing field directly — the setter's release path
* would clobber the terminal state the UI needs.
*/
#consumeIntent(kind: MediaCaptureSourceKind): void {
#consumeIntent(kind: MediaCaptureSourceKind | 'mic'): void {
if (kind === 'camera') this.#cameraActive = false;
else this.#screenShareActive = false;
else if (kind === 'screen') this.#screenShareActive = false;
else this.#micActive = false;
this.dispatchEvent(new Event('capturesourcechange'));
// Every involuntary-release caller (acquire failure, track-ended) routes
// through here, so this is the one place that can see "no source left" —
// a live session with nothing capturing must not stay live.
if (
!this.#cameraActive &&
!this.#screenShareActive &&
(this.#publishState === 'connecting' || this.#publishState === 'live')
) {
this.unpublish();
// through here — a live session with nothing capturing must not stay live.
this.#unpublishWithoutSources();
}

/** Shared by intent consumption and the consumer-release setters: no intent left means no session. */
#unpublishWithoutSources(): void {
if (this.#cameraActive || this.#screenShareActive || this.#micActive) return;
if (this.#publishState === 'connecting' || this.#publishState === 'live') this.unpublish();
}

/** The explicit audio-only pipeline behind `micActive`; video capture keeps merging its own mic audio. */
async #acquireMic(): Promise<void> {
const token = ++this.#micAcquireToken;
this.#updateMicState(() => {
this.#releaseMicStream();
this.#micAcquiring = true;
this.#micDenied = false;
});

try {
const stream = await navigator.mediaDevices.getUserMedia({
audio: this.#audioInputDeviceId ? { deviceId: this.#audioInputDeviceId } : true,
});
if (token !== this.#micAcquireToken) {
for (const track of stream.getTracks()) track.stop();
return;
}
for (const track of stream.getAudioTracks()) track.enabled = !this.#micMuted;
this.#updateMicState(() => {
this.#micAcquiring = false;
this.#micEnded = false;
this.#micStream = stream;
});
this.#watchMicTracks(stream);
// Device labels stay redacted until a grant, so refresh after acquiring.
void this.#refreshDevices();
} catch (error) {
if (token !== this.#micAcquireToken) return;
this.#updateMicState(() => {
this.#micAcquiring = false;
this.#micDenied = error instanceof DOMException && error.name === 'NotAllowedError';
});
this.#consumeIntent('mic');
}
}

/**
* Mirrors `#setCaptureState`'s dedupe for the derived mic lifecycle:
* merged video audio can hold `micState` at 'active' across explicit
* pipeline changes, and an unchanged state must not fire the event.
*/
#updateMicState(mutate: () => void): void {
const before = this.micState;
mutate();
if (this.micState !== before) this.dispatchEvent(new Event('capturestatechange'));
}

#watchMicTracks(stream: MediaStream): void {
for (const track of stream.getTracks()) {
track.addEventListener(
'ended',
() => {
if (this.#micStream !== stream) return;
this.#updateMicState(() => {
this.#releaseMicStream();
this.#micEnded = true;
});
this.#consumeIntent('mic');
},
{ once: true }
);
}
}

#releaseMicStream(): void {
const stream = this.#micStream;
if (!stream) return;
this.#micStream = null;
for (const track of stream.getTracks()) track.stop();
}

async #getCameraStream(): Promise<MediaStream> {
return navigator.mediaDevices.getUserMedia({
video: this.#videoInputDeviceId ? { deviceId: this.#videoInputDeviceId } : true,
Expand Down Expand Up @@ -397,7 +508,7 @@ export class FakePublishMedia
else this.#screenShareStream = null;
for (const track of stream.getTracks()) track.stop();
// The last source is gone — clean slate, back to 'idle' rather than a stale 'ended' outliving the capture that produced it.
if (!this.#cameraStream && !this.#screenShareStream) this.#micEnded = false;
if (!this.#cameraStream && !this.#screenShareStream && !this.#micStream) this.#micEnded = false;
this.dispatchEvent(new Event('capturestreamchange'));
this.#syncPreview();
}
Expand Down Expand Up @@ -438,6 +549,12 @@ export class FakePublishMedia
void this.#acquire('camera');
}

#reacquireMic(): void {
if (!this.#micActive) return;
if (!this.#micStream && !this.#micAcquiring) return;
void this.#acquireMic();
}

async #refreshDevices(): Promise<void> {
if (!navigator.mediaDevices?.enumerateDevices) return;
const devices = await navigator.mediaDevices.enumerateDevices();
Expand Down Expand Up @@ -495,13 +612,18 @@ export class FakePublishMedia
let droppedFrames = 0;

this.#statsTimer = setInterval(() => {
const videoBitrate = FAKE_VIDEO_BITRATE * (0.92 + Math.random() * 0.16);
const audioBitrate = FAKE_AUDIO_BITRATE;
bytesSent += Math.round((videoBitrate + audioBitrate) / 8);
if (Math.random() < 0.1) droppedFrames += 1;
// Source-aware per sample, so a mic-only session reports no video
// traffic (the stats contract uses NaN for absent metrics) and a
// video source joining mid-session starts counting.
const hasVideo = this.cameraStream !== null || this.screenShareStream !== null;
const hasAudio = this.micState === 'active';
const videoBitrate = hasVideo ? FAKE_VIDEO_BITRATE * (0.92 + Math.random() * 0.16) : Number.NaN;
const audioBitrate = hasAudio ? FAKE_AUDIO_BITRATE : Number.NaN;
bytesSent += Math.round(((hasVideo ? videoBitrate : 0) + (hasAudio ? audioBitrate : 0)) / 8);
if (hasVideo && Math.random() < 0.1) droppedFrames += 1;

this.#publishStats = {
encodedFps: Math.round(29 + Math.random() * 2),
encodedFps: hasVideo ? Math.round(29 + Math.random() * 2) : Number.NaN,
videoBitrate,
audioBitrate,
droppedFrames,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export interface CapturePlaceholderState extends ButtonState {
* Core state machine for the placeholder shown over the preview area before
* any capture source is active. Its label is an enable-devices CTA while
* `idle` or `ended`, a progress hint while `acquiring`, permission guidance
* when `denied`, and empty once either source is `active`.
* when `denied`, and empty once any counted source is `active`.
*/
export class CapturePlaceholderCore {
static readonly defaultProps: NonNullableObject<CapturePlaceholderProps> = {
Expand Down Expand Up @@ -67,7 +67,7 @@ export class CapturePlaceholderCore {

getState(): CapturePlaceholderState {
const media = this.#media!;
this.state.patch({ captureState: aggregateCaptureState(media.cameraState, media.screenShareState) });
this.state.patch({ captureState: aggregateCaptureState(media) });
this.state.patch({ label: resolveText(this.getLabel(this.state.current)) });

return this.state.current;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,15 @@ function createMediaState(overrides: Partial<MediaCaptureSourceState> = {}): Med
return {
cameraActive: false,
screenShareActive: false,
micActive: false,
micExplicit: false,
cameraState: 'idle',
screenShareState: 'idle',
micState: 'idle',
screenShareAvailability: 'available',
toggleCamera: vi.fn(() => true),
toggleScreenShare: vi.fn(() => true),
toggleMic: vi.fn(() => true),
...overrides,
};
}
Expand Down Expand Up @@ -44,6 +47,35 @@ describe('CapturePlaceholderCore', () => {
expect(core.getState().captureState).toBe('active');
});

it('clears the placeholder for a mic-only capture', () => {
const core = new CapturePlaceholderCore();
core.setMedia(createMediaState({ micState: 'active', micActive: true, micExplicit: true }));

const state = core.getState();
expect(state.captureState).toBe('active');
expect(state.label).toBe('');
});

it('keeps the CTA up when only an implied mic is live', () => {
const core = new CapturePlaceholderCore();
core.setMedia(createMediaState({ micState: 'active', micExplicit: false }));

const state = core.getState();
expect(state.captureState).toBe('idle');
expect(state.label).toBe('Enable camera and microphone');
});

it('shows permission guidance after a mic-only denial consumes the intent', () => {
// The acquire pipeline sets micActive back to false on denial while
// parking micState — the guidance must survive that consumption.
const core = new CapturePlaceholderCore();
core.setMedia(createMediaState({ micState: 'denied', micExplicit: true }));

const state = core.getState();
expect(state.captureState).toBe('denied');
expect(state.label).toBe('Camera and microphone access is blocked. Update your browser permissions to continue.');
});

it('prefers the more in-progress status when the two disagree', () => {
const core = new CapturePlaceholderCore();
core.setMedia(createMediaState({ cameraState: 'idle', screenShareState: 'acquiring' }));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ export class EnableDevicesButtonCore {
const media = this.#media!;

this.state.patch({
captureState: aggregateCaptureState(media.cameraState, media.screenShareState),
captureState: aggregateCaptureState(media),
// Disabled only while the CAMERA pipeline is mid-acquisition — the
// sources are independently acquirable, so an open screen-share
// picker must not grey out the camera CTA (matching activate()'s
Expand Down
Loading
Loading