diff --git a/.papercuts/troubleshooting.md b/.papercuts/troubleshooting.md index 9893aa1f5..6da2a690f 100644 --- a/.papercuts/troubleshooting.md +++ b/.papercuts/troubleshooting.md @@ -525,3 +525,6 @@ symlink with this checkout's own npm ci. Full type-check and lint then passed. - Live cue tests must flush passive React effects after microphone and stop state updates before asserting audio feedback. The repo has no local Prettier binary; use its configured ESLint validation instead. - Real signed Live session reached open/microphone-ready/input-first-packet, then Google emitted an empty top-level envelope at 00:20:01 UTC on 2026-09-17. Rejecting `{}` caused the observed silent disconnect. Admit exact empty envelopes as rate-limited no-ops without extending idle timeout; keep unknown populated fields rejected. Terminal error HUDs must remain visible after active becomes false. - The Mac's default input was Bose Mini II while output was MacBook speakers. Packet flow alone does not prove intelligible user speech or audible playback; retain separate operator verification. +- Pullfrog completed only after GitHub had already accepted the exact-head merge, and it found release-blocking screen-capture races. Keep publication cancellable until external review finishes, even when first-party CI and the merge gate are green. +- Electron exposes setters but no getters for session permission handlers. Scope the Live handler to the lifetime of exact display bindings, explicitly admit only display capture and microphone for that bound document, then restore the default handlers when the last binding disappears. +- Electron's `media` permission covers camera as well as microphone; check `mediaType` on permission checks and exact `mediaTypes: ["audio"]` on permission requests instead of treating the permission name as audio-only. diff --git a/docs/plans/gemini-live-assistant-plan.md b/docs/plans/gemini-live-assistant-plan.md index f9f60cccd..f227fe395 100644 --- a/docs/plans/gemini-live-assistant-plan.md +++ b/docs/plans/gemini-live-assistant-plan.md @@ -3,6 +3,8 @@ Status: Partial — Phases 0–4 plus the beta-labeled Aiden Live orb/setup shell and dedicated metadata-only session threads are implemented; authorized macOS screen capture and real Google beta receipts remain operator-owned Date: 2026-09-15 Continuous orb follow-up (2026-09-17): active listening/thinking/speaking/acting now share stable Listening + Weaving canvas layers with a slow complementary blend and connection/rest crossfades. Removed visible dock status text (retained screen-reader status). First active click reveals Stop; the second stops, Escape restores the orb, and closing disables duplicate clicks. Sharing/error surfaces remain visible. Added mapping/click-policy regressions; all 230 Live tests, type-check, lint, and standalone browser interaction/motion checks pass. Downloads duplex-review.html uses the actual orb component; the installed notarized app has not been rebuilt for this change. + +Release hardening follow-up (2026-09-17): Pullfrog's post-merge review blocked 0.41.4 publication until screen authority is generation-fenced. Exact binding tokens now prevent stale releases from revoking replacements, late picker streams stop after teardown, rejected frame sends revoke capture, and temporary Electron permission handlers admit only audio-only media for the exact document before restoring defaults when the final binding closes. Source Change/Remove is locked across pending provider startup and start failure releases the exact current binding. Adversarial coverage was added for each path. Combined artifact verification (2026-09-17): Apple accepted notarization submission `5b7f74d8-4740-47b9-82d8-66704cb75ec7`; the Downloads Combined Notarized app has a stapled ticket and passes the repository notarized-package/Gatekeeper check. Opened with the preserved isolated Test Profile and verified its renderer and Start Aiden Live control. Full Live suites, type-check, lint, audio/settings tests, and Electron selector-layout regression passed. Native-picker and real-provider acceptance remain separate and pending. Combined build (2026-09-17): integrated the screen-sharing work from the 2f30 worktree into this audio-fix checkout without changing the source worktree. Preserved extended-thinking model selection, dedicated session threads, async thread finalization, direct actions, diagnostic markers, empty-envelope handling, device routing, and dropdown layout. Screen capture remains separately gated and off in the ordinary test launcher pending the exact-build native-picker acceptance receipt. Notarization and attended acceptance are separate gates; this is not a public release. Audio selector layout follow-up: scoped single-line trigger styling and actual Radix value-span ellipsis fix long device labels pushing chevrons outside the control. Real Electron geometry regression passes at 1280, 600, and 390px; the full label remains available through the dropdown and hover title. diff --git a/main/handlers/assistant-live-parse.test.ts b/main/handlers/assistant-live-parse.test.ts index 28d80b662..86035bb33 100644 --- a/main/handlers/assistant-live-parse.test.ts +++ b/main/handlers/assistant-live-parse.test.ts @@ -5,6 +5,7 @@ import { parseAssistantLiveAudioIntent, parseAssistantLiveEmptyIntent, parseAssistantLiveFrameIntent, + parseAssistantLiveDisplayReleaseIntent, parseAssistantLiveStartIntent, parseAssistantLiveStopIntent, } from "./assistant-live-parse.js"; @@ -74,8 +75,8 @@ test("Assistant Live frame admission accepts only one bounded copied JPEG byte r ]) assert.throws(() => parseAssistantLiveFrameIntent(value), /frame request/u); }); -test("Assistant Live display bind/release accept only an exact empty record", () => { - for (const name of ["display-bind", "display-release"]) { +test("Assistant Live display bind accepts only an exact empty record", () => { + for (const name of ["display-bind"]) { assert.doesNotThrow(() => parseAssistantLiveEmptyIntent({}, name)); assert.throws( () => parseAssistantLiveEmptyIntent({ sessionId: "forged" }, name), @@ -87,3 +88,15 @@ test("Assistant Live display bind/release accept only an exact empty record", () ); } }); + +test("Assistant Live display release requires one bounded binding token", () => { + assert.deepEqual(parseAssistantLiveDisplayReleaseIntent({ bindingId: "binding-1" }), { + bindingId: "binding-1", + }); + for (const value of [{}, { bindingId: "" }, { bindingId: "x", extra: true }, null]) { + assert.throws( + () => parseAssistantLiveDisplayReleaseIntent(value), + /Invalid Assistant Live display-release/u, + ); + } +}); diff --git a/main/handlers/assistant-live-parse.ts b/main/handlers/assistant-live-parse.ts index 11bcb2c86..6c43026b3 100644 --- a/main/handlers/assistant-live-parse.ts +++ b/main/handlers/assistant-live-parse.ts @@ -87,3 +87,18 @@ export function parseAssistantLiveEmptyIntent( throw new Error(`Invalid Assistant Live ${name} request.`); } } + +export function parseAssistantLiveDisplayReleaseIntent( + value: unknown, +): { bindingId: string } { + if ( + !isRecord(value) || + !exactKeys(value, ["bindingId"]) || + typeof value.bindingId !== "string" || + value.bindingId.length < 1 || + value.bindingId.length > 128 + ) { + throw new Error("Invalid Assistant Live display-release request."); + } + return { bindingId: value.bindingId }; +} diff --git a/main/handlers/assistant-live.ts b/main/handlers/assistant-live.ts index 8a27d8116..27c0d205e 100644 --- a/main/handlers/assistant-live.ts +++ b/main/handlers/assistant-live.ts @@ -1,4 +1,3 @@ -import { session } from "electron"; import { ipcMain } from "../platform.js"; import { authorizeAidenLiveComputerUse, @@ -7,13 +6,13 @@ import { import { rendererDocumentOwner } from "../services/renderer-document-owner.js"; import { bindGeminiLiveDisplayMediaDocument, - installGeminiLiveDisplayMediaGuards, } from "../services/gemini-live/display-media-contract.js"; import { parseAssistantLiveStartIntent, parseAssistantLiveAudioIntent, parseAssistantLiveEmptyIntent, parseAssistantLiveFrameIntent, + parseAssistantLiveDisplayReleaseIntent, parseAssistantLiveStopIntent, } from "./assistant-live-parse.js"; import { invokeAssistantLiveStart } from "./assistant-live-start.js"; @@ -53,18 +52,11 @@ export function registerAssistantLiveHandlers(): void { ipcMain.handle("assistant-live:display-bind", (event, input: unknown) => { parseAssistantLiveEmptyIntent(input, "display-bind"); const binding = bindGeminiLiveDisplayMediaDocument(event); - if (!geminiLiveService.bindDisplayMedia(owner(event), binding)) return false; - // The guards consult the live binding set, so install them only after this - // document's binding is registered and only once per Electron session. - installGeminiLiveDisplayMediaGuards(session.defaultSession, () => - geminiLiveService.displayMediaBindings(), - ); - return true; + return geminiLiveService.bindDisplayMedia(owner(event), binding); }); ipcMain.handle("assistant-live:display-release", (event, input: unknown) => { - parseAssistantLiveEmptyIntent(input, "display-release"); - geminiLiveService.releaseDisplayMedia(owner(event)); - return true; + const intent = parseAssistantLiveDisplayReleaseIntent(input); + return geminiLiveService.releaseDisplayMedia(owner(event), intent.bindingId); }); ipcMain.handle("assistant-live:frame", (event, input: unknown) => { const intent = parseAssistantLiveFrameIntent(input); diff --git a/main/services/gemini-live/display-media-contract.test.ts b/main/services/gemini-live/display-media-contract.test.ts index 7e9482899..46aed4dba 100644 --- a/main/services/gemini-live/display-media-contract.test.ts +++ b/main/services/gemini-live/display-media-contract.test.ts @@ -96,6 +96,15 @@ test("binds both custom-picker and system-picker permission admission to one exa binding.allowsPermissionRequest(sender as unknown as Electron.WebContents, "media", { isMainFrame: true, requestingUrl: frame.url, + mediaType: "audio", + }), + true, + ); + assert.equal( + binding.allowsPermissionRequest(sender as unknown as Electron.WebContents, "media", { + isMainFrame: true, + requestingUrl: frame.url, + mediaType: "video", }), false, ); @@ -158,14 +167,21 @@ test("session guards gate display-capture only and install exactly once", () => const frame = new FakeFrame(10, 20, "document-one", "file:///Aiden/main-window.html"); const sender = new FakeWebContents(7, frame); const bindings: GeminiLiveDisplayMediaBinding[] = []; - installGeminiLiveDisplayMediaGuards(electronSession, () => bindings); - installGeminiLiveDisplayMediaGuards(electronSession, () => bindings); + const dispose = installGeminiLiveDisplayMediaGuards(electronSession, () => bindings); + assert.equal( + installGeminiLiveDisplayMediaGuards(electronSession, () => bindings), + dispose, + ); assert.equal(installed.checks, 1, "re-installation must not stack handlers"); assert.equal(installed.requests, 1); assert.equal(installed.displays, 1); assert.deepEqual(installed.opts, { useSystemPicker: true }); const details = { isMainFrame: true, requestingUrl: frame.url } as Electron.PermissionRequest; + const audioCheckDetails = { ...details, mediaType: "audio" } as Electron.PermissionCheckHandlerHandlerDetails; + const videoCheckDetails = { ...details, mediaType: "video" } as Electron.PermissionCheckHandlerHandlerDetails; + const audioRequestDetails = { ...details, mediaTypes: ["audio"] } as Electron.MediaAccessPermissionRequest; + const videoRequestDetails = { ...details, mediaTypes: ["video"] } as Electron.MediaAccessPermissionRequest; // Without a Live binding every display-capture path denies. assert.equal( installed.check?.( @@ -187,8 +203,9 @@ test("session guards gate display-capture only and install exactly once", () => ); assert.equal(granted, false); - // A bound document admits display-capture; every other permission keeps the - // session's default allow so the guards never shrink unrelated authority. + // A bound document admits only its display capture and microphone. Unrelated + // contents and unrelated permission types remain denied while the temporary + // guard owns the session policy. bindings.push(bindGeminiLiveDisplayMediaDocument(invokeEvent(sender, frame))); assert.equal( installed.check?.( @@ -200,15 +217,29 @@ test("session guards gate display-capture only and install exactly once", () => true, ); assert.equal( - installed.check?.(null, "media", "file:///Aiden/", details), + installed.check?.(sender as unknown as Electron.WebContents, "media", "file:///Aiden/", audioCheckDetails), true, - "non-display permissions keep the default policy", + "the exact bound document keeps microphone access", ); granted = null; installed.request?.(sender as unknown as Electron.WebContents, "media", (next) => { granted = next; - }, details); + }, audioRequestDetails); assert.equal(granted, true); + assert.equal( + installed.check?.(sender as unknown as Electron.WebContents, "media", "file:///Aiden/", videoCheckDetails), + false, + ); + granted = null; + installed.request?.(sender as unknown as Electron.WebContents, "media", (next) => { + granted = next; + }, videoRequestDetails); + assert.equal(granted, false); + assert.equal( + installed.check?.(sender as unknown as Electron.WebContents, "notifications", "file:///Aiden/", details), + false, + ); + assert.equal(installed.check?.(null, "media", "file:///Aiden/", details), false); // Any non-picker dispatch to the fallback handler is denied outright. const streams: Electron.Streams[] = []; @@ -226,6 +257,10 @@ test("session guards gate display-capture only and install exactly once", () => ), false, ); + dispose(); + assert.equal(installed.check, null); + assert.equal(installed.request, null); + assert.equal(installed.display, null); }); test("navigation, replacement frames, and unrelated WebContents fail closed", () => { diff --git a/main/services/gemini-live/display-media-contract.ts b/main/services/gemini-live/display-media-contract.ts index bf5d9b678..d5486c1e0 100644 --- a/main/services/gemini-live/display-media-contract.ts +++ b/main/services/gemini-live/display-media-contract.ts @@ -1,4 +1,5 @@ import { rendererDocumentOwner, type RendererDocumentOwner } from "../renderer-document-owner.js"; +import { randomUUID } from "node:crypto"; export const GEMINI_LIVE_SYSTEM_PICKER_OPTIONS: Electron.DisplayMediaRequestHandlerOpts = { useSystemPicker: true, @@ -7,9 +8,12 @@ export const GEMINI_LIVE_SYSTEM_PICKER_OPTIONS: Electron.DisplayMediaRequestHand interface DisplayPermissionDetails { isMainFrame: boolean; requestingUrl: string; + mediaType?: "video" | "audio" | "unknown"; + mediaTypes?: Array<"video" | "audio">; } export interface GeminiLiveDisplayMediaBinding { + readonly bindingId: string; readonly documentId: string; readonly owner: RendererDocumentOwner; allowsDisplayRequest(request: Electron.DisplayMediaRequestHandlerHandlerRequest): boolean; @@ -68,6 +72,7 @@ export function bindGeminiLiveDisplayMediaDocument( }; return { + bindingId: randomUUID(), documentId: owner.documentId, owner, allowsDisplayRequest: (request) => @@ -78,12 +83,18 @@ export function bindGeminiLiveDisplayMediaDocument( request.frame !== null && liveFrame(request.frame) && sameFrame(request.frame, frame), - allowsPermissionRequest: (webContents, permission, details) => - current() && - webContents === sender && - permission === "display-capture" && - details.isMainFrame === true && - details.requestingUrl === requestingUrl, + allowsPermissionRequest: (webContents, permission, details) => { + const audioOnly = + details.mediaType === "audio" || + (details.mediaTypes?.length === 1 && details.mediaTypes[0] === "audio"); + return ( + current() && + webContents === sender && + (permission === "display-capture" || (permission === "media" && audioOnly)) && + details.isMainFrame === true && + details.requestingUrl === requestingUrl + ); + }, }; } @@ -115,43 +126,53 @@ interface DisplayMediaGuardSession { ): void; } -const guardedSessions = new WeakSet(); +const guardedSessions = new WeakMap void>(); /** * Installs the display-capture boundary once per Electron session. Every - * permission other than display-capture keeps Electron's default allow, and a - * display request succeeds only while a currently bound Live document admits - * it. The system-picker session never dispatches to the fallback handler; any + * display or microphone permission succeeds only for a currently bound Live + * document. Every unrelated permission is denied while this temporary guard + * owns the session policy, and disposal restores Electron's default handlers. + * The system-picker session never dispatches to the fallback handler; any * non-picker dispatch is denied rather than trusted to select a source. */ export function installGeminiLiveDisplayMediaGuards( electronSession: DisplayMediaGuardSession, getBindings: () => readonly GeminiLiveDisplayMediaBinding[], -): void { - if (guardedSessions.has(electronSession)) return; - guardedSessions.add(electronSession); +): () => void { + const installed = guardedSessions.get(electronSession); + if (installed) return installed; + let active = true; + const dispose = () => { + if (!active || guardedSessions.get(electronSession) !== dispose) return; + active = false; + guardedSessions.delete(electronSession); + electronSession.setPermissionCheckHandler(null); + electronSession.setPermissionRequestHandler(null); + electronSession.setDisplayMediaRequestHandler(null); + }; + guardedSessions.set(electronSession, dispose); electronSession.setPermissionCheckHandler( (webContents, permission, _requestingOrigin, details) => { - if (String(permission) !== "display-capture" || !webContents) return true; + if (!webContents) return false; return getBindings().some((binding) => - binding.allowsPermissionRequest(webContents, "display-capture", { + binding.allowsPermissionRequest(webContents, String(permission), { isMainFrame: details.isMainFrame === true, requestingUrl: details.requestingUrl ?? "", + mediaType: details.mediaType, }), ); }, ); electronSession.setPermissionRequestHandler( (webContents, permission, callback, details) => { - if (String(permission) !== "display-capture") { - callback(true); - return; - } + const mediaDetails = details as Electron.MediaAccessPermissionRequest; callback( getBindings().some((binding) => - binding.allowsPermissionRequest(webContents, "display-capture", { + binding.allowsPermissionRequest(webContents, String(permission), { isMainFrame: details.isMainFrame === true, requestingUrl: details.requestingUrl ?? "", + mediaTypes: mediaDetails.mediaTypes, }), ), ); @@ -163,4 +184,5 @@ export function installGeminiLiveDisplayMediaGuards( }, GEMINI_LIVE_SYSTEM_PICKER_OPTIONS, ); + return dispose; } diff --git a/main/services/gemini-live/service-main.ts b/main/services/gemini-live/service-main.ts index 952ae6b70..c2d444c45 100644 --- a/main/services/gemini-live/service-main.ts +++ b/main/services/gemini-live/service-main.ts @@ -15,6 +15,8 @@ import * as path from "node:path"; import { randomUUID } from "node:crypto"; import type { RendererDocumentOwner } from "../renderer-document-owner.js"; import { Behavior } from "@google/genai"; +import { session } from "electron"; +import { installGeminiLiveDisplayMediaGuards } from "./display-media-contract.js"; const aidenLiveThreadStore = new AidenLiveThreadStore(() => path.join(app.getPath("userData"), "aiden-live"), @@ -70,6 +72,8 @@ export async function authorizeAidenLiveComputerUse( const LIVE_COMPUTER_USE_DESCRIPTION = "Use Aiden's Computer Use controller during this user-started Live session. Capture an exact window first. You may operate Aiden itself to focus its main composer, choose the current web model or Actions menu, send a prompt, and create or review scheduled tasks. Execute the user's requested actions directly without per-action approval prompts. Keep speaking naturally while work is in progress. Stop when the session ends or the user cancels. Never claim success before the tool result confirms it."; +let disposeDisplayMediaGuards: (() => void) | null = null; + /** * The acceptance-gated beta resolves only the recorded * `gemini-3.8-live-extended-thinking` model; it @@ -95,6 +99,18 @@ export const geminiLiveService = new GeminiLiveService({ }, resolveModel: () => experimentalGeminiLiveModel(), screenShareEnabled: () => geminiLiveScreenEnabled(), + onDisplayBindingsChanged: (bindings) => { + if (bindings.length > 0 && !disposeDisplayMediaGuards) { + disposeDisplayMediaGuards = installGeminiLiveDisplayMediaGuards( + session.defaultSession, + () => geminiLiveService.displayMediaBindings(), + ); + } else if (bindings.length === 0 && disposeDisplayMediaGuards) { + const dispose = disposeDisplayMediaGuards; + disposeDisplayMediaGuards = null; + dispose(); + } + }, createConnector: (apiKey) => createOwnedGoogleGenAIConnector({ apiKey }), prepareComputerUse: async ({ authorization, owner, sessionId, signal }) => { if (!authorization || signal.aborted || owner.isDestroyed()) return null; diff --git a/main/services/gemini-live/service.test.ts b/main/services/gemini-live/service.test.ts index 77af4b9ff..fcc17cf93 100644 --- a/main/services/gemini-live/service.test.ts +++ b/main/services/gemini-live/service.test.ts @@ -168,8 +168,10 @@ test("shutdown waits for an in-flight thread begin and finalizes its record", as */ function fakeDisplayBinding( owner: FakeOwner, + bindingId = `binding-${owner.documentId}`, ): GeminiLiveDisplayMediaBinding { return { + bindingId, documentId: owner.documentId, owner, allowsDisplayRequest: () => true, @@ -961,7 +963,7 @@ test("screen frames are admitted only for a bound, gated, open session", async ( const subject = serviceHarness(undefined, { screenShareEnabled: () => true }); const owner = new FakeOwner(35, "35:1:doc"); const binding = fakeDisplayBinding(owner); - assert.equal(subject.service.bindDisplayMedia(owner, binding), true); + assert.equal(subject.service.bindDisplayMedia(owner, binding), binding.bindingId); const session = await subject.service.start(owner, { microphone: false, screen: true, @@ -1023,17 +1025,28 @@ test("screen frames are admitted only for a bound, gated, open session", async ( test("a display binding dies with its document and never survives shutdown", async () => { const subject = serviceHarness(undefined, { screenShareEnabled: () => true }); const owner = new FakeOwner(37, "37:1:doc"); - assert.equal(subject.service.bindDisplayMedia(owner, fakeDisplayBinding(owner)), true); + assert.equal(subject.service.bindDisplayMedia(owner, fakeDisplayBinding(owner)), `binding-${owner.documentId}`); owner.navigate(); assert.deepEqual(subject.service.displayMediaBindings(), []); const next = new FakeOwner(38, "38:1:doc"); - assert.equal(subject.service.bindDisplayMedia(next, fakeDisplayBinding(next)), true); + assert.equal(subject.service.bindDisplayMedia(next, fakeDisplayBinding(next)), `binding-${next.documentId}`); subject.service.shutdown(); assert.deepEqual(subject.service.displayMediaBindings(), []); assert.equal( subject.service.bindDisplayMedia(next, fakeDisplayBinding(next)), - false, + null, "a shut-down service never binds new capture authority", ); }); + +test("an older display release cannot revoke its replacement binding", () => { + const subject = serviceHarness(undefined, { screenShareEnabled: () => true }); + const owner = new FakeOwner(39, "39:1:doc"); + assert.equal(subject.service.bindDisplayMedia(owner, fakeDisplayBinding(owner, "old")), "old"); + assert.equal(subject.service.bindDisplayMedia(owner, fakeDisplayBinding(owner, "new")), "new"); + assert.equal(subject.service.releaseDisplayMedia(owner, "old"), false); + assert.equal(subject.service.displayMediaBindings()[0]?.bindingId, "new"); + assert.equal(subject.service.releaseDisplayMedia(owner, "new"), true); + assert.deepEqual(subject.service.displayMediaBindings(), []); +}); diff --git a/main/services/gemini-live/service.ts b/main/services/gemini-live/service.ts index f588343dd..de4965038 100644 --- a/main/services/gemini-live/service.ts +++ b/main/services/gemini-live/service.ts @@ -101,6 +101,7 @@ export interface GeminiLiveServiceOptions { * for this build. When absent or false, every screen intent is rejected. */ screenShareEnabled?(): boolean; + onDisplayBindingsChanged?(bindings: readonly GeminiLiveDisplayMediaBinding[]): void; createSessionId?: () => string; acceptanceEvidence?: GeminiLiveAcceptanceEvidenceRecorder | null; threads?: { @@ -139,6 +140,7 @@ interface OwnedLiveSession { state: AssistantLiveSnapshot["state"]; microphone: boolean; screen: boolean; + displayBindingId: string | null; resumptionAudio: Uint8Array[]; resumptionFrame: Uint8Array | null; model?: string; @@ -278,29 +280,35 @@ export class GeminiLiveService { bindDisplayMedia( owner: RendererDocumentOwner, binding: GeminiLiveDisplayMediaBinding, - ): boolean { + ): string | null { if ( this.shuttingDown || this.options.screenShareEnabled?.() !== true || owner.isDestroyed() ) { - return false; + return null; } const key = documentKey(owner); this.displayBindings.get(key)?.dispose(); const dispose = owner.onInvalidated(() => { if (this.displayBindings.get(key)?.binding === binding) { this.displayBindings.delete(key); + this.options.onDisplayBindingsChanged?.(this.displayMediaBindings()); } }); this.displayBindings.set(key, { binding, dispose }); - return true; + this.options.onDisplayBindingsChanged?.(this.displayMediaBindings()); + return binding.bindingId; } - releaseDisplayMedia(owner: RendererDocumentOwner): void { - const entry = this.displayBindings.get(documentKey(owner)); + releaseDisplayMedia(owner: RendererDocumentOwner, bindingId: string): boolean { + const key = documentKey(owner); + const entry = this.displayBindings.get(key); + if (!entry || entry.binding.bindingId !== bindingId) return false; entry?.dispose(); - this.displayBindings.delete(documentKey(owner)); + this.displayBindings.delete(key); + this.options.onDisplayBindingsChanged?.(this.displayMediaBindings()); + return true; } /** Live bindings consulted by the session-level display-media guards. */ @@ -351,6 +359,7 @@ export class GeminiLiveService { throw new GeminiLiveStartError("live_start_failed"); } + const displayBinding = this.displayBindings.get(documentKey(owner))?.binding; const session: OwnedLiveSession = { abort: new AbortController(), documentKey: documentKey(owner), @@ -361,6 +370,7 @@ export class GeminiLiveService { state: "connecting", microphone: _intent.microphone, screen: _intent.screen === true, + displayBindingId: _intent.screen ? (displayBinding?.bindingId ?? null) : null, resumptionAudio: [], resumptionFrame: null, computerUse: null, @@ -566,6 +576,7 @@ export class GeminiLiveService { this.shuttingDown = true; for (const entry of this.displayBindings.values()) entry.dispose(); this.displayBindings.clear(); + this.options.onDisplayBindingsChanged?.([]); for (const session of this.sessions.values()) this.closeSession(session, false, "stopped"); await this.waitForStarts(); const sessions = [...this.pendingThreadFinalization]; @@ -691,7 +702,10 @@ export class GeminiLiveService { session.resumptionFrame = null; // Stopping or replacing a session also revokes this document's authority // to admit display capture; a restart re-binds through the picker flow. - this.releaseDisplayMedia(session.owner); + if (session.displayBindingId) { + this.releaseDisplayMedia(session.owner, session.displayBindingId); + session.displayBindingId = null; + } if (notifyRenderer && !session.owner.isDestroyed()) { try { session.owner.send(ASSISTANT_LIVE_EVENT_CHANNEL, { diff --git a/renderer/components/assistant/assistant-live.tsx b/renderer/components/assistant/assistant-live.tsx index b45cbeb40..008473968 100644 --- a/renderer/components/assistant/assistant-live.tsx +++ b/renderer/components/assistant/assistant-live.tsx @@ -177,7 +177,7 @@ export function AssistantLiveSetupDialog({ : "Optional. Share one screen or window so Aiden can see what you see." } ready={Boolean(live.screenSourceLabel)} - busy={live.screenBusy} + busy={live.screenBusy || live.busy} action="Choose…" onAction={() => void live.chooseScreenSource()} trailing={ @@ -188,7 +188,7 @@ export function AssistantLiveSetupDialog({ variant="transparent" className={ASSISTANT_LIVE_FOCUS_CLASS} onClick={() => void live.chooseScreenSource()} - disabled={live.screenBusy} + disabled={live.screenBusy || live.busy} > Change @@ -197,6 +197,7 @@ export function AssistantLiveSetupDialog({ variant="transparent" className={ASSISTANT_LIVE_FOCUS_CLASS} onClick={live.releaseScreen} + disabled={live.busy} > Remove diff --git a/renderer/components/assistant/assistant-ui.test.tsx b/renderer/components/assistant/assistant-ui.test.tsx index d89d1080d..8ef87629c 100644 --- a/renderer/components/assistant/assistant-ui.test.tsx +++ b/renderer/components/assistant/assistant-ui.test.tsx @@ -278,6 +278,8 @@ test("screen sharing is opt-in, source-labelled, and visibly active in the HUD", assert.match(live, /title="Screen share"/u); assert.match(live, /live\.chooseScreenSource/u); assert.match(live, /live\.releaseScreen/u); + assert.match(live, /busy=\{live\.screenBusy \|\| live\.busy\}/u); + assert.match(live, /disabled=\{live\.screenBusy \|\| live\.busy\}/u); assert.match(live, /Sharing \{live\.screenSourceLabel \?\? "screen"\}/u); assert.match(live, /role="alert"[\s\S]*live\.screenError/u); }); diff --git a/renderer/components/assistant/use-assistant-live.test.tsx b/renderer/components/assistant/use-assistant-live.test.tsx index 9eb6a74fa..7ce47c3b7 100644 --- a/renderer/components/assistant/use-assistant-live.test.tsx +++ b/renderer/components/assistant/use-assistant-live.test.tsx @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; import { LiveAudioDeviceError } from "../../lib/live-audio-devices.js"; +import type { DisplayMediaStream } from "../../lib/gemini-live-media-core.js"; test("Live device preparation failure prevents provider start and surfaces recovery", async () => { const fixture = await mountHook({ prepareAudioSession: async () => { throw new LiveAudioDeviceError("Choose another output in Aiden Live settings."); } }); @@ -1478,7 +1479,8 @@ interface ScreenFixtureOptions { startCalls: Array<{ screen: boolean }>; framesSent: Array<{ sessionId: string; frame: Uint8Array }>; binds: number[]; - releases: number[]; + releases: string[]; + admitFrames: boolean; displayTrack: FakeDisplayTrack; frameSource: { captures: number; @@ -1505,6 +1507,7 @@ function screenDependencies(): { framesSent: [], binds: [], releases: [], + admitFrames: true, displayTrack, frameSource: { captures: 0, @@ -1536,15 +1539,15 @@ function screenDependencies(): { sendAudio: async () => true, bindDisplay: async () => { state.binds.push(1); - return true; + return `binding-${state.binds.length}`; }, - releaseDisplay: async () => { - state.releases.push(1); + releaseDisplay: async (bindingId) => { + state.releases.push(bindingId); return true; }, sendFrame: async (sessionId, frame) => { state.framesSent.push({ sessionId, frame }); - return true; + return state.admitFrames; }, onEvent: () => () => undefined, }; @@ -1639,6 +1642,70 @@ test("a picker failure that is not cancellation surfaces an accessible error", a await fixture.unmount(); }); +test("releasing setup fences and stops a display stream returned by a late picker", async () => { + const { dependencies, state } = screenDependencies(); + const picker = deferred(); + const fixture = await mountHook({ + ...dependencies, + getDisplayMedia: () => picker.promise, + }); + const choosing = fixture.controller().chooseScreenSource(); + await settle(); + assert.deepEqual(state.binds, [1]); + + fixture.controller().releaseScreen(); + picker.resolve({ getTracks: () => [state.displayTrack] }); + await choosing; + await settle(); + + assert.equal(state.displayTrack.stops, 1, "the late stream is stopped immediately"); + assert.deepEqual(state.releases, ["binding-1"]); + assert.equal(fixture.controller().screenSourceLabel, null); + await fixture.unmount(); +}); + +test("a rejected screen frame stops capture and releases its exact binding", async () => { + const { dependencies, state } = screenDependencies(); + state.admitFrames = false; + const fixture = await mountHook(dependencies); + await fixture.controller().chooseScreenSource(); + await fixture.controller().start(); + await settle(); + await settle(); + + assert.equal(state.framesSent.length, 1); + assert.equal(state.frameSource.stops, 1); + assert.equal(state.displayTrack.stops, 1); + assert.deepEqual(state.releases, ["binding-1"]); + assert.equal(fixture.controller().screenActive, false); + assert.equal(fixture.controller().screenSourceLabel, null); + await fixture.unmount(); +}); + +test("a pending Live start blocks display-source replacement", async () => { + const { dependencies, state } = screenDependencies(); + const opening = deferred(); + dependencies.api!.start = async (intent) => { + state.startCalls.push({ screen: intent.screen === true }); + return opening.promise; + }; + const fixture = await mountHook(dependencies); + await fixture.controller().chooseScreenSource(); + const starting = fixture.controller().start(); + await settle(); + assert.equal(fixture.controller().busy, true); + + await fixture.controller().chooseScreenSource(); + assert.deepEqual(state.binds, [1], "busy start owns the selected binding"); + + opening.reject(new Error("provider rejected start")); + await starting; + await settle(); + assert.deepEqual(state.releases, ["binding-1"]); + assert.equal(fixture.controller().screenSourceLabel, null); + await fixture.unmount(); +}); + test("an externally ended display source stops the Live session instead of streaming a dead feed", async () => { const { dependencies, state } = screenDependencies(); const fixture = await mountHook(dependencies); diff --git a/renderer/components/assistant/use-assistant-live.ts b/renderer/components/assistant/use-assistant-live.ts index 5b22e046a..bb50aa817 100644 --- a/renderer/components/assistant/use-assistant-live.ts +++ b/renderer/components/assistant/use-assistant-live.ts @@ -309,8 +309,8 @@ interface AssistantLiveApi { }): Promise; stop(): Promise; sendAudio(sessionId: string, pcm: Uint8Array): Promise; - bindDisplay?(): Promise; - releaseDisplay?(): Promise; + bindDisplay?(): Promise; + releaseDisplay?(bindingId: string): Promise; sendFrame?(sessionId: string, frame: Uint8Array): Promise; onEvent(handler: (event: AssistantLiveRendererEvent) => void): () => void; } @@ -493,6 +493,8 @@ export function useAssistantLiveWithDependencies( cleanup: () => Promise; } | null>(null); const screenStreamRef = React.useRef(null); + const screenBindingRef = React.useRef(null); + const screenPickerGeneration = React.useRef(0); const screenFinishRef = React.useRef<(() => void) | null>(null); const screenPumpRef = React.useRef<{ sessionId: string; @@ -580,6 +582,15 @@ export function useAssistantLiveWithDependencies( } }, []); + const releaseDisplayAuthority = React.useCallback(() => { + screenPickerGeneration.current += 1; + const bindingId = screenBindingRef.current; + screenBindingRef.current = null; + if (bindingId) { + void dependencies.api.releaseDisplay?.(bindingId).catch(() => undefined); + } + }, [dependencies.api]); + const teardownMedia = React.useCallback( async (expected?: { sessionId: string; generation: number }) => { const record = mediaCleanupRef.current; @@ -704,9 +715,9 @@ export function useAssistantLiveWithDependencies( sessionRef.current = null; void teardownMedia(); void dependencies.api.stop().catch(() => undefined); - void dependencies.api.releaseDisplay?.().catch(() => undefined); + releaseDisplayAuthority(); }; - }, [acceptEvent, dependencies.api, dependencies.geminiLive, teardownMedia]); + }, [acceptEvent, dependencies.api, dependencies.geminiLive, releaseDisplayAuthority, teardownMedia]); React.useEffect(() => { const generation = ++availabilityGeneration.current; @@ -841,27 +852,44 @@ export function useAssistantLiveWithDependencies( const releaseScreen = React.useCallback(() => { stopScreenCapture(); - void dependencies.api.releaseDisplay?.().catch(() => undefined); - }, [dependencies.api, stopScreenCapture]); + releaseDisplayAuthority(); + }, [releaseDisplayAuthority, stopScreenCapture]); const chooseScreenSource = React.useCallback(async () => { if ( screenBusy || + busy || active || snapshot.screenShareAllowed !== true || !dependencies.getDisplayMedia || !dependencies.api.bindDisplay ) return; + const generation = ++screenPickerGeneration.current; setScreenBusy(true); setScreenError(null); + let bindingId: string | null = null; try { - if (!(await dependencies.api.bindDisplay())) { + bindingId = await dependencies.api.bindDisplay(); + if (!bindingId) { throw new Error("Screen sharing is unavailable for this window."); } + if (!mounted.current || screenPickerGeneration.current !== generation) { + void dependencies.api.releaseDisplay?.(bindingId).catch(() => undefined); + return; + } + screenBindingRef.current = bindingId; const stream = await dependencies.getDisplayMedia(); - if (!mounted.current || sessionRef.current !== null) { + if ( + !mounted.current || + screenPickerGeneration.current !== generation || + sessionRef.current !== null + ) { for (const track of stream.getTracks()) track.stop(); + if (screenBindingRef.current === bindingId) { + screenBindingRef.current = null; + void dependencies.api.releaseDisplay?.(bindingId).catch(() => undefined); + } return; } stopScreenCapture(); @@ -880,8 +908,14 @@ export function useAssistantLiveWithDependencies( } catch (cause) { // A cancelled picker keeps any previously chosen source; only a failed // first pick releases this document's binding authority. - if (!screenStreamRef.current) { - void dependencies.api.releaseDisplay?.().catch(() => undefined); + if ( + bindingId && + screenPickerGeneration.current === generation && + !screenStreamRef.current && + screenBindingRef.current === bindingId + ) { + screenBindingRef.current = null; + void dependencies.api.releaseDisplay?.(bindingId).catch(() => undefined); } if (mounted.current) { const name = cause instanceof DOMException ? cause.name : ""; @@ -892,10 +926,13 @@ export function useAssistantLiveWithDependencies( ); } } finally { - if (mounted.current) setScreenBusy(false); + if (mounted.current && screenPickerGeneration.current === generation) { + setScreenBusy(false); + } } }, [ active, + busy, dependencies.api, dependencies.getDisplayMedia, screenBusy, @@ -910,8 +947,7 @@ export function useAssistantLiveWithDependencies( const isCurrent = () => mounted.current && !signal.aborted && sessionRef.current === sessionId; if (!dependencies.createDisplayFrameSource || !dependencies.api.sendFrame) { - stopScreenCapture(); - void dependencies.api.releaseDisplay?.().catch(() => undefined); + releaseScreen(); return; } try { @@ -927,9 +963,10 @@ export function useAssistantLiveWithDependencies( try { const frame = await source.capture(); if (frame && isCurrent()) { - await dependencies.api + const admitted = await dependencies.api .sendFrame?.(sessionId, frame) .catch(() => false); + if (admitted !== true && isCurrent()) releaseScreen(); } } catch { // A skipped frame is never fatal; the next interval sends fresh. @@ -945,11 +982,10 @@ export function useAssistantLiveWithDependencies( setScreenActive(true); void sendOnce(); } catch { - stopScreenCapture(); - void dependencies.api.releaseDisplay?.().catch(() => undefined); + releaseScreen(); } }, - [dependencies, stopScreenCapture], + [dependencies, releaseScreen], ); const startMicrophone = React.useCallback( @@ -1173,6 +1209,7 @@ export function useAssistantLiveWithDependencies( await dependencies.api.stop().catch(() => undefined); sessionRef.current = null; await teardownMedia(); + releaseDisplayAuthority(); if (mounted.current && !setupAbort.signal.aborted) setError(assistantLiveStartErrorDetail(startError)); } @@ -1193,6 +1230,7 @@ export function useAssistantLiveWithDependencies( snapshot.available, startMicrophone, startScreenShare, + releaseDisplayAuthority, teardownMedia, ]); @@ -1208,7 +1246,7 @@ export function useAssistantLiveWithDependencies( await teardownMedia(); try { await dependencies.api.stop(); - void dependencies.api.releaseDisplay?.().catch(() => undefined); + releaseDisplayAuthority(); const next = await dependencies.api.status(); if (mounted.current && operationGeneration.current === generation) { sessionRef.current = activeSnapshot(next) ? (next.sessionId ?? null) : null; @@ -1255,7 +1293,7 @@ export function useAssistantLiveWithDependencies( setBusy(false); } } - }, [dependencies, teardownMedia]); + }, [dependencies, releaseDisplayAuthority, teardownMedia]); const cancelSetup = React.useCallback(async () => { setSetupOpenState(false); diff --git a/renderer/lib/ipc.ts b/renderer/lib/ipc.ts index e12aae6a9..6c6a60c0c 100644 --- a/renderer/lib/ipc.ts +++ b/renderer/lib/ipc.ts @@ -317,8 +317,9 @@ export const assistantLiveApi = { stop: () => invoke("assistant-live:stop", {}), sendAudio: (sessionId: string, pcm: Uint8Array) => invoke("assistant-live:audio", { sessionId, pcm }), - bindDisplay: () => invoke("assistant-live:display-bind", {}), - releaseDisplay: () => invoke("assistant-live:display-release", {}), + bindDisplay: () => invoke("assistant-live:display-bind", {}), + releaseDisplay: (bindingId: string) => + invoke("assistant-live:display-release", { bindingId }), sendFrame: (sessionId: string, frame: Uint8Array) => invoke("assistant-live:frame", { sessionId, frame }), onEvent: (handler: (event: AssistantLiveRendererEvent) => void) =>