diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fcac1feb..c8507f3a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +### Typing into a Bot's browser no longer triggers the app's own shortcuts + +While somebody drives a Bot's browser, every keystroke is sent to it. The app's shortcuts listen for +keystrokes too, and they heard each one first, so typing a capital N into the Bot's browser, as in +"New York", started a new chat and took the person away from the Bot mid-word, and Ctrl+B there +also showed or hid the sidebar. A keystroke sent to the Bot's browser now reaches only the Bot's +browser. Escape still closes the view, and the paste shortcut still pastes. ### The Python LangGraph Bot on Anthropic answers after a skill was picked A skill somebody picks reaches the Bot as a system message just ahead of their message, and it diff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsx index ce85392c9..70ad22143 100644 --- a/app/src/components/computer/live-screen.tsx +++ b/app/src/components/computer/live-screen.tsx @@ -279,6 +279,13 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { * * Listen on window because canvas cannot hold focus. `preventDefault` keeps Tab and typing directed * at the remote page while takeover is active. + * + * The keydown in the capture phase, and stopped as well as prevented, because a keystroke sent to + * the Bot's browser is not also this page's. The app's own shortcuts listen on this window too, + * and they were bound first, when the signed-in app mounted, so they saw every keystroke before + * this did: a capital N typed into the remote page started a new chat, and Ctrl+B there toggled + * the sidebar here. Escape and the paste shortcut are not stopped, because both are meant for this + * page. */ useEffect(() => { if (!driving) return; @@ -289,6 +296,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { return; } event.preventDefault(); + event.stopPropagation(); send({ type: "key", event: "down", @@ -324,11 +332,11 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { send({ type: "text", text }); }; - window.addEventListener("keydown", onKeyDown); + window.addEventListener("keydown", onKeyDown, true); window.addEventListener("keyup", onKeyUp); window.addEventListener("paste", onPaste); return () => { - window.removeEventListener("keydown", onKeyDown); + window.removeEventListener("keydown", onKeyDown, true); window.removeEventListener("keyup", onKeyUp); window.removeEventListener("paste", onPaste); localKeyUps.current.clear(); diff --git a/app/tests/live-screen-app-shortcuts.test.tsx b/app/tests/live-screen-app-shortcuts.test.tsx new file mode 100644 index 000000000..8a563ee5a --- /dev/null +++ b/app/tests/live-screen-app-shortcuts.test.tsx @@ -0,0 +1,174 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { + createMemoryHistory, + createRootRoute, + createRoute, + createRouter, + Outlet, + RouterProvider, +} from "@tanstack/react-router"; +import { act, cleanup, render, waitFor } from "@testing-library/react"; +import { LiveScreen } from "@/components/computer/live-screen"; +import { SidebarProvider, useSidebar } from "@/components/ui/sidebar"; +import { AppHotkeys } from "@/lib/hotkeys/app-hotkeys"; + +/** + * The app's own shortcuts while a person drives a Bot's browser. + * + * Every keystroke then belongs to the Bot's browser: LiveScreen forwards it and prevents its default. + * The app's shortcuts listen on the same window, and they were bound first, when the signed-in app + * mounted, so they saw each keystroke before LiveScreen did and acted on it as well. Typing a + * capital N into a page, "New York" in a search box, started a new chat and took the person away + * from the Bot mid-word; Ctrl+B, bold in a document, showed or hid the sidebar here as well. + */ + +class SocketDouble { + static readonly OPEN = 1; + static readonly CLOSED = 3; + static latest: SocketDouble | undefined; + + readyState = SocketDouble.OPEN; + onopen: (() => void) | null = null; + onmessage: (() => void) | null = null; + onerror: (() => void) | null = null; + onclose: (() => void) | null = null; + readonly sent: Record[] = []; + + constructor(_url: string) { + SocketDouble.latest = this; + queueMicrotask(() => this.onopen?.()); + } + + send(payload: string) { + this.sent.push(JSON.parse(payload) as Record); + } + + close() { + this.readyState = SocketDouble.CLOSED; + this.onclose?.(); + } +} + +let originalWebSocket: typeof WebSocket; + +beforeAll(() => { + GlobalRegistrator.register(); + originalWebSocket = globalThis.WebSocket; + globalThis.WebSocket = SocketDouble as unknown as typeof WebSocket; +}); + +afterEach(() => { + cleanup(); + SocketDouble.latest = undefined; +}); + +afterAll(() => { + globalThis.WebSocket = originalWebSocket; + GlobalRegistrator.unregister(); +}); + +function SidebarState() { + return {useSidebar().open ? "open" : "closed"}; +} + +/** The signed-in shell's shortcuts and sidebar, around a conversation showing a Bot's screen. */ +async function renderDriving(driving: boolean) { + const rootRoute = createRootRoute({ + component: () => ( + + + + + + ), + }); + const conversation = createRoute({ + getParentRoute: () => rootRoute, + path: "/", + component: () => ( + + ), + }); + const newChat = createRoute({ + getParentRoute: () => rootRoute, + path: "/channel/new", + component: () =>

New chat

, + }); + const router = createRouter({ + routeTree: rootRoute.addChildren([conversation, newChat]), + history: createMemoryHistory({ initialEntries: ["/"] }), + }); + const view = render(); + await waitFor(() => + expect(view.container.querySelector("canvas")).not.toBeNull(), + ); + if (driving) await waitFor(() => expect(SocketDouble.latest).toBeDefined()); + return { + router, + sidebar: () => view.container.querySelector("output")?.textContent, + /** A keystroke where the browser sends one while the page has no focused field: at the body. */ + press(init: KeyboardEventInit) { + act(() => { + document.body.dispatchEvent( + new KeyboardEvent("keydown", { + ...init, + bubbles: true, + cancelable: true, + }), + ); + }); + }, + }; +} + +test("a capital N typed into the Bot's browser goes to it, and does not start a new chat", async () => { + const screen = await renderDriving(true); + + screen.press({ key: "N", code: "KeyN", keyCode: 78, shiftKey: true }); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(screen.router.state.location.pathname).toBe("/"); + expect(SocketDouble.latest?.sent).toEqual([ + { + type: "key", + event: "down", + key: "N", + code: "KeyN", + text: "N", + windowsVirtualKeyCode: 78, + modifiers: 8, + }, + ]); +}); + +test("Ctrl+B typed into the Bot's browser goes to it, and leaves the sidebar alone", async () => { + const screen = await renderDriving(true); + + screen.press({ key: "b", code: "KeyB", keyCode: 66, ctrlKey: true }); + + expect(screen.sidebar()).toBe("open"); + expect(SocketDouble.latest?.sent).toEqual([ + { + type: "key", + event: "down", + key: "b", + code: "KeyB", + text: "b", + windowsVirtualKeyCode: 66, + modifiers: 2, + }, + ]); +}); + +test("the same shortcuts still work while nobody is driving", async () => { + const screen = await renderDriving(false); + + screen.press({ key: "b", code: "KeyB", keyCode: 66, ctrlKey: true }); + expect(screen.sidebar()).toBe("closed"); + + screen.press({ key: "N", code: "KeyN", keyCode: 78, shiftKey: true }); + await waitFor(() => + expect(screen.router.state.location.pathname).toBe("/channel/new"), + ); +});