diff --git a/CHANGELOG.md b/CHANGELOG.md index 6590bfa62..2854ac92b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +### A long control name or value in a page snapshot is cut between characters + +The computer's page snapshot keeps the first 200 UTF-16 code units of each control's accessible +name and value. When that limit fell between the two halves of an emoji, the Bot was handed text +ending on half a character, which reads as U+FFFD: a broken character that is not on the page, often +at the end of a message the Bot had just typed into a text box. The cut now stops one code unit +short in that case, the same rule tool results and relayed answers already follow. ### A Bot's shell can no longer read the deployment's keys from a neighbouring process In the all-in-one image the API and the browser ran under one account, and a Bot's shell — a child diff --git a/agent-computer/src/aria-snapshot.ts b/agent-computer/src/aria-snapshot.ts index 3c2856c7d..d15781bba 100644 --- a/agent-computer/src/aria-snapshot.ts +++ b/agent-computer/src/aria-snapshot.ts @@ -156,6 +156,20 @@ export function parseDescriptor(text: string): Descriptor | null { return { role, name, flags }; } +/** + * The first `limit` UTF-16 code units of `text`, one fewer when the cut would split a character. + * + * `slice` counts code units and an emoji is two, so a limit landing between the halves leaves a lone + * high surrogate last: JSON carries it as a bare `\ud83d` and UTF-8 as U+FFFD, and the Bot reads a + * broken character that is not on the page. The server's `cutAtCodeUnits` is the same rule; this + * process shares no code with the server, so it is repeated here rather than imported. + */ +function cutAtCodeUnits(text: string, limit: number): string { + const sliced = text.slice(0, limit); + const last = sliced.charCodeAt(sliced.length - 1); + return last >= 0xd800 && last <= 0xdbff ? sliced.slice(0, -1) : sliced; +} + /** Build an element from a descriptor and whatever YAML gave as its value, or null if not actionable. */ function toElement( descriptor: Descriptor, @@ -170,13 +184,13 @@ function toElement( const element: SnapshotElement = { ref, role: descriptor.role, - name: descriptor.name.slice(0, 200), + name: cutAtCodeUnits(descriptor.name, 200), }; // Values arrive as text, with quoting and escapes already resolved. if (typeof value === "string") { const text = value.trim(); - if (text) element.value = text.slice(0, 200); + if (text) element.value = cutAtCodeUnits(text, 200); } if (descriptor.flags.has("disabled")) element.disabled = true; diff --git a/agent-computer/tests/aria-snapshot.test.ts b/agent-computer/tests/aria-snapshot.test.ts index 50dd87043..f95619460 100644 --- a/agent-computer/tests/aria-snapshot.test.ts +++ b/agent-computer/tests/aria-snapshot.test.ts @@ -167,6 +167,39 @@ describe("parseAriaSnapshot, against captured output", () => { }); }); +/** + * A name or value longer than the 200 kept is cut, and the cut counts UTF-16 code units. + * + * An emoji is two of them. With its first half as the 200th unit, a plain `slice` leaves a lone high + * surrogate as the last character of what the Bot is handed: JSON carries it as a bare `\ud83d` and + * UTF-8 as U+FFFD, so the Bot reads back a broken character that is not on the page, in text it may + * well have typed itself. + */ +describe("a name or value too long to keep whole", () => { + const CUT_THROUGH = `${"a".repeat(199)}😀 and the rest`; + const ENDS_INSIDE = `${"a".repeat(198)}😀 and the rest`; + + test("a name cut inside a character loses the whole character, not half of it", () => { + const { elements } = parseAriaSnapshot(`- link "${CUT_THROUGH}" [ref=e1]`); + expect(elements[0]?.name).toBe("a".repeat(199)); + }); + + test("a value cut inside a character loses the whole character, not half of it", () => { + const { elements } = parseAriaSnapshot( + `- textbox "Message" [ref=e1]: ${CUT_THROUGH}`, + ); + expect(elements[0]?.value).toBe("a".repeat(199)); + }); + + test("a character that ends at the cut is kept", () => { + const { elements } = parseAriaSnapshot( + `- textbox "${ENDS_INSIDE}" [ref=e1]: ${ENDS_INSIDE}`, + ); + expect(elements[0]?.name).toBe(`${"a".repeat(198)}😀`); + expect(elements[0]?.value).toBe(`${"a".repeat(198)}😀`); + }); +}); + describe("values a real parser handles and a pattern got wrong", () => { test("a quoted numeric value is not left with its quotes", () => { // Numeric-looking text remains a string, so one-time codes are not coerced.