From 2ac2a22424a495e0419cbd8c39eeabfbc65b332d Mon Sep 17 00:00:00 2001 From: maximus792 <08093.claver@gmail.com> Date: Mon, 7 Sep 2026 16:05:51 +0200 Subject: [PATCH 1/3] fix(computer): preserve punctuation and paste input --- agent-computer/src/screencast.ts | 10 +- agent-computer/tests/live-screen.test.ts | 56 ++++++++ app/src/components/computer/live-screen.tsx | 9 ++ app/tests/live-screen-keyboard.test.tsx | 141 ++++++++++++++++++++ 4 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 app/tests/live-screen-keyboard.test.tsx diff --git a/agent-computer/src/screencast.ts b/agent-computer/src/screencast.ts index 9bc27c11f..82511c38a 100644 --- a/agent-computer/src/screencast.ts +++ b/agent-computer/src/screencast.ts @@ -39,6 +39,8 @@ export type InputMessage = key: string; code: string; text?: string; + /** Legacy browser keyCode, which CDP uses as its Windows virtual key code. */ + windowsVirtualKeyCode?: number; modifiers?: number; } | { type: "text"; text: string }; @@ -184,7 +186,13 @@ export async function startScreencast( } if (message.type === "key") { - const code = virtualKeyCode(message.key); + const offeredCode = message.windowsVirtualKeyCode; + const code = + Number.isInteger(offeredCode) && + (offeredCode ?? 0) > 0 && + (offeredCode ?? 0) <= 255 + ? (offeredCode as number) + : virtualKeyCode(message.key); await client.send("Input.dispatchKeyEvent", { // `keyDown` only when there is text to insert; otherwise `rawKeyDown`, which is what Chrome // expects for keys that do not produce a character. Sending keyDown with no text makes diff --git a/agent-computer/tests/live-screen.test.ts b/agent-computer/tests/live-screen.test.ts index ee306ce7d..e7f098daa 100644 --- a/agent-computer/tests/live-screen.test.ts +++ b/agent-computer/tests/live-screen.test.ts @@ -75,6 +75,13 @@ const TYPING_PAGE = "start", ); +/** A focused field whose value becomes page text, so character insertion is observable via /read. */ +const TEXT_FIELD_PAGE = + "data:text/html," + + encodeURIComponent( + "empty", + ); + let root = ""; let closing: Array<() => void> = []; @@ -205,6 +212,7 @@ afterAll(async () => { "reset-viewer", "wont-launch", "still-starting", + "punctuation", ]) { await api("/computers/stop", botId, { method: "POST" }).catch( () => undefined, @@ -316,6 +324,54 @@ describe.skipIf(!asked)("a superseded socket closing later", () => { }, 30_000); }); +describe.skipIf(!asked)("printable punctuation from the live screen", () => { + test("inserts a period using the browser key code sent by the surface", async () => { + const botId = "punctuation"; + await api("/navigate", botId, { + method: "POST", + body: JSON.stringify({ url: TEXT_FIELD_PAGE }), + }); + const viewer = watch(botId); + await viewer.casting; + await api("/control/take", botId, { method: "POST" }); + + viewer.socket.send( + JSON.stringify({ + type: "key", + event: "down", + key: ".", + code: "Period", + text: ".", + windowsVirtualKeyCode: 190, + modifiers: 0, + }), + ); + viewer.socket.send( + JSON.stringify({ + type: "key", + event: "up", + key: ".", + code: "Period", + windowsVirtualKeyCode: 190, + modifiers: 0, + }), + ); + + let landed = ""; + await until( + () => landed.includes("."), + 5_000, + "the period to be inserted into the focused field", + async () => { + const read = await api("/read", botId); + landed = ((await read.json()) as { text: string }).text; + }, + ); + + expect(viewer.errors).toEqual([]); + }, 30_000); +}); + describe.skipIf(!asked)( "the wheel, with the ownership check in front of it", () => { diff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsx index 370231122..348ed79a7 100644 --- a/app/src/components/computer/live-screen.tsx +++ b/app/src/components/computer/live-screen.tsx @@ -31,6 +31,11 @@ function modifierBits(event: { ); } +/** Let the local browser create a paste event, whose clipboard text is forwarded separately. */ +function isPasteShortcut(event: KeyboardEvent): boolean { + return (event.ctrlKey || event.metaKey) && event.key.toLowerCase() === "v"; +} + type Props = { /** * Computer identity is part of the stream URL so input and frames stay scoped to the active Bot. @@ -196,12 +201,14 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { if (!driving) return; const onKeyDown = (event: KeyboardEvent) => { if (event.key === "Escape") return; // Escape still closes the view. + if (isPasteShortcut(event)) return; event.preventDefault(); send({ type: "key", event: "down", key: event.key, code: event.code, + windowsVirtualKeyCode: event.keyCode, // Only a printable character carries text. Sending text for Backspace makes Chrome insert a // character instead of deleting one. ...(event.key.length === 1 ? { text: event.key } : {}), @@ -210,12 +217,14 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { }; const onKeyUp = (event: KeyboardEvent) => { if (event.key === "Escape") return; + if (isPasteShortcut(event)) return; event.preventDefault(); send({ type: "key", event: "up", key: event.key, code: event.code, + windowsVirtualKeyCode: event.keyCode, modifiers: modifierBits(event), }); }; diff --git a/app/tests/live-screen-keyboard.test.tsx b/app/tests/live-screen-keyboard.test.tsx new file mode 100644 index 000000000..58c493fef --- /dev/null +++ b/app/tests/live-screen-keyboard.test.tsx @@ -0,0 +1,141 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { cleanup, render, waitFor } from "@testing-library/react"; +import { LiveScreen } from "@/components/computer/live-screen"; + +class SocketDouble { + static readonly OPEN = 1; + static readonly CLOSED = 3; + static latest: SocketDouble | undefined; + + readyState = SocketDouble.OPEN; + onopen: (() => void) | null = null; + onmessage: (() => void) | null = null; + onerror: (() => void) | null = null; + onclose: (() => void) | null = null; + readonly sent: Record[] = []; + + constructor(_url: string) { + SocketDouble.latest = this; + queueMicrotask(() => this.onopen?.()); + } + + send(payload: string) { + this.sent.push(JSON.parse(payload) as Record); + } + + close() { + this.readyState = SocketDouble.CLOSED; + this.onclose?.(); + } +} + +let originalWebSocket: typeof WebSocket; + +beforeAll(() => { + GlobalRegistrator.register(); + originalWebSocket = globalThis.WebSocket; + globalThis.WebSocket = SocketDouble as unknown as typeof WebSocket; +}); + +afterEach(() => { + cleanup(); + SocketDouble.latest = undefined; +}); + +afterAll(() => { + globalThis.WebSocket = originalWebSocket; + GlobalRegistrator.unregister(); +}); + +async function liveSocket(): Promise { + render(); + await waitFor(() => expect(SocketDouble.latest).toBeDefined()); + return SocketDouble.latest as SocketDouble; +} + +for (const [name, modifier] of [ + ["Ctrl+V", { ctrlKey: true }], + ["Cmd+V", { metaKey: true }], +] as const) { + test(`${name} stays in the local page so it can produce a paste event`, async () => { + const socket = await liveSocket(); + const shortcut = new KeyboardEvent("keydown", { + key: "v", + code: "KeyV", + ...modifier, + bubbles: true, + cancelable: true, + }); + + window.dispatchEvent(shortcut); + + expect(shortcut.defaultPrevented).toBe(false); + expect(socket.sent).toEqual([]); + }); +} + +test("a period carries the browser's virtual key code to the remote screen", async () => { + const socket = await liveSocket(); + window.dispatchEvent( + new KeyboardEvent("keydown", { + key: ".", + code: "Period", + keyCode: 190, + bubbles: true, + cancelable: true, + }), + ); + + expect(socket.sent).toEqual([ + { + type: "key", + event: "down", + key: ".", + code: "Period", + text: ".", + windowsVirtualKeyCode: 190, + modifiers: 0, + }, + ]); +}); + +test("Ctrl+A remains a remote keyboard shortcut", async () => { + const socket = await liveSocket(); + const shortcut = new KeyboardEvent("keydown", { + key: "a", + code: "KeyA", + keyCode: 65, + ctrlKey: true, + bubbles: true, + cancelable: true, + }); + + window.dispatchEvent(shortcut); + + expect(shortcut.defaultPrevented).toBe(true); + expect(socket.sent).toEqual([ + { + type: "key", + event: "down", + key: "a", + code: "KeyA", + text: "a", + windowsVirtualKeyCode: 65, + modifiers: 2, + }, + ]); +}); + +test("the paste event sends clipboard text without forwarding it through a key event", async () => { + const socket = await liveSocket(); + const paste = new Event("paste", { bubbles: true, cancelable: true }); + Object.defineProperty(paste, "clipboardData", { + value: { getData: () => "." }, + }); + + window.dispatchEvent(paste); + + expect(paste.defaultPrevented).toBe(true); + expect(socket.sent).toEqual([{ type: "text", text: "." }]); +}); From 7ab867bd65bfcb13f914d252239d8747d6ce14e5 Mon Sep 17 00:00:00 2001 From: maximus792 <08093.claver@gmail.com> Date: Mon, 7 Sep 2026 16:14:11 +0200 Subject: [PATCH 2/3] fix(computer): keep paste keyup local --- app/src/components/computer/live-screen.tsx | 12 +++++++++-- app/tests/live-screen-keyboard.test.tsx | 23 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/app/src/components/computer/live-screen.tsx b/app/src/components/computer/live-screen.tsx index 348ed79a7..356e0f2b2 100644 --- a/app/src/components/computer/live-screen.tsx +++ b/app/src/components/computer/live-screen.tsx @@ -50,6 +50,8 @@ type Props = { export function LiveScreen({ computerId, driving, onProblem }: Props) { const canvasRef = useRef(null); const socketRef = useRef(null); + /** Keydowns handled locally whose matching keyup must not leak to the remote browser. */ + const localKeyUps = useRef(new Set()); /** The size of the frames Chrome is sending, which is what input coordinates are relative to. */ const frameSize = useRef<{ width: number; height: number } | null>(null); const [connected, setConnected] = useState(false); @@ -201,7 +203,10 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { if (!driving) return; const onKeyDown = (event: KeyboardEvent) => { if (event.key === "Escape") return; // Escape still closes the view. - if (isPasteShortcut(event)) return; + if (isPasteShortcut(event)) { + localKeyUps.current.add(event.code); + return; + } event.preventDefault(); send({ type: "key", @@ -217,7 +222,9 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { }; const onKeyUp = (event: KeyboardEvent) => { if (event.key === "Escape") return; - if (isPasteShortcut(event)) return; + if (localKeyUps.current.delete(event.code) || isPasteShortcut(event)) { + return; + } event.preventDefault(); send({ type: "key", @@ -243,6 +250,7 @@ export function LiveScreen({ computerId, driving, onProblem }: Props) { window.removeEventListener("keydown", onKeyDown); window.removeEventListener("keyup", onKeyUp); window.removeEventListener("paste", onPaste); + localKeyUps.current.clear(); }; }, [driving, send]); diff --git a/app/tests/live-screen-keyboard.test.tsx b/app/tests/live-screen-keyboard.test.tsx index 58c493fef..d7cd677d5 100644 --- a/app/tests/live-screen-keyboard.test.tsx +++ b/app/tests/live-screen-keyboard.test.tsx @@ -75,6 +75,29 @@ for (const [name, modifier] of [ }); } +test("a paste keyup stays local when the modifier was released first", async () => { + const socket = await liveSocket(); + window.dispatchEvent( + new KeyboardEvent("keydown", { + key: "v", + code: "KeyV", + ctrlKey: true, + bubbles: true, + cancelable: true, + }), + ); + window.dispatchEvent( + new KeyboardEvent("keyup", { + key: "v", + code: "KeyV", + bubbles: true, + cancelable: true, + }), + ); + + expect(socket.sent).toEqual([]); +}); + test("a period carries the browser's virtual key code to the remote screen", async () => { const socket = await liveSocket(); window.dispatchEvent( From b01c6de9d9e26496c657ff00d02e692474e0d27e Mon Sep 17 00:00:00 2001 From: maximus792 <08093.claver@gmail.com> Date: Mon, 7 Sep 2026 16:29:04 +0200 Subject: [PATCH 3/3] fix(computer): support punctuation from older surfaces --- agent-computer/src/screencast.ts | 11 +++++++++++ agent-computer/tests/live-screen.test.ts | 25 ++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/agent-computer/src/screencast.ts b/agent-computer/src/screencast.ts index 82511c38a..6b998188e 100644 --- a/agent-computer/src/screencast.ts +++ b/agent-computer/src/screencast.ts @@ -80,6 +80,17 @@ const VIRTUAL_KEY_CODES: Record = { ArrowRight: 39, ArrowDown: 40, Delete: 46, + ";": 186, + "=": 187, + ",": 188, + "-": 189, + ".": 190, + "/": 191, + "`": 192, + "[": 219, + "\\": 220, + "]": 221, + "'": 222, }; function virtualKeyCode(key: string): number { diff --git a/agent-computer/tests/live-screen.test.ts b/agent-computer/tests/live-screen.test.ts index e7f098daa..c3bc40c7a 100644 --- a/agent-computer/tests/live-screen.test.ts +++ b/agent-computer/tests/live-screen.test.ts @@ -356,12 +356,33 @@ describe.skipIf(!asked)("printable punctuation from the live screen", () => { modifiers: 0, }), ); + // Older OpenBot surfaces did not send the browser keyCode. Keep their punctuation usable while + // a deployment rolls the frontend and computer images independently. + viewer.socket.send( + JSON.stringify({ + type: "key", + event: "down", + key: ".", + code: "Period", + text: ".", + modifiers: 0, + }), + ); + viewer.socket.send( + JSON.stringify({ + type: "key", + event: "up", + key: ".", + code: "Period", + modifiers: 0, + }), + ); let landed = ""; await until( - () => landed.includes("."), + () => landed.includes(".."), 5_000, - "the period to be inserted into the focused field", + "periods from current and legacy surfaces to be inserted into the focused field", async () => { const read = await api("/read", botId); landed = ((await read.json()) as { text: string }).text;