Add optional headed browser mode

.env.example

@@ -258,6 +258,11 @@ COMPUTER_TOKEN=
 # docker-compose.yml. Pointing it somewhere unmounted disables login persistence.
 # PROFILES_DIR=/profiles
 
+# Browser process used by a Bot's computer. `headless` preserves the smaller default. `headed` runs
+# full Chromium on a private virtual display; the existing live screen and take-the-wheel controls
+# are still how a person sees and drives it.
+# COMPUTER_BROWSER_MODE=headless
+
 # Which Bot this computer belongs to, and therefore which profile directory it uses.
 # COMPUTER_BOT_ID=shared
 
@@ -333,4 +338,3 @@ AGENT_TOOL_TOKEN=
 # for a deployment that has not stood up a worker. Set for one that has: openssl rand -base64 32.
 # Do not accept a default in production.
 WORKER_SHARED_SECRET=
-

agent-computer/src/browser-mode.ts

@@ -0,0 +1,10 @@
+export type BrowserMode = "headless" | "headed";
+
+/** Choose the browser process an operator asked for, preserving the existing default. */
+export function browserModeFromEnv(raw: string | undefined): BrowserMode {
+  const mode = raw?.trim() || "headless";
+  if (mode === "headless" || mode === "headed") return mode;
+  throw new Error(
+    `COMPUTER_BROWSER_MODE must be headless or headed, not ${JSON.stringify(mode)}.`,
+  );
+}

agent-computer/src/index.ts

@@ -1,6 +1,7 @@
 import { serve } from "bun";
 import type { Page } from "playwright";
 import { parseAriaSnapshot, type SnapshotElement } from "./aria-snapshot";
+import { browserModeFromEnv } from "./browser-mode";
 import {
   actsOnTheComputer,
   isOpenPath,
@@ -22,6 +23,7 @@ import { parseExecTimeout, parseNavigateUrl } from "./request-validation";
 import { type InputMessage, startScreencast } from "./screencast";
 import { createShell } from "./shell";
 import { createViewerSlot, type ViewerSlot } from "./viewer";
+import { startVirtualDisplay } from "./virtual-display";
 import {
   createWorkspace,
   WorkspaceFileError,
@@ -73,6 +75,17 @@ if (!COMPUTER_TOKEN) {
   process.exit(1);
 }
 
+const BROWSER_MODE = browserModeFromEnv(process.env.COMPUTER_BROWSER_MODE);
+const VIRTUAL_DISPLAY = await startVirtualDisplay(BROWSER_MODE);
+if (VIRTUAL_DISPLAY) process.env.DISPLAY = VIRTUAL_DISPLAY.name;
+console.info(
+  JSON.stringify({
+    type: "computer-browser-mode",
+    mode: BROWSER_MODE,
+    display: VIRTUAL_DISPLAY?.name ?? null,
+  }),
+);
+
 const PORT = numberFromEnv("PORT", 4100);
 const NAVIGATION_TIMEOUT_MS = numberFromEnv("NAVIGATION_TIMEOUT_MS", 30000);
 
@@ -750,6 +763,7 @@ serve<StreamData>({
         // deployment without it, not a failure, and it is reported rather than omitted so the
         // difference between "no identity here" and "identity broken" is visible.
         identity: await identity(),
+        browserMode: BROWSER_MODE,
       });
     }
 
@@ -1234,12 +1248,36 @@ console.info(`agent-computer listening on http://localhost:${PORT}`);
  *
  * `stop_grace_period` in docker-compose.yml is what gives this time to run.
  */
+let shuttingDown = false;
+
+async function shutDown(reason: string, exitCode: number): Promise<void> {
+  if (shuttingDown) return;
+  // Set before the first await: a display exiting while Chromium flushes is part of this shutdown,
+  // not a second failure racing it.
+  shuttingDown = true;
+  console.info(`${reason}: closing the browser so its profile is flushed`);
+  await profiles.closeAll();
+  await VIRTUAL_DISPLAY?.stop();
+  process.exit(exitCode);
+}
+
+if (VIRTUAL_DISPLAY) {
+  void VIRTUAL_DISPLAY.terminated.then(({ code, expected }) => {
+    if (expected || shuttingDown) return;
+    console.error(
+      JSON.stringify({
+        type: "computer-virtual-display-exited",
+        exitCode: code,
+      }),
+    );
+    // A headed Chromium cannot recover without its display. Let the container restart policy build
+    // the pair together again instead of advertising a healthy service whose next browser fails.
+    void shutDown("virtual display exited", 1);
+  });
+}
+
 for (const signal of ["SIGTERM", "SIGINT"] as const) {
   process.on(signal, () => {
-    void (async () => {
-      console.info(`${signal}: closing the browser so its profile is flushed`);
-      await profiles.closeAll();
-      process.exit(0);
-    })();
+    void shutDown(signal, 0);
   });
 }

agent-computer/src/profiles.ts

@@ -37,6 +37,7 @@ import { readdir, rm } from "node:fs/promises";
 import { join } from "node:path";
 import { type BrowserContext, chromium, type Page } from "playwright";
 import { profileDirectoryFor } from "./bot-id";
+import { browserModeFromEnv } from "./browser-mode";
 import { chooseEvictions, chooseIdle } from "./browser-eviction";
 import { egressFor, egressLabel } from "./egress";
 import { numberFromEnv, settleWithin } from "./env";
@@ -91,6 +92,7 @@ const SINGLETON_FILES = ["SingletonLock", "SingletonSocket", "SingletonCookie"];
  * whether the browser rendering the open internet is sandboxed.
  */
 const SANDBOX_ENABLED = process.env.COMPUTER_SANDBOX === "on";
+const BROWSER_MODE = browserModeFromEnv(process.env.COMPUTER_BROWSER_MODE);
 
 const LAUNCH_ARGS = [
   ...(SANDBOX_ENABLED ? [] : ["--no-sandbox"]),
@@ -380,6 +382,7 @@ export function createProfiles(root: string, onClosed: BrowserClosed) {
         await sweepLocks(dir);
         const proxy = egressFor(botId, process.env);
         const context = await chromium.launchPersistentContext(dir, {
+          headless: BROWSER_MODE === "headless",
           args: LAUNCH_ARGS,
           // Playwright launches with `--enable-automation`, which sets `navigator.webdriver` and the
           // "controlled by automated software" banner. Dropped for the same reason as the flag above:

agent-computer/src/virtual-display.ts

@@ -0,0 +1,143 @@
+import { spawn } from "node:child_process";
+import type { Readable } from "node:stream";
+import type { BrowserMode } from "./browser-mode";
+
+export type DisplayProcess = {
+  /** The display allocated by this exact Xvfb process, reported through -displayfd. */
+  ready: Promise<string>;
+  exited: Promise<number>;
+  kill: (signal?: NodeJS.Signals) => boolean;
+};
+
+export type DisplayRuntime = {
+  spawn: (command: string, args: string[]) => DisplayProcess;
+  wait: (milliseconds: number) => Promise<void>;
+};
+
+export type VirtualDisplay = {
+  name: string;
+  /** Resolves for both normal shutdown and a display that failed while the computer was running. */
+  terminated: Promise<{ code: number; expected: boolean }>;
+  stop: () => Promise<void>;
+};
+
+const READY_BUDGET_MS = 5_000;
+const STOP_BUDGET_MS = 2_000;
+
+async function allocatedDisplay(stream: Readable): Promise<string> {
+  let response = "";
+  for await (const chunk of stream) {
+    response += String(chunk);
+    if (response.length > 32) {
+      throw new Error("Xvfb returned an invalid display number.");
+    }
+    if (response.includes("\n")) break;
+  }
+
+  const number = response.trim();
+  if (!/^\d+$/.test(number)) {
+    throw new Error("Xvfb returned an invalid display number.");
+  }
+  return `:${number}`;
+}
+
+const systemRuntime: DisplayRuntime = {
+  spawn(command, args) {
+    const child = spawn(command, args, {
+      // fd 3 is private to this child. Xvfb writes its selected display there only after that display
+      // is ready, so a stale socket or another X server can never satisfy our readiness check.
+      stdio: ["ignore", "ignore", "inherit", "pipe"],
+    });
+    const exited = new Promise<number>((resolve) => {
+      child.once("exit", (code) => resolve(code ?? 1));
+      child.once("error", () => resolve(1));
+    });
+    const displayFd = child.stdio[3] as Readable | null;
+    if (!displayFd)
+      throw new Error("Xvfb did not expose its display descriptor.");
+    return {
+      ready: allocatedDisplay(displayFd),
+      exited,
+      kill: (signal) => child.kill(signal),
+    };
+  },
+  wait: (milliseconds) =>
+    new Promise((resolve) => setTimeout(resolve, milliseconds)),
+};
+
+async function stop(
+  process: DisplayProcess,
+  runtime: DisplayRuntime,
+): Promise<void> {
+  process.kill("SIGTERM");
+  const stopped = await Promise.race([
+    process.exited.then(() => true),
+    runtime.wait(STOP_BUDGET_MS).then(() => false),
+  ]);
+  if (!stopped) {
+    process.kill("SIGKILL");
+    await process.exited;
+  }
+}
+
+/** Start the local-only X display a headed computer needs. */
+export async function startVirtualDisplay(
+  mode: BrowserMode,
+  runtime: DisplayRuntime = systemRuntime,
+): Promise<VirtualDisplay | null> {
+  if (mode === "headless") return null;
+
+  const displayProcess = runtime.spawn("Xvfb", [
+    "-displayfd",
+    "3",
+    "-screen",
+    "0",
+    "1280x800x24",
+    "-nolisten",
+    "tcp",
+    "-ac",
+  ]);
+  let stopping = false;
+  let exitCode: number | undefined;
+  const exited = displayProcess.exited.then((code) => {
+    exitCode = code;
+    return code;
+  });
+  const terminated = exited.then((code) => ({ code, expected: stopping }));
+
+  let name: string;
+  try {
+    name = await Promise.race([
+      displayProcess.ready,
+      exited.then((code) => {
+        throw new Error(
+          `The virtual display exited before it became ready (exit ${code}).`,
+        );
+      }),
+      runtime.wait(READY_BUDGET_MS).then(() => {
+        throw new Error(
+          `The virtual display did not become ready within ${READY_BUDGET_MS}ms.`,
+        );
+      }),
+    ]);
+    if (!/^:\d+$/.test(name)) {
+      throw new Error("Xvfb returned an invalid display number.");
+    }
+  } catch (error) {
+    if (exitCode === undefined) {
+      stopping = true;
+      await stop(displayProcess, runtime);
+    }
+    throw error;
+  }
+
+  return {
+    name,
+    terminated,
+    stop: async () => {
+      if (stopping || exitCode !== undefined) return;
+      stopping = true;
+      await stop(displayProcess, runtime);
+    },
+  };
+}

agent-computer/tests/browser-mode.test.ts

@@ -0,0 +1,20 @@
+import { describe, expect, test } from "bun:test";
+import { browserModeFromEnv } from "../src/browser-mode";
+
+describe("choosing the browser people take over", () => {
+  test("keeps the existing headless browser when no mode is configured", () => {
+    expect(browserModeFromEnv(undefined)).toBe("headless");
+    expect(browserModeFromEnv("")).toBe("headless");
+  });
+
+  test("runs the full browser only when headed is requested", () => {
+    expect(browserModeFromEnv("headed")).toBe("headed");
+    expect(browserModeFromEnv("headless")).toBe("headless");
+  });
+
+  test("refuses a typo rather than silently changing browser behavior", () => {
+    expect(() => browserModeFromEnv("visible")).toThrow(
+      'COMPUTER_BROWSER_MODE must be headless or headed, not "visible".',
+    );
+  });
+});

agent-computer/tests/headed-browser.test.ts

@@ -0,0 +1,89 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
+import { createServer } from "node:net";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+/**
+ * The optional full browser, driven as the container drives it.
+ *
+ * Asked for explicitly because it needs a virtual display and a real Chromium:
+ *
+ *   OPENBOT_HEADED_BROWSER=1 bun test tests/headed-browser.test.ts
+ *
+ * The computer starts and owns Xvfb itself. Wrapping this command in xvfb-run would hide display
+ * allocation bugs by giving Chromium a second display it does not own.
+ */
+const asked = process.env.OPENBOT_HEADED_BROWSER === "1";
+const TOKEN = "headed-browser-test-token";
+const BOT = "headed-browser";
+
+async function freePort(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const probe = createServer();
+    probe.on("error", reject);
+    probe.listen(0, "127.0.0.1", () => {
+      const address = probe.address();
+      if (!address || typeof address === "string") {
+        reject(new Error("The port probe did not return a TCP address."));
+        return;
+      }
+      probe.close(() => resolve(address.port));
+    });
+  });
+}
+
+let base = "";
+let root = "";
+
+function api(path: string, init?: RequestInit) {
+  return fetch(`${base}${path}`, {
+    ...init,
+    headers: {
+      "content-type": "application/json",
+      "x-openbot-bot-id": BOT,
+      "x-openbot-computer-token": TOKEN,
+      ...(init?.headers ?? {}),
+    },
+  });
+}
+
+beforeAll(async () => {
+  if (!asked) return;
+  root = await mkdtemp(join(tmpdir(), "agent-computer-headed-"));
+  const port = await freePort();
+  base = `http://127.0.0.1:${port}`;
+  process.env.COMPUTER_TOKEN = TOKEN;
+  process.env.COMPUTER_BROWSER_MODE = "headed";
+  process.env.PORT = String(port);
+  process.env.PROFILES_DIR = join(root, "profiles");
+  process.env.WORKSPACE_DIR = join(root, "workspace");
+  await mkdir(process.env.PROFILES_DIR, { recursive: true });
+  await import(`../src/index?headed=${Date.now()}`);
+});
+
+afterAll(async () => {
+  if (!asked) return;
+  await api("/computers/stop", { method: "POST" }).catch(() => undefined);
+  await rm(root, { recursive: true, force: true });
+}, 30_000);
+
+describe.skipIf(!asked)("a browser a person can take over", () => {
+  test("is full Chromium rather than the headless shell", async () => {
+    const page =
+      "data:text/html," +
+      encodeURIComponent(
+        "<body></body><script>document.body.textContent=navigator.userAgent+'\\nwebdriver='+navigator.webdriver</script>",
+      );
+    const navigated = await api("/navigate", {
+      method: "POST",
+      body: JSON.stringify({ url: page }),
+    });
+    expect(navigated.status).toBe(200);
+
+    const read = await api("/read");
+    const body = (await read.json()) as { text?: string };
+    expect(body.text).not.toContain("HeadlessChrome");
+    expect(body.text).toContain("webdriver=false");
+  }, 30_000);
+});