From 61ad18d965741cdbe56496d1ec101005da527e5e Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 1 Oct 2026 08:00:56 +0000 Subject: [PATCH 1/3] feat(provider): reuse uploaded screenshot references Externalize Cloud Responses images before gateway JSON serialization, preserving tool history with bounded run caches, expiry recovery, ownership proofs and disposal cleanup. --- packages/opencode/src/provider/image-files.ts | 265 ++++++++++ packages/opencode/src/provider/provider.ts | 24 +- .../test/provider/image-files.test.ts | 480 ++++++++++++++++++ 3 files changed, 768 insertions(+), 1 deletion(-) create mode 100644 packages/opencode/src/provider/image-files.ts create mode 100644 packages/opencode/test/provider/image-files.test.ts diff --git a/packages/opencode/src/provider/image-files.ts b/packages/opencode/src/provider/image-files.ts new file mode 100644 index 0000000000..b1ebda1c11 --- /dev/null +++ b/packages/opencode/src/provider/image-files.ts @@ -0,0 +1,265 @@ +import type { LanguageModelV3, LanguageModelV3CallOptions } from "@ai-sdk/provider" + +type Uploaded = { file_id: string; expires_at: number; signature: string } +type Options = { baseURL: string; apiKey: string; fetch?: typeof fetch; now?: () => number } +type RunCache = { + cache: Map> + refreshed: Set + retired: Uploaded[] + waiting: Array<() => void> + active: number + closed: boolean +} +const runCaches = new Map() + +export function withImageFiles(model: LanguageModelV3, options: Options) { + let state: RunCache = { cache: new Map(), refreshed: new Set(), retired: [], waiting: [], active: 0, closed: false } + const now = options.now ?? Date.now + const endpoint = options.baseURL.replace(/\/$/, "") + "/image-files" + const send = options.fetch ?? fetch + let scope: string | undefined + let routeHeaders: Record = {} + const enabled = + Boolean(options.apiKey) && + /^\/(?:_v4\/bcode-relay\/[^/]+\/)?api\/v4\/llm\/openai\/v1\/?$/.test(new URL(options.baseURL).pathname) + + async function prepare(params: LanguageModelV3CallOptions): Promise { + if (!enabled) return params + const hasImages = params.prompt.some( + (message) => + (message.role === "user" && + message.content.some( + (part) => + part.type === "file" && + part.mediaType.startsWith("image/") && + (!(part.data instanceof URL) || part.data.protocol === "data:"), + )) || + (message.role === "tool" && + message.content.some( + (part) => + part.type === "tool-result" && + part.output.type === "content" && + part.output.value.some((item) => item.type === "image-data"), + )), + ) + if (!hasImages) return params + if (state.closed) throw new Error("Screenshot file cache is closed") + const previous = params.providerOptions?.openai?.previousResponseId + const legacy = params.prompt.some( + (message) => + message.role === "assistant" && + message.content.some( + (part) => + part.type === "reasoning" && + String(part.providerOptions?.openai?.reasoningEncryptedContent ?? "").startsWith("gAAAAA"), + ), + ) + const currentHeaders = { + Authorization: `Bearer ${options.apiKey}`, + ...(typeof previous === "string" ? { "x-bu-previous-response-id": previous } : {}), + ...(legacy ? { "x-bu-openai-reasoning": "1" } : {}), + } + const signal = AbortSignal.any([...(params.abortSignal ? [params.abortSignal] : []), AbortSignal.timeout(30_000)]) + const capability = await send(`${endpoint}/capability?model=${encodeURIComponent(model.modelId)}`, { + headers: currentHeaders, + signal, + }) + // Older gateways have no upload contract. Never fall back after selecting reference mode. + if (capability.status === 404 && !scope) return params + if (!capability.ok) throw new Error(`Screenshot capability failed (${capability.status})`) + const route = (await capability.json()) as { supported: boolean; scope?: string } + if (!route.supported && !scope) return params + if (!route.supported || !route.scope || (scope && scope !== route.scope)) + throw new Error("Screenshot route or account changed; start a new run") + if (state.closed) throw new Error("Screenshot file cache is closed") + scope = route.scope + if (!runCaches.has(scope) && runCaches.size >= 128) throw new Error("Too many active screenshot caches") + state = runCaches.get(scope) ?? state + runCaches.set(scope, state) + routeHeaders = currentHeaders + const headers = { ...currentHeaders, "x-bu-image-scope": scope } + const proofs: Record = {} + async function upload(data: string | Uint8Array | URL, mediaType: string) { + if (state.active >= 2) await new Promise((resolve) => state.waiting.push(resolve)) + else state.active++ + try { + signal.throwIfAborted() + if (state.closed) throw new Error("Screenshot file cache is closed") + const raw = data instanceof URL ? data.toString().split(",", 2)[1] : data + const bytes = typeof raw === "string" ? Buffer.from(raw, "base64") : raw + if (bytes.byteLength > 5 * 1024 * 1024) throw new Error("Screenshot exceeds upload limit") + const hash = new Bun.CryptoHasher("sha256").update(bytes).digest("hex") + const key = `${scope}:${mediaType}:${hash}` + let existing = state.cache.get(key) + if (existing) { + const file = await existing + if (state.cache.get(key) !== existing) existing = state.cache.get(key) + else if (file.expires_at <= now() / 1000 + 30) { + if (state.refreshed.has(key)) throw new Error("Screenshot reference expired again; start a new run") + state.refreshed.add(key) + state.retired.push(file) + existing = undefined + } + } + if (!existing && !state.cache.has(key) && state.cache.size >= 64) + throw new Error("Screenshot file limit reached; start a new run") + const pending = + existing ?? + (async () => { + const response = await send(`${endpoint}?model=${encodeURIComponent(model.modelId)}`, { + method: "POST", + headers: { ...headers, "content-type": mediaType }, + body: new Uint8Array(bytes), + signal, + }) + if (!response.ok) throw new Error(`Screenshot upload failed (${response.status})`) + const uploaded = (await response.json()) as Uploaded + if (!uploaded.file_id?.startsWith("file-")) throw new Error("Invalid screenshot file ID") + if (!(uploaded.expires_at > now() / 1000 + 30)) throw new Error("Screenshot reference expired") + return uploaded + })() + state.cache.set(key, pending) + const uploaded = await pending + if (uploaded.expires_at <= now() / 1000 + 30) throw new Error("Screenshot reference expired; start a new run") + signal.throwIfAborted() + if (state.closed) throw new Error("Screenshot file cache is closed") + proofs[uploaded.file_id] = { expires_at: uploaded.expires_at, signature: uploaded.signature } + return uploaded.file_id + } finally { + const next = state.waiting.shift() + if (next) next() + else state.active-- + } + } + const prompt = await Promise.all( + params.prompt.map(async (message) => { + if (message.role === "tool") + return { + ...message, + content: await Promise.all( + message.content.map(async (part) => { + if (part.type !== "tool-result" || part.output.type !== "content") return part + return { + ...part, + output: { + ...part.output, + value: await Promise.all( + part.output.value.map(async (item) => { + if (item.type !== "image-data") return item + return { + type: "image-url" as const, + url: `bu-openai-file:${await upload(item.data, item.mediaType)}`, + providerOptions: item.providerOptions, + } + }), + ), + }, + } + }), + ), + } + if (message.role !== "user") return message + return { + ...message, + content: await Promise.all( + message.content.map(async (part) => { + if ( + part.type !== "file" || + !part.mediaType.startsWith("image/") || + (part.data instanceof URL && part.data.protocol !== "data:") + ) + return part + return { ...part, data: await upload(part.data, part.mediaType) } + }), + ), + } + }), + ) + return { + ...params, + prompt, + headers: { ...params.headers, "x-bu-image-scope": scope, "x-bu-image-proofs": JSON.stringify(proofs) }, + } + } + + async function invoke( + params: LanguageModelV3CallOptions, + call: (prepared: LanguageModelV3CallOptions) => PromiseLike, + ): Promise { + try { + return await call(await prepare(params)) + } catch (error) { + const message = error instanceof Error ? error.message : "" + if (!/file.*(not found|does not exist|expired|deleted)/i.test(message)) throw error + const entries = await Promise.all([...state.cache].map(async ([key, pending]) => ({ key, file: await pending }))) + const missing = entries.filter(({ file }) => message.includes(file.file_id)) + if (!missing.length || missing.some(({ key }) => state.refreshed.has(key))) throw error + for (const { key, file } of missing) { + state.refreshed.add(key) + state.retired.push(file) + state.cache.delete(key) + } + return await call(await prepare(params)) + } + } + + return { + specificationVersion: model.specificationVersion, + provider: model.provider, + modelId: model.modelId, + supportedUrls: model.supportedUrls, + async doGenerate(params: LanguageModelV3CallOptions) { + return invoke(params, (prepared) => model.doGenerate(prepared)) + }, + async doStream(params: LanguageModelV3CallOptions) { + return invoke(params, (prepared) => model.doStream(prepared)) + }, + async closeImageFiles() { + state.closed = true + if (scope) runCaches.delete(scope) + const entries = await Promise.allSettled([ + ...state.cache.values(), + ...state.retired.map((file) => Promise.resolve(file)), + ]) + await Promise.allSettled( + entries + .filter((entry) => entry.status === "fulfilled") + .map((entry) => + send(`${endpoint}/${encodeURIComponent(entry.value.file_id)}?model=${encodeURIComponent(model.modelId)}`, { + method: "DELETE", + headers: { ...routeHeaders, "x-bu-image-scope": scope! }, + signal: AbortSignal.timeout(10_000), + }), + ), + ) + state.cache.clear() + }, + } +} + +// The pinned SDK lacks tool-result file IDs; only small placeholders enter its JSON serializer. +export function imageFileRequest(init?: RequestInit): RequestInit | undefined { + if (!init || !new Headers(init.headers).has("x-bu-image-scope") || typeof init.body !== "string") return init + const body = JSON.parse(init.body) + if (Array.isArray(body.input)) + body.input = body.input.map((item: Record) => { + if (!item || typeof item !== "object") return item + const field = + item.type === "function_call_output" || item.type === "custom_tool_call_output" ? "output" : "content" + if (!Array.isArray(item[field])) return item + return { + ...item, + [field]: item[field].map((part: Record) => { + if ( + part?.type !== "input_image" || + typeof part.image_url !== "string" || + !part.image_url.startsWith("bu-openai-file:file-") + ) + return part + const { image_url, ...rest } = part + return { ...rest, file_id: image_url.slice("bu-openai-file:".length) } + }), + } + }) + return { ...init, body: JSON.stringify(body) } +} diff --git a/packages/opencode/src/provider/provider.ts b/packages/opencode/src/provider/provider.ts index 73a8080d1e..b9d702eb1f 100644 --- a/packages/opencode/src/provider/provider.ts +++ b/packages/opencode/src/provider/provider.ts @@ -1,3 +1,4 @@ +import { imageFileRequest, withImageFiles } from "./image-files" import { LayerNode } from "@opencode-ai/core/effect/layer-node" import os from "os" import { ConfigV1 } from "@opencode-ai/core/v1/config/config" @@ -1158,6 +1159,7 @@ export interface Interface { } interface State { + imageFiles: Set<() => Promise> models: Map providers: Record catalog: Record @@ -1344,6 +1346,12 @@ const layer = Layer.effect( const providers: Record = {} as Record const languages = new Map() + const imageFiles = new Set<() => Promise>() + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + await Promise.allSettled([...imageFiles].map((close) => close())) + }), + ) const modelLoaders: { [providerID: string]: CustomModelLoader } = {} @@ -1653,6 +1661,7 @@ const layer = Layer.effect( return { models: languages, + imageFiles, providers, catalog, sdk, @@ -1752,7 +1761,7 @@ const layer = Layer.effect( if (combined) opts.signal = combined const res = await fetchFn(input, { - ...opts, + ...imageFileRequest(opts), // @ts-ignore see here: https://github.com/oven-sh/bun/issues/16682 timeout: false, }).finally(() => headerTimeoutCtl?.clear()) @@ -1768,6 +1777,19 @@ const layer = Layer.effect( name: model.providerID, ...options, }) + if ( + model.api.npm === "@ai-sdk/openai" && + loaded.responses && + typeof options.apiKey === "string" && + typeof baseURL === "string" + ) { + const responses = loaded.responses.bind(loaded) + loaded.responses = (id: string) => { + const wrapped = withImageFiles(responses(id), { baseURL, apiKey: options.apiKey, fetch: options.fetch }) + s.imageFiles.add(wrapped.closeImageFiles) + return wrapped + } + } s.sdk.set(key, loaded) return loaded as SDK } diff --git a/packages/opencode/test/provider/image-files.test.ts b/packages/opencode/test/provider/image-files.test.ts new file mode 100644 index 0000000000..36b133dac6 --- /dev/null +++ b/packages/opencode/test/provider/image-files.test.ts @@ -0,0 +1,480 @@ +import { expect, test } from "bun:test" +import { generateText } from "ai" +import { SessionV1 } from "@opencode-ai/core/v1/session" +import type { Provider } from "../../src/provider/provider" +import { MessageV2 } from "../../src/session/message-v2" +import { createOpenAI } from "@ai-sdk/openai" +import { imageFileRequest, withImageFiles } from "../../src/provider/image-files" + +const screenshot = Buffer.alloc(1024 * 1024, 42) +const prompt = [ + { + role: "user" as const, + content: [ + { type: "text" as const, text: "Read this screenshot" }, + { + type: "file" as const, + mediaType: "image/png", + data: screenshot, + providerOptions: { openai: { imageDetail: "high" } }, + }, + ], + }, + { + role: "assistant" as const, + content: [{ type: "tool-call" as const, toolCallId: "call_1", toolName: "browse", input: {} }], + }, + { + role: "tool" as const, + content: [ + { + type: "tool-result" as const, + toolCallId: "call_1", + toolName: "browse", + output: { type: "text" as const, value: "done" }, + }, + ], + }, +] + +test("one screenshot reused in three turns is uploaded once before gateway JSON", async () => { + const requests: Array<{ input: Array> }> = [] + let uploads = 0 + const testScope = crypto.randomUUID() + const server = Bun.serve({ + port: 0, + async fetch(req) { + const path = new URL(req.url).pathname + if (path.endsWith("/image-files/capability")) return Response.json({ supported: true, scope: testScope }) + if (path.endsWith("/image-files")) { + expect(req.headers.get("content-type")).toBe("image/png") + expect(Buffer.from(await req.arrayBuffer()).equals(screenshot)).toBe(true) + uploads++ + return Response.json({ file_id: "file-screenshot", signature: "proof", expires_at: Date.now() / 1000 + 3600 }) + } + requests.push(await req.json()) + return Response.json({ + id: "resp_test", + object: "response", + created_at: 1, + model: "gpt-test", + output: [ + { + type: "message", + id: "msg_1", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "ok", annotations: [] }], + }, + ], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + status: "completed", + }) + }, + }) + try { + const baseURL = `${server.url}api/v4/llm/openai/v1` + const native = createOpenAI({ + baseURL, + apiKey: "v4rt_test", + fetch: Object.assign((url: RequestInfo | URL, init?: RequestInit) => fetch(url, imageFileRequest(init)), { + preconnect: fetch.preconnect, + }), + }).responses("gpt-test") + const model = process.env.IMAGE_FILES_BASELINE ? native : withImageFiles(native, { baseURL, apiKey: "v4rt_test" }) + for (let i = 0; i < 3; i++) + await model.doGenerate({ prompt: [...prompt, { role: "user", content: [{ type: "text", text: `turn ${i}` }] }] }) + if (process.env.IMAGE_FILES_EVIDENCE) + await Bun.write( + process.env.IMAGE_FILES_EVIDENCE, + JSON.stringify({ + uploads, + bodies: requests, + originalHistoryIntact: JSON.stringify(prompt).includes(JSON.stringify(screenshot)), + }), + ) + expect(uploads).toBe(1) + expect(requests).toHaveLength(3) + for (const body of requests) { + expect(JSON.stringify(body)).not.toContain(screenshot.toString("base64")) + expect(body.input[0].content).toEqual([ + { type: "input_text", text: "Read this screenshot" }, + { type: "input_image", file_id: "file-screenshot", detail: "high" }, + ]) + expect(body.input[1]).toMatchObject({ type: "function_call", call_id: "call_1", name: "browse", arguments: "{}" }) + expect(body.input[2]).toMatchObject({ type: "function_call_output", call_id: "call_1", output: "done" }) + } + expect(prompt[0].content[1]).toMatchObject({ data: screenshot }) + } finally { + server.stop(true) + } +}) + +test("actual BrowserCode screenshot history is externalized without changing persisted history", async () => { + const history = [ + { + info: { + id: "msg-a", + parentID: "msg-u", + sessionID: "session", + role: "assistant", + modelID: "gpt-test", + providerID: "openai", + time: { created: 0 }, + }, + parts: [ + { + id: "part", + sessionID: "session", + messageID: "msg-a", + type: "tool", + tool: "browser_execute", + callID: "call-1", + state: { + status: "completed", + input: { code: "screenshot()" }, + output: "Captured", + title: "Screenshot", + metadata: {}, + time: { start: 0, end: 1 }, + attachments: [ + { + id: "file", + sessionID: "session", + messageID: "msg-a", + type: "file", + mime: "image/png", + url: `data:image/png;base64,${screenshot.toString("base64")}`, + }, + ], + }, + }, + ], + }, + ] as unknown as SessionV1.WithParts[] + const original = JSON.stringify(history) + const spec = { + id: "gpt-test", + providerID: "openai", + api: { id: "gpt-test", npm: "@ai-sdk/openai" }, + capabilities: { input: { image: true } }, + } as Provider.Model + let uploads = 0 + const bodies: Record[] = [] + const testScope = crypto.randomUUID() + const server = Bun.serve({ + port: 0, + async fetch(req) { + if (req.url.includes("/capability")) return Response.json({ supported: true, scope: testScope }) + if (req.url.includes("/image-files")) { + uploads++ + return Response.json({ file_id: "file-image", signature: "proof", expires_at: Date.now() / 1000 + 3600 }) + } + bodies.push(await req.json()) + return Response.json({ + id: "resp_a", + created_at: 1, + model: "gpt-test", + output: [ + { + type: "message", + id: "msg-a", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "ok", annotations: [] }], + }, + ], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + status: "completed", + }) + }, + }) + try { + const baseURL = `${server.url}api/v4/llm/openai/v1` + const native = createOpenAI({ + baseURL, + apiKey: "v4rt_test", + fetch: Object.assign((url: RequestInfo | URL, init?: RequestInit) => fetch(url, imageFileRequest(init)), { + preconnect: fetch.preconnect, + }), + }).responses("gpt-test") + const model = process.env.IMAGE_FILES_BASELINE ? native : withImageFiles(native, { baseURL, apiKey: "v4rt_test" }) + for (let turn = 0; turn < 3; turn++) + await generateText({ + model, + messages: [...(await MessageV2.toModelMessages(history, spec)), { role: "user", content: `turn ${turn}` }], + maxRetries: 0, + }) + expect(uploads).toBe(1) + for (const body of bodies) { + const input = body.input as Array> + expect(input[0]).toMatchObject({ type: "function_call", call_id: "call-1", name: "browser_execute" }) + expect(input[1]).toMatchObject({ + type: "function_call_output", + call_id: "call-1", + output: [ + { type: "input_text", text: "Captured" }, + { type: "input_image", file_id: "file-image" }, + ], + }) + expect(JSON.stringify(body)).toContain('"file_id":"file-image"') + expect(JSON.stringify(body)).not.toContain(screenshot.toString("base64")) + } + expect(JSON.stringify(history)).toBe(original) + } finally { + server.stop(true) + } +}) + +function fixture( + options: { supported?: boolean; missing?: number; uploadDelay?: number; uploadStatus?: number; relay?: boolean } = {}, +) { + const state = { + uploads: 0, + responses: 0, + deletes: 0, + scope: String(crypto.randomUUID()), + now: Date.now(), + active: 0, + peak: 0, + bodies: [] as string[], + } + const testScope = crypto.randomUUID() + const server = Bun.serve({ + port: 0, + async fetch(req) { + if (req.url.includes("/capability")) + return Response.json({ supported: options.supported ?? true, scope: state.scope }) + if (req.method === "DELETE") { + state.deletes++ + return Response.json({ deleted: true }) + } + if (req.url.includes("/image-files")) { + state.uploads++ + state.active++ + state.peak = Math.max(state.peak, state.active) + if (options.uploadDelay) await Bun.sleep(options.uploadDelay) + state.active-- + if (options.uploadStatus) return new Response("failed", { status: options.uploadStatus }) + return Response.json({ + file_id: `file-${state.uploads}`, + expires_at: Math.floor(state.now / 1000) + 3600, + signature: "proof", + }) + } + state.responses++ + const body = await req.text() + state.bodies.push(body) + if (state.responses <= (options.missing ?? 0)) + return Response.json( + { + error: { + message: `File file-${state.uploads} not found`, + type: "invalid_request_error", + code: "invalid_image", + }, + }, + { status: 400 }, + ) + return Response.json({ + id: "resp_a", + created_at: 1, + model: "gpt-test", + output: [ + { + type: "message", + id: "msg-a", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "ok", annotations: [] }], + }, + ], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + status: "completed", + }) + }, + }) + const baseURL = `${server.url}${options.relay ? "_v4/bcode-relay/relay-123/" : ""}api/v4/llm/openai/v1` + const native = createOpenAI({ + baseURL, + apiKey: "v4rt_test", + fetch: Object.assign((url: RequestInfo | URL, init?: RequestInit) => fetch(url, imageFileRequest(init)), { + preconnect: fetch.preconnect, + }), + }).responses("gpt-test") + const model = withImageFiles(native, { baseURL, apiKey: "v4rt_test", now: () => state.now }) + return { state, model, stop: () => server.stop(true) } +} + +test("unsupported actual route keeps inline behavior", async () => { + const f = fixture({ supported: false }) + try { + await f.model.doGenerate({ prompt }) + expect(f.state.uploads).toBe(0) + expect(f.state.bodies[0]).toContain(screenshot.toString("base64")) + } finally { + f.stop() + } +}) + +test("concurrent repeated screenshot has one upload and cleanup deletes it", async () => { + const f = fixture({ uploadDelay: 30 }) + try { + await Promise.all([f.model.doGenerate({ prompt }), f.model.doGenerate({ prompt }), f.model.doGenerate({ prompt })]) + expect(f.state.uploads).toBe(1) + await f.model.closeImageFiles() + expect(f.state.deletes).toBe(1) + } finally { + f.stop() + } +}) + +test("deleted reference recovers once with original previous_response_id and no inline retry", async () => { + const f = fixture({ missing: 1 }) + try { + await f.model.doGenerate({ prompt, providerOptions: { openai: { previousResponseId: "resp_prior", store: true } } }) + expect(f.state.uploads).toBe(2) + expect(f.state.responses).toBe(2) + for (const body of f.state.bodies) { + expect(body).not.toContain(screenshot.toString("base64")) + expect(JSON.parse(body).previous_response_id).toBe("resp_prior") + } + } finally { + f.stop() + } +}) + +test("repeated invalid references and failed upload stay bounded", async () => { + const f = fixture({ missing: 10 }) + try { + await expect(f.model.doGenerate({ prompt })).rejects.toThrow("not found") + expect(f.state.uploads).toBe(2) + expect(f.state.responses).toBe(2) + await expect(f.model.doGenerate({ prompt })).rejects.toThrow("not found") + expect(f.state.uploads).toBe(2) + } finally { + f.stop() + } + const failed = fixture({ uploadStatus: 503 }) + try { + await expect(failed.model.doGenerate({ prompt })).rejects.toThrow("upload failed") + expect(failed.state.responses).toBe(0) + expect(failed.state.uploads).toBe(1) + } finally { + failed.stop() + } +}) + +test("account rotation fails before forwarding any inline screenshot", async () => { + const f = fixture() + try { + await f.model.doGenerate({ prompt }) + f.state.scope = "another-account" + await expect(f.model.doGenerate({ prompt })).rejects.toThrow("account changed") + expect(f.state.uploads).toBe(1) + expect(f.state.responses).toBe(1) + } finally { + f.stop() + } +}) + +test("producer bounds concurrent image uploads", async () => { + const f = fixture({ uploadDelay: 10 }) + try { + await f.model.doGenerate({ + prompt: [ + { + role: "user", + content: Array.from({ length: 6 }, (_, i) => ({ + type: "file", + mediaType: "image/png", + data: Buffer.alloc(1024, i), + })), + }, + ], + }) + expect(f.state.uploads).toBe(6) + expect(f.state.peak).toBeLessThanOrEqual(2) + } finally { + f.stop() + } +}) + +test("aborted upload never falls back to a large request", async () => { + const f = fixture({ uploadDelay: 100 }) + try { + const abort = new AbortController() + const pending = f.model.doGenerate({ prompt, abortSignal: abort.signal }) + await Bun.sleep(20) + abort.abort() + await expect(pending).rejects.toThrow() + expect(f.state.responses).toBe(0) + } finally { + f.stop() + } +}) + +test("placeholder conversion only changes image parts in native input arrays", () => { + const unrelated = { type: "input_image", image_url: "bu-openai-file:file-meta" } + const result = imageFileRequest({ + headers: { "x-bu-image-scope": "run" }, + body: JSON.stringify({ + metadata: unrelated, + input: [ + { + type: "function_call_output", + output: [ + { type: "input_image", image_url: "bu-openai-file:file-a", detail: "high" }, + { type: "input_text", text: "bu-openai-file:file-a" }, + ], + }, + ], + }), + }) + expect(JSON.parse(result!.body as string)).toEqual({ + metadata: unrelated, + input: [ + { + type: "function_call_output", + output: [ + { type: "input_image", file_id: "file-a", detail: "high" }, + { type: "input_text", text: "bu-openai-file:file-a" }, + ], + }, + ], + }) +}) + +test("reusable worker relay URL uses upload references", async () => { + const f = fixture({ relay: true }) + try { + await f.model.doGenerate({ prompt }) + expect(f.state.uploads).toBe(1) + expect(f.state.bodies[0]).not.toContain(screenshot.toString("base64")) + } finally { + f.stop() + } +}) + +test("expired duplicate references coalesce even with a full 64-image run cache", async () => { + const f = fixture() + try { + const images = Array.from({ length: 64 }, (_, i) => ({ + type: "file" as const, + mediaType: "image/png", + data: Buffer.alloc(1024, i), + })) + await f.model.doGenerate({ prompt: [{ role: "user", content: images }] }) + expect(f.state.uploads).toBe(64) + f.state.now += 3600_000 + await f.model.doGenerate({ prompt: [{ role: "user", content: [images[0], images[0]] }] }) + expect(f.state.uploads).toBe(65) + f.state.now += 3600_000 + await expect(f.model.doGenerate({ prompt: [{ role: "user", content: [images[0]] }] })).rejects.toThrow( + "expired again", + ) + expect(f.state.uploads).toBe(65) + } finally { + f.stop() + } +}) From c5951b9071c603210d3f69b860f02418ee3e1a46 Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 1 Oct 2026 08:48:16 +0000 Subject: [PATCH 2/3] feat(provider): upload Anthropic screenshots before gateway serialization --- packages/opencode/src/provider/image-files.ts | 33 ++- packages/opencode/src/provider/provider.ts | 16 ++ .../provider/anthropic-image-files.test.ts | 241 ++++++++++++++++++ 3 files changed, 286 insertions(+), 4 deletions(-) create mode 100644 packages/opencode/test/provider/anthropic-image-files.test.ts diff --git a/packages/opencode/src/provider/image-files.ts b/packages/opencode/src/provider/image-files.ts index b1ebda1c11..2e034b2fa1 100644 --- a/packages/opencode/src/provider/image-files.ts +++ b/packages/opencode/src/provider/image-files.ts @@ -19,9 +19,12 @@ export function withImageFiles(model: LanguageModelV3, options: Options) { const send = options.fetch ?? fetch let scope: string | undefined let routeHeaders: Record = {} + const anthropic = /\/anthropic\/v1\/?$/.test(new URL(options.baseURL).pathname) const enabled = Boolean(options.apiKey) && - /^\/(?:_v4\/bcode-relay\/[^/]+\/)?api\/v4\/llm\/openai\/v1\/?$/.test(new URL(options.baseURL).pathname) + /^\/(?:_v4\/bcode-relay\/[^/]+\/)?api\/v4\/llm\/(?:openai|anthropic)\/v1\/?$/.test( + new URL(options.baseURL).pathname, + ) async function prepare(params: LanguageModelV3CallOptions): Promise { if (!enabled) return params @@ -114,7 +117,8 @@ export function withImageFiles(model: LanguageModelV3, options: Options) { }) if (!response.ok) throw new Error(`Screenshot upload failed (${response.status})`) const uploaded = (await response.json()) as Uploaded - if (!uploaded.file_id?.startsWith("file-")) throw new Error("Invalid screenshot file ID") + if (!uploaded.file_id?.startsWith(anthropic ? "file_" : "file-")) + throw new Error("Invalid screenshot file ID") if (!(uploaded.expires_at > now() / 1000 + 30)) throw new Error("Screenshot reference expired") return uploaded })() @@ -148,7 +152,7 @@ export function withImageFiles(model: LanguageModelV3, options: Options) { if (item.type !== "image-data") return item return { type: "image-url" as const, - url: `bu-openai-file:${await upload(item.data, item.mediaType)}`, + url: `bu-${anthropic ? "anthropic" : "openai"}-file:${await upload(item.data, item.mediaType)}`, providerOptions: item.providerOptions, } }), @@ -169,7 +173,8 @@ export function withImageFiles(model: LanguageModelV3, options: Options) { (part.data instanceof URL && part.data.protocol !== "data:") ) return part - return { ...part, data: await upload(part.data, part.mediaType) } + const id = await upload(part.data, part.mediaType) + return { ...part, data: anthropic ? new URL(`bu-anthropic-file:${id}`) : id } }), ), } @@ -261,5 +266,25 @@ export function imageFileRequest(init?: RequestInit): RequestInit | undefined { }), } }) + if (Array.isArray(body.messages)) { + const rewrite = (content: Array>): Array> => + content.map((part) => { + if (part?.type === "tool_result" && Array.isArray(part.content)) + return { ...part, content: rewrite(part.content) } + const source = part?.source as { type?: string; url?: string } | undefined + if (part?.type !== "image" || source?.type !== "url" || !source.url?.startsWith("bu-anthropic-file:file_")) + return part + return { ...part, source: { type: "file", file_id: source.url.slice("bu-anthropic-file:".length) } } + }) + body.messages = body.messages.map((message: Record) => + Array.isArray(message.content) ? { ...message, content: rewrite(message.content) } : message, + ) + const headers = new Headers(init.headers) + headers.set( + "anthropic-beta", + [...new Set([...(headers.get("anthropic-beta")?.split(",") ?? []), "files-api-2025-04-14"])].join(","), + ) + return { ...init, headers, body: JSON.stringify(body) } + } return { ...init, body: JSON.stringify(body) } } diff --git a/packages/opencode/src/provider/provider.ts b/packages/opencode/src/provider/provider.ts index b9d702eb1f..ff4a6a9d62 100644 --- a/packages/opencode/src/provider/provider.ts +++ b/packages/opencode/src/provider/provider.ts @@ -1790,6 +1790,22 @@ const layer = Layer.effect( return wrapped } } + if ( + model.api.npm === "@ai-sdk/anthropic" && + typeof options.apiKey === "string" && + typeof baseURL === "string" + ) { + const languageModel = loaded.languageModel.bind(loaded) + loaded.languageModel = (id: string) => { + const wrapped = withImageFiles(languageModel(id), { + baseURL, + apiKey: options.apiKey, + fetch: options.fetch, + }) + s.imageFiles.add(wrapped.closeImageFiles) + return wrapped + } + } s.sdk.set(key, loaded) return loaded as SDK } diff --git a/packages/opencode/test/provider/anthropic-image-files.test.ts b/packages/opencode/test/provider/anthropic-image-files.test.ts new file mode 100644 index 0000000000..4d98fdcff1 --- /dev/null +++ b/packages/opencode/test/provider/anthropic-image-files.test.ts @@ -0,0 +1,241 @@ +import { expect, test } from "bun:test" +import { createAnthropic } from "@ai-sdk/anthropic" +import { generateText } from "ai" +import { SessionV1 } from "@opencode-ai/core/v1/session" +import type { Provider } from "../../src/provider/provider" +import { MessageV2 } from "../../src/session/message-v2" +import { imageFileRequest, withImageFiles } from "../../src/provider/image-files" + +const screenshot = Buffer.alloc(1024 * 1024, 42) +const history = [ + { + info: { + id: "msg-a", + parentID: "msg-u", + sessionID: "session", + role: "assistant", + modelID: "claude-test", + providerID: "anthropic", + time: { created: 0 }, + }, + parts: [ + { + id: "part", + sessionID: "session", + messageID: "msg-a", + type: "tool", + tool: "browser_execute", + callID: "call-1", + state: { + status: "completed", + input: { code: "screenshot()" }, + output: "Captured", + title: "Screenshot", + metadata: {}, + time: { start: 0, end: 1 }, + attachments: [ + { + id: "file", + sessionID: "session", + messageID: "msg-a", + type: "file", + mime: "image/png", + url: `data:image/png;base64,${screenshot.toString("base64")}`, + }, + ], + }, + }, + ], + }, +] as unknown as SessionV1.WithParts[] +const spec = { + id: "claude-test", + providerID: "anthropic", + api: { id: "claude-test", npm: "@ai-sdk/anthropic" }, + capabilities: { input: { image: true } }, +} as Provider.Model +function fixture(options: { relay?: boolean; supported?: boolean; missing?: number; uploadStatus?: number } = {}) { + const state = { + uploads: 0, + deletes: 0, + calls: 0, + scope: crypto.randomUUID() as string, + bodies: [] as Array<{ messages: Array<{ content: Array> }> }>, + headers: [] as Headers[], + } + const server = Bun.serve({ + port: 0, + async fetch(req) { + if (req.url.includes("/capability")) + return Response.json({ supported: options.supported ?? true, scope: state.scope }) + if (req.method === "DELETE") { + state.deletes++ + return Response.json({ deleted: true }) + } + if (req.url.includes("/image-files")) { + expect(req.headers.get("content-type")).toBe("image/png") + expect(Buffer.from(await req.arrayBuffer()).equals(screenshot)).toBe(true) + state.uploads++ + if (options.uploadStatus) return new Response("failed", { status: options.uploadStatus }) + return Response.json({ + file_id: `file_${state.uploads}`, + signature: "proof", + expires_at: Date.now() / 1000 + 3600, + }) + } + state.calls++ + state.bodies.push(await req.json()) + state.headers.push(req.headers) + if (state.calls <= (options.missing ?? 0)) + return Response.json( + { type: "error", error: { type: "not_found_error", message: `File file_${state.uploads} not found` } }, + { status: 404 }, + ) + return Response.json({ + id: "msg_test", + type: "message", + role: "assistant", + model: "claude-test", + content: [{ type: "text", text: "ok" }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, + }) + }, + }) + const baseURL = `${server.url}${options.relay ? "_v4/bcode-relay/worker-1/" : ""}api/v4/llm/anthropic/v1` + const native = createAnthropic({ + baseURL, + apiKey: "v4rt_test", + headers: { "anthropic-beta": "interleaved-thinking-2025-05-14,fine-grained-tool-streaming-2025-05-14" }, + fetch: Object.assign((url: RequestInfo | URL, init?: RequestInit) => fetch(url, imageFileRequest(init)), { + preconnect: fetch.preconnect, + }), + }).languageModel("claude-test") + const adapted = withImageFiles(native, { baseURL, apiKey: "v4rt_test" }) + const model = process.env.ANTHROPIC_FILES_BASELINE ? native : adapted + return { + state, + model, + async close() { + await adapted.closeImageFiles() + server.stop(true) + }, + } +} +const prompt = [ + { + role: "user" as const, + content: [ + { type: "text" as const, text: "Read screenshot" }, + { + type: "file" as const, + mediaType: "image/png", + data: screenshot, + providerOptions: { anthropic: { cacheControl: { type: "ephemeral" } } }, + }, + ], + }, +] +for (const relay of [false, true]) + test(`actual BrowserCode Anthropic screenshot history three turns (relay=${relay})`, async () => { + const f = fixture({ relay }) + const original = JSON.stringify(history) + try { + for (let turn = 0; turn < 3; turn++) + await generateText({ + model: f.model, + messages: [...(await MessageV2.toModelMessages(history, spec)), { role: "user", content: `turn ${turn}` }], + maxRetries: 0, + }) + if (process.env.ANTHROPIC_FILES_EVIDENCE) + await Bun.write( + process.env.ANTHROPIC_FILES_EVIDENCE + (relay ? "-relay" : ""), + JSON.stringify({ + uploads: f.state.uploads, + bodies: f.state.bodies, + historyIntact: JSON.stringify(history) === original, + }), + ) + expect(f.state.uploads).toBe(1) + expect(f.state.bodies).toHaveLength(3) + for (const body of f.state.bodies) { + expect(body.messages[0].content).toEqual([ + { type: "tool_use", id: "call-1", name: "browser_execute", input: { code: "screenshot()" } }, + ]) + expect(body.messages[1].content[0]).toEqual({ + type: "tool_result", + tool_use_id: "call-1", + content: [ + { type: "text", text: "Captured" }, + { type: "image", source: { type: "file", file_id: "file_1" } }, + ], + }) + expect(JSON.stringify(body)).not.toContain(screenshot.toString("base64")) + expect(JSON.stringify(body)).not.toContain("bu-anthropic-file:") + } + expect(JSON.stringify(history)).toBe(original) + expect(f.state.headers[0].get("anthropic-beta")).toContain("files-api-2025-04-14") + expect(f.state.headers[0].get("anthropic-beta")).toContain("interleaved-thinking-2025-05-14") + expect(f.state.headers[0].get("anthropic-beta")).toContain("fine-grained-tool-streaming-2025-05-14") + } finally { + await f.close() + } + }) +test("Anthropic user images preserve cache control and source history", async () => { + const f = fixture() + try { + await f.model.doGenerate({ prompt }) + expect(f.state.uploads).toBe(1) + expect(f.state.bodies[0].messages[0].content[1]).toEqual({ + type: "image", + source: { type: "file", file_id: "file_1" }, + cache_control: { type: "ephemeral" }, + }) + expect(prompt[0].content[1]).toMatchObject({ data: screenshot }) + } finally { + await f.close() + } +}) +test("unsupported Anthropic routes retain legacy behavior before selection", async () => { + const f = fixture({ supported: false }) + try { + await f.model.doGenerate({ prompt }) + expect(f.state.uploads).toBe(0) + expect(f.state.bodies[0].messages[0].content[1]).toMatchObject({ source: { type: "base64" } }) + } finally { + await f.close() + } +}) +test("Anthropic upload failure never silently falls back", async () => { + const f = fixture({ uploadStatus: 503 }) + try { + await expect(f.model.doGenerate({ prompt })).rejects.toThrow("upload failed") + expect(f.state.calls).toBe(0) + } finally { + await f.close() + } +}) +test("Anthropic scope rotation fails before another inference", async () => { + const f = fixture() + try { + await f.model.doGenerate({ prompt }) + f.state.scope = "rotated" + await expect(f.model.doGenerate({ prompt })).rejects.toThrow("account changed") + expect(f.state.calls).toBe(1) + } finally { + await f.close() + } +}) +for (const missing of [1, 2]) + test(`Anthropic missing file gets one bounded refresh (${missing})`, async () => { + const f = fixture({ missing }) + try { + if (missing === 1) await f.model.doGenerate({ prompt }) + else await expect(f.model.doGenerate({ prompt })).rejects.toThrow() + expect(f.state.uploads).toBe(2) + expect(f.state.calls).toBe(2) + } finally { + await f.close() + } + }) From e701221d8f58eb7ae53d7244dc7a798da53e1dff Mon Sep 17 00:00:00 2001 From: "exe.dev user" Date: Thu, 1 Oct 2026 09:13:15 +0000 Subject: [PATCH 3/3] fix(provider): retain live screenshot cache owners and retry failed uploads --- packages/opencode/src/provider/image-files.ts | 33 +++++++++++--- .../test/provider/image-files.test.ts | 43 ++++++++++++++++++- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/packages/opencode/src/provider/image-files.ts b/packages/opencode/src/provider/image-files.ts index 2e034b2fa1..850fb8b832 100644 --- a/packages/opencode/src/provider/image-files.ts +++ b/packages/opencode/src/provider/image-files.ts @@ -3,6 +3,7 @@ import type { LanguageModelV3, LanguageModelV3CallOptions } from "@ai-sdk/provid type Uploaded = { file_id: string; expires_at: number; signature: string } type Options = { baseURL: string; apiKey: string; fetch?: typeof fetch; now?: () => number } type RunCache = { + owners: Set cache: Map> refreshed: Set retired: Uploaded[] @@ -13,7 +14,17 @@ type RunCache = { const runCaches = new Map() export function withImageFiles(model: LanguageModelV3, options: Options) { - let state: RunCache = { cache: new Map(), refreshed: new Set(), retired: [], waiting: [], active: 0, closed: false } + let state: RunCache = { + owners: new Set(), + cache: new Map(), + refreshed: new Set(), + retired: [], + waiting: [], + active: 0, + closed: false, + } + const owner = Symbol() + let closed = false const now = options.now ?? Date.now const endpoint = options.baseURL.replace(/\/$/, "") + "/image-files" const send = options.fetch ?? fetch @@ -46,7 +57,7 @@ export function withImageFiles(model: LanguageModelV3, options: Options) { )), ) if (!hasImages) return params - if (state.closed) throw new Error("Screenshot file cache is closed") + if (closed || state.closed) throw new Error("Screenshot file cache is closed") const previous = params.providerOptions?.openai?.previousResponseId const legacy = params.prompt.some( (message) => @@ -74,10 +85,11 @@ export function withImageFiles(model: LanguageModelV3, options: Options) { if (!route.supported && !scope) return params if (!route.supported || !route.scope || (scope && scope !== route.scope)) throw new Error("Screenshot route or account changed; start a new run") - if (state.closed) throw new Error("Screenshot file cache is closed") + if (closed || state.closed) throw new Error("Screenshot file cache is closed") scope = route.scope if (!runCaches.has(scope) && runCaches.size >= 128) throw new Error("Too many active screenshot caches") state = runCaches.get(scope) ?? state + state.owners.add(owner) runCaches.set(scope, state) routeHeaders = currentHeaders const headers = { ...currentHeaders, "x-bu-image-scope": scope } @@ -87,7 +99,7 @@ export function withImageFiles(model: LanguageModelV3, options: Options) { else state.active++ try { signal.throwIfAborted() - if (state.closed) throw new Error("Screenshot file cache is closed") + if (closed || state.closed) throw new Error("Screenshot file cache is closed") const raw = data instanceof URL ? data.toString().split(",", 2)[1] : data const bytes = typeof raw === "string" ? Buffer.from(raw, "base64") : raw if (bytes.byteLength > 5 * 1024 * 1024) throw new Error("Screenshot exceeds upload limit") @@ -123,10 +135,13 @@ export function withImageFiles(model: LanguageModelV3, options: Options) { return uploaded })() state.cache.set(key, pending) - const uploaded = await pending + const uploaded = await pending.catch((error) => { + if (state.cache.get(key) === pending) state.cache.delete(key) + throw error + }) if (uploaded.expires_at <= now() / 1000 + 30) throw new Error("Screenshot reference expired; start a new run") signal.throwIfAborted() - if (state.closed) throw new Error("Screenshot file cache is closed") + if (closed || state.closed) throw new Error("Screenshot file cache is closed") proofs[uploaded.file_id] = { expires_at: uploaded.expires_at, signature: uploaded.signature } return uploaded.file_id } finally { @@ -220,8 +235,12 @@ export function withImageFiles(model: LanguageModelV3, options: Options) { return invoke(params, (prepared) => model.doStream(prepared)) }, async closeImageFiles() { + if (closed) return + closed = true + state.owners.delete(owner) + if (state.owners.size) return state.closed = true - if (scope) runCaches.delete(scope) + if (scope && runCaches.get(scope) === state) runCaches.delete(scope) const entries = await Promise.allSettled([ ...state.cache.values(), ...state.retired.map((file) => Promise.resolve(file)), diff --git a/packages/opencode/test/provider/image-files.test.ts b/packages/opencode/test/provider/image-files.test.ts index 36b133dac6..ff10e70ece 100644 --- a/packages/opencode/test/provider/image-files.test.ts +++ b/packages/opencode/test/provider/image-files.test.ts @@ -303,7 +303,12 @@ function fixture( }), }).responses("gpt-test") const model = withImageFiles(native, { baseURL, apiKey: "v4rt_test", now: () => state.now }) - return { state, model, stop: () => server.stop(true) } + return { + state, + model, + wrap: () => withImageFiles(native, { baseURL, apiKey: "v4rt_test", now: () => state.now }), + stop: () => server.stop(true), + } } test("unsupported actual route keeps inline behavior", async () => { @@ -478,3 +483,39 @@ test("expired duplicate references coalesce even with a full 64-image run cache" f.stop() } }) + +test("transient upload rejection permits a later explicit retry", async () => { + const options: { uploadStatus?: number } = { uploadStatus: 503 } + const f = fixture(options) + try { + await expect(f.model.doGenerate({ prompt })).rejects.toThrow("upload failed") + expect(f.state.responses).toBe(0) + options.uploadStatus = undefined + await f.model.doGenerate({ prompt }) + expect(f.state.uploads).toBe(2) + expect(f.state.responses).toBe(1) + } finally { + await f.model.closeImageFiles() + f.stop() + } +}) + +test("closing one wrapper preserves references owned by a sibling", async () => { + const f = fixture() + const sibling = f.wrap() + try { + await f.model.doGenerate({ prompt }) + await sibling.doGenerate({ prompt }) + await f.model.closeImageFiles() + expect(f.state.deletes).toBe(0) + await expect(f.model.doGenerate({ prompt })).rejects.toThrow("closed") + await sibling.doGenerate({ prompt }) + expect(f.state.uploads).toBe(1) + await sibling.closeImageFiles() + expect(f.state.deletes).toBe(1) + } finally { + await f.model.closeImageFiles() + await sibling.closeImageFiles() + f.stop() + } +})