From dc4c6eb75c4dabe75245a5f87e90b64a94d4a6db Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Tue, 6 Oct 2026 18:28:18 +0000 Subject: [PATCH] websearch: Ceramic, Exa and Linkup with their own keys, no Cloudflare credits needed (0.63.0) Cloudflare never applied the AI Gateway credits, so the three providers it fronts always 402. Each now accepts its own API key (cli-tools config set ceramic / exa / linkup) and is called directly: api.ceramic.ai/search, api.exa.ai/search with highlights, api.linkup.so/v1/search (fast, searchResults). A provider with its own key never goes through Cloudflare; without one it falls back to the gateway as before. The 402 message now says how to fix it. Co-Authored-By: Claude Opus 5.5 --- README.md | 6 +-- bin/websearch.ts | 19 ++++++++- package.json | 2 +- src/credentials.ts | 5 +++ src/websearch.ts | 91 +++++++++++++++++++++++++++++++++++++++++- test/websearch.test.ts | 31 ++++++++++++++ 6 files changed, 147 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index bc2357b..fea4a9a 100644 --- a/README.md +++ b/README.md @@ -1158,9 +1158,9 @@ with a ranking of our own. In the pit it is `/search`. Five providers: | Letter | Provider | How it is called | Key | | --- | --- | --- | --- | -| C | Ceramic | [Cloudflare Web Search](https://developers.cloudflare.com/web-search/providers/) | `cloudflare` (or email + global key) | -| E | Exa | Cloudflare Web Search | same | -| L | Linkup | Cloudflare Web Search | same | +| C | Ceramic | its own key, else [Cloudflare Web Search](https://developers.cloudflare.com/web-search/providers/) | `ceramic` ([platform.ceramic.ai/keys](https://platform.ceramic.ai/keys)), else `cloudflare` | +| E | Exa | its own key, else Cloudflare Web Search | `exa` ([dashboard.exa.ai](https://dashboard.exa.ai)), else `cloudflare` | +| L | Linkup | its own key, else Cloudflare Web Search | `linkup` ([app.linkup.so](https://app.linkup.so)), else `cloudflare` | | P | Perplexity (its own index) | directly, Perplexity Search API | `perplexity` | | S | Serper (Google results) | directly, serper.dev | `serper` | diff --git a/bin/websearch.ts b/bin/websearch.ts index adb2f9c..6592c8d 100755 --- a/bin/websearch.ts +++ b/bin/websearch.ts @@ -23,7 +23,10 @@ import { authCandidates, cloudflareCaller, costOf, + ceramicCaller, discoverAccountId, + exaCaller, + linkupCaller, formatResults, perplexitySearchCaller, rank, @@ -58,8 +61,12 @@ Options: Keys, stored once (any subset works; providers without one are skipped): + cli-tools config set ceramic # CERAMIC_API_KEY platform.ceramic.ai/keys + cli-tools config set exa # EXA_API_KEY dashboard.exa.ai + cli-tools config set linkup # LINKUP_API_KEY app.linkup.so cli-tools config set perplexity # PERPLEXITY_API_KEY cli-tools config set serper # SERPER_API_KEY (serper.dev) + # Without their own keys, Ceramic, Exa and Linkup go through Cloudflare: cli-tools config set cloudflare # CLOUDFLARE_API_TOKEN, Workers AI + # AI Gateway Read cli-tools config set cloudflare_email # with cloudflare_global, the @@ -105,9 +112,17 @@ if (isMain(import.meta.url)) { const serperKey = credentials['SERPER_API_KEY']; if (perplexityKey) direct.perplexity = perplexitySearchCaller(perplexityKey, timeout); if (serperKey) direct.serper = serperCaller(serperKey, timeout); + // A provider's own key beats going through Cloudflare: no gateway credits + // needed, and it is the same index either way. + const ceramicKey = credentials['CERAMIC_API_KEY']; + const exaKey = credentials['EXA_API_KEY']; + const linkupKey = credentials['LINKUP_API_KEY']; + if (ceramicKey) direct.ceramic = ceramicCaller(ceramicKey, timeout); + if (exaKey) direct.exa = exaCaller(exaKey, timeout); + if (linkupKey) direct.linkup = linkupCaller(linkupKey, timeout); const isCloudflare = (p: Provider) => (CLOUDFLARE_PROVIDERS as readonly string[]).includes(p); - const configured = PROVIDERS.filter((p) => (isCloudflare(p) ? auths.length > 0 : !!direct[p])); + const configured = PROVIDERS.filter((p) => !!direct[p] || (isCloudflare(p) && auths.length > 0)); const providers = (asked.length ? [...new Set(asked)] : configured) as Provider[]; if (providers.length === 0) { throw new UsageError( @@ -119,7 +134,7 @@ if (isMain(import.meta.url)) { // The account id is needed only when a Cloudflare provider is asked. If it // cannot be found, those providers fail and the direct ones still answer. let accountId = values.get('--account') ?? credentials['CLOUDFLARE_ACCOUNT_ID'] ?? ''; - if (!accountId && auths.length > 0 && providers.some(isCloudflare)) { + if (!accountId && auths.length > 0 && providers.some((p) => isCloudflare(p) && !direct[p])) { for (const auth of auths) { try { accountId = await discoverAccountId(auth, timeout); diff --git a/package.json b/package.json index 7c587d1..498190c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/cli-tools", - "version": "0.62.0", + "version": "0.63.0", "private": true, "description": "Local command-line tools, in TypeScript, exposed on PATH.", "type": "module", diff --git a/src/credentials.ts b/src/credentials.ts index 887e184..622ed32 100644 --- a/src/credentials.ts +++ b/src/credentials.ts @@ -61,6 +61,11 @@ export const KNOWN_KEYS: Record = { cloudflare_account: 'CLOUDFLARE_ACCOUNT_ID', cloudflare_email: 'CLOUDFLARE_EMAIL', cloudflare_global: 'CLOUDFLARE_GLOBAL_API_KEY', + // Read by `websearch`: each provider's own key, called directly instead of + // through Cloudflare's AI Gateway. + ceramic: 'CERAMIC_API_KEY', + exa: 'EXA_API_KEY', + linkup: 'LINKUP_API_KEY', // Read by `websearch`: Google results from serper.dev, called directly. serper: 'SERPER_API_KEY', }; diff --git a/src/websearch.ts b/src/websearch.ts index 8450d40..b3e4afa 100644 --- a/src/websearch.ts +++ b/src/websearch.ts @@ -186,7 +186,7 @@ export function describeError(status: number, text: string): SearchError { const effective = Number(gw.status ?? status); if (code === 'web_search_payment_required') { return new SearchError( - 'no AI Gateway credits on this account — add some under AI > AI Gateway > Credits (402)', + 'no Cloudflare AI Gateway credits (402) — give each provider its own key instead: `cli-tools config set ceramic` / `exa` / `linkup`', effective, ); } @@ -265,6 +265,95 @@ export function parseSerper(body: any): Hit[] { })); } +/** Ceramic: `result.results[]` of { title, url, description }. */ +export function parseCeramic(body: any): Hit[] { + const results: any[] = Array.isArray(body?.result?.results) ? body.result.results : Array.isArray(body?.results) ? body.results : []; + return results + .filter((r) => typeof r?.url === 'string' && r.url) + .map((r) => ({ + url: r.url.trim(), + title: typeof r.title === 'string' ? r.title.trim() : '', + description: typeof r.description === 'string' ? r.description.trim() : '', + })); +} + +/** Exa: `results[]` of { title, url, highlights[] }; the highlights are the snippet. */ +export function parseExa(body: any): Hit[] { + const results: any[] = Array.isArray(body?.results) ? body.results : []; + return results + .filter((r) => typeof r?.url === 'string' && r.url) + .map((r) => ({ + url: r.url.trim(), + title: typeof r.title === 'string' ? r.title.trim() : '', + description: Array.isArray(r.highlights) + ? r.highlights.filter((h: unknown) => typeof h === 'string').join(' … ').trim() + : typeof r.text === 'string' + ? r.text.trim().slice(0, 1000) + : '', + })); +} + +/** Linkup searchResults: `results[]` of { name, url, content }. */ +export function parseLinkup(body: any): Hit[] { + const results: any[] = Array.isArray(body?.results) ? body.results : []; + return results + .filter((r) => typeof r?.url === 'string' && r.url) + .map((r) => ({ + url: r.url.trim(), + title: typeof r.name === 'string' ? r.name.trim() : '', + // Linkup returns page content, sometimes the whole page; the ranker only + // needs enough to match the query against. + description: typeof r.content === 'string' ? r.content.trim().slice(0, 1000) : '', + })); +} + +export function ceramicCaller(apiKey: string, timeoutMs: number): DirectCaller { + return async (query, limit) => { + const { body, latencyMs } = await postJson( + 'https://api.ceramic.ai/search', + { Authorization: `Bearer ${apiKey}` }, + { query }, + timeoutMs, + ); + return { hits: parseCeramic(body).slice(0, limit), latencyMs }; + }; +} + +export function exaCaller(apiKey: string, timeoutMs: number): DirectCaller { + return async (query, limit) => { + const { body, latencyMs } = await postJson( + 'https://api.exa.ai/search', + { 'x-api-key': apiKey }, + { query, type: 'auto', numResults: limit, contents: { highlights: true } }, + timeoutMs, + ); + return { hits: parseExa(body), latencyMs }; + }; +} + +export function linkupCaller(apiKey: string, timeoutMs: number): DirectCaller { + return async (query, limit) => { + const started = Date.now(); + const params = new URLSearchParams({ q: query, depth: 'fast', outputType: 'searchResults', maxResults: String(limit) }); + const response = await fetch(`https://api.linkup.so/v1/search?${params}`, { + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(timeoutMs), + }); + const text = await response.text(); + if (!response.ok) { + let message = text.trim().slice(0, 200) || `HTTP ${response.status}`; + try { + const parsed = JSON.parse(text); + message = parsed?.error?.message ?? parsed?.message ?? message; + } catch { + // keep the raw text + } + throw new SearchError(`${message} (${response.status})`, response.status); + } + return { hits: parseLinkup(JSON.parse(text)).slice(0, limit), latencyMs: Date.now() - started }; + }; +} + export function perplexitySearchCaller(apiKey: string, timeoutMs: number): DirectCaller { return async (query, limit) => { const { body, latencyMs } = await postJson( diff --git a/test/websearch.test.ts b/test/websearch.test.ts index 1451ee3..4ab77c9 100644 --- a/test/websearch.test.ts +++ b/test/websearch.test.ts @@ -13,7 +13,10 @@ import { describeError, formatResults, matchScore, + parseCeramic, + parseExa, parseHits, + parseLinkup, parsePerplexity, parseSerper, rank, @@ -227,3 +230,31 @@ describe('direct providers', () => { expect(result!.error).toMatch(/no serper key/); }); }); + +describe('direct keys for the Cloudflare providers', () => { + it('reads Ceramic, Exa and Linkup responses', () => { + expect(parseCeramic({ result: { results: [{ title: 'C', url: 'https://c.example', description: 'd' }] } })).toEqual([ + hit('https://c.example', 'C', 'd'), + ]); + expect(parseExa({ results: [{ title: 'E', url: 'https://e.example', highlights: ['one', 'two'] }] })).toEqual([ + hit('https://e.example', 'E', 'one … two'), + ]); + expect(parseLinkup({ results: [{ type: 'text', name: 'L', url: 'https://l.example', content: 'x' }] })).toEqual([ + hit('https://l.example', 'L', 'x'), + ]); + }); + + // A direct key must win over Cloudflare, which is why the keys exist. + it('calls a provider directly when it has its own key, not through Cloudflare', async () => { + let cloudflareCalled = false; + const caller: Caller = async () => { + cloudflareCalled = true; + throw new SearchError('no credits', 402); + }; + const [result] = await searchAll('q', ['exa'], caller, [{ kind: 'token', token: 't' }], { + direct: { exa: async () => ({ hits: [hit('https://e.example')], latencyMs: 1 }) }, + }); + expect(cloudflareCalled).toBe(false); + expect(result!.error).toBeNull(); + }); +});