diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 1dcedf9..9ca21e6 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -266,6 +266,12 @@ import { jobPostRoutes } from './job-posts'; import { ideaRoutes } from './ideas'; import { leadRoutes } from './buyer-leads'; import { linkPostRoutes } from './link-posts'; +import { + campaignSourceStatus, + checkCampaignSource, + normaliseSourceUrl, + parsePostNetworks, +} from './campaign-source'; import type { LeadMonitor } from '@outreachgraph/pipeline'; import type { FeedSource } from '@outreachgraph/providers'; import type { ChovyConfig, Fetcher as RedditFetcher } from '@outreachgraph/ideas'; @@ -1787,7 +1793,57 @@ export function createApp(options: AppOptions): Hono { const actor = c.get('actor'); const campaign = await repo.getCampaign(c.get('db'), actor.workspaceId, c.req.param('id')); if (!campaign) throw ApiError.notFound('campaign'); - return c.json({ campaign }); + const source = await campaignSourceStatus(c.get('db'), campaign.id); + return c.json({ + campaign, + // The URL the campaign reads on a schedule, and the networks it drafts for. + source: { + url: source?.source_url ?? null, + postNetworks: parsePostNetworks(source?.post_networks ?? '[]'), + checkedAt: source?.source_checked_at ?? null, + error: source?.source_error ?? null, + }, + }); + }); + + /** Reads the campaign's source now instead of waiting for the schedule. */ + api.post('/campaigns/:id/source/check', async (c) => { + const actor = c.get('actor'); + const db = c.get('db'); + if (!canApprove(actor)) throw ApiError.forbidden('reading a campaign source'); + if (!options.model) { + throw new ApiError( + 503, + 'no_model', + 'no language model is configured, so posts cannot be drafted', + ); + } + const row = await queryOne<{ + id: string; + workspace_id: string; + offering_id: string; + voice_profile_id: string | null; + brief: string | null; + source: string | null; + post_networks: string; + source_fingerprint: string | null; + }>( + db, + `SELECT c.id, c.workspace_id, c.offering_id, c.voice_profile_id, c.brief, + COALESCE(c.source_url, o.url) AS source, c.post_networks, c.source_fingerprint + FROM campaigns c JOIN offerings o ON o.id = c.offering_id + WHERE c.id = ? AND c.workspace_id = ?`, + [c.req.param('id'), actor.workspaceId], + ); + if (!row) throw ApiError.notFound('campaign'); + if (!row.source) + throw ApiError.badRequest('this campaign has no source URL; set sourceUrl first'); + const result = await checkCampaignSource( + { db, model: options.model, fetchImpl: options.linkFetch, lookup: options.linkLookup }, + { ...row, source: row.source }, + new Date(), + ); + return c.json({ result }); }); /** @@ -1891,6 +1947,31 @@ export function createApp(options: AppOptions): Hono { const body = safeJson(await c.req.raw.text()); const input = typeof body.input === 'string' ? body.input : ''; + // Every campaign reads a URL of its own on a schedule: the one given, else + // the product's site. A campaign with neither has nothing to post about. + const givenSource = + body.sourceUrl === undefined ? undefined : normaliseSourceUrl(body.sourceUrl); + if (body.sourceUrl !== undefined && !givenSource) { + throw ApiError.badRequest('sourceUrl must be a web address, like https://example.com/blog'); + } + if (!givenSource) { + const product = await queryOne<{ url: string | null }>( + db, + typeof body.offeringId === 'string' && body.offeringId + ? 'SELECT url FROM offerings WHERE id = ? AND workspace_id = ?' + : 'SELECT url FROM offerings WHERE workspace_id = ? ORDER BY created_at LIMIT 1', + typeof body.offeringId === 'string' && body.offeringId + ? [body.offeringId, actor.workspaceId] + : [actor.workspaceId], + ); + if (product && !product.url) { + throw ApiError.badRequest( + 'a campaign needs a URL to read: pass sourceUrl, or give the product a website', + { sourceUrl: ['required'] }, + ); + } + } + let result; try { result = await createCampaignFromIntake(db, actor, input, { @@ -1911,6 +1992,21 @@ export function createApp(options: AppOptions): Hono { throw error; } + if (givenSource || body.postNetworks !== undefined) { + const networks = parsePostNetworks(JSON.stringify(body.postNetworks ?? ['linkedin'])); + await db.execute({ + sql: `UPDATE campaigns SET source_url = COALESCE(?, source_url), post_networks = ? + WHERE id = ? AND workspace_id = ?`, + args: [givenSource ?? null, JSON.stringify(networks), result.campaignId, actor.workspaceId], + }); + } else { + await db.execute({ + sql: `UPDATE campaigns SET source_url = (SELECT url FROM offerings WHERE id = campaigns.offering_id) + WHERE id = ? AND source_url IS NULL`, + args: [result.campaignId], + }); + } + await repo.audit(db, { workspaceId: actor.workspaceId, actorKind: 'user', @@ -2027,6 +2123,36 @@ export function createApp(options: AppOptions): Hono { touched = true; } + if (body.sourceUrl !== undefined || body.postNetworks !== undefined) { + const sets: string[] = []; + const args: (string | null)[] = []; + if (body.sourceUrl !== undefined) { + const url = normaliseSourceUrl(body.sourceUrl); + if (!url) + throw ApiError.badRequest( + 'sourceUrl must be a web address, like https://example.com/blog', + ); + // A new URL is a new source: forget what the old one looked like. + sets.push('source_url = ?', 'source_fingerprint = NULL', 'source_checked_at = NULL'); + args.push(url); + changed.sourceUrl = url; + } + if (body.postNetworks !== undefined) { + const networks = parsePostNetworks(JSON.stringify(body.postNetworks), false); + if (networks.length === 0) + throw ApiError.badRequest('postNetworks needs at least one network'); + sets.push('post_networks = ?'); + args.push(JSON.stringify(networks)); + changed.postNetworks = networks; + } + const updated = await db.execute({ + sql: `UPDATE campaigns SET ${sets.join(', ')} WHERE id = ? AND workspace_id = ?`, + args: [...args, campaignId, actor.workspaceId], + }); + if (Number(updated.rowsAffected ?? 0) === 0) throw ApiError.notFound('campaign'); + touched = true; + } + if (typeof body.name === 'string') { if (!(await renameCampaign(db, actor.workspaceId, campaignId, body.name))) { throw ApiError.notFound('campaign'); diff --git a/apps/api/src/campaign-source.test.ts b/apps/api/src/campaign-source.test.ts new file mode 100644 index 0000000..d5dd1b1 --- /dev/null +++ b/apps/api/src/campaign-source.test.ts @@ -0,0 +1,239 @@ +/** + * A campaign's source URL, read on a schedule into drafted posts. + * + * What has to hold: a feed's first read drafts only its newest item; later + * reads draft only new items; a plain page drafts when its title or + * description changes, never on the first read; the daily cap holds; the + * campaign's voice and target customer reach the writer; and the API requires + * a source when the product has no site. + */ + +import { afterEach, describe, expect, setDefaultTimeout, test } from 'bun:test'; +import { StubModel } from '@outreachgraph/ai'; +import { now, queryAll, type Client } from '@outreachgraph/db'; +import type { FetchLike, HostLookup } from '@outreachgraph/providers'; +import { createApp } from './app'; +import { BATCHES_PER_CAMPAIGN_PER_DAY, feedLink, runCampaignSources } from './campaign-source'; +import type { RequestActor } from './context'; +import { seedDatabase, SEED, type SeededDatabase } from './test-seed'; + +setDefaultTimeout(30_000); + +let active: SeededDatabase | undefined; +afterEach(() => { + active?.cleanup(); + active = undefined; +}); + +const OWNER: RequestActor = { + userId: SEED.userId, + workspaceId: SEED.workspaceId, + organizationId: SEED.organizationId, + role: 'owner', +}; + +const SITE = 'https://acme.example.com/'; +const FEED = 'https://acme.example.com/feed.xml'; +const article = (n: number) => `https://acme.example.com/blog/post-${n}`; + +const lookup: HostLookup = async () => ['93.184.216.34']; + +function rss(items: number[]): string { + return `Acme${items + .map( + (n) => + `Post ${n}${article(n)}Mon, 0${n} Oct 2026 10:00:00 GMT`, + ) + .join('')}`; +} + +const ARTICLE_HTML = (n: number) => + `

Post ${n} explains how we cut costs by half with caching.

`; + +function site(state: { items: number[]; withFeed: boolean; title: string }): FetchLike { + return async (input) => { + const url = String(input instanceof Request ? input.url : input); + if (url === SITE) { + const link = state.withFeed + ? `` + : ''; + return new Response( + `${state.title}${link}

Home

`, + { headers: { 'content-type': 'text/html' } }, + ); + } + if (url === FEED) { + return new Response(rss(state.items), { headers: { 'content-type': 'application/rss+xml' } }); + } + const match = /post-(\d+)$/.exec(url); + if (match) { + return new Response(ARTICLE_HTML(Number(match[1])), { + headers: { 'content-type': 'text/html' }, + }); + } + return new Response('nope', { status: 404 }); + }; +} + +const POSTS = JSON.stringify({ + posts: [{ network: 'linkedin', text: 'We cut costs by half with caching.' }], +}); + +async function setup(label: string): Promise { + active = await seedDatabase(label); + const db = active.db; + await db.execute({ + sql: `UPDATE campaigns SET source_url = ?, status = 'active' WHERE id = ?`, + args: [SITE, SEED.campaignId], + }); + return db; +} + +async function batches(db: Client) { + return queryAll<{ batch_id: string; url: string }>( + db, + 'SELECT DISTINCT batch_id, url FROM link_posts WHERE campaign_id = ? ORDER BY url', + [SEED.campaignId], + ); +} + +const later = (hours: number) => new Date(Date.now() + hours * 3_600_000); + +describe('campaign sources', () => { + test('a feed: the first read drafts only the newest item, later reads only new ones', async () => { + const db = await setup('campaign-source-feed'); + const state = { items: [3, 2, 1], withFeed: true, title: 'Acme' }; + const model = new StubModel([POSTS, POSTS, POSTS, POSTS]); + const deps = { db, model, fetchImpl: site(state), lookup }; + + const first = await runCampaignSources(deps); + expect(first[0]).toMatchObject({ mode: 'feed', newItems: 3, drafted: 1 }); + expect((await batches(db)).map((b) => b.url)).toEqual([article(3)]); + + // Not due again until the interval has passed. + expect(await runCampaignSources(deps)).toEqual([]); + + state.items = [4, 3, 2, 1]; + const second = await runCampaignSources({ ...deps, now: later(7) }); + expect(second[0]).toMatchObject({ mode: 'feed', newItems: 1, drafted: 1 }); + expect((await batches(db)).map((b) => b.url)).toEqual([article(3), article(4)]); + }); + + test("the campaign's voice and target customer reach the writer", async () => { + const db = await setup('campaign-source-voice'); + const stamp = now(); + await db.execute({ + sql: `INSERT INTO voice_profiles (id, workspace_id, name, style, instructions, created_at, updated_at) + VALUES ('vp_dry', ?, 'Dry', 'dry and specific', 'No exclamation marks.', ?, ?)`, + args: [SEED.workspaceId, stamp, stamp], + }); + await db.execute({ + sql: `UPDATE campaigns SET voice_profile_id = 'vp_dry' WHERE id = ?`, + args: [SEED.campaignId], + }); + await db.execute({ + sql: `INSERT INTO campaign_filters (campaign_id, titles, industries, updated_at) + VALUES (?, '["CTO","VP Engineering"]', '["fintech"]', ?) + ON CONFLICT (campaign_id) DO UPDATE SET titles = excluded.titles, + industries = excluded.industries`, + args: [SEED.campaignId, stamp], + }); + const model = new StubModel([POSTS]); + await runCampaignSources({ + db, + model, + fetchImpl: site({ items: [1], withFeed: true, title: 'Acme' }), + lookup, + }); + + const prompt = model.calls[0]!.user; + expect(prompt).toContain('dry and specific'); + expect(prompt).toContain('No exclamation marks.'); + expect(prompt).toContain('CTO, VP Engineering in fintech'); + }); + + test('a plain page drafts when its title changes, never on the first read', async () => { + const db = await setup('campaign-source-page'); + const state = { items: [], withFeed: false, title: 'Acme' }; + const model = new StubModel([POSTS, POSTS]); + const deps = { db, model, fetchImpl: site(state), lookup }; + + expect((await runCampaignSources(deps))[0]).toMatchObject({ mode: 'page', drafted: 0 }); + expect((await runCampaignSources({ ...deps, now: later(7) }))[0]).toMatchObject({ + mode: 'page', + drafted: 0, + }); + + state.title = 'Acme: now with caching'; + expect((await runCampaignSources({ ...deps, now: later(14) }))[0]).toMatchObject({ + mode: 'page', + newItems: 1, + }); + expect(await batches(db)).toHaveLength(1); + }); + + test('the daily cap holds however much is new', async () => { + const db = await setup('campaign-source-cap'); + const state = { items: [1], withFeed: true, title: 'Acme' }; + const model = new StubModel(Array.from({ length: 12 }, () => POSTS)); + const deps = { db, model, fetchImpl: site(state), lookup }; + + await runCampaignSources(deps); + state.items = [9, 8, 7, 6, 5, 4, 3, 2, 1]; + const second = await runCampaignSources({ ...deps, now: later(7) }); + // Same UTC day unless the test straddles midnight. + if (later(7).toISOString().slice(0, 10) === new Date().toISOString().slice(0, 10)) { + expect(second[0]!.drafted).toBe(BATCHES_PER_CAMPAIGN_PER_DAY - 1); + } + }); + + test('a page advertises its feed', () => { + expect(feedLink('', SITE)).toBe( + 'https://acme.example.com/atom', + ); + expect(feedLink('', SITE)).toBeUndefined(); + }); +}); + +describe('campaign source API', () => { + async function harness(label: string) { + const seeded = await seedDatabase(label); + active = seeded; + const app = createApp({ db: seeded.db, authenticate: async () => OWNER }); + const call = async (method: string, path: string, body?: unknown) => { + const res = await app.request(`/api/v1${path}`, { + method, + headers: { 'content-type': 'application/json' }, + ...(body !== undefined ? { body: JSON.stringify(body) } : {}), + }); + return { status: res.status, body: (await res.json()) as Record }; + }; + return { call, db: seeded.db }; + } + + test('a source URL and networks are set and read back', async () => { + const { call } = await harness('campaign-source-api'); + const patched = await call('PATCH', `/campaigns/${SEED.campaignId}`, { + sourceUrl: 'acme.example.com/blog', + postNetworks: ['linkedin', 'x', 'nonsense'], + }); + expect(patched.status).toBe(200); + + const read = await call('GET', `/campaigns/${SEED.campaignId}`); + expect(read.body.source).toMatchObject({ + url: 'https://acme.example.com/blog', + postNetworks: ['linkedin', 'x'], + }); + + expect( + (await call('PATCH', `/campaigns/${SEED.campaignId}`, { sourceUrl: 'not a url' })).status, + ).toBe(400); + }); + + test('starting a campaign needs a URL when the product has no site', async () => { + const { call } = await harness('campaign-source-required'); + const res = await call('POST', '/campaigns', { input: 'CTOs at fintech startups' }); + expect(res.status).toBe(400); + expect(JSON.stringify(res.body)).toContain('sourceUrl'); + }); +}); diff --git a/apps/api/src/campaign-source.ts b/apps/api/src/campaign-source.ts new file mode 100644 index 0000000..9528e99 --- /dev/null +++ b/apps/api/src/campaign-source.ts @@ -0,0 +1,350 @@ +/** + * The campaign's own URL, read on a schedule and turned into drafted posts. + * + * Drafting used to wait for someone to paste a link and press Draft. Every + * campaign now has a source URL (its product's site unless set otherwise), + * and the worker reads it every `CHECK_EVERY_MS`: + * + * - a feed, or a page that links one (``): each item + * not seen before becomes one batch of cards, newest first; + * - a plain page: a change to its title or description becomes one batch. + * + * The first read only learns what is there (a feed's newest item is drafted so + * the campaign has something to show), so turning this on never floods the + * queue with a site's back catalogue. Batches are capped per campaign and per + * workspace per UTC day, because each one is a model call and a card a person + * has to post. Posts are written in the campaign's voice for its target + * customer, and nothing is posted by the product. + */ + +import type { TextModel } from '@outreachgraph/ai'; +import { queryAll, queryOne, type Client } from '@outreachgraph/db'; +import { LINK_POST_NETWORKS, type LinkPostNetwork } from '@outreachgraph/domain'; +import { + assertPublicUrl, + looksLikeFeed, + parseFeedItems, + type FetchLike, + type HostLookup, +} from '@outreachgraph/providers'; +import { extractLinkPage, readLinkPage, storeDraftedBatch } from './link-posts'; + +export const CHECK_EVERY_MS = 6 * 3_600_000; +export const BATCHES_PER_CAMPAIGN_PER_DAY = 3; +export const BATCHES_PER_WORKSPACE_PER_DAY = 12; +const CAMPAIGNS_PER_RUN = 5; +const ITEMS_READ = 10; + +export interface CampaignSourceDeps { + readonly db: Client; + readonly model: TextModel; + readonly fetchImpl?: FetchLike | undefined; + readonly lookup?: HostLookup | undefined; + readonly now?: Date; +} + +export interface CampaignSourceResult { + readonly campaignId: string; + readonly mode: 'feed' | 'page' | 'error'; + readonly newItems: number; + readonly drafted: number; + readonly error?: string; +} + +interface CampaignRow { + readonly id: string; + readonly workspace_id: string; + readonly offering_id: string; + readonly voice_profile_id: string | null; + readonly brief: string | null; + readonly source: string; + readonly post_networks: string; + readonly source_fingerprint: string | null; +} + +/** Reads the campaigns that are due, a few per call. */ +export async function runCampaignSources( + deps: CampaignSourceDeps, +): Promise { + const at = deps.now ?? new Date(); + const due = await queryAll( + deps.db, + `SELECT c.id, c.workspace_id, c.offering_id, c.voice_profile_id, c.brief, + COALESCE(c.source_url, o.url) AS source, c.post_networks, c.source_fingerprint + FROM campaigns c JOIN offerings o ON o.id = c.offering_id + WHERE c.status IN ('active', 'running') + AND COALESCE(c.source_url, o.url) IS NOT NULL + AND (c.source_checked_at IS NULL OR c.source_checked_at < ?) + ORDER BY c.source_checked_at IS NOT NULL, c.source_checked_at + LIMIT ?`, + [new Date(at.getTime() - CHECK_EVERY_MS).toISOString(), CAMPAIGNS_PER_RUN], + ); + + const results: CampaignSourceResult[] = []; + for (const campaign of due) { + results.push(await checkCampaignSource(deps, campaign, at)); + } + return results; +} + +/** One campaign: read its source, remember what is new, draft what the caps allow. */ +export async function checkCampaignSource( + deps: CampaignSourceDeps, + campaign: CampaignRow, + at: Date, +): Promise { + const { db } = deps; + const source = withScheme(campaign.source); + const stamp = at.toISOString(); + + const finish = async (fields: { fingerprint?: string | null; error?: string | null }) => { + await db.execute({ + sql: `UPDATE campaigns SET source_checked_at = ?, + source_fingerprint = COALESCE(?, source_fingerprint), source_error = ? + WHERE id = ?`, + args: [stamp, fields.fingerprint ?? null, fields.error ?? null, campaign.id], + }); + }; + + let body: string; + let contentUrl = source; + try { + body = await fetchText(source, deps); + } catch (error) { + const message = (error as Error).message.slice(0, 300); + await finish({ error: message }); + return { campaignId: campaign.id, mode: 'error', newItems: 0, drafted: 0, error: message }; + } + + // A page that advertises a feed is read through the feed. + let feed: string | undefined = looksLikeFeed(body) ? body : undefined; + if (!feed) { + const advertised = feedLink(body, source); + if (advertised) { + try { + const text = await fetchText(advertised, deps); + if (looksLikeFeed(text)) { + feed = text; + contentUrl = advertised; + } + } catch { + // The page itself is still worth watching. + } + } + } + + const room = await draftRoom(db, campaign, at); + + if (feed) { + const items = parseFeedItems(feed).slice(0, ITEMS_READ); + const seen = await queryOne<{ n: number }>( + db, + 'SELECT count(*) AS n FROM campaign_source_items WHERE campaign_id = ?', + [campaign.id], + ); + const firstRead = Number(seen?.n ?? 0) === 0; + + const fresh: typeof items = []; + for (const item of items) { + const inserted = await db.execute({ + sql: `INSERT INTO campaign_source_items (campaign_id, url, title, seen_at) + VALUES (?, ?, ?, ?) ON CONFLICT DO NOTHING`, + args: [campaign.id, item.url, item.title ?? null, stamp], + }); + if (Number(inserted.rowsAffected ?? 0) > 0) fresh.push(item); + } + + // First read: the newest item only, never the back catalogue. + const toDraft = (firstRead ? fresh.slice(0, 1) : fresh).slice(0, room); + let drafted = 0; + for (const item of toDraft) { + const batchId = await draftFor(deps, campaign, item.url); + if (batchId) { + drafted += 1; + await db.execute({ + sql: `UPDATE campaign_source_items SET drafted_batch_id = ? + WHERE campaign_id = ? AND url = ?`, + args: [batchId, campaign.id, item.url], + }); + } + } + await finish({ fingerprint: `feed:${contentUrl}` }); + return { campaignId: campaign.id, mode: 'feed', newItems: fresh.length, drafted }; + } + + // A plain page: its title and description are what "changed" means. + const page = extractLinkPage(body); + const fingerprint = `page:${page.title ?? ''}|${page.description ?? ''}`; + const changed = + campaign.source_fingerprint !== null && campaign.source_fingerprint !== fingerprint; + let drafted = 0; + if (changed && room > 0 && (await draftFor(deps, campaign, source))) drafted = 1; + await finish({ fingerprint }); + return { campaignId: campaign.id, mode: 'page', newItems: changed ? 1 : 0, drafted }; +} + +/** Batches this campaign may still draft today, under both caps. */ +async function draftRoom(db: Client, campaign: CampaignRow, at: Date): Promise { + const day = `${at.toISOString().slice(0, 10)}T00:00:00.000Z`; + const counts = await queryOne<{ mine: number; all: number }>( + db, + `SELECT COUNT(DISTINCT CASE WHEN campaign_id = ? THEN batch_id END) AS mine, + COUNT(DISTINCT batch_id) AS "all" + FROM link_posts + WHERE workspace_id = ? AND campaign_id IS NOT NULL AND created_at >= ?`, + [campaign.id, campaign.workspace_id, day], + ); + return Math.max( + 0, + Math.min( + BATCHES_PER_CAMPAIGN_PER_DAY - Number(counts?.mine ?? 0), + BATCHES_PER_WORKSPACE_PER_DAY - Number(counts?.all ?? 0), + ), + ); +} + +async function draftFor( + deps: CampaignSourceDeps, + campaign: CampaignRow, + url: string, +): Promise { + const page = await readLinkPage(url, { + ...(deps.fetchImpl ? { fetchImpl: deps.fetchImpl } : {}), + ...(deps.lookup ? { lookup: deps.lookup } : {}), + }); + if (!page.read) return undefined; + try { + const { batchId } = await storeDraftedBatch(deps.db, deps.model, { + workspaceId: campaign.workspace_id, + url, + page, + networks: parsePostNetworks(campaign.post_networks), + offeringId: campaign.offering_id, + voiceProfileId: campaign.voice_profile_id ?? undefined, + audience: await audienceFor(deps.db, campaign), + campaignId: campaign.id, + createdBy: 'campaign_source', + }); + return batchId; + } catch (error) { + console.error(`campaign source ${campaign.id}: drafting ${url} failed`, error); + return undefined; + } +} + +/** The campaign's target customer profile, as a sentence a writer can use. */ +export async function audienceFor(db: Client, campaign: CampaignRow): Promise { + const filters = await queryOne>( + db, + `SELECT titles, seniorities, industries, countries, keywords, + employee_count_min, employee_count_max + FROM campaign_filters WHERE campaign_id = ?`, + [campaign.id], + ); + const list = (value: unknown): string[] => { + try { + const parsed = JSON.parse(String(value ?? '[]')) as unknown; + return Array.isArray(parsed) ? parsed.map(String).filter(Boolean) : []; + } catch { + return []; + } + }; + const parts: string[] = []; + const titles = [...list(filters?.seniorities), ...list(filters?.titles)]; + if (titles.length) parts.push(titles.slice(0, 8).join(', ')); + const industries = list(filters?.industries); + if (industries.length) parts.push(`in ${industries.slice(0, 6).join(', ')}`); + const min = filters?.employee_count_min; + const max = filters?.employee_count_max; + if (min || max) parts.push(`at companies of ${min ?? 1}-${max ?? '∞'} people`); + const countries = list(filters?.countries); + if (countries.length) parts.push(`(${countries.slice(0, 5).join(', ')})`); + const keywords = list(filters?.keywords); + if (keywords.length) parts.push(`who care about ${keywords.slice(0, 6).join(', ')}`); + if (campaign.brief?.trim()) parts.push(`Brief: ${campaign.brief.trim().slice(0, 300)}`); + return parts.length ? parts.join(' ') : undefined; +} + +/** The networks a campaign drafts for; LinkedIn when none are valid, unless `fallback` is off. */ +export function parsePostNetworks(raw: string, fallback = true): LinkPostNetwork[] { + let valid: LinkPostNetwork[] = []; + try { + const parsed = JSON.parse(raw) as unknown; + valid = Array.isArray(parsed) + ? [ + ...new Set( + parsed.filter((n): n is LinkPostNetwork => + (LINK_POST_NETWORKS as readonly string[]).includes(String(n)), + ), + ), + ] + : []; + } catch { + valid = []; + } + return valid.length || !fallback ? valid : ['linkedin']; +} + +/** A source URL as stored, or undefined when it is not a web address. */ +export function normaliseSourceUrl(raw: unknown): string | undefined { + if (typeof raw !== 'string' || !raw.trim()) return undefined; + const trimmed = raw.trim(); + try { + const url = new URL(/^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`); + if (!url.hostname.includes('.')) return undefined; + url.hash = ''; + return url.toString(); + } catch { + return undefined; + } +} + +function withScheme(url: string): string { + return /^https?:\/\//i.test(url) ? url : `https://${url}`; +} + +/** The feed a page advertises, absolute. */ +export function feedLink(html: string, base: string): string | undefined { + for (const tag of html.match(/]*>/gi) ?? []) { + if (!/rel\s*=\s*["']alternate["']/i.test(tag)) continue; + if (!/type\s*=\s*["']application\/(rss|atom)\+xml["']/i.test(tag)) continue; + const href = /href\s*=\s*["']([^"']+)["']/i.exec(tag)?.[1]; + if (!href) continue; + try { + return new URL(href.replace(/&/g, '&'), base).toString(); + } catch { + continue; + } + } + return undefined; +} + +async function fetchText(url: string, deps: CampaignSourceDeps): Promise { + await assertPublicUrl(url, { allowHttp: true, ...(deps.lookup ? { lookup: deps.lookup } : {}) }); + const response = await (deps.fetchImpl ?? fetch)(url, { + headers: { + 'user-agent': 'OutreachGraph/1.0 (+https://outreachgraph.com)', + accept: 'application/rss+xml, application/atom+xml, text/html;q=0.9, */*;q=0.5', + }, + redirect: 'follow', + signal: AbortSignal.timeout(12_000), + }); + if (!response.ok) throw new Error(`${url} answered ${response.status}`); + return (await response.text()).slice(0, 2_000_000); +} + +/** For the campaign page: what the source last said. */ +export async function campaignSourceStatus(db: Client, campaignId: string) { + return queryOne<{ + source_url: string | null; + post_networks: string; + source_checked_at: string | null; + source_error: string | null; + }>( + db, + `SELECT COALESCE(c.source_url, o.url) AS source_url, c.post_networks, c.source_checked_at, + c.source_error + FROM campaigns c JOIN offerings o ON o.id = c.offering_id WHERE c.id = ?`, + [campaignId], + ); +} diff --git a/apps/api/src/link-posts.ts b/apps/api/src/link-posts.ts index 5074981..cd1ff21 100644 --- a/apps/api/src/link-posts.ts +++ b/apps/api/src/link-posts.ts @@ -298,6 +298,121 @@ export async function listSavedLinks(db: Client, workspaceId: string): Promise { + const base = await voiceFor(db, input.workspaceId, input.offeringId); + const campaignVoice = input.voiceProfileId + ? await queryOne<{ style: string; instructions: string | null }>( + db, + 'SELECT style, instructions FROM voice_profiles WHERE id = ? AND workspace_id = ?', + [input.voiceProfileId, input.workspaceId], + ) + : undefined; + const voice = campaignVoice + ? { + style: campaignVoice.style, + ...(campaignVoice.instructions ? { instructions: campaignVoice.instructions } : {}), + } + : base.voice; + + const result = await draftLinkPosts(model, { + page: input.page, + networks: input.networks, + voice, + brand: base.brand, + audience: input.audience, + notes: input.notes, + }); + if (result.posts.length === 0) { + throw new ApiError(502, 'draft_failed', 'the model did not return any posts; try again'); + } + + const batchId = newId('linkPost'); + const at = now(); + const rows: LinkPostRow[] = result.posts.map((post) => ({ + id: newId('linkPost'), + workspace_id: input.workspaceId, + batch_id: batchId, + offering_id: base.offeringId ?? null, + url: input.url, + page_title: input.page.title ?? null, + page_description: input.page.description ?? null, + page_text: input.page.text ?? null, + notes: input.notes?.trim() || null, + network: post.network, + title: post.title ?? null, + body: post.text, + subreddit: post.subreddit ?? null, + mastodon_instance: input.mastodonInstance?.trim() || null, + status: 'open', + posted_url: null, + model: result.model ?? null, + regenerations: 0, + created_at: at, + updated_at: at, + done_at: null, + })); + + await db.batch( + rows.map((row) => ({ + sql: `INSERT INTO link_posts (id, workspace_id, batch_id, offering_id, url, page_title, + page_description, page_text, notes, network, title, body, subreddit, + mastodon_instance, status, model, regenerations, created_by, created_at, updated_at, + campaign_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', ?, 0, ?, ?, ?, ?)`, + args: [ + row.id, + row.workspace_id, + row.batch_id, + row.offering_id, + row.url, + row.page_title, + row.page_description, + row.page_text, + row.notes, + row.network, + row.title, + row.body, + row.subreddit, + row.mastodon_instance, + row.model, + input.createdBy, + row.created_at, + row.updated_at, + input.campaignId ?? null, + ], + })), + ); + + return { batchId, rows, at }; +} + /* ---------------------------------------------------------------- routes */ const networkEnum = z.enum(LINK_POST_NETWORKS); @@ -486,76 +601,16 @@ export function linkPostRoutes(deps: LinkPostRouteDeps): Hono { ); } - const { offeringId, voice, brand } = await voiceFor( - db, - actor.workspaceId, - input.offeringId ?? (await productForLink(db, actor.workspaceId, url)), - ); - const result = await draftLinkPosts(deps.model, { + const { batchId, rows, at } = await storeDraftedBatch(db, deps.model, { + workspaceId: actor.workspaceId, + url, page, networks, - voice, - brand, + offeringId: input.offeringId ?? (await productForLink(db, actor.workspaceId, url)), notes: input.notes, + mastodonInstance: input.mastodonInstance, + createdBy: actor.userId, }); - if (result.posts.length === 0) { - throw new ApiError(502, 'draft_failed', 'the model did not return any posts; try again'); - } - - const batchId = newId('linkPost'); - const at = now(); - const rows: LinkPostRow[] = result.posts.map((post) => ({ - id: newId('linkPost'), - workspace_id: actor.workspaceId, - batch_id: batchId, - offering_id: offeringId ?? null, - url, - page_title: page.title ?? null, - page_description: page.description ?? null, - page_text: page.text ?? null, - notes: input.notes?.trim() || null, - network: post.network, - title: post.title ?? null, - body: post.text, - subreddit: post.subreddit ?? null, - mastodon_instance: input.mastodonInstance?.trim() || null, - status: 'open', - posted_url: null, - model: result.model ?? null, - regenerations: 0, - created_at: at, - updated_at: at, - done_at: null, - })); - - await db.batch( - rows.map((row) => ({ - sql: `INSERT INTO link_posts (id, workspace_id, batch_id, offering_id, url, page_title, - page_description, page_text, notes, network, title, body, subreddit, - mastodon_instance, status, model, regenerations, created_by, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', ?, 0, ?, ?, ?)`, - args: [ - row.id, - row.workspace_id, - row.batch_id, - row.offering_id, - row.url, - row.page_title, - row.page_description, - row.page_text, - row.notes, - row.network, - row.title, - row.body, - row.subreddit, - row.mastodon_instance, - row.model, - actor.userId, - row.created_at, - row.updated_at, - ], - })), - ); await saveLink(db, actor.workspaceId, url, page.title, at); diff --git a/apps/cli/src/commands.ts b/apps/cli/src/commands.ts index 4ac8a25..1692eae 100644 --- a/apps/cli/src/commands.ts +++ b/apps/cli/src/commands.ts @@ -815,7 +815,7 @@ async function runBuyers({ client, args, flags }: CommandContext): Promise [--to linkedin,x,reddit,hackernews,facebook,bluesky,mastodon,threads] [--product ] [--notes "angle"] | list [--status open|done|skipped|all] | show | regen [--notes "angle"] | done [--url ] | skip '; + 'og linkpost [--to linkedin,x,reddit,hackernews,facebook,bluesky,mastodon,threads] [--product ] [--notes "angle"] | list [--status open|done|skipped|all] | show | regen [--notes "angle"] | done [--url ] | skip | source [--url ] [--to linkedin,x] [--check]'; /** * `og linkpost` — draft posts about a link, one per network, for you to post. @@ -851,6 +851,37 @@ async function runLinkPost({ client, args, flags }: CommandContext): Promise --to linkedin,x'; } + // The campaign's own source: read every 6 hours, new items drafted automatically. + if (verb === 'source') { + if (!target) + throw new Error('og linkpost source [--url ] [--to a,b] [--check]'); + const path = `/campaigns/${encodeURIComponent(target)}`; + const url = flagString(flags, 'url'); + const to = flagString(flags, 'to'); + if (url || to) { + if (!client.patch) throw new Error('this client cannot edit campaigns'); + await client.patch(path, { + ...(url ? { sourceUrl: url } : {}), + ...(to ? { postNetworks: to.split(',').map((n) => n.trim()) } : {}), + }); + } + if (flags.check) { + const { result } = (await client.post(`${path}/source/check`, {})) as { + result: Record; + }; + return result.error + ? `Could not read it: ${text(result, 'error')}` + : `Read the ${text(result, 'mode')}: ${text(result, 'newItems')} new, ${text(result, 'drafted')} drafted. See: og linkpost list`; + } + const { source } = (await client.get(path)) as { source: Record }; + const nets = Array.isArray(source.postNetworks) ? (source.postNetworks as string[]) : []; + return [ + `source: ${text(source, 'url') || '(none)'}`, + `networks: ${nets.join(', ')}`, + `last read: ${text(source, 'checkedAt') || 'never'}${source.error ? ` (failed: ${text(source, 'error')})` : ''}`, + ].join('\n'); + } + if (verb === 'show') { if (!target) throw new Error('og linkpost show '); const { post } = (await client.get(`/link-posts/${encodeURIComponent(target)}`)) as { diff --git a/apps/mcp/src/tools.ts b/apps/mcp/src/tools.ts index d0a4871..4eaceb6 100644 --- a/apps/mcp/src/tools.ts +++ b/apps/mcp/src/tools.ts @@ -1457,6 +1457,53 @@ export const TOOLS: readonly ToolDefinition[] = [ ...(str(args, 'notes') ? { notes: str(args, 'notes') } : {}), }), }, + { + name: 'set_campaign_source', + title: 'Set the URL a campaign posts from', + description: + "Every campaign reads a source URL every 6 hours (its product's site unless set): new items in its " + + 'feed (or the feed the page links to), or a change to a plain page, are drafted into hand-off posts ' + + "in the campaign's voice for its target customer. Set the URL and the networks to draft for; " + + 'check: true reads it now. Nothing is posted: a person posts each card.', + readOnly: false, + inputSchema: { + type: 'object', + properties: { + campaignId: { type: 'string' }, + sourceUrl: { type: 'string', description: 'A blog, feed or page URL' }, + postNetworks: { + type: 'array', + items: { + type: 'string', + enum: [ + 'linkedin', + 'x', + 'reddit', + 'hackernews', + 'facebook', + 'bluesky', + 'mastodon', + 'threads', + ], + }, + }, + check: { type: 'boolean', description: 'Read the source now' }, + }, + required: ['campaignId'], + }, + run: async (client, args) => { + const path = `/campaigns/${encodeURIComponent(require(args, 'campaignId'))}`; + if (str(args, 'sourceUrl') || Array.isArray(args.postNetworks)) { + if (!client.patch) throw new Error('this client cannot edit campaigns'); + await client.patch(path, { + ...(str(args, 'sourceUrl') ? { sourceUrl: str(args, 'sourceUrl') } : {}), + ...(Array.isArray(args.postNetworks) ? { postNetworks: args.postNetworks } : {}), + }); + } + if (args.check === true) return client.post(`${path}/source/check`, {}); + return client.get(path); + }, + }, { name: 'list_link_posts', title: 'List drafted posts about links', diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts index 851cbce..e8ce6a9 100644 --- a/apps/server/src/index.ts +++ b/apps/server/src/index.ts @@ -36,6 +36,7 @@ import { createApp } from '../../api/src/app'; import { prunePasswordResetTokens, pruneSessions } from '../../api/src/auth'; import { verifyOpenAccessBearer } from '../../api/src/openaccess'; import { runBootstrapProductJob } from '../../api/src/bulk-products'; +import { runCampaignSources } from '../../api/src/campaign-source'; import { routesToApi } from './routing'; import { drainQueue, @@ -1085,6 +1086,24 @@ async function tick(): Promise { } } + // Each campaign's source URL, read every few hours: new feed items or a + // changed page become drafted hand-off cards. A few campaigns per tick, + // capped per campaign and per workspace a day inside the function. + if (model) { + try { + for (const result of await runCampaignSources({ db, model })) { + if (result.drafted > 0 || result.error) { + console.log( + `campaign source ${result.campaignId}: ${result.mode}, ${result.newItems} new, ` + + `${result.drafted} drafted${result.error ? `, ${result.error}` : ''}`, + ); + } + } + } catch (error) { + console.error('campaign sources failed', error); + } + } + // A face for the people about to be written to. Bounded per tick and per // day inside the sweep; this loop only decides who is looked at. if (photoFinder) { diff --git a/apps/web/app/(app)/products/[id]/page.tsx b/apps/web/app/(app)/products/[id]/page.tsx index 9ef49ef..05ba4a2 100644 --- a/apps/web/app/(app)/products/[id]/page.tsx +++ b/apps/web/app/(app)/products/[id]/page.tsx @@ -3,6 +3,7 @@ import { notFound, redirect } from 'next/navigation'; import { Avatar } from '../../../../components/avatar'; import { AddLeads } from '../../../../components/add-leads'; import { CampaignControls } from '../../../../components/campaign-controls'; +import { CampaignSource } from '../../../../components/campaign-source'; import { LeadEnrichment } from '../../../../components/lead-enrichment'; import { ScreenedLeads } from '../../../../components/screened-leads'; import { @@ -80,6 +81,9 @@ export default async function ProductPage({ params }: { params: Promise<{ id: st

No campaign yet.

)} + {campaign && campaign.status !== 'archived' ? ( + + ) : null}

Buyer leads diff --git a/apps/web/components/campaign-source.tsx b/apps/web/components/campaign-source.tsx new file mode 100644 index 0000000..b478dbf --- /dev/null +++ b/apps/web/components/campaign-source.tsx @@ -0,0 +1,158 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { LINK_POST_FORMATS, LINK_POST_NETWORKS, type LinkPostNetwork } from '@outreachgraph/domain'; + +interface SourceView { + url: string | null; + postNetworks: LinkPostNetwork[]; + checkedAt: string | null; + error: string | null; +} + +/** + * The URL a campaign reads on a schedule, and the networks it drafts for. + * + * New feed items, or a change to the page, become drafted posts under + * Approve → Manual posts, in this campaign's voice for its target customer. + */ +export function CampaignSource({ campaignId }: { campaignId: string }) { + const [source, setSource] = useState(); + const [url, setUrl] = useState(''); + const [networks, setNetworks] = useState(['linkedin']); + const [busy, setBusy] = useState(false); + const [message, setMessage] = useState(); + + useEffect(() => { + void fetch(`/api/v1/campaigns/${encodeURIComponent(campaignId)}`, { + credentials: 'same-origin', + }) + .then((response) => response.json()) + .then((payload: { source?: SourceView }) => { + if (!payload.source) return; + setSource(payload.source); + setUrl(payload.source.url ?? ''); + setNetworks(payload.source.postNetworks); + }) + .catch(() => undefined); + }, [campaignId]); + + async function save(): Promise { + setBusy(true); + setMessage(undefined); + try { + const response = await fetch(`/api/v1/campaigns/${encodeURIComponent(campaignId)}`, { + method: 'PATCH', + headers: { 'content-type': 'application/json' }, + credentials: 'same-origin', + body: JSON.stringify({ sourceUrl: url, postNetworks: networks }), + }); + const payload = (await response.json().catch(() => ({}))) as { + error?: { message?: string }; + }; + setMessage( + response.ok ? 'Saved.' : (payload.error?.message ?? `that failed (${response.status})`), + ); + } finally { + setBusy(false); + } + } + + async function checkNow(): Promise { + setBusy(true); + setMessage(undefined); + try { + const response = await fetch( + `/api/v1/campaigns/${encodeURIComponent(campaignId)}/source/check`, + { method: 'POST', credentials: 'same-origin' }, + ); + const payload = (await response.json().catch(() => ({}))) as { + result?: { mode: string; newItems: number; drafted: number; error?: string }; + error?: { message?: string }; + }; + if (!response.ok || !payload.result) { + setMessage(payload.error?.message ?? `that failed (${response.status})`); + return; + } + const r = payload.result; + setMessage( + r.error + ? `Could not read it: ${r.error}` + : `Read the ${r.mode}: ${r.newItems} new, ${r.drafted} drafted into Manual posts.`, + ); + } finally { + setBusy(false); + } + } + + function toggle(network: LinkPostNetwork): void { + setNetworks((current) => + current.includes(network) + ? current.filter((n) => n !== network) + : LINK_POST_NETWORKS.filter((n) => n === network || current.includes(n)), + ); + } + + return ( +

+

Posts from

+

+ Read every 6 hours. New posts in its feed, or a change to the page, are drafted in this + campaign’s voice for its customers and wait under Approve → Manual posts. +

+ setUrl(event.target.value)} + placeholder="https://yoursite.com/blog" + aria-label="URL this campaign posts from" + className="border-border bg-surface mt-2 w-full rounded-xl border px-3 py-2 text-sm" + /> +
+ {LINK_POST_NETWORKS.map((network) => { + const on = networks.includes(network); + return ( + + ); + })} +
+
+ + + + {message ?? + (source?.error + ? `Last read failed: ${source.error}` + : source?.checkedAt + ? `Last read ${new Date(source.checkedAt).toLocaleString()}` + : 'Not read yet')} + +
+
+ ); +} diff --git a/migrations-pg/0066_campaign_source.sql b/migrations-pg/0066_campaign_source.sql new file mode 100644 index 0000000..55f7073 --- /dev/null +++ b/migrations-pg/0066_campaign_source.sql @@ -0,0 +1,20 @@ +-- 0066_campaign_source.sql (Postgres). See migrations/0066_campaign_source.sql for the reasoning. +ALTER TABLE campaigns ADD COLUMN IF NOT EXISTS source_url TEXT; +ALTER TABLE campaigns ADD COLUMN IF NOT EXISTS post_networks TEXT NOT NULL DEFAULT '["linkedin"]'; +ALTER TABLE campaigns ADD COLUMN IF NOT EXISTS source_checked_at TEXT; +ALTER TABLE campaigns ADD COLUMN IF NOT EXISTS source_fingerprint TEXT; +ALTER TABLE campaigns ADD COLUMN IF NOT EXISTS source_error TEXT; + +UPDATE campaigns SET source_url = (SELECT url FROM offerings WHERE offerings.id = campaigns.offering_id) + WHERE source_url IS NULL; + +CREATE TABLE IF NOT EXISTS campaign_source_items ( + campaign_id TEXT NOT NULL REFERENCES campaigns(id) ON DELETE CASCADE, + url TEXT NOT NULL, + title TEXT, + seen_at TEXT NOT NULL, + drafted_batch_id TEXT, + PRIMARY KEY (campaign_id, url) +); + +ALTER TABLE link_posts ADD COLUMN IF NOT EXISTS campaign_id TEXT; diff --git a/migrations/0066_campaign_source.sql b/migrations/0066_campaign_source.sql new file mode 100644 index 0000000..d922625 --- /dev/null +++ b/migrations/0066_campaign_source.sql @@ -0,0 +1,27 @@ +-- 0066_campaign_source.sql: every campaign reads a URL of its own, on a schedule. +-- +-- Posts were drafted only when someone pasted a link and pressed Draft. A +-- campaign now carries the URL it is about (its product's site unless set), +-- which the worker reads every few hours: each new item of its feed, or a +-- change to the page itself, becomes drafted hand-off cards written in the +-- campaign's voice for its target customer. A person still posts every one. +ALTER TABLE campaigns ADD COLUMN source_url TEXT; +ALTER TABLE campaigns ADD COLUMN post_networks TEXT NOT NULL DEFAULT '["linkedin"]'; +ALTER TABLE campaigns ADD COLUMN source_checked_at TEXT; +ALTER TABLE campaigns ADD COLUMN source_fingerprint TEXT; +ALTER TABLE campaigns ADD COLUMN source_error TEXT; + +UPDATE campaigns SET source_url = (SELECT url FROM offerings WHERE offerings.id = campaigns.offering_id) + WHERE source_url IS NULL; + +-- What the source has already shown, so only new items are drafted. +CREATE TABLE IF NOT EXISTS campaign_source_items ( + campaign_id TEXT NOT NULL REFERENCES campaigns(id) ON DELETE CASCADE, + url TEXT NOT NULL, + title TEXT, + seen_at TEXT NOT NULL, + drafted_batch_id TEXT, + PRIMARY KEY (campaign_id, url) +); + +ALTER TABLE link_posts ADD COLUMN campaign_id TEXT; diff --git a/packages/ai/src/link-post.ts b/packages/ai/src/link-post.ts index 78d7f5c..834390c 100644 --- a/packages/ai/src/link-post.ts +++ b/packages/ai/src/link-post.ts @@ -50,6 +50,8 @@ export interface LinkPostRequest { readonly brand?: LinkPostBrand | undefined; /** What the poster wants said: an angle, a call to action, a correction. */ readonly notes?: string | undefined; + /** Who the posts are for: the campaign's target customer, in words. */ + readonly audience?: string | undefined; } export interface LinkPostDraft { @@ -87,7 +89,7 @@ function host(url: string | undefined): string | undefined { /** The user prompt. Exported for tests: what the model is told is part of the contract. */ export function linkPostPrompt(request: LinkPostRequest): string { - const { page, brand, voice, notes } = request; + const { page, brand, voice, notes, audience } = request; const own = brand?.url && host(brand.url) && host(brand.url) === host(page.url); const lines: string[] = [ @@ -120,6 +122,15 @@ export function linkPostPrompt(request: LinkPostRequest): string { ); } + if (audience?.trim()) { + lines.push( + 'AUDIENCE', + `Written for: ${audience.trim().slice(0, 600)}`, + 'Pick the angle this reader cares about; do not name them or the targeting.', + '', + ); + } + if (notes?.trim()) lines.push('NOTES FROM THE POSTER', notes.trim().slice(0, 1000), ''); lines.push('NETWORKS'); diff --git a/packages/providers/src/feeds/index.ts b/packages/providers/src/feeds/index.ts index 3fb50c7..d27b41b 100644 --- a/packages/providers/src/feeds/index.ts +++ b/packages/providers/src/feeds/index.ts @@ -36,7 +36,7 @@ export { redditPostId, type WebDiscussionSourceOptions, } from './web-discussions'; -export { RssSource, type RssSourceOptions } from './rss'; +export { looksLikeFeed, parseFeedItems, RssSource, type RssSourceOptions } from './rss'; export { BlueskyFeedSource, type BlueskyFeedSourceOptions } from './bluesky'; export { NostrSource, diff --git a/packages/providers/src/feeds/rss.ts b/packages/providers/src/feeds/rss.ts index 48f5247..6318b28 100644 --- a/packages/providers/src/feeds/rss.ts +++ b/packages/providers/src/feeds/rss.ts @@ -189,3 +189,28 @@ function parseDate(raw: string | undefined): string { const parsed = Date.parse(textOf(raw)); return Number.isNaN(parsed) ? new Date().toISOString() : new Date(parsed).toISOString(); } + +/** A feed document's items, newest first as the feed lists them. */ +export function parseFeedItems( + xml: string, +): Array<{ url: string; title?: string; publishedAt: string }> { + const out: Array<{ url: string; title?: string; publishedAt: string }> = []; + for (const item of splitItems(xml)) { + const url = itemLink(item)?.trim(); + if (!url || !/^https?:\/\//i.test(url)) continue; + const title = textOf(firstTag(item, 'title') ?? ''); + out.push({ + url, + ...(title ? { title } : {}), + publishedAt: parseDate( + firstTag(item, 'pubDate') ?? firstTag(item, 'published') ?? firstTag(item, 'updated'), + ), + }); + } + return out; +} + +/** True when a body is an RSS or Atom document rather than a web page. */ +export function looksLikeFeed(body: string): boolean { + return /^\s*(<\?xml[^>]*>\s*)?(\s*)*<(rss|feed|rdf:RDF)[\s>]/i.test(body); +} diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts index 7d65ba0..a93fcb4 100644 --- a/packages/providers/src/index.ts +++ b/packages/providers/src/index.ts @@ -239,6 +239,8 @@ export { DEFAULT_DISCUSSION_SITES, RedditSource, RssSource, + looksLikeFeed, + parseFeedItems, suggestSubreddits, DEFAULT_NOSTR_RELAYS, REDDIT_API,