explee: add more leads to an Explee project from a CSV (0.61.0)

README.md

@@ -43,6 +43,7 @@ TypeScript, installed as executables on `PATH`.
 | [`scorecard`](#scorecard) | One weekly number for the whole fleet: traffic, channels, posts and ads, week over week |
 | [`users-dump`](#users-dump) | Every user account across the fleet, as one CSV |
 | [`user-export`](#user-export) | Every user across many databases, as one CSV |
+| [`explee`](#explee) | Add more of your own leads to an Explee project from a CSV |
 | [`email-cleaner`](#email-cleaner) | Clean a mailing list: drop bad, role, disposable, duplicate and unlikely addresses |
 
 One thing here is not a `PATH` command and does not need Node:
@@ -805,6 +806,28 @@ goes through OpenRDAP. Either way `dig` adds records, hosts, reverse lookups and
 per-nameserver AXFR attempts. Errors are JSON too — a tool whose output gets
 parsed should not change shape when it fails.
 
+### `explee`
+
+Explee's app takes your CSV once, when a campaign is created; there is no
+"import more leads" on an existing campaign, in the app or the API. `explee
+import` does the next best thing: a new campaign in the same project, with the
+brief (first email, follow-ups, language) copied from the campaign you name.
+Explee drops anyone the project already contacted, so overlapping CSVs are safe.
+
+```sh
+user-export --clean --no-resend --format explee -o more.csv        # or any CSV with these columns
+explee import more.csv --project 38837 --brief-from 222092 --dry-run   # check the file
+explee import more.csv --project 38837 --brief-from 222092             # import and wait
+explee status <task_id>
+```
+
+The project and campaign ids are in the app URL: `/p/<project>/segments/<campaign>`.
+Columns by name: `email`, `first_name`, `last_name`, `company_domain`, `job_title`
+(required; rows without them are listed and left out, since Explee would skip
+them silently), `linkedin_url`, `company_name`. The key is `$EXPLEE_API_KEY`, or
+a reference in `$EXPLEE_API_KEY_REF` (`vault:<team>/<project>/<env>/<KEY>`).
+Importing is free; sending is billed by Explee.
+
 ### `fe` and `dealsubs`
 
 `fe` manages Forward Email aliases (profullstack.com, c0upons.com) with an account

bin/explee.ts

@@ -0,0 +1,123 @@
+#!/usr/bin/env node
+/**
+ * explee — add more of your own leads to an Explee project.
+ *
+ *   explee import more.csv --project 38837 --brief-from 222092
+ *   explee campaigns --project 38837
+ *   explee status <task_id>
+ *
+ * Explee's app takes a CSV once, when a campaign is made. Every later import
+ * creates a new campaign, so this copies the brief of the campaign you name
+ * into the new one and lets Explee drop anyone the project already contacted.
+ * src/explee.ts has the rest.
+ */
+
+import { readFileSync } from 'node:fs';
+
+import { UsageError, parseArgs } from '../src/args.ts';
+import { isMain } from '../src/is-main.ts';
+import { ExpleeError, briefOf, client, formatResult, leadsFromCsv, waitForImport } from '../src/explee.ts';
+import { secretResolver } from '../src/user-export.ts';
+
+const USAGE = `Usage:
+  explee import FILE.csv --project ID [--brief-from CAMPAIGN_ID] [--name NAME] [--no-wait] [--dry-run]
+  explee campaigns [--project ID]
+  explee status TASK_ID
+
+import   creates a new campaign in the project from your CSV (Explee has no
+         "add to this campaign"; every import is a new campaign). Columns by
+         name: email, first_name, last_name, company_domain, job_title
+         (required by Explee, rows without them are reported and left out),
+         linkedin_url and company_name (optional). user-export --format explee
+         writes exactly this. --brief-from copies that campaign's first-email
+         brief, follow-up brief and language, so the new leads get the same
+         pitch. Leads the project already contacted are dropped by Explee.
+         Waits for the import and prints what Explee kept and skipped.
+
+Options:
+      --project ID        project to import into (from the app URL: /p/<ID>/)
+      --brief-from ID     campaign whose brief to copy (the app URL's segments/<ID>)
+      --name NAME         campaign name (default: "<brief campaign> + YYYY-MM-DD")
+      --no-wait           print the task id and return
+      --dry-run           parse and check the CSV, send nothing
+      --json              machine-readable output
+  -h, --help
+
+The API key comes from $EXPLEE_API_KEY, or from a secret reference in
+$EXPLEE_API_KEY_REF (env:NAME, vault:<team>/<project>/<env>/<KEY>, cmd:<shell>).
+Create one at https://explee.com/app-auto-gtm/api-keys. Importing is free;
+sending is billed by Explee as usual.
+`;
+
+const id = (value: string | undefined, flag: string): number | undefined => {
+  if (value === undefined) return undefined;
+  if (!/^\d+$/.test(value)) throw new UsageError(`${flag} takes a number`);
+  return Number(value);
+};
+
+if (isMain(import.meta.url)) {
+  try {
+    const { flags, values, positional } = parseArgs(process.argv.slice(2), {
+      boolean: ['-h', '--help', '--no-wait', '--dry-run', '--json'],
+      string: ['--project', '--brief-from', '--name'],
+    });
+    const [command, arg] = positional;
+    if (!command || flags.has('-h') || flags.has('--help')) {
+      process.stdout.write(USAGE);
+      process.exit(command ? 0 : 1);
+    }
+    const json = flags.has('--json');
+    const print = (value: unknown, text: string) => process.stdout.write(json ? `${JSON.stringify(value, null, 2)}\n` : `${text}\n`);
+    const apiKey = () => {
+      const ref = process.env.EXPLEE_API_KEY_REF || 'env:EXPLEE_API_KEY';
+      return secretResolver(process.env)(ref, 'Explee API key');
+    };
+
+    if (command === 'import') {
+      if (!arg) throw new UsageError('import needs a CSV file');
+      const project = id(values.get('--project'), '--project');
+      if (!project) throw new UsageError('import needs --project ID (the number after /p/ in the app URL)');
+      const { leads, missing } = leadsFromCsv(readFileSync(arg, 'utf8'));
+      for (const m of missing) process.stderr.write(`row ${m.row} ${m.email || '(no email)'}: no ${m.missing.join(', ')}\n`);
+      process.stderr.write(`${leads.length} leads ready, ${missing.length} rows left out for a missing required column\n`);
+      if (leads.length === 0) throw new ExpleeError('no importable leads in the CSV');
+      if (leads.length > 30_000) throw new ExpleeError(`Explee takes at most 30,000 leads per import, the CSV has ${leads.length}`);
+      if (flags.has('--dry-run')) process.exit(0);
+
+      const explee = client(apiKey());
+      const from = id(values.get('--brief-from'), '--brief-from');
+      const source = from ? await explee.campaign(from) : undefined;
+      if (source && source.project_id !== project) {
+        throw new UsageError(`campaign ${from} is in project ${source.project_id}, not ${project}`);
+      }
+      const name = values.get('--name') || `${source?.name ?? 'Import'} + ${new Date().toISOString().slice(0, 10)}`;
+      const { task_id } = await explee.startImport({ project_id: project, name, leads, ...(source ? briefOf(source) : {}) });
+      process.stderr.write(`import started: task ${task_id}${source ? `, brief copied from campaign ${from}` : ''}\n`);
+      if (flags.has('--no-wait')) {
+        print({ task_id }, task_id);
+        process.exit(0);
+      }
+      const status = await waitForImport(explee, task_id, {
+        onProgress: (s) => s.progress && !json && process.stderr.write(`  ${s.progress.stage} ${s.progress.done}/${s.progress.total}\n`),
+      });
+      if (!status.result) throw new ExpleeError(`import ${task_id} ${status.status}: ${status.error ?? 'no result'}`);
+      print(status.result, formatResult(status.result));
+    } else if (command === 'campaigns') {
+      const result = await client(apiKey()).campaigns(id(values.get('--project'), '--project'));
+      print(result, JSON.stringify(result, null, 2));
+    } else if (command === 'status') {
+      if (!arg) throw new UsageError('status needs a task id');
+      const status = await client(apiKey()).importStatus(arg);
+      print(status, status.result ? formatResult(status.result) : `${status.status}${status.error ? `: ${status.error}` : ''}`);
+    } else {
+      throw new UsageError(`unknown command: ${command}`);
+    }
+  } catch (error) {
+    if (error instanceof UsageError || error instanceof ExpleeError) {
+      process.stderr.write(`explee: ${error.message}\n`);
+      process.exit(1);
+    }
+    process.stderr.write(`explee: ${error instanceof Error ? error.message : error}\n`);
+    process.exit(2);
+  }
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/cli-tools",
-  "version": "0.60.0",
+  "version": "0.61.0",
   "private": true,
   "description": "Local command-line tools, in TypeScript, exposed on PATH.",
   "type": "module",

src/explee.ts

@@ -0,0 +1,176 @@
+/**
+ * Explee's auto-GTM API, for the one thing its app cannot do: add more of your
+ * own leads to a project after the first CSV.
+ *
+ * Explee has no "import into this campaign". Every import, in the app or over
+ * the API, creates a new campaign. So "upload another CSV" becomes: read the
+ * existing campaign's brief, create a new campaign in the same project with the
+ * same brief and the new leads, and wait for Explee to finish. Explee drops
+ * anyone the project has already contacted, so a CSV that overlaps the first
+ * one is safe to send.
+ *
+ * Network access is injectable so the tests never touch it.
+ */
+
+import { parseCsv } from './user-export.ts';
+
+export const EXPLEE_API = 'https://api.explee.com';
+const BASE = '/public/api/v1/autogtm';
+
+export class ExpleeError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'ExpleeError';
+  }
+}
+
+export interface Lead {
+  email: string;
+  first_name: string;
+  last_name: string;
+  company_domain: string;
+  job_title: string;
+  linkedin_url?: string;
+  company_name?: string;
+}
+
+/** The fields of a campaign that make up its copy brief. */
+export interface Brief {
+  instructions?: string | null;
+  followup_instructions?: string | null;
+  language?: string | null;
+}
+
+export interface Campaign extends Brief {
+  id: number;
+  project_id: number;
+  name: string;
+  status?: string;
+}
+
+export interface ImportResult {
+  campaign_id: number;
+  project_id: number;
+  name: string;
+  leads_total: number;
+  leads_imported: number;
+  deduped: number;
+  enriched_from_base: number;
+  skipped_missing: number;
+  skipped_invalid_email: number;
+  flagged_freemail: number;
+  skipped?: unknown;
+}
+
+export interface ImportStatus {
+  status: string;
+  error: string | null;
+  progress: { stage: string; done: number; total: number } | null;
+  result: ImportResult | null;
+}
+
+export const REQUIRED = ['email', 'first_name', 'last_name', 'company_domain', 'job_title'] as const;
+const OPTIONAL = ['linkedin_url', 'company_name'] as const;
+
+export interface LeadsFromCsv {
+  leads: Lead[];
+  /** Rows Explee would skip anyway, with the required columns they lack. */
+  missing: { row: number; email: string; missing: string[] }[];
+}
+
+/** Leads out of a CSV with a header row, by column name (case-insensitive). */
+export function leadsFromCsv(text: string): LeadsFromCsv {
+  const [header, ...records] = parseCsv(text.replace(/^/, ''));
+  if (!header) throw new ExpleeError('the CSV is empty');
+  const index = new Map(header.map((h, i) => [h.trim().toLowerCase(), i]));
+  const absent = REQUIRED.filter((c) => !index.has(c));
+  if (absent.length) throw new ExpleeError(`the CSV has no ${absent.join(', ')} column${absent.length > 1 ? 's' : ''}`);
+
+  const out: LeadsFromCsv = { leads: [], missing: [] };
+  const seen = new Set<string>();
+  records.forEach((values, n) => {
+    const get = (c: string) => (index.has(c) ? (values[index.get(c)!] ?? '').trim() : '');
+    const email = get('email').toLowerCase();
+    const gaps = REQUIRED.filter((c) => !get(c));
+    if (gaps.length) {
+      out.missing.push({ row: n + 2, email, missing: [...gaps] });
+      return;
+    }
+    if (seen.has(email)) return;
+    seen.add(email);
+    const lead: Lead = {
+      email,
+      first_name: get('first_name'),
+      last_name: get('last_name'),
+      company_domain: get('company_domain'),
+      job_title: get('job_title'),
+    };
+    for (const c of OPTIONAL) if (get(c)) lead[c] = get(c);
+    out.leads.push(lead);
+  });
+  return out;
+}
+
+type Fetch = (url: string, init?: { method?: string; headers?: Record<string, string>; body?: string }) => Promise<{
+  ok: boolean;
+  status: number;
+  text(): Promise<string>;
+}>;
+
+export function client(apiKey: string, { api = EXPLEE_API, fetchImpl = fetch as unknown as Fetch } = {}) {
+  const call = async <T>(method: string, path: string, body?: unknown): Promise<T> => {
+    const res = await fetchImpl(`${api}${BASE}${path}`, {
+      method,
+      headers: { 'X-API-Key': apiKey, accept: 'application/json', ...(body ? { 'content-type': 'application/json' } : {}) },
+      ...(body ? { body: JSON.stringify(body) } : {}),
+    });
+    const text = await res.text();
+    if (!res.ok) throw new ExpleeError(`Explee ${method} ${path}: HTTP ${res.status} ${text.slice(0, 300)}`);
+    return (text ? JSON.parse(text) : null) as T;
+  };
+  return {
+    projects: () => call<unknown>('GET', '/projects'),
+    campaigns: (projectId?: number) => call<unknown>('GET', `/campaigns${projectId ? `?project_id=${projectId}` : ''}`),
+    campaign: (id: number) => call<Campaign>('GET', `/campaigns/${id}`),
+    startImport: (payload: { project_id: number; name: string; leads: Lead[] } & Brief) =>
+      call<{ task_id: string }>('POST', '/campaigns/import', payload),
+    importStatus: (taskId: string) => call<ImportStatus>('GET', `/campaigns/import/${encodeURIComponent(taskId)}`),
+  };
+}
+
+export type ExpleeClient = ReturnType<typeof client>;
+
+/** Only the brief fields that are set; Explee falls back to the project description for the rest. */
+export function briefOf(campaign: Brief): Brief {
+  const brief: Brief = {};
+  if (campaign.instructions) brief.instructions = campaign.instructions;
+  if (campaign.followup_instructions) brief.followup_instructions = campaign.followup_instructions;
+  if (campaign.language) brief.language = campaign.language;
+  return brief;
+}
+
+/** Poll an import until it completes or fails. */
+export async function waitForImport(
+  explee: Pick<ExpleeClient, 'importStatus'>,
+  taskId: string,
+  { intervalMs = 3000, timeoutMs = 15 * 60_000, onProgress = (_s: ImportStatus) => {} } = {},
+): Promise<ImportStatus> {
+  const deadline = Date.now() + timeoutMs;
+  for (;;) {
+    const status = await explee.importStatus(taskId);
+    onProgress(status);
+    if (status.status === 'completed' || status.status === 'failed' || status.error) return status;
+    if (Date.now() > deadline) throw new ExpleeError(`import ${taskId} still ${status.status} after ${timeoutMs / 60_000} min`);
+    await new Promise((r) => setTimeout(r, intervalMs));
+  }
+}
+
+export function formatResult(r: ImportResult): string {
+  return [
+    `campaign ${r.campaign_id} "${r.name}" in project ${r.project_id}`,
+    `  ${r.leads_imported} of ${r.leads_total} leads imported`,
+    `  ${r.deduped} already contacted in this project (dropped)`,
+    `  ${r.skipped_missing} missing a required column, ${r.skipped_invalid_email} invalid email`,
+    `  ${r.flagged_freemail} on a free mail domain (flagged), ${r.enriched_from_base} enriched from Explee's base`,
+  ].join('\n');
+}

src/registry.ts

@@ -63,6 +63,7 @@ const SUMMARIES: Record<string, string> = {
   'gh-pulse': 'What moved on GitHub, last hour to all time, ranked, with traffic: by email, TUI or JSON',
   hqtui: 'Every vital of this box, in the terminal: sockets, HTTP, sessions, services',
   porkbun: 'Read and change DNS at Porkbun, and un-park a domain',
+  explee: 'Add more of your own leads to an Explee project: a CSV becomes a campaign with the same brief',
   fe: 'Forward Email aliases without the dashboard: list, ensure (idempotent), remove',
   dealsubs: 'Subscribe an inbox to coupon and deal newsletters, throttled, through TronBrowser',
   proxy: 'Fetch through our paid proxies (Proxiware, Webshare); serve, MCP and TUI too',