From 28a37c2e97f04470528fdc3be61012a604f4f3f5 Mon Sep 17 00:00:00 2001 From: Agent Date: Sun, 20 Sep 2026 10:39:05 +0000 Subject: [PATCH] feat(cli): report anonymous install telemetry After a successful pgflow install, send one strict allowlisted contribution with the fixed package version, fresh/update/no-op result, and bucketed copied-migration count. CI, tests, DO_NOT_TRACK, and PGFLOW_TELEMETRY_DISABLED suppress the event; request failures never affect installation. Reuse the existing telemetry worker and retention policy, extend its allowlist and tests, and expand the single telemetry reference page with an upfront no-identifiers guarantee, exact CLI payload, and both CLI and database opt-out controls. Raw migration timestamps, filenames, paths, errors, and project values never leave the machine. --- .changeset/quiet-install-signals.md | 5 ++ apps/telemetry-worker/src/index.test.ts | 18 +++++ apps/telemetry-worker/src/index.ts | 3 + .../install/report-install-telemetry.test.ts | 56 +++++++++++++++ pkgs/cli/scripts/test-install | 1 + pkgs/cli/scripts/test-install-duplicates | 1 + .../src/commands/install/copy-migrations.ts | 20 ++++-- pkgs/cli/src/commands/install/index.ts | 15 +++- .../install/report-install-telemetry.ts | 67 ++++++++++++++++++ .../src/content/docs/reference/telemetry.mdx | 69 +++++++++++++++---- 10 files changed, 234 insertions(+), 21 deletions(-) create mode 100644 .changeset/quiet-install-signals.md create mode 100644 pkgs/cli/__tests__/commands/install/report-install-telemetry.test.ts create mode 100644 pkgs/cli/src/commands/install/report-install-telemetry.ts diff --git a/.changeset/quiet-install-signals.md b/.changeset/quiet-install-signals.md new file mode 100644 index 000000000..79db61176 --- /dev/null +++ b/.changeset/quiet-install-signals.md @@ -0,0 +1,5 @@ +--- +"pgflow": patch +--- + +Report anonymous fresh, update, or no-op install completion telemetry with the pgflow version and a bucketed migration count. CI, tests, `DO_NOT_TRACK`, and `PGFLOW_TELEMETRY_DISABLED` disable the event. diff --git a/apps/telemetry-worker/src/index.test.ts b/apps/telemetry-worker/src/index.test.ts index f67deefa1..be87c9bc2 100644 --- a/apps/telemetry-worker/src/index.test.ts +++ b/apps/telemetry-worker/src/index.test.ts @@ -50,6 +50,24 @@ describe('telemetry ingest', () => { expect(points).toHaveLength(3); }); + it.each([ + 'cli_install_fresh', + 'cli_install_update', + 'cli_install_noop', + ])('accepts the %s event with version and migration-count bucket', async (metric) => { + const { env, points } = makeEnv(); + const res = await post({ + schema: 1, + contributions: [{ metric, bucket: '0.18.0', count: '4-7' }], + }, env); + expect(res.status).toBe(204); + expect(points).toEqual([{ + indexes: [metric], + blobs: ['0.18.0', '4-7'], + doubles: [1], + }]); + }); + it('stores the count bucket as a second blob', async () => { const { env, points } = makeEnv(); await post(valid, env); diff --git a/apps/telemetry-worker/src/index.ts b/apps/telemetry-worker/src/index.ts index 794994e3d..c41617973 100644 --- a/apps/telemetry-worker/src/index.ts +++ b/apps/telemetry-worker/src/index.ts @@ -29,6 +29,9 @@ type BucketKind = | 'yes' | 'semver'; const METRICS: Record = { + cli_install_fresh: 'semver', + cli_install_update: 'semver', + cli_install_noop: 'semver', active_db_day: 'yes', workers_by_version: 'semver', version_changed: 'yes', diff --git a/pkgs/cli/__tests__/commands/install/report-install-telemetry.test.ts b/pkgs/cli/__tests__/commands/install/report-install-telemetry.test.ts new file mode 100644 index 000000000..88185dbef --- /dev/null +++ b/pkgs/cli/__tests__/commands/install/report-install-telemetry.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it, vi } from 'vitest'; +import { reportInstallTelemetry } from '../../../src/commands/install/report-install-telemetry'; + +const result = { kind: 'update' as const, copied: 5 }; + +describe('reportInstallTelemetry', () => { + it.each([ + ['fresh', 12, '8-15'], + ['update', 5, '4-7'], + ['noop', 0, '0'], + ] as const)('reports a successful %s install', async (kind, copied, count) => { + const send = vi.fn().mockResolvedValue(new Response(null, { status: 204 })); + + await reportInstallTelemetry( + { kind, copied }, + { env: {}, version: '0.18.0', send }, + ); + + expect(send).toHaveBeenCalledOnce(); + const [url, request] = send.mock.calls[0] as [string, RequestInit]; + expect(url).toBe('https://pgflow-telemetry.workers.dev/'); + expect(request.method).toBe('POST'); + expect(JSON.parse(request.body as string)).toEqual({ + schema: 1, + contributions: [{ + metric: `cli_install_${kind}`, + bucket: '0.18.0', + count, + }], + }); + }); + + it.each([ + { CI: 'true' }, + { NODE_ENV: 'test' }, + { DO_NOT_TRACK: '1' }, + { PGFLOW_TELEMETRY_DISABLED: 'true' }, + ])('does not report when disabled by $env', async (env) => { + const send = vi.fn(); + await reportInstallTelemetry(result, { env, version: '0.18.0', send }); + expect(send).not.toHaveBeenCalled(); + }); + + it('does not report an invalid version', async () => { + const send = vi.fn(); + await reportInstallTelemetry(result, { env: {}, version: 'unknown', send }); + expect(send).not.toHaveBeenCalled(); + }); + + it('never fails installation when the request fails', async () => { + const send = vi.fn().mockRejectedValue(new Error('offline')); + await expect( + reportInstallTelemetry(result, { env: {}, version: '0.18.0', send }), + ).resolves.toBeUndefined(); + }); +}); diff --git a/pkgs/cli/scripts/test-install b/pkgs/cli/scripts/test-install index 12127dac6..b5efba9a8 100755 --- a/pkgs/cli/scripts/test-install +++ b/pkgs/cli/scripts/test-install @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -e +export PGFLOW_TELEMETRY_DISABLED=1 # Script to test pgflow CLI install functionality # Uses a temp directory to avoid conflicts with other tests diff --git a/pkgs/cli/scripts/test-install-duplicates b/pkgs/cli/scripts/test-install-duplicates index 72643adf8..0758fc1ef 100755 --- a/pkgs/cli/scripts/test-install-duplicates +++ b/pkgs/cli/scripts/test-install-duplicates @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -e +export PGFLOW_TELEMETRY_DISABLED=1 # Script to test pgflow CLI install duplicate prevention # Uses a temp directory to avoid conflicts with other tests diff --git a/pkgs/cli/src/commands/install/copy-migrations.ts b/pkgs/cli/src/commands/install/copy-migrations.ts index 298cbc1d0..2a8028cd6 100644 --- a/pkgs/cli/src/commands/install/copy-migrations.ts +++ b/pkgs/cli/src/commands/install/copy-migrations.ts @@ -155,13 +155,18 @@ function generateNewTimestamp( // Find the migrations directory const sourcePath = findMigrationsDirectory(); +export type MigrationInstallResult = { + kind: 'fresh' | 'update' | 'noop'; + copied: number; +}; + export async function copyMigrations({ supabasePath, autoConfirm = false, }: { supabasePath: string; autoConfirm?: boolean; -}): Promise { +}): Promise { const migrationsPath = path.join(supabasePath, 'migrations'); if (!fs.existsSync(migrationsPath)) { @@ -180,7 +185,7 @@ export async function copyMigrations({ log.info( 'If running in development mode, try building the core package first with: nx build core' ); - return false; + return null; } // Get all existing migrations in user's directory @@ -231,10 +236,10 @@ export async function copyMigrations({ } } - // If no files to copy, show message and return false (no changes made) + // If no files need copying, this is a successful no-op install. if (filesToCopy.length === 0) { log.success('Migrations already up to date'); - return false; + return { kind: 'noop', copied: 0 }; } // Generate new timestamps for migrations to install @@ -267,7 +272,7 @@ export async function copyMigrations({ if (confirmResult !== true) { log.warn('Migration installation skipped'); - return false; + return null; } } @@ -281,5 +286,8 @@ export async function copyMigrations({ log.success(`Installed ${filesToCopy.length} migration${filesToCopy.length !== 1 ? 's' : ''}`); - return true; // Return true to indicate migrations were copied + return { + kind: skippedFiles.length === 0 ? 'fresh' : 'update', + copied: filesToCopy.length, + }; } diff --git a/pkgs/cli/src/commands/install/index.ts b/pkgs/cli/src/commands/install/index.ts index 5e020f9f7..cfc0d2fb4 100644 --- a/pkgs/cli/src/commands/install/index.ts +++ b/pkgs/cli/src/commands/install/index.ts @@ -6,6 +6,7 @@ import { updateConfigToml } from './update-config-toml.js'; import { createFlowsDirectory } from './create-flows-directory.js'; import { createExampleWorker } from './create-example-worker.js'; import { supabasePathPrompt } from './supabase-path-prompt.js'; +import { reportInstallTelemetry } from './report-install-telemetry.js'; export default (program: Command) => { program @@ -38,6 +39,10 @@ export default (program: Command) => { ` • Create ${chalk.cyan('supabase/functions/greet-user-worker/')} ${chalk.dim('(example worker)')}`, '', ` ${chalk.green('✓ Safe to re-run - completed steps will be skipped')}`, + '', + chalk.dim( + 'Anonymous telemetry (no identifiers or project values). Opt out: PGFLOW_TELEMETRY_DISABLED=1 · pgflow.dev/reference/telemetry' + ), ].join('\n'); log.info(summaryMsg); @@ -85,8 +90,10 @@ export default (program: Command) => { // Step 4: Show completion message const outroMessages: string[] = []; + const migrationsChanged = + migrations?.kind === 'fresh' || migrations?.kind === 'update'; - if (migrations || configUpdate || flowsDirectory || exampleWorker) { + if (migrationsChanged || configUpdate || flowsDirectory || exampleWorker) { outroMessages.push(chalk.green.bold('✓ Installation complete!')); } else { outroMessages.push( @@ -107,7 +114,7 @@ export default (program: Command) => { stepNumber++; } - if (migrations) { + if (migrationsChanged) { outroMessages.push( ` ${stepNumber}. Apply migrations: ${chalk.cyan('supabase migrations up')}` ); @@ -119,5 +126,9 @@ export default (program: Command) => { ); outro(outroMessages.join('\n')); + + if (migrations) { + await reportInstallTelemetry(migrations); + } }); }; diff --git a/pkgs/cli/src/commands/install/report-install-telemetry.ts b/pkgs/cli/src/commands/install/report-install-telemetry.ts new file mode 100644 index 000000000..473a3e42a --- /dev/null +++ b/pkgs/cli/src/commands/install/report-install-telemetry.ts @@ -0,0 +1,67 @@ +import { getVersion } from '../../utils/get-version.js'; +import type { MigrationInstallResult } from './copy-migrations.js'; + +const ENDPOINT = 'https://pgflow-telemetry.workers.dev/'; +const SEMVER_RE = /^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?$/; + +type Environment = Record; +type Send = typeof globalThis.fetch; + +const isSet = (value: string | undefined) => { + const normalized = value?.toLowerCase(); + return normalized !== undefined + && normalized !== '' + && normalized !== '0' + && normalized !== 'false'; +}; + +function telemetryDisabled(env: Environment): boolean { + return env.NODE_ENV === 'test' + || isSet(env.CI) + || isSet(env.DO_NOT_TRACK) + || isSet(env.PGFLOW_TELEMETRY_DISABLED); +} + +function bucketCount(value: number): string { + if (value === 0) return '0'; + if (value === 1) return '1'; + if (value <= 3) return '2-3'; + if (value <= 7) return '4-7'; + if (value <= 15) return '8-15'; + if (value <= 31) return '16-31'; + if (value <= 63) return '32-63'; + if (value <= 127) return '64-127'; + if (value <= 255) return '128-255'; + return '256+'; +} + +export async function reportInstallTelemetry( + result: MigrationInstallResult, + { + env = process.env, + version = getVersion(), + send = globalThis.fetch, + }: { env?: Environment; version?: string; send?: Send } = {}, +): Promise { + if (telemetryDisabled(env) || version.length > 32 || !SEMVER_RE.test(version)) { + return; + } + + try { + await send(ENDPOINT, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + schema: 1, + contributions: [{ + metric: `cli_install_${result.kind}`, + bucket: version, + count: bucketCount(result.copied), + }], + }), + signal: AbortSignal.timeout(500), + }); + } catch { + // Telemetry must never delay or fail installation. + } +} diff --git a/pkgs/website/src/content/docs/reference/telemetry.mdx b/pkgs/website/src/content/docs/reference/telemetry.mdx index c03e507eb..9517b9954 100644 --- a/pkgs/website/src/content/docs/reference/telemetry.mdx +++ b/pkgs/website/src/content/docs/reference/telemetry.mdx @@ -3,17 +3,48 @@ title: Telemetry description: What anonymous usage data pgflow collects, how to inspect it, and how to disable it --- -pgflow includes anonymous, opt-out telemetry. It answers one question for the maintainers: **is pgflow actually used, and which features matter?** Your data stays yours — nothing below can identify you, your project, or your users. +pgflow includes anonymous, opt-out telemetry. It answers one question for the maintainers: **is pgflow actually used, and which features matter?** + +**pgflow never sends or stores information that can identify you, your project, or your users.** The only variable exact value from your installation that is stored as telemetry is a pgflow semantic version such as `0.18.0`. Everything else is a fixed label defined in the source code or a coarse bucket. No exact usage count, duration, source timestamp, migration filename, or other exact project value leaves your machine or database. + +Two narrowly scoped signals are sent: + +- After `pgflow install` completes successfully outside CI and tests, the CLI sends one installation-result event. +- Once per active day, a non-local database sends one aggregate usage report. Nothing is sent per run, step, or task, so telemetry never touches the execution path. ## What is collected -Once per day, your database aggregates yesterday's activity into coarse buckets and sends one small JSON document. Nothing is sent per run, per step, or per task — telemetry never touches the execution path. +### Installation command -The payload contains only: +A successful `pgflow install` sends only: + +- The exact pgflow CLI version +- A fixed result label: `fresh`, `update`, or `noop` +- The number of copied pgflow migrations as a coarse bucket (`0`, `1`, `2-3`, `4-7`, and so on) + +`fresh` means that no recognized pgflow migration existed, `update` means that older pgflow migrations existed and new ones were copied, and `noop` means that all bundled pgflow migrations were already present. Raw migration timestamps, filenames, project migration counts, paths, and installation errors are never sent. The event looks like this: + +```json +{ + "schema": 1, + "contributions": [ + { + "metric": "cli_install_update", + "bucket": "0.18.0", + "count": "4-7" + } + ] +} +``` + +### Daily database report + +Once per day, your database aggregates yesterday's activity into coarse buckets and sends one small JSON document. The payload contains only: - Whether any runs happened that day - Which pgflow versions started workers, and whether a version changed - Coarse counts: runs started, run outcomes, workers started/stopped, registered worker functions +- Registered worker start modes (`http` or `process`) - Flow shape in buckets: steps per flow, which features appear (map steps, conditions, retries, delays, graceful failure, step queues) - Coarse durations and task counts in fixed ranges @@ -39,11 +70,13 @@ A payload never exceeds 64 contributions or 2 KB. On an unusually varied day the ## What is never collected -No slugs, flow or step names, queue names, function names, run or task IDs, inputs, outputs, error messages, URLs, hostnames, environment names, IP addresses, or any free text. No cookies. No account. Nothing that could identify you, your project, or your users. +pgflow collects no slugs, flow or step names, queue names, function names, migration names or timestamps, run or task IDs, inputs, outputs, error messages, URLs, file paths, hostnames, environment names, operating-system details, IP addresses, or free text. It creates no account, persistent installation ID, machine ID, project ID, or cookie. + +The telemetry application cannot connect two reports to the same installation. It cannot count unique users or reconstruct one project's history. ## Inspect exactly what is sent -Every payload is stored before it leaves, so you can audit the real bytes: +The installation event shape is shown above and is fixed in the open-source CLI. Every daily database payload is stored before it leaves, so you can audit its exact bytes: ```sql select jsonb_pretty(payload) @@ -61,11 +94,21 @@ select pgflow_telemetry.preview('2026-09-10'); -- any past day ## Disable or re-enable -Telemetry is opt-out and runs as a daily `pg_cron` job. The job's presence is the switch: +Telemetry is opt-out. The CLI shows a disclosure before installation and does not ask for permission. + +Disable the installation event for one command: + +```bash frame="none" +PGFLOW_TELEMETRY_DISABLED=1 npx pgflow@latest install +``` + +The CLI also honors the conventional `DO_NOT_TRACK=1` environment variable. It never sends installation telemetry in CI or tests. Set either variable in your shell profile to disable CLI telemetry permanently on that machine. + +Daily database telemetry uses a `pg_cron` job. The job's presence is the switch: ```sql -select pgflow_telemetry.disable(); -- stop reporting -select pgflow_telemetry.enable(); -- resume reporting +select pgflow_telemetry.disable(); -- stop daily database reports +select pgflow_telemetry.enable(); -- resume daily database reports ``` `disable()` errors instead of reporting success when it cannot prove the job absent — for example when the job was scheduled by a different database role and `cron.job` row security hides it from yours. In that case run `disable()` as the role that installed pgflow (the migration or `enable()` caller). On a vanilla PostgreSQL install where pgflow's migration owner is neither superuser, `BYPASSRLS`, nor a role that inherits the privileges of the `cron.job` owner, the same row security also hides the job from the privileged check itself; the error then names the one-line fix, reassigning `pgflow_telemetry.job_is_scheduled()` to a capable owner (on Supabase the migration owner `postgres` already qualifies, so this never applies there). Calling `disable()` when no job exists is a safe no-op. @@ -74,9 +117,9 @@ Local development databases (`supabase start`) never send telemetry, and days wi ## Privacy -- Data goes to `https://pgflow-telemetry.workers.dev`, a [Cloudflare Worker](https://workers.cloudflare.com/) backed by [Workers Analytics Engine](https://developers.cloudflare.com/analytics/analytics-engine/). -- On the ingest side, stored data is retained for three months, then expires automatically. +- Both signals go to `https://pgflow-telemetry.workers.dev`, a [Cloudflare Worker](https://workers.cloudflare.com/) backed by [Workers Analytics Engine](https://developers.cloudflare.com/analytics/analytics-engine/). +- Stored data expires automatically after three months. - The `sent_reports` audit table in your database is separate: its 90-day prune runs only as part of a successful daily report. Disabled databases and databases with inactive days keep their audit rows until you delete them yourself. -- pgflow sends no identifiers and the ingest application stores no transport metadata (no IPs, no user agents). Cloudflare necessarily processes source IPs as network transport. -- Counts are undercounts in practice: opt-outs, blocked egress, and failed sends are invisible. The maintainers treat every number as a lower bound and as directional product evidence only. -- The sender is the open-source SQL in this repository; the receiver is the open-source Worker in `apps/telemetry-worker/`. +- pgflow sends no identifiers and the telemetry application stores no transport metadata, including IP addresses or user agents. Cloudflare necessarily processes source IPs to carry the network request. +- Counts are undercounts in practice: opt-outs, CI exclusion, blocked egress, and failed sends are invisible. The maintainers treat every number as a lower bound and as directional product evidence only. +- Both senders and the receiver are open source: the CLI is in `pkgs/cli/`, the daily database sender is in `pkgs/core/schemas/`, and the receiver is in `apps/telemetry-worker/`.