ci(health): hourly /healthz probe that pages Paperclip on red

.github/workflows/health-probe.yml

@@ -0,0 +1,192 @@
+name: Health probe
+
+# Hourly is the interval: six silent days was the failure mode, and catching
+# it the same day is the acceptance bar. The 6h ingest-age threshold is
+# independent — a quiet night is not an alert; six hours without a span is.
+#
+# Two halves, neither sufficient alone. The loopback half runs on the deploy
+# host and sees the application; it is blind to its own host being down. The
+# edge half runs on a GitHub runner and sees host, tunnel, DNS and Access; it
+# cannot see /healthz at all until the Access service token is allowed on the
+# cotel.aignite.pl application, so it reports that as a warning rather than an
+# outage.
+on:
+  schedule:
+    - cron: "17 * * * *"
+  workflow_dispatch:
+    inputs:
+      url:
+        description: "Override the edge /healthz URL (empty = production dashboard)"
+        required: false
+        default: ""
+      loopback_url:
+        description: "Override the on-host /healthz URL (empty = http://127.0.0.1:8080/healthz)"
+        required: false
+        default: ""
+      page:
+        description: "Create or resolve a Paperclip alert for this run"
+        type: boolean
+        default: false
+  pull_request:
+  push:
+    branches: [main]
+    paths:
+      - "scripts/probe-healthz.sh"
+      - "scripts/probe-healthz_test.sh"
+      - "scripts/page-cotel-health.sh"
+      - "scripts/page-cotel-health_test.sh"
+      - ".github/workflows/health-probe.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Probe script tests
+    if: github.event_name == 'pull_request' || github.event_name == 'push'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - name: Run probe-healthz tests
+        run: bash scripts/probe-healthz_test.sh
+      - name: Run pager tests
+        run: bash scripts/page-cotel-health_test.sh
+
+  probe-loopback:
+    name: Probe /healthz on the deploy host
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: [self-hosted, flopsstuff, docker]
+    # This job cannot report its own host being down: with the runner offline
+    # the run sits in the queue, and timeout-minutes does not bound queue time
+    # — it starts counting once a runner picks the job up. So an absent host
+    # produces no colour at all, which is what the edge half is for. The cap
+    # only bounds a job that is running and stuck (6h by default), and the
+    # concurrency group keeps each hour superseding the last queued attempt
+    # instead of piling up a day's worth of them.
+    timeout-minutes: 5
+    concurrency:
+      group: health-probe-loopback
+      cancel-in-progress: true
+    env:
+      STALE_AFTER_SECONDS: "21600"
+      CONNECT_TIMEOUT: "15"
+      PC_API_URL: ${{ secrets.PAPERCLIP_API_URL }}
+      PC_API_TOKEN: ${{ secrets.PAPERCLIP_API_TOKEN }}
+      PC_COMPANY_ID: ${{ secrets.PAPERCLIP_COMPANY_ID }}
+      GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+    steps:
+      - uses: actions/checkout@v5
+
+      # The pager is the only thing here that wakes anyone, and this runner is
+      # a developer Mac rather than a managed image. Check its interpreters on
+      # every green hour, so a missing one surfaces before an incident needs
+      # them.
+      - name: Check pager prerequisites
+        run: |
+          for bin in jq python3; do
+            command -v "$bin" >/dev/null \
+              || echo "::warning title=Pager prerequisite missing::${bin} is not on this runner's PATH; a red hour would not page"
+          done
+
+      # No CF_ACCESS_* here: loopback has no Cloudflare in front of it, and the
+      # probe sends the Access headers whenever both are set.
+      - name: Probe /healthz
+        env:
+          HEALTHZ_URL: ${{ inputs.loopback_url || 'http://127.0.0.1:8080/healthz' }}
+        run: |
+          set -o pipefail
+          mkdir -p "$RUNNER_TEMP"
+          PROBE_OUT="$RUNNER_TEMP/probe-loopback.out"
+          set +e
+          scripts/probe-healthz.sh | tee "$PROBE_OUT"
+          rc=${PIPESTATUS[0]}
+          set -e
+          echo "PROBE_OUT=$PROBE_OUT" >> "$GITHUB_ENV"
+          echo "PROBE_RC=$rc" >> "$GITHUB_ENV"
+
+      # Paging failures are a warning, never the job's verdict: this job's
+      # red/green must mean "production is red/green" and nothing else.
+      - name: Page Paperclip on red
+        if: env.PROBE_RC != '0' && (github.event_name == 'schedule' || inputs.page)
+        env:
+          CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }}
+          CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}
+        run: |
+          scripts/page-cotel-health.sh raise "$PROBE_OUT" \
+            || echo "::warning title=Pager failed::could not raise the Paperclip alert; nobody was woken for this red hour"
+
+      - name: Resolve Paperclip alert on green
+        if: env.PROBE_RC == '0' && (github.event_name == 'schedule' || inputs.page)
+        env:
+          CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }}
+          CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}
+        run: |
+          scripts/page-cotel-health.sh resolve \
+            || echo "::warning title=Pager failed::could not resolve the standing Paperclip alert"
+
+      - name: Fail the job when the probe is red
+        if: env.PROBE_RC != '0'
+        run: |
+          echo "probe-healthz exited ${PROBE_RC}"
+          cat "$PROBE_OUT"
+          exit 1
+
+  probe-edge:
+    name: Probe /healthz through Cloudflare
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    env:
+      HEALTHZ_URL: ${{ inputs.url || 'https://cotel.aignite.pl/healthz' }}
+      STALE_AFTER_SECONDS: "21600"
+      CONNECT_TIMEOUT: "15"
+      CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }}
+      CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}
+      PC_API_URL: ${{ secrets.PAPERCLIP_API_URL }}
+      PC_API_TOKEN: ${{ secrets.PAPERCLIP_API_TOKEN }}
+      PC_COMPANY_ID: ${{ secrets.PAPERCLIP_COMPANY_ID }}
+      # A separate dedup marker from the loopback half. On one marker the two
+      # halves would fight: a green loopback hour would resolve the alert the
+      # edge half had just raised.
+      PC_ORIGIN_ID: cotel-health-probe-edge
+      GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Probe /healthz
+        run: |
+          set -o pipefail
+          mkdir -p "$RUNNER_TEMP"
+          PROBE_OUT="$RUNNER_TEMP/probe-edge.out"
+          set +e
+          scripts/probe-healthz.sh | tee "$PROBE_OUT"
+          rc=${PIPESTATUS[0]}
+          set -e
+          echo "PROBE_OUT=$PROBE_OUT" >> "$GITHUB_ENV"
+          echo "PROBE_RC=$rc" >> "$GITHUB_ENV"
+
+      - name: Page Paperclip on red
+        if: env.PROBE_RC != '0' && env.PROBE_RC != '4' && (github.event_name == 'schedule' || inputs.page)
+        run: |
+          scripts/page-cotel-health.sh raise "$PROBE_OUT" \
+            || echo "::warning title=Pager failed::could not raise the Paperclip alert; nobody was woken for this red hour"
+
+      - name: Resolve Paperclip alert on green
+        if: env.PROBE_RC == '0' && (github.event_name == 'schedule' || inputs.page)
+        run: |
+          scripts/page-cotel-health.sh resolve \
+            || echo "::warning title=Pager failed::could not resolve the standing Paperclip alert"
+
+      # Exit 4 is "cannot see /healthz", not "cotel is down". Failing on it
+      # would paint this job red every hour until the Access service token is
+      # allowed on the application, and an alarm that is always on is one
+      # nobody reads. It stays a warning until it can actually observe.
+      - name: Report the probe verdict
+        if: env.PROBE_RC != '0'
+        run: |
+          cat "$PROBE_OUT"
+          if [ "$PROBE_RC" = "4" ]; then
+            echo "::warning title=Edge probe cannot observe /healthz::Cloudflare Access blocked the probe, so this half is not watching anything. Allow the CF_ACCESS service token on the cotel.aignite.pl application."
+            exit 0
+          fi
+          echo "probe-healthz exited ${PROBE_RC}"
+          exit 1

docs/.vitepress/config.js

@@ -29,6 +29,7 @@ export default defineConfig({
             { text: 'JSON API Reference', link: '/operations/api-reference' },
             { text: 'Cloudflare Tunnel — Token Mode', link: '/operations/cloudflare-tunnel-remote' },
             { text: 'Cloudflare Tunnel — Local Config', link: '/operations/cloudflare-tunnel-local' },
+            { text: 'Production /healthz probe', link: '/operations/health-probe' },
             { text: 'Export / Import', link: '/operations/export-import' },
             { text: 'DuckDB Recovery', link: '/operations/duckdb-recovery' },
             { text: 'README Screenshots', link: '/operations/screenshots' },

docs/index.md

@@ -100,3 +100,4 @@ Single Go binary. Single DuckDB file. Single named volume. No sidecars.
 
 - [Architecture Decisions](/decisions/) — ADRs explaining key technical choices
 - [Design Docs](/design/) — UI information architecture, page specs, components, tokens
+- [Production /healthz probe](/operations/health-probe) — hourly check of production `/healthz` and who is woken when it is red

docs/operations/health-probe.md

@@ -0,0 +1,158 @@
+# Production /healthz probe
+
+An hourly GitHub Actions workflow asks the **dashboard** `/healthz` on the
+production instance and pages a Paperclip agent when the answer is not healthy.
+It exists because a dead or silent cotel otherwise has no one looking: `Deploy`
+only runs on a push to `main`, the Cloudflare tunnel stays green while the
+process is 503 or not listening, and a red Actions run in this company does not
+wake anyone on its own.
+
+## Two halves, neither sufficient alone
+
+The workflow probes the same endpoint from two places, because the two vantage
+points are blind to different things.
+
+| | `probe-loopback` | `probe-edge` |
+|---|---|---|
+| Runner | `[self-hosted, flopsstuff, docker]` — the deploy host | `ubuntu-latest` |
+| Target | `http://127.0.0.1:8080/healthz` | `https://cotel.aignite.pl/healthz` |
+| Cloudflare in the path | no | yes (tunnel + Access) |
+| Catches | process dead, crash loop, 503 DuckDB, stale/empty ingest | all of that, **plus** host down, tunnel down, DNS, Access misconfigured |
+| Blind to | **its own host being down** — with the runner off the job queues, producing no colour at all | nothing in the path, but see the Access caveat below |
+| Access service token | not needed | required |
+| Alert dedup marker | `[cotel-health-probe]` | `[cotel-health-probe-edge]` |
+
+The loopback half is the one that would have caught the 2026-09-28 incident:
+the container was crash-looping and `/healthz` refused the connection on
+localhost. It needs no secret and no Cloudflare, which is why it is the half
+that works today.
+
+The edge half is not a luxury. A self-hosted job cannot report that its own
+host is down — if the runner is off or offline, GitHub queues the run rather
+than failing it, and `timeout-minutes` does **not** bound queue time: it starts
+counting once a runner picks the job up. So an absent host yields no colour at
+all. Nothing goes red, nobody is paged, and the only trace is a run sitting in
+`queued`, which is also what a busy runner looks like. `timeout-minutes: 5`
+bounds a different case — a job that is running but stuck, instead of hanging
+for the 6h default — and the job-level `concurrency` group with
+`cancel-in-progress` keeps each hour superseding the last queued attempt rather
+than stacking a day's worth of them. Only a probe from outside the host can
+tell "host down" from "no signal", and only the edge half sees the tunnel, DNS
+and Access at all.
+
+The two halves use **different dedup markers** on purpose. On a single marker
+they would fight: a green loopback hour would mark the alert the edge half had
+just raised as done.
+
+## What the probe checks
+
+The probe classifies failure modes with distinct text, because they send the
+on-call looking in different places:
+
+| Probe verdict | Exit | Typical cause |
+|---|---|---|
+| `unreachable` | 1 | Process not listening, host down, or the tunnel is not forwarding |
+| `HTTP 503 database unreadable` | 2 | Process is up; DuckDB is not |
+| `HTTP <other>` | 2 | Crash loop, proxy error, unexpected handler |
+| `ingest stale` / `empty database` | 3 | Process is up and the DB reads; spans are not being accepted |
+| `cloudflare access blocked` | 4 | Probe config, **not** an outage — see below |
+
+A 200 with `ok: true` is not enough. Staleness is a body field
+(`newest_span_age_seconds`); the endpoint keeps 200 on a quiet instance so the
+container HEALTHCHECK does not flap. The probe applies its own threshold:
+**6 hours**. A present JSON `null` is an empty database (never ingested), which
+is not the same as `0` (ingested just now) and not the same as a missing key.
+
+If `last_ingest_at` / `newest_span_age_seconds` are absent, the probe degrades
+to liveness (HTTP 200) so it keeps working before that contract is on
+production.
+
+### The Access caveat (why the edge half warns instead of failing)
+
+Cloudflare Access sits in front of the dashboard. The workflow sends the same
+Access service-token headers as `paperclip-issue-sync.yml`, but that token is
+**not currently allowed on the `cotel.aignite.pl` application**: the request
+comes back as a 302 to the Access login, exactly as it does with no token at
+all.
+
+So `probe-edge` reports exit 4 as a **warning, and the job stays green**. That
+is deliberate. Failing on exit 4 would paint the workflow red every hour until
+the token is allowed, and a probe that is always red is one nobody reads —
+worse than no probe, because it also buries a genuine red. Exit 4 never pages
+either: it says nothing about whether cotel is up.
+
+The price of that choice: while the token is unauthorized, **the edge half is
+not watching anything**, and the only thing saying so is a warning annotation
+on an otherwise-green run. To switch it on, allow the service token on the
+`cotel.aignite.pl` Access application (or bypass `/healthz` only). No code
+change is needed — the half starts observing on the next scheduled hour.
+
+## Schedule
+
+`17 * * * *` UTC, hourly. Six silent days was too long; catching an outage the
+same day is the bar. The 6h ingest-age threshold is independent of the poll
+interval: a quiet night is not an alert.
+
+Manual run: Actions → **Health probe** → **Run workflow**. Each half has its
+own optional URL override (`url` for the edge, `loopback_url` for the host) for
+demonstrating a red run against a dead endpoint; leave **page** unchecked
+unless you intend to open a Paperclip alert.
+
+The schedule runs only from the default branch. On a public repository GitHub
+disables scheduled workflows after 60 days with no repository activity. The
+notice for that goes to GitHub notifications, which do not wake anyone here —
+the same silence this probe exists to close. A push is repository activity and
+resets that 60-day clock. If the repository sits idle long enough for GitHub
+to disable the schedule, this probe goes quiet with it.
+
+## Who is woken, and how
+
+A red GitHub Actions run is **not** the page. Notifications on the Fl0p
+account are unproven (no `notifications` API scope, no public mailbox, agent
+identities are not GitHub users), and this company has already watched red
+Actions sit unnoticed.
+
+On red, the workflow opens a Paperclip issue titled
+`cotel prod /healthz is red [<marker>]`, assigned to Daedalus. That assignment
+is the wake. A later red hour searches `q=<marker>` and comments on the open
+issue whose **title** contains it. Search also matches comments and
+descriptions, so the first hit is not the alert, and a longer marker such as
+`[cotel-health-probe-selftest]` is not this one. The create request does not
+send `originId`: the issues API drops unknown fields, and list search does not
+query that column.
+
+A green hour marks that same issue done. If the assignee still has the alert
+checked out, the status change comes back as a run-ownership conflict. The
+probe leaves the issue open and does not fail the job, because that assignee
+is already awake. The next green hour closes it once the checkout is released.
+
+**A paging failure is a warning, never the job's verdict.** Each job's red or
+green means "production is red or green" and nothing else; if the pager itself
+cannot reach Paperclip, the run carries a `Pager failed` annotation saying
+nobody was woken. The loopback job also checks for `jq` and `python3` on every
+run — that runner is a developer machine, not a managed image, so a missing
+interpreter should surface on a green hour rather than during an incident.
+
+Paperclip budget is spent only when a probe is red (create or comment) or
+when it recovers. A green hour with no open alert does not write. A scheduled
+Paperclip routine that fires every hour regardless of health is the more
+expensive alternative (24 heartbeats a day). It is not enabled.
+
+## Local use
+
+```sh
+# the deploy host, from the deploy host — no Access token involved
+HEALTHZ_URL=http://127.0.0.1:8080/healthz scripts/probe-healthz.sh
+
+# through Cloudflare (needs an Access service token allowed on the app)
+scripts/probe-healthz.sh
+
+# a closed local port — the failure demo
+scripts/probe-healthz.sh http://127.0.0.1:1/healthz
+
+# the classification tests, including 503 vs stale vs empty vs refused
+bash scripts/probe-healthz_test.sh
+```
+
+Exit codes: `0` healthy, `1` unreachable, `2` HTTP non-200, `3` stale/empty,
+`4` Access blocked.