diff --git a/.github/workflows/health-probe.yml b/.github/workflows/health-probe.yml index 19583a5..2c32bbc 100644 --- a/.github/workflows/health-probe.yml +++ b/.github/workflows/health-probe.yml @@ -1,18 +1,17 @@ name: Health probe -# Hourly is the interval: six silent days was the failure mode, and catching -# it the same day is the acceptance bar. The 6h ingest-age threshold is -# independent — a quiet night is not an alert; six hours without a span is. +# This workflow no longer schedules anything. `cron: "17 * * * *"` sat on the +# default branch for nine hours and produced one run: GitHub schedules public +# repositories on a best-effort basis and *drops* ticks rather than delaying +# them, so the detection delay it bought was unbounded — the same silence the +# probe exists to close. The scheduled probe is now a systemd timer on the Pi +# (`~/ops/cotel-healthz.sh`, every 10 minutes), which owns the production alert +# marker. See docs/operations/health-probe.md and ADR-0022. # -# Two halves, neither sufficient alone. The loopback half runs on the deploy -# host and sees the application; it is blind to its own host being down. The -# edge half runs on a GitHub runner and sees host, tunnel, DNS and Access; it -# cannot see /healthz at all until the Access service token is allowed on the -# cotel.aignite.pl application, so it reports that as a warning rather than an -# outage. +# What is left here is dispatch-only: a drill of the pager, and the only +# vantage point outside the LAN. Both jobs therefore page the *drill* markers, +# never production. on: - schedule: - - cron: "17 * * * *" workflow_dispatch: inputs: url: @@ -54,15 +53,15 @@ jobs: probe-loopback: name: Probe /healthz on the deploy host - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + if: github.event_name == 'workflow_dispatch' runs-on: [self-hosted, flopsstuff, docker] # This job cannot report its own host being down: with the runner offline # the run sits in the queue, and timeout-minutes does not bound queue time # — it starts counting once a runner picks the job up. So an absent host - # produces no colour at all, which is what the edge half is for. The cap - # only bounds a job that is running and stuck (6h by default), and the - # concurrency group keeps each hour superseding the last queued attempt - # instead of piling up a day's worth of them. + # produces no colour at all, which is why the Pi timer probes this host + # over the LAN instead. The cap only bounds a job that is running and stuck + # (6h by default), and the concurrency group keeps a re-dispatch superseding + # the last queued attempt instead of stacking them. timeout-minutes: 5 concurrency: group: health-probe-loopback @@ -78,20 +77,20 @@ jobs: # drill can neither mint an alert that reads like an outage nor act on a # standing real one. Set explicitly rather than left to the script # default: inheriting the default is what made the drills invisible. - PC_ORIGIN_ID: ${{ github.event_name == 'workflow_dispatch' && 'cotel-health-probe-drill' || 'cotel-health-probe' }} + PC_ORIGIN_ID: cotel-health-probe-drill GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} steps: - uses: actions/checkout@v5 # The pager is the only thing here that wakes anyone, and this runner is # a developer Mac rather than a managed image. Check its interpreters on - # every green hour, so a missing one surfaces before an incident needs - # them. + # every run, so a missing one surfaces during a drill rather than when an + # incident needs them. - name: Check pager prerequisites run: | for bin in jq python3; do command -v "$bin" >/dev/null \ - || echo "::warning title=Pager prerequisite missing::${bin} is not on this runner's PATH; a red hour would not page" + || echo "::warning title=Pager prerequisite missing::${bin} is not on this runner's PATH; a red probe would not page" done # No CF_ACCESS_* here: loopback has no Cloudflare in front of it, and the @@ -114,22 +113,22 @@ jobs: # Paging failures are a warning, never the job's verdict: this job's # red/green must mean "production is red/green" and nothing else. - name: Page Paperclip on red - if: env.PROBE_RC != '0' && (github.event_name == 'schedule' || inputs.page) + if: env.PROBE_RC != '0' && inputs.page env: CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }} CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }} run: | scripts/page-cotel-health.sh raise "$PROBE_OUT" \ - || echo "::warning title=Pager failed::could not raise the Paperclip alert; nobody was woken for this red hour" + || echo "::warning title=Pager failed::could not raise the Paperclip alert; nobody was woken for this red probe" - name: Wake the alert's assignee on green - if: env.PROBE_RC == '0' && (github.event_name == 'schedule' || inputs.page) + if: env.PROBE_RC == '0' && inputs.page env: CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }} CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }} run: | scripts/page-cotel-health.sh resolve "$PROBE_OUT" \ - || echo "::warning title=Pager failed::could not wake the alert's assignee; the standing Paperclip alert will keep deduping later red hours" + || echo "::warning title=Pager failed::could not wake the alert's assignee; the standing Paperclip alert will keep deduping later red probes" - name: Fail the job when the probe is red if: env.PROBE_RC != '0' @@ -140,7 +139,7 @@ jobs: probe-edge: name: Probe /healthz through Cloudflare - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest env: HEALTHZ_URL: ${{ inputs.url || 'https://cotel.aignite.pl/healthz' }} @@ -155,7 +154,7 @@ jobs: # halves would fight: a green loopback hour would resolve the alert the # edge half had just raised. A dispatched run is an exercise and gets a # separate marker again, for the same reason the loopback half does. - PC_ORIGIN_ID: ${{ github.event_name == 'workflow_dispatch' && 'cotel-health-probe-drill-edge' || 'cotel-health-probe-edge' }} + PC_ORIGIN_ID: cotel-health-probe-drill-edge GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} steps: - uses: actions/checkout@v5 @@ -173,19 +172,19 @@ jobs: echo "PROBE_RC=$rc" >> "$GITHUB_ENV" - name: Page Paperclip on red - if: env.PROBE_RC != '0' && env.PROBE_RC != '4' && (github.event_name == 'schedule' || inputs.page) + if: env.PROBE_RC != '0' && env.PROBE_RC != '4' && inputs.page run: | scripts/page-cotel-health.sh raise "$PROBE_OUT" \ - || echo "::warning title=Pager failed::could not raise the Paperclip alert; nobody was woken for this red hour" + || echo "::warning title=Pager failed::could not raise the Paperclip alert; nobody was woken for this red probe" - name: Wake the alert's assignee on green - if: env.PROBE_RC == '0' && (github.event_name == 'schedule' || inputs.page) + if: env.PROBE_RC == '0' && inputs.page run: | scripts/page-cotel-health.sh resolve "$PROBE_OUT" \ - || echo "::warning title=Pager failed::could not wake the alert's assignee; the standing Paperclip alert will keep deduping later red hours" + || echo "::warning title=Pager failed::could not wake the alert's assignee; the standing Paperclip alert will keep deduping later red probes" # Exit 4 is "cannot see /healthz", not "cotel is down". Failing on it - # would paint this job red every hour until the Access service token is + # would paint this job red on every run until the Access service token is # allowed on the application, and an alarm that is always on is one # nobody reads. It stays a warning until it can actually observe. - name: Report the probe verdict diff --git a/docs/.vitepress/config.js b/docs/.vitepress/config.js index ba639b3..b4c42a7 100644 --- a/docs/.vitepress/config.js +++ b/docs/.vitepress/config.js @@ -61,6 +61,7 @@ export default defineConfig({ { text: 'ADR-0019 — CI Never Mutates an Issue (superseded)', link: '/decisions/0019-ci-never-mutates-an-issue' }, { text: 'ADR-0020 — Recovery Arrives as a New Issue (superseded)', link: '/decisions/0020-recovery-arrives-as-a-new-issue' }, { text: "ADR-0021 — Recovery Wakes the Alert's Assignee", link: '/decisions/0021-recovery-wakes-the-alerts-assignee' }, + { text: 'ADR-0022 — Health Probe Scheduler Outside This Repo', link: '/decisions/0022-health-probe-scheduler-outside-github' }, ], }, ], diff --git a/docs/decisions/0022-health-probe-scheduler-outside-github.md b/docs/decisions/0022-health-probe-scheduler-outside-github.md new file mode 100644 index 0000000..ac1c2c7 --- /dev/null +++ b/docs/decisions/0022-health-probe-scheduler-outside-github.md @@ -0,0 +1,95 @@ +# ADR 0022 — The health probe's scheduler lives outside this repo + +**Date:** 2026-10-05 +**Status:** Accepted +**Deciders:** Prospero (CEO, mechanism choice), Daedalus (CTO, implementation) + +--- + +## Context + +The probe and the pager work. Both halves were exercised live: a red probe +creates the alert, the assignment wakes its assignee, and the recovery routes +through a wake that the assignee acts on +([ADR-0021](./0021-recovery-wakes-the-alerts-assignee)). What did not work was +the leg nobody had measured — the schedule. + +`cron: "17 * * * *"` landed on `main` at 00:44Z on 2026-10-05. By 09:46Z it had +produced **one** run instead of about nine, and `gh run list +--workflow=health-probe.yml --limit 200` showed exactly one `event=schedule` run +in the workflow's whole history. Two innocent explanations were checked and +excluded: a cancelled run would still be listed as `cancelled`, and the 60-day +deactivation of schedules on idle public repositories had not fired (the +repository was active that day). GitHub schedules public repositories on a +best-effort basis and **drops** ticks rather than delaying them. + +That matters more here than it would elsewhere. This probe exists because +production sat dead for six days with nobody looking; a watcher whose detection +delay is unbounded is a quieter version of the same silence, and nothing watches +whether the watcher ran. + +A second gap compounded it. The loopback half runs on a self-hosted runner on +the deploy host itself, so when that host is off the job does not go red — it +queues, and `timeout-minutes` does not bound queue time. The failure mode most +worth catching produced no colour at all. + +## Options considered + +1. **A systemd timer on the Pi** (`~/ops`), probing `robmini` over the LAN and + calling this repo's existing pager. +2. **A scheduled Paperclip routine** — a reliable scheduler that wakes an agent + by construction. +3. **Leave the GitHub schedule as it is** and accept best-effort detection. + +## Decision + +**Option 1.** `cotel-healthz.timer` on the Pi, every 10 minutes, paging on the +second consecutive failure. It owns the production alert marker +`[cotel-health-probe]`. The GitHub workflow keeps `workflow_dispatch` and its +pull-request tests but no longer schedules anything; both of its probe jobs page +drill markers only. + +Reasons, in the order they decided it: + +- **systemd does not drop ticks**, and `Persistent=true` makes up a tick missed + across a reboot. That is the entire defect being fixed. +- **Probing from the Pi over the LAN catches the host being off**, which the job + on that host structurally cannot report. One prober now covers both failure + classes: dead process (non-200, refused) and dead host (timeout, refused). +- **It costs nothing recurring.** Option 2 was rejected on exactly that: ~24 + full agent runs a day to perform a check that is one `curl`, right after this + company cut idle heartbeat burn by moving the interval from 1h to 6h. A probe + should be cheap enough that its cadence is a free variable — at 10 minutes + this one spends a loopback HTTP request and nothing else while production is + healthy. +- **Option 3 was rejected** because unbounded detection delay is the thing the + parent ticket exists to remove. + +Consequences accepted with it: + +- **The scheduler is not in this repo, and cannot be.** The repo documents where + it is (`docs/operations/health-probe.md` → `~/ops/README.md`) and the timer + materializes this repo's scripts from `origin/main` at each tick, so the code + path stays reviewed here and reaches the timer with no sync step. A reader of + this repo alone cannot see the schedule; a reader of either runbook can. +- **One machine, one prober.** If the Pi is off, nothing probes — the same shape + as before, with a different single point. The Pi is where the Paperclip + instance and every agent already run, so an off Pi has no one to wake anyway. +- **Edge coverage is now unscheduled.** Tunnel, DNS and Cloudflare Access are + outside the LAN and therefore outside what the Pi can see. This is a real loss + only on paper: the edge probe cannot observe `/healthz` at all today, because + its Access service token is not allowed on the application, so it returns exit + 4 and watches a login redirect. Allowing that token is the prerequisite for an + edge schedule being worth anything, and it is a board action, not a code + change. +- **The pager's defaults are GitHub's.** The alert description tells the woken + agent how to re-probe, and the default text names a workflow dispatch — which + for a Pi-raised alert would point at a runner on the possibly-dead host. The + pager therefore takes `PC_SOURCE_LINE` and `PC_REPROBE_HINT` overrides, and + the timer sets them to the local command. Any future non-Actions caller must + do the same. +- **The credential is wider than ideal.** The timer uses the board ops key + already on that machine, because minting a narrow agent key is board-only. + Dropping an agent key at `~/.secrets/cotel-health-pager.token` narrows it with + no code change. ADR-0021's refusal of a board key stands where it was aimed — + a public repository's CI — and does not extend to the owner's own host. diff --git a/docs/decisions/index.md b/docs/decisions/index.md index 9caa6b7..17d15a7 100644 --- a/docs/decisions/index.md +++ b/docs/decisions/index.md @@ -29,3 +29,4 @@ New ADRs go in this directory as `NNNN-short-title.md`, numbered sequentially. | [ADR-0019](./0019-ci-never-mutates-an-issue) | CI never mutates an issue: recovery wakes the alert's assignee | Superseded by ADR-0021 | | [ADR-0020](./0020-recovery-arrives-as-a-new-issue) | Recovery arrives as a new issue, and dedup is time-bounded | Superseded by ADR-0021 | | [ADR-0021](./0021-recovery-wakes-the-alerts-assignee) | Recovery wakes the alert's assignee, and dedup is time-bounded | Accepted | +| [ADR-0022](./0022-health-probe-scheduler-outside-github) | The health probe's scheduler lives outside this repo | Accepted | diff --git a/docs/index.md b/docs/index.md index dde0d41..13780ca 100644 --- a/docs/index.md +++ b/docs/index.md @@ -100,4 +100,4 @@ Single Go binary. Single DuckDB file. Single named volume. No sidecars. - [Architecture Decisions](/decisions/) — ADRs explaining key technical choices - [Design Docs](/design/) — UI information architecture, page specs, components, tokens -- [Production /healthz probe](/operations/health-probe) — hourly check of production `/healthz` and who is woken when it is red +- [Production /healthz probe](/operations/health-probe) — the 10-minute check of production `/healthz`, where its scheduler lives, and who is woken when it is red diff --git a/docs/operations/health-probe.md b/docs/operations/health-probe.md index 691c841..55448fc 100644 --- a/docs/operations/health-probe.md +++ b/docs/operations/health-probe.md @@ -1,50 +1,56 @@ # Production /healthz probe -An hourly GitHub Actions workflow asks the **dashboard** `/healthz` on the -production instance and pages a Paperclip agent when the answer is not healthy. -It exists because a dead or silent cotel otherwise has no one looking: `Deploy` -only runs on a push to `main`, the Cloudflare tunnel stays green while the -process is 503 or not listening, and a red Actions run in this company does not -wake anyone on its own. - -## Two halves, neither sufficient alone - -The workflow probes the same endpoint from two places, because the two vantage -points are blind to different things. - -| | `probe-loopback` | `probe-edge` | -|---|---|---| -| Runner | `[self-hosted, flopsstuff, docker]` — the deploy host | `ubuntu-latest` | -| Target | `http://127.0.0.1:8080/healthz` | `https://cotel.aignite.pl/healthz` | -| Cloudflare in the path | no | yes (tunnel + Access) | -| Catches | process dead, crash loop, 503 DuckDB, stale/empty ingest | all of that, **plus** host down, tunnel down, DNS, Access misconfigured | -| Blind to | **its own host being down** — with the runner off the job queues, producing no colour at all | nothing in the path, but see the Access caveat below | -| Access service token | not needed | required | -| Alert dedup marker (scheduled) | `[cotel-health-probe]` | `[cotel-health-probe-edge]` | -| Alert dedup marker (dispatched) | `[cotel-health-probe-drill]` | `[cotel-health-probe-drill-edge]` | - -The loopback half is the one that would have caught the 2026-09-28 incident: -the container was crash-looping and `/healthz` refused the connection on -localhost. It needs no secret and no Cloudflare, which is why it is the half -that works today. - -The edge half is not a luxury. A self-hosted job cannot report that its own -host is down — if the runner is off or offline, GitHub queues the run rather -than failing it, and `timeout-minutes` does **not** bound queue time: it starts -counting once a runner picks the job up. So an absent host yields no colour at -all. Nothing goes red, nobody is paged, and the only trace is a run sitting in -`queued`, which is also what a busy runner looks like. `timeout-minutes: 5` -bounds a different case — a job that is running but stuck, instead of hanging -for the 6h default — and the job-level `concurrency` group with -`cancel-in-progress` keeps each hour superseding the last queued attempt rather -than stacking a day's worth of them. Only a probe from outside the host can -tell "host down" from "no signal", and only the edge half sees the tunnel, DNS -and Access at all. - -The two halves use **different dedup markers** on purpose. On a single marker -they would fight: a green loopback hour would ask for the close of the alert -the edge half had just raised. A dispatched run is separated from the schedule -the same way — see [Running a drill](#running-a-drill). +A **systemd timer on the Pi** asks the **dashboard** `/healthz` on the +production instance every 10 minutes and pages a Paperclip agent once two +consecutive ticks fail. It exists because a dead or silent cotel otherwise has +no one looking: `Deploy` only runs on a push to `main`, the Cloudflare tunnel +stays green while the process is 503 or not listening, and a red Actions run in +this company does not wake anyone on its own. + +The scheduler used to be GitHub Actions `cron`, and that is the part that did +not work: see [Why the schedule is not in this repo](#why-the-schedule-is-not-in-this-repo) +and [ADR-0022](../decisions/0022-health-probe-scheduler-outside-github.md). The +workflow still exists, dispatch-only, for drills and for the one vantage point +outside the LAN. + +## The three vantage points + +The same endpoint is asked from three places, because each is blind to +something the others see. + +| | `cotel-healthz.timer` (the Pi) | `probe-loopback` (dispatch) | `probe-edge` (dispatch) | +|---|---|---|---| +| Scheduled | **yes, every 10 min** | no | no | +| Runner | the Pi, `~/ops/cotel-healthz.sh` | `[self-hosted, flopsstuff, docker]` — the deploy host | `ubuntu-latest` | +| Target | `http://robmini.local:8080/healthz` (LAN) | `http://127.0.0.1:8080/healthz` | `https://cotel.aignite.pl/healthz` | +| Cloudflare in the path | no | no | yes (tunnel + Access) | +| Catches | process dead, crash loop, 503 DuckDB, stale/empty ingest, **and the host being off** | all of that except host-off | all of that **plus** tunnel down, DNS, Access misconfigured | +| Blind to | the tunnel, DNS and Access — a LAN-healthy cotel unreachable from the internet reads green | **its own host being down** — with the runner off the job queues, producing no colour at all | nothing in the path, but see the Access caveat below | +| Access service token | not needed | not needed | required | +| Alert dedup marker | `[cotel-health-probe]` | `[cotel-health-probe-drill]` | `[cotel-health-probe-drill-edge]` | + +The Pi timer is the authoritative path, and it is the only one that pages the +production marker. Probing over the LAN rather than from the deploy host closes +the gap that the self-hosted job cannot: a job on that host cannot report the +host being off, because GitHub queues the run rather than failing it, and +`timeout-minutes` does **not** bound queue time — it starts counting once a +runner picks the job up. An absent host yields no colour at all, and the only +trace is a run sitting in `queued`, which is also what a busy runner looks like. +From the Pi, an absent host is a refused connection or a timeout, which is a +verdict. + +What the Pi cannot see is everything Cloudflare adds: tunnel, DNS and Access. +That half is unscheduled today, and it is also **not observing anything** — the +Access service token is not allowed on the application, so the edge probe +reports exit 4 (see the Access caveat below). Allowing that token is what would +make an edge schedule worth having; until then an hourly edge run would watch a +login redirect. + +The three use **different dedup markers** on purpose. On a single marker they +would fight: a green tick on one would ask for the close of the alert another +had just raised. Dispatched runs are separated from the Pi timer for the +stronger version of the same reason — a drill must not touch a real alert; see +[Running a drill](#running-a-drill). ## What the probe checks @@ -78,7 +84,7 @@ comes back as a 302 to the Access login, exactly as it does with no token at all. So `probe-edge` reports exit 4 as a **warning, and the job stays green**. That -is deliberate. Failing on exit 4 would paint the workflow red every hour until +is deliberate. Failing on exit 4 would paint the workflow red on every run until the token is allowed, and a probe that is always red is one nobody reads — worse than no probe, because it also buries a genuine red. Exit 4 never pages either: it says nothing about whether cotel is up. @@ -91,35 +97,86 @@ change is needed — the half starts observing on the next scheduled hour. ## Schedule -`17 * * * *` UTC, hourly. Six silent days was too long; catching an outage the -same day is the bar. The 6h ingest-age threshold is independent of the poll -interval: a quiet night is not an alert. +Every **10 minutes**, from `cotel-healthz.timer` on the Pi +(`OnCalendar=*:0/10`, `Persistent=true`). It pages on the **second consecutive** +failure, so the worst-case detection delay is about 20 minutes and a single +blinked tick wakes nobody. The 6h ingest-age threshold is independent of the +poll interval: a quiet night is not an alert. -Manual run: Actions → **Health probe** → **Run workflow**. Each half has its -own optional URL override (`url` for the edge, `loopback_url` for the host) for -demonstrating a red run against a dead endpoint; leave **page** unchecked -unless you intend to open a Paperclip alert — see -[Running a drill](#running-a-drill) for what a dispatch with **page** checked -does. +The timer, its unit files and its runbook live in `~/ops` on the Pi +(`~/ops/README.md`, section *cotel health probe*) — not in this repo, because a +scheduler inside the repo is exactly what did not work. The scripts it runs +**are** this repo's: each tick materializes `scripts/probe-healthz.sh` and +`scripts/page-cotel-health.sh` from `origin/main` with `git show`, so a fix here +reaches the timer with no sync step and the timer does not care which branch the +shared checkout has yanked. -The schedule runs only from the default branch. On a public repository GitHub -disables scheduled workflows after 60 days with no repository activity. The -notice for that goes to GitHub notifications, which do not wake anyone here — -the same silence this probe exists to close. A push is repository activity and -resets that 60-day clock. If the repository sits idle long enough for GitHub -to disable the schedule, this probe goes quiet with it. +```sh +# on the Pi +~/ops/cotel-healthz.sh --status # streak, credential, next elapse, last 20 ticks +~/ops/cotel-healthz.sh --probe-only # probe now, page nothing, touch no state +systemctl --user list-timers cotel-healthz.timer +journalctl --user -u cotel-healthz.service -n 50 +``` + +A tick exits non-zero **only when the watcher itself is broken** (no +credential, the scripts cannot be materialized, the pager cannot reach +Paperclip). Production being red is a successful tick — it pages Paperclip. That +split is what makes `OnFailure=ops-alert@cotel-healthz.service.service` mean +"nobody is watching" rather than "cotel is down": the first needs the owner at +this machine, the second needs an agent. + +### Why the schedule is not in this repo + +`cron: "17 * * * *"` was merged to `main` at 00:44Z on 2026-10-05 and had +produced **one** run by 09:46Z — one tick of an expected nine. `gh run list +--workflow=health-probe.yml --limit 200` showed exactly one `event=schedule` run +in the workflow's entire history; it was not a cancellation (those stay in the +list as `cancelled`) and not the 60-day public-repo deactivation (the repo was +active that day). GitHub schedules public repositories on a best-effort basis +and **drops** ticks rather than delaying them, so the detection delay the cron +bought was not "an hour" but unbounded — a weaker version of the six-day silence +this probe was built to end. + +`systemd` does not drop ticks, and `Persistent=true` makes up a tick missed +across a reboot. A scheduled Paperclip routine was the other candidate and was +rejected on cost: ~24 full agent runs a day to do what `curl` does. See +[ADR-0022](../decisions/0022-health-probe-scheduler-outside-github.md). + +### Dispatching the workflow + +Actions → **Health probe** → **Run workflow**. Each half has its own optional +URL override (`url` for the edge, `loopback_url` for the host) for demonstrating +a red run against a dead endpoint; leave **page** unchecked unless you intend to +open a Paperclip alert — see [Running a drill](#running-a-drill) for what a +dispatch with **page** checked does. + +A dispatch is the only way the workflow's probe jobs run now. The `test` job +still runs on every pull request and on pushes that touch the probe scripts, so +the classification and pager tests remain a merge gate. ## Running a drill -A dispatched run pages a **drill** marker, never the production one. The -workflow derives `PC_ORIGIN_ID` from `github.event_name`: +Every rehearsal pages a **drill** marker, never the production one. Only the Pi +timer uses `cotel-health-probe`; the workflow's jobs pin +`cotel-health-probe-drill` and `cotel-health-probe-drill-edge` unconditionally, +and a drill from the Pi has to pass the drill marker itself: -| Event | `probe-loopback` | `probe-edge` | -|---|---|---| -| `schedule` | `cotel-health-probe` | `cotel-health-probe-edge` | -| `workflow_dispatch` | `cotel-health-probe-drill` | `cotel-health-probe-drill-edge` | +```sh +# on the Pi — a red pair against a closed port, on the drill marker +PC_ORIGIN_ID=cotel-health-probe-drill COTEL_HEALTHZ_URL=http://127.0.0.1:9/healthz \ + ~/ops/cotel-healthz.sh # first tick: streak 1, pages nobody +PC_ORIGIN_ID=cotel-health-probe-drill COTEL_HEALTHZ_URL=http://127.0.0.1:9/healthz \ + ~/ops/cotel-healthz.sh # second tick: raises the drill alert +PC_ORIGIN_ID=cotel-health-probe-drill ~/ops/cotel-healthz.sh # green: routes the close +``` -Dedup is per marker, so the two sets never see each other: a red drill cannot +Two things to put back afterwards: the failure streak in +`~/ops/reports/cotel-healthz/state` is shared with production (a green tick +resets it to 0, which the third command above does), and the drill alert is a +real issue someone has to close. + +Dedup is per marker, so the sets never see each other: a red drill cannot attach itself to a standing real alert, and — the half that actually bites — a green drill cannot clear or file recovery against one nobody has read yet. @@ -128,10 +185,10 @@ URL, so a woken reader can tell an exercise from an outage without opening Actions — the probe text cannot say, since a closed-port drill and a dead process produce the same line. -To exercise the pager end to end, dispatch twice with **page** checked: once -with `loopback_url=http://127.0.0.1:9` (a closed port — raises the alert, whose -assignment wakes its assignee), then once with the default (green — wakes that -assignee again, who closes the alert from their own run). Never break +To exercise the pager through GitHub instead, dispatch twice with **page** +checked: once with `loopback_url=http://127.0.0.1:9` (a closed port — raises the +alert, whose assignment wakes its assignee), then once with the default (green — +wakes that assignee again, who closes the alert from their own run). Never break production to get a red run. **Leave a gap between the two halves.** Wait for the alert's *assignment* run to @@ -184,15 +241,17 @@ drill is over, and say in the close that they were drill artifacts. ## Who is woken, and how -A red GitHub Actions run is **not** the page. Notifications on the Fl0p -account are unproven (no `notifications` API scope, no public mailbox, agent -identities are not GitHub users), and this company has already watched red -Actions sit unnoticed. +A red GitHub Actions run is **not** the page, and neither is a non-zero exit of +the Pi timer. Notifications on the Fl0p account are unproven (no +`notifications` API scope, no public mailbox, agent identities are not GitHub +users), this company has already watched red Actions sit unnoticed, and +`~/ops/ALERT` plus a desktop `notify-send` reach whoever is at the Pi — which +is the owner, not an agent. -On red, the workflow opens a Paperclip issue titled +On red, the prober opens a Paperclip issue titled `cotel prod /healthz is red []`, assigned to Daedalus. That assignment is the wake, and it is the only tracker issue the pager ever creates. A later -hour searches `q=` and acts on the open issue whose **title** contains +tick searches `q=` and acts on the open issue whose **title** contains it. Search also matches comments and descriptions, so the first hit is not the alert, and a longer marker such as `[cotel-health-probe-selftest]` is not this one. The create request does not send `originId`: the issues API drops unknown @@ -233,7 +292,7 @@ assignment-born run are indistinguishable in the field that decides authority. See [ADR-0021](../decisions/0021-recovery-wakes-the-alerts-assignee.md) for the options and the rule it sets. -| Probe hour | Alert state | What the pager does | +| Probe tick | Alert state | What the pager does | |---|---|---| | red | none open | **creates** the alert, assigned to `PC_ASSIGNEE_AGENT_ID` | | red | one open, created within `PC_ALERT_MAX_AGE_H` (default 6 hours) | **wakes** its assignee — "still red, add this probe output" | @@ -245,16 +304,17 @@ options and the rule it sets. An open alert older than `PC_ALERT_MAX_AGE_H` hours is not a dedup target. Everything that closes an alert is a write that can fail, and a failed close -leaves the alert open forever; the next red hour would fold into it and wake +leaves the alert open forever; the next red tick would fold into it and wake nobody. The window is the backstop that cannot be refused, because it performs no write: the stale alert is not closed, not commented on, and not patched. `raise` opens a new issue, assigned as usual, which is the wake, and logs one -line naming both alerts so the Actions run shows why a second one exists. +line naming both alerts, so the tick log (or the Actions run) shows why a +second one exists. A creation timestamp that is missing or not a timestamp is treated as inside the window. A parse failure must not mint a duplicate alert. `PC_ALERT_MAX_AGE_H=0` disables the window, so a drill always dedups. The -window applies only to that raise decision. A green hour still wakes the +window applies only to that raise decision. A green tick still wakes the assignee of whatever alert is open, however old it is. The tracker returns a recent open issue with the same title instead of @@ -270,15 +330,18 @@ rather than reporting an alert nobody was assigned. | `PC_API_TOKEN` | required | Agent API key. Create-only on issues | | `PC_COMPANY_ID` | required | Company the alert is opened in | | `PC_ASSIGNEE_AGENT_ID` | Daedalus | Assignee of a new alert. The assignment is the wake | -| `PC_ORIGIN_ID` | trigger-derived | Dedup marker: `cotel-health-probe{,-edge}` on schedule, `cotel-health-probe-drill{,-edge}` on dispatch | -| `PC_ALERT_MAX_AGE_H` | `6` | Hours an open alert stays a dedup target on a red hour. `0` always dedups | +| `PC_ORIGIN_ID` | `cotel-health-probe` | Dedup marker. The Pi timer keeps the default; both workflow jobs pin `cotel-health-probe-drill{,-edge}` | +| `PC_ALERT_MAX_AGE_H` | `6` | Hours an open alert stays a dedup target on a red tick. `0` always dedups | | `PC_RUN_ID` | unset | Sent as `X-Paperclip-Run-Id` when set | | `CF_ACCESS_CLIENT_ID`, `CF_ACCESS_CLIENT_SECRET` | unset | Same pair as issue-sync, when the API is behind Access | | `GITHUB_RUN_URL` | unset | Attached to the alert and the wake reason when set | | `GITHUB_EVENT_NAME` | unset | Quoted into a new alert's body when set | | `GITHUB_ACTOR` | unset | Quoted into a new alert's body when set | | `HEALTHZ_URL` | unset | Named in the alert so the assignee re-probes the same URL | -| `GITHUB_RUN_ID` | unset | Recovery idempotency key. `manual` when unset | +| `GITHUB_RUN_ID` | unset | Fallback for `PC_CALL_ID` | +| `PC_CALL_ID` | `GITHUB_RUN_ID`, then `manual` | Unique half of the recovery idempotency key. A caller with no run id should pass a coarse time bucket; the Pi timer sends `pi-` | +| `PC_SOURCE_LINE` | names Actions | First paragraph of a new alert: who opened it. Override it if you are not the workflow | +| `PC_REPROBE_HINT` | names an Actions dispatch | How the woken agent re-probes. The Pi timer replaces it with `~/ops/cotel-healthz.sh --probe-only` | ### The woken agent re-probes, and the alert tells it to @@ -318,24 +381,31 @@ ordinary production shape, where an alert standing for an hour has no live run, and it is swallowed in exactly the case these drills keep landing in. The protocol has to live in the description, which survives both. -CI cannot put the recovery in the alert thread either: a comment on an existing -issue is the same refused write as the status flip. What CI *does* always have -is the `create`, so **the alert's description carries the protocol** — it tells -the agent that this description always reads red, that the wake carries no probe -output, and how to check production before acting: dispatch **Health probe** -with `page` unchecked and read its verdict. Green means close the issue citing -that run; still red means add its output. +Neither prober can put the recovery in the alert thread either: a comment on an +existing issue is the same refused write as the status flip. What they *do* +always have is the `create`, so **the alert's description carries the protocol** +— it tells the agent that this description always reads red, that the wake +carries no probe output, and how to check production before acting. Green means +close the issue citing that check; still red means add its output. -The instruction is a dispatch rather than a `curl` for two reasons. A bare +The instruction is a re-probe rather than a `curl` for two reasons. A bare request only proves liveness, while the probe also classifies 503, stale ingest -and an empty database; and for the loopback half the URL is the *deploy host's* -`127.0.0.1`, which nothing but that runner can reach. The description names the -probed URL so the agent re-checks the same endpoint, and warns against passing a -`loopback_url` override while checking a real alert. +and an empty database; and the probed URL may be a host's own loopback, which +nothing but that host can reach. The description names the URL that was probed +so the agent re-checks the same endpoint. + +**Which re-probe depends on who paged.** The default text in +`page-cotel-health.sh` names the GitHub dispatch, which is right for a drill +and wrong for the Pi timer: a dispatch of the loopback half runs on the host +that may itself be the thing that is down, and would queue forever. So the +timer overrides two strings, `PC_SOURCE_LINE` and `PC_REPROBE_HINT`, and its +alerts tell the agent to run `~/ops/cotel-healthz.sh --probe-only` on the Pi +instead. Any future caller that is not GitHub Actions must do the same — the +defaults are the workflow's, not neutral. That costs the woken agent one extra probe, by design. A live re-probe is better -evidence than a payload minted an hour earlier, and it needs no passthrough the -tracker does not offer. Both dispatch drills show the assignee doing exactly +evidence than a payload minted ten minutes earlier, and it needs no passthrough +the tracker does not offer. Every drill so far shows the assignee doing exactly this by hand before closing; the description is what stops it being folklore. Three more details matter in operation: @@ -349,7 +419,8 @@ Three more details matter in operation: the wake was going to report. A started wake answers with the run object, so `status: "skipped"` is what distinguishes the two, and `GET /api/issues//diagnostics/wakes` shows a coalesced wake sharing the - live run's id. Correct in production, where red and green are an hour apart; + live run's id. Correct in production, where red and green are at least a + tick apart; **in a drill it is a trap** — see [Running a drill](#running-a-drill). The "and ticket" is what makes this safe, and it is a property of the @@ -359,13 +430,16 @@ Three more details matter in operation: the recovery — the alert has no lock holder, and the wake proceeds as a run of its own. Daedalus being busy elsewhere therefore cannot lose a recovery. - **Idempotency keys differ by path.** Recovery uses - `cotel-health-recovery::`, so a re-dispatched or - retried job cannot mint a second heartbeat. The still-red wake buckets on a - coarse window instead — `cotel-health-still-red::` — - so a multi-day outage spends about four heartbeats a day rather than - twenty-four. -- **An alert assigned to anyone else is a pager failure, by name.** The API - allows self-wake only, so the pager reports + `cotel-health-recovery::`, where `PC_CALL_ID` defaults + to `GITHUB_RUN_ID`, so a re-dispatched or retried job cannot mint a second + heartbeat. The Pi timer has no run id and would otherwise send the same key + forever — one wake per alert, with a dropped wake leaving a resolved alert + standing — so it passes a 6h bucket (`pi-`) instead. The + still-red wake buckets on the same window — + `cotel-health-still-red::` — so a multi-day outage + spends about four heartbeats a day rather than one per tick. +- **An alert assigned to anyone else is a pager failure, by name.** An agent + API key may wake only its own agent, so the pager reports `alert wake: HTTP 403, alert is assigned to an agent this credential cannot wake` rather than stepping over it. In normal operation this cannot happen — the pager assigns the alert itself — but a reassigned alert must not @@ -381,23 +455,49 @@ Every pager failure line names the call that produced it (`issue search`, `issue create`, `alert wake`), because these calls share status codes and a bare `HTTP 403` is not debuggable. -**A paging failure is a warning, never the job's verdict.** Each job's red or -green means "production is red or green" and nothing else; if the pager itself -cannot reach Paperclip, the run carries a `Pager failed` annotation saying -nobody was woken. The loopback job also checks for `jq` and `python3` on every -run — that runner is a developer machine, not a managed image, so a missing -interpreter should surface on a green hour rather than during an incident. +**A paging failure is never the probe's verdict.** A workflow job's red or +green means "production is red or green" and nothing else; if the pager cannot +reach Paperclip, the run carries a `Pager failed` annotation saying nobody was +woken. The Pi timer splits it the other way round, for the same reason: a red +probe is a *successful* tick, and the only thing that fails the unit is the +watcher being broken — no credential, scripts unavailable, or the pager +erroring. That failure raises `~/ops/ALERT` and a desktop notification through +`OnFailure=ops-alert@`, which is the right audience for "nobody is watching". +The loopback job also checks for `jq` and `python3` on every run — that runner +is a developer machine, not a managed image, so a missing interpreter surfaces +in a drill rather than during an incident. Paperclip budget is spent only on a state change: the create on the first red -hour, a create when the open alert has aged out of the dedup window, and a -wake on a further red inside the window or on a recovery. A green hour with no -open alert spends nothing. A scheduled Paperclip routine that fires every hour -regardless of health is the more expensive alternative (24 heartbeats a day). -It is not enabled. +tick past the streak threshold, a create when the open alert has aged out of +the dedup window, and a wake on a further red inside the window or on a +recovery. A green tick with no open alert spends nothing — it costs one +loopback search against the local instance. A scheduled Paperclip routine that +fires regardless of health was the rejected alternative (~24 full agent runs a +day); see [ADR-0022](../decisions/0022-health-probe-scheduler-outside-github.md). + +### Credentials + +| Prober | Credential | Authority it needs | +|---|---|---| +| Pi timer | `~/.secrets/paperclip-board-ops.token` (board key `ops: paperclip-deploy health`, already on the box for the Paperclip updater) | create an issue, wake the assignee | +| workflow (both jobs) | `secrets.PAPERCLIP_API_TOKEN` — agent API key `github-actions` on Daedalus | the same, and nothing more: an agent key cannot wake anyone else | + +The board key is wider than this needs. A dedicated agent key would be the +narrow fit, but minting one is board-only — an agent key gets +`Board access required` on `/api/agents/{id}/keys` — and the key is used here on +the owner's own machine, not handed to a public repository's CI, which is the +case that was refused in [ADR-0021](../decisions/0021-recovery-wakes-the-alerts-assignee.md). +To narrow it later, drop an agent key at +`~/.secrets/cotel-health-pager.token`: `cotel-healthz.sh` prefers that path and +needs no other change. ## Local use ```sh +# on the Pi — the authoritative prober, over the LAN +~/ops/cotel-healthz.sh --probe-only +~/ops/cotel-healthz.sh --status + # the deploy host, from the deploy host — no Access token involved HEALTHZ_URL=http://127.0.0.1:8080/healthz scripts/probe-healthz.sh @@ -410,7 +510,8 @@ scripts/probe-healthz.sh http://127.0.0.1:1/healthz # the classification tests, including 503 vs stale vs empty vs refused bash scripts/probe-healthz_test.sh -# the pager against a fake curl — dedup, the staleness window, per-call failure labels +# the pager against a fake curl — dedup, the staleness window, per-call failure +# labels, and the non-GitHub caller's overrides bash scripts/page-cotel-health_test.sh ``` diff --git a/scripts/page-cotel-health.sh b/scripts/page-cotel-health.sh index 49c3cb9..f25a629 100755 --- a/scripts/page-cotel-health.sh +++ b/scripts/page-cotel-health.sh @@ -32,6 +32,17 @@ # target on the raise path; 0 disables that window. # GITHUB_RUN_URL, GITHUB_EVENT_NAME, GITHUB_ACTOR and HEALTHZ_URL are quoted # into a new alert when set, so the reader can place the run without Actions. +# +# Callers outside GitHub Actions — the systemd timer on the Pi is the +# authoritative one — override three things, because the defaults below name +# Actions as both the origin of the alert and the way to re-check it: +# PC_SOURCE_LINE who opened the alert, and that a red run is not the page +# PC_REPROBE_HINT how the woken agent re-probes before acting +# PC_CALL_ID the unique thing making this call, for the recovery +# idempotency key (defaults to GITHUB_RUN_ID, then +# "manual"); a caller with no run id should pass a coarse +# time bucket so a dropped recovery wake is retried +# instead of leaving a resolved alert standing forever. set -euo pipefail @@ -289,14 +300,28 @@ RUN_LINE="" RUN_SUFFIX="" # Named in the alert so the woken agent re-checks the same endpoint. A bare # curl is not the check: the probe also classifies 503, stale and empty ingest. -# And this URL may be the deploy host's own loopback, reachable only from the -# runner — which is why the re-probe is a dispatch, not a local command. +# And this URL may be a host's own loopback, reachable only from there — which +# is why the default re-probe is a dispatch, not a local command. PROBED_URL="${HEALTHZ_URL:-the URL in the probe output above}" if [ -n "${GITHUB_RUN_URL:-}" ]; then RUN_LINE="GitHub Actions run: ${GITHUB_RUN_URL}" RUN_SUFFIX=" (${GITHUB_RUN_URL})" fi +# The unique thing making this call. Only the recovery key uses it: a caller +# that repeats the same id repeats one wake, which is correct for a CI job and +# wrong for a timer that keeps ticking while the alert stands. +CALL_ID="${PC_CALL_ID:-${GITHUB_RUN_ID:-manual}}" + +# Both of these name GitHub Actions, and both are wrong for a caller that is +# not it — so both are overridable, and the defaults stay what the workflow +# needs. The re-probe hint is the load-bearing half: an agent woken by the Pi +# timer must not be told to dispatch a workflow whose runner is on the host +# that may itself be down. +SOURCE_LINE="${PC_SOURCE_LINE:-The health probe in Flopsstuff/cotel opened this issue so an agent is woken. Do not treat a red GitHub Actions run as the page — that channel does not wake anyone here.}" +DEFAULT_REPROBE="Re-probe by dispatching **Health probe** in Flopsstuff/cotel with **page unchecked** (Actions → Health probe → Run workflow), and read its verdict. Do not just \`curl\` the URL: the probe also classifies 503, stale ingest and an empty database, and \`${PROBED_URL}\` may be the deploy host's own loopback, which only that runner can reach. Never pass a \`loopback_url\` override while checking a real alert — that probes something else." +REPROBE_HINT="${PC_REPROBE_HINT:-$DEFAULT_REPROBE}" + # The woken reader must be able to tell an exercise from an outage without # opening Actions: a dispatched run against an overridden URL is a drill, the # hourly schedule against the real endpoint is not. The probe text alone does @@ -310,6 +335,13 @@ if [ -n "${HEALTHZ_URL:-}" ]; then CONTEXT_BLOCK="${CONTEXT_BLOCK} Probe URL: ${HEALTHZ_URL}" fi +# Last line of the same block rather than a paragraph of its own: a caller +# outside Actions has no run URL, and a lone empty paragraph is what that looked +# like in the alert body. +if [ -n "$RUN_LINE" ]; then + CONTEXT_BLOCK="${CONTEXT_BLOCK} +${RUN_LINE}" +fi TITLE="cotel prod /healthz is red ${MARKER}" @@ -345,7 +377,7 @@ case "$ACTION" in exit 0 fi fi - description="$(printf 'Production cotel /healthz probe is red.\n\n```\n%s\n```\n\n%s\n\n%s\n\nThe hourly probe in Flopsstuff/cotel opened this issue so an agent is woken. Do not treat a red GitHub Actions run as the page — that channel does not wake anyone here.\n\n## If you are woken on this issue again, re-probe before acting\n\n**This description always reads red.** The probe wrote it once, when the outage was detected, and cannot edit it afterwards; the wake that brought you here carries no probe output either. So nothing above tells you the state of production right now — check it yourself before you act.\n\nRe-probe by dispatching **Health probe** in Flopsstuff/cotel with **page unchecked** (Actions → Health probe → Run workflow), and read its verdict. Do not just `curl` the URL: the probe also classifies 503, stale ingest and an empty database, and `%s` may be the deploy host'"'"'s own loopback, which only that runner can reach. Never pass a `loopback_url` override while checking a real alert — that probes something else.\n\n- **Green** — the outage is over. Close this issue as done, citing the run you probed with.\n- **Still red** — the outage continues. Add that run'"'"'s output here as the current evidence.\n' "$reason" "$CONTEXT_BLOCK" "$RUN_LINE" "$PROBED_URL")" + description="$(printf 'Production cotel /healthz probe is red.\n\n```\n%s\n```\n\n%s\n\n%s\n\n## If you are woken on this issue again, re-probe before acting\n\n**This description always reads red.** The probe wrote it once, when the outage was detected, and cannot edit it afterwards; the wake that brought you here carries no probe output either. So nothing above tells you the state of production right now — check it yourself before you act.\n\n%s\n\n- **Green** — the outage is over. Close this issue as done, citing the probe you checked with.\n- **Still red** — the outage continues. Add that output here as the current evidence.\n' "$reason" "$CONTEXT_BLOCK" "$SOURCE_LINE" "$REPROBE_HINT")" # Same-title creates collapse into a recent open issue unless this # asks not to. The stale alert is that issue, so without the opt-out # the response is the alert we just refused to dedup into. @@ -409,7 +441,7 @@ case "$ACTION" in "Production /healthz is green again. Close this alert as done, with the green run URL in the closing comment." \ "$green")" wake_assignee \ - "cotel-health-recovery:${EXISTING_ID}:${GITHUB_RUN_ID:-manual}" \ + "cotel-health-recovery:${EXISTING_ID}:${CALL_ID}" \ "cotel prod /healthz recovered${RUN_SUFFIX} — close alert ${EXISTING_IDENT:-$EXISTING_ID} as done" \ "$payload" ;; diff --git a/scripts/page-cotel-health_test.sh b/scripts/page-cotel-health_test.sh index 2df267b..ff72cca 100755 --- a/scripts/page-cotel-health_test.sh +++ b/scripts/page-cotel-health_test.sh @@ -798,6 +798,70 @@ else pass "missing action exits non-zero" fi +# 13. A caller that is not GitHub Actions replaces both Actions-specific halves +# of the alert body. The re-probe hint is the load-bearing one: telling an agent +# to dispatch a workflow whose runner lives on the possibly-dead host is advice +# that cannot be followed during the outage it describes. +write_fixture <<'JSON' +[] +JSON +reset_log +out="$(run_page 200 raise "$PROBE_FILE" 2>"$TMP/err")" || { fail "default raise — exit $?"; cat "$TMP/err"; } +assert_log_has "default body keeps the Actions origin line" "The health probe in Flopsstuff/cotel opened this issue" +assert_log_has "default body keeps the dispatch re-probe hint" "Re-probe by dispatching **Health probe**" + +reset_log +out="$( + export PC_SOURCE_LINE="The systemd timer cotel-healthz.timer on the Pi opened this issue." + export PC_REPROBE_HINT="Re-probe with \`~/ops/cotel-healthz.sh --probe-only\` on the Pi." + run_page 200 raise "$PROBE_FILE" +)" +case "$out" in + "page-cotel-health: opened ALT-9 new-id") pass "overridden raise creates" ;; + *) fail "overridden raise creates — output: $out" ;; +esac +assert_log_has "overridden body names the timer as the origin" "cotel-healthz.timer on the Pi opened this issue" +assert_log_has "overridden body names the local re-probe" "cotel-healthz.sh --probe-only" +assert_log_lacks "overridden body drops the dispatch hint" "Re-probe by dispatching" +assert_log_lacks "overridden body drops the Actions origin line" "The health probe in Flopsstuff/cotel" + +# 14. PC_CALL_ID is the recovery key's unique half for a caller with no run id. +# A timer repeating one key would wake the assignee once for an alert it keeps +# finding open; repeating a coarse bucket retries a dropped wake a few times a +# day instead. +write_fixture <<'JSON' +[ + { + "id": "alert-id", + "identifier": "ALT-1", + "title": "cotel prod /healthz is red [cotel-health-probe]", + "status": "in_progress", + "assigneeAgentId": "agent-on-call" + } +] +JSON +reset_log +out="$(export PC_CALL_ID="pi-2026-10-05T12"; run_page 200 resolve "$GREEN_FILE" 2>"$TMP/err")" \ + || { fail "recovery wake with PC_CALL_ID — exit $?"; cat "$TMP/err"; } +python3 - "$(wake_body)" <<'PY2' +import json, sys +body = json.loads(sys.argv[1]) +if body.get("idempotencyKey") != "cotel-health-recovery:alert-id:pi-2026-10-05T12": + raise SystemExit("idempotencyKey=" + str(body.get("idempotencyKey"))) +PY2 +pass "PC_CALL_ID is the recovery key's unique half" + +reset_log +out="$(export PC_CALL_ID="pi-bucket" GITHUB_RUN_ID="4242"; run_page 200 resolve "$GREEN_FILE" 2>"$TMP/err")" \ + || { fail "PC_CALL_ID precedence — exit $?"; cat "$TMP/err"; } +python3 - "$(wake_body)" <<'PY2' +import json, sys +body = json.loads(sys.argv[1]) +if body.get("idempotencyKey") != "cotel-health-recovery:alert-id:pi-bucket": + raise SystemExit("idempotencyKey=" + str(body.get("idempotencyKey"))) +PY2 +pass "PC_CALL_ID wins over GITHUB_RUN_ID" + echo echo "passed=$PASS failed=$FAIL" if [ "$FAIL" -ne 0 ]; then