From 0eea21df8c9567e3d0b250826459da030d49f45e Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sat, 19 Sep 2026 12:56:24 +0200 Subject: [PATCH 1/3] fix(agentic-engineering): define classifier output contract --- .claude-plugin/marketplace.json | 2 +- .github/plugin/marketplace.json | 2 +- .../.claude-plugin/plugin.json | 2 +- .../agents/portfolio-surveyor.agent.md | 14 ++++++++++++++ plugins/agentic-engineering/plugin.json | 2 +- .../provider-neutral.desired-state.json | 2 +- .../scripts/portfolio-surveyor-agent.test.sh | 17 +++++++++++++++++ 7 files changed, 36 insertions(+), 5 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 70a625d..64bc2e8 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -27,7 +27,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "5.1.4", + "version": "5.1.5", "source": "./plugins/agentic-engineering" }, { diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index 70a625d..64bc2e8 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -27,7 +27,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "5.1.4", + "version": "5.1.5", "source": "./plugins/agentic-engineering" }, { diff --git a/plugins/agentic-engineering/.claude-plugin/plugin.json b/plugins/agentic-engineering/.claude-plugin/plugin.json index a1d2346..a32765a 100644 --- a/plugins/agentic-engineering/.claude-plugin/plugin.json +++ b/plugins/agentic-engineering/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "5.1.4", + "version": "5.1.5", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md index ea73ce6..54f3ea6 100644 --- a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md +++ b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md @@ -502,6 +502,20 @@ by a successful consumer of partial output and the read-only role never writes a The bundled `forge-readonly-guard.sh` recognises only this exact installed sibling; offline `--input` remains denied. Exit 0 is a complete classification; exit 2 means `unknown`, never green. +**Read the verdict from the helper output, never by appending the guard-denied `; echo "EXIT=$?"` idiom.** +The guard rejects shell chaining before the helper runs, while the tool result already surfaces both +stdout and stderr. Classify the complete output with this closed set: + +| Output | Meaning | +|---|---| +| completely empty | exit 0 with no red runs → that branch is **green** | +| **well-formed TSV rows** — exactly eight tab-separated fields in helper order: numeric `workflow_id`, red `conclusion` (`failure`, `timed_out`, or `startup_failure`), `html_url`, `name`, supported `event`, `path`, valid `created_at`, numeric `run_id` | those are the **red runs** | +| anything else, including mixed valid and malformed rows | the helper FAILED → **`QUERY-UNKNOWN`**; never `nothing_on_fire: true` | + +Every nonempty line must match the complete eight-field row shape. Do not accept a numeric first +field, a diagnostic prefix, or any other partial predicate as sufficient, and never treat merely +nonempty output as red runs. The empty result is the only green case. + Invoke it from a process that already has `GH_TELEMETRY=0` (or `false`) in the environment, or rely on the helper's own export of `GH_TELEMETRY=0` before its remote `gh api` GET — GitHub CLI 2.96.0 otherwise writes `gh/device-id` on a certified read. Do not prefix the helper with diff --git a/plugins/agentic-engineering/plugin.json b/plugins/agentic-engineering/plugin.json index a1d2346..a32765a 100644 --- a/plugins/agentic-engineering/plugin.json +++ b/plugins/agentic-engineering/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "5.1.4", + "version": "5.1.5", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json index 05d35ec..25080e9 100644 --- a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json +++ b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json @@ -66,7 +66,7 @@ "portfolio-surveyor": { "enabled": true, "mode": "delegated-read-only", - "definitionSha256": "53dfcda04e4e404b6d13fb465aa76c574a79401d1b2be1058beae1ccc9bc6425" + "definitionSha256": "5ed8ba4017289e3b535e042e0a17ca80e8c35a4bd745f8c1748b75fb1de75ee7" }, "agent-improver": { "enabledWhen": "Both optional consumer contract sections are present", diff --git a/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh b/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh index fef1d4b..cc4cdb3 100755 --- a/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh +++ b/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh @@ -23,6 +23,23 @@ if grep -Fq -- '--json issueType,blockedBy,assignees' "$SURVEYOR"; then fail 'dependency reads must not enumerate blocker nodes through gh issue view' fi +CI_STEP=$(sed -n \ + '/^### 4\. CI red on the default branch/,/^### 5\. Triage, stale, and advance signals/p' \ + "$SURVEYOR" | tr '\n' ' ' | tr -s '[:space:]' ' ') +[ -n "$CI_STEP" ] || fail 'could not extract the default-branch CI step' +# shellcheck disable=SC2016 # The asserted agent text contains a literal shell idiom. +grep -Fq 'Read the verdict from the helper output, never by appending the guard-denied `; echo "EXIT=$?"` idiom.' \ + <<<"$CI_STEP" || + fail 'default-branch CI must prescribe output reading instead of denied exit capture' +# shellcheck disable=SC2016 # Backticks belong to the asserted Markdown contract. +grep -Fq '| **well-formed TSV rows** — exactly eight tab-separated fields in helper order: numeric `workflow_id`, red `conclusion` (`failure`, `timed_out`, or `startup_failure`), `html_url`, `name`, supported `event`, `path`, valid `created_at`, numeric `run_id` | those are the **red runs** |' \ + <<<"$CI_STEP" || + fail 'the complete eight-field TSV predicate must map to the red verdict as one table row' +# shellcheck disable=SC2016 # Backticks belong to the asserted Markdown contract. +grep -Fq '| anything else, including mixed valid and malformed rows | the helper FAILED → **`QUERY-UNKNOWN`**; never `nothing_on_fire: true` |' \ + <<<"$CI_STEP" || + fail 'malformed or mixed classifier output must fail closed as QUERY-UNKNOWN' + JQ_FILTER=$(sed -n \ "/issueDependenciesSummary{blockedBy totalBlockedBy}/{n;s/^[[:space:]]*--jq '\\(.*\\)'$/\\1/p;}" \ "$SURVEYOR") From 31000a988369a3afdc775e289045569bf09912a5 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sat, 19 Sep 2026 13:08:25 +0200 Subject: [PATCH 2/3] test(agentic-engineering): pin empty classifier output --- .../scripts/portfolio-surveyor-agent.test.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh b/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh index cc4cdb3..b29572a 100755 --- a/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh +++ b/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh @@ -31,6 +31,9 @@ CI_STEP=$(sed -n \ grep -Fq 'Read the verdict from the helper output, never by appending the guard-denied `; echo "EXIT=$?"` idiom.' \ <<<"$CI_STEP" || fail 'default-branch CI must prescribe output reading instead of denied exit capture' +grep -Fq '| completely empty | exit 0 with no red runs → that branch is **green** |' \ + <<<"$CI_STEP" || + fail 'completely empty classifier output must be the explicit green case' # shellcheck disable=SC2016 # Backticks belong to the asserted Markdown contract. grep -Fq '| **well-formed TSV rows** — exactly eight tab-separated fields in helper order: numeric `workflow_id`, red `conclusion` (`failure`, `timed_out`, or `startup_failure`), `html_url`, `name`, supported `event`, `path`, valid `created_at`, numeric `run_id` | those are the **red runs** |' \ <<<"$CI_STEP" || From bca79d8e146ac27e5c9f23270f48322229a2a49b Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sat, 19 Sep 2026 13:22:37 +0200 Subject: [PATCH 3/3] fix(agentic-engineering): require successful classifier status --- .../agents/portfolio-surveyor.agent.md | 17 +++++++++-------- .../provider-neutral.desired-state.json | 2 +- .../scripts/portfolio-surveyor-agent.test.sh | 14 +++++++------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md index 54f3ea6..b441451 100644 --- a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md +++ b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md @@ -502,19 +502,20 @@ by a successful consumer of partial output and the read-only role never writes a The bundled `forge-readonly-guard.sh` recognises only this exact installed sibling; offline `--input` remains denied. Exit 0 is a complete classification; exit 2 means `unknown`, never green. -**Read the verdict from the helper output, never by appending the guard-denied `; echo "EXIT=$?"` idiom.** -The guard rejects shell chaining before the helper runs, while the tool result already surfaces both -stdout and stderr. Classify the complete output with this closed set: +**Read the verdict from the helper's native tool result, never by appending the guard-denied `; echo +"EXIT=$?"` idiom.** The guard rejects shell chaining before the helper runs, while the tool result +already surfaces stdout, stderr, and the process status. Classify that complete result with this closed +set: -| Output | Meaning | +| Native process status and output | Meaning | |---|---| -| completely empty | exit 0 with no red runs → that branch is **green** | -| **well-formed TSV rows** — exactly eight tab-separated fields in helper order: numeric `workflow_id`, red `conclusion` (`failure`, `timed_out`, or `startup_failure`), `html_url`, `name`, supported `event`, `path`, valid `created_at`, numeric `run_id` | those are the **red runs** | -| anything else, including mixed valid and malformed rows | the helper FAILED → **`QUERY-UNKNOWN`**; never `nothing_on_fire: true` | +| observed native process status 0 and completely empty output | no red runs → that branch is **green** | +| observed native process status 0 and **well-formed TSV rows** — exactly eight tab-separated fields in helper order: numeric `workflow_id`, red `conclusion` (`failure`, `timed_out`, or `startup_failure`), `html_url`, `name`, supported `event`, `path`, valid `created_at`, numeric `run_id` | those are the **red runs** | +| any nonzero or unavailable native process status; or any other output, including mixed valid and malformed rows | the helper FAILED → **`QUERY-UNKNOWN`**; never `nothing_on_fire: true` | Every nonempty line must match the complete eight-field row shape. Do not accept a numeric first field, a diagnostic prefix, or any other partial predicate as sufficient, and never treat merely -nonempty output as red runs. The empty result is the only green case. +nonempty output as red runs. Status 0 plus empty output is the only green case. Invoke it from a process that already has `GH_TELEMETRY=0` (or `false`) in the environment, or rely on the helper's own export of `GH_TELEMETRY=0` before its remote `gh api` GET — GitHub CLI diff --git a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json index 25080e9..9756634 100644 --- a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json +++ b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json @@ -66,7 +66,7 @@ "portfolio-surveyor": { "enabled": true, "mode": "delegated-read-only", - "definitionSha256": "5ed8ba4017289e3b535e042e0a17ca80e8c35a4bd745f8c1748b75fb1de75ee7" + "definitionSha256": "8e31a79bb60e0c68f92f95c5f4fbf2abda78e719dd40c04392f9e2f74de91275" }, "agent-improver": { "enabledWhen": "Both optional consumer contract sections are present", diff --git a/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh b/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh index b29572a..d74cef9 100755 --- a/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh +++ b/plugins/agentic-engineering/scripts/portfolio-surveyor-agent.test.sh @@ -28,20 +28,20 @@ CI_STEP=$(sed -n \ "$SURVEYOR" | tr '\n' ' ' | tr -s '[:space:]' ' ') [ -n "$CI_STEP" ] || fail 'could not extract the default-branch CI step' # shellcheck disable=SC2016 # The asserted agent text contains a literal shell idiom. -grep -Fq 'Read the verdict from the helper output, never by appending the guard-denied `; echo "EXIT=$?"` idiom.' \ +grep -Fq 'Read the verdict from the helper'"'"'s native tool result, never by appending the guard-denied `; echo "EXIT=$?"` idiom.' \ <<<"$CI_STEP" || - fail 'default-branch CI must prescribe output reading instead of denied exit capture' -grep -Fq '| completely empty | exit 0 with no red runs → that branch is **green** |' \ + fail 'default-branch CI must prescribe the native tool result instead of denied shell exit capture' +grep -Fq '| observed native process status 0 and completely empty output | no red runs → that branch is **green** |' \ <<<"$CI_STEP" || - fail 'completely empty classifier output must be the explicit green case' + fail 'green must require both observed native process status 0 and completely empty classifier output' # shellcheck disable=SC2016 # Backticks belong to the asserted Markdown contract. -grep -Fq '| **well-formed TSV rows** — exactly eight tab-separated fields in helper order: numeric `workflow_id`, red `conclusion` (`failure`, `timed_out`, or `startup_failure`), `html_url`, `name`, supported `event`, `path`, valid `created_at`, numeric `run_id` | those are the **red runs** |' \ +grep -Fq '| observed native process status 0 and **well-formed TSV rows** — exactly eight tab-separated fields in helper order: numeric `workflow_id`, red `conclusion` (`failure`, `timed_out`, or `startup_failure`), `html_url`, `name`, supported `event`, `path`, valid `created_at`, numeric `run_id` | those are the **red runs** |' \ <<<"$CI_STEP" || fail 'the complete eight-field TSV predicate must map to the red verdict as one table row' # shellcheck disable=SC2016 # Backticks belong to the asserted Markdown contract. -grep -Fq '| anything else, including mixed valid and malformed rows | the helper FAILED → **`QUERY-UNKNOWN`**; never `nothing_on_fire: true` |' \ +grep -Fq '| any nonzero or unavailable native process status; or any other output, including mixed valid and malformed rows | the helper FAILED → **`QUERY-UNKNOWN`**; never `nothing_on_fire: true` |' \ <<<"$CI_STEP" || - fail 'malformed or mixed classifier output must fail closed as QUERY-UNKNOWN' + fail 'nonzero or unavailable status and malformed or mixed output must fail closed as QUERY-UNKNOWN' JQ_FILTER=$(sed -n \ "/issueDependenciesSummary{blockedBy totalBlockedBy}/{n;s/^[[:space:]]*--jq '\\(.*\\)'$/\\1/p;}" \