fix(docs-sync): scope edits to owning pages; add replay evals

.gitignore

@@ -52,3 +52,6 @@ next-env.d.ts
 
 # PR preview screenshots (uploaded via draft GitHub releases, never committed)
 /screenshots/
+
+# docs-sync eval output (scripts/sync-from-base-std/eval)
+/.sync-eval/

scripts/sync-from-base-std/README.md

@@ -101,6 +101,10 @@ Configuration knobs are optional positive numbers:
 - `CLAUDE_MAX_TOKENS` and `CLAUDE_MODEL`
 - The bounded release manifest/selection settings documented in `index.mjs`
 
+Before changing prompts, routing, or the route table, run the evals with and
+without your change (`eval/run-eval.mjs --runs 3`, then `--code-ref master`)
+and compare. See the script header for options.
+
 ## Source PR attribution
 
 For `code-change` dispatches the workflow @mentions the source PR author in

scripts/sync-from-base-std/__tests__/code-sample-guard.test.mjs

@@ -0,0 +1,51 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { manifestChangesSignatures, restoreCodeSamples } from "../release-utils.mjs";
+
+const page = (fn, prose = "Send payouts.", diagram = "A-->B") => [
+  "# Send a payout",
+  "",
+  prose,
+  "",
+  "```ts",
+  "const simulation = await publicClient.simulateContract({",
+  `  functionName: "${fn}",`,
+  "});",
+  "```",
+  "",
+  "```mermaid",
+  `flowchart TD`,
+  `  ${diagram}`,
+  "```",
+  "",
+].join("\n");
+
+test("restoreCodeSamples: reverts a changed sample, keeps prose and diagram edits", () => {
+  const current = page("sendPayouts");
+  const updated = page("simulateContract", "Send payouts. Recipients cannot be the token itself.", "A-->C");
+  const r = restoreCodeSamples(current, updated);
+  assert.equal(r.restored, 1);
+  assert.equal(r.content, page("sendPayouts", "Send payouts. Recipients cannot be the token itself.", "A-->C"));
+});
+
+test("restoreCodeSamples: block count mismatch leaves output alone", () => {
+  const current = page("sendPayouts");
+  const updated = current + "\n```bash\nnpm i\n```\n";
+  const r = restoreCodeSamples(current, updated);
+  assert.equal(r.mismatched, true);
+  assert.equal(r.content, updated);
+});
+
+test("manifestChangesSignatures", () => {
+  assert.equal(manifestChangesSignatures([{ kind: "revert_added" }]), false);
+  assert.equal(manifestChangesSignatures([{ kind: "signature_change" }]), true);
+});
+
+test("normalizeForNoop: table re-padding is a noop, a cell edit is not", async () => {
+  const { normalizeForNoop } = await import("../release-utils.mjs");
+  const a = "| `NotSoleAdmin()`    | renounce while others exist |\n";
+  const b = "| `NotSoleAdmin()`                | renounce while others exist |\n";
+  const c = "| `NotSoleAdmin()` | renounce while other admins exist |\n";
+  assert.equal(normalizeForNoop(a), normalizeForNoop(b));
+  assert.notEqual(normalizeForNoop(a), normalizeForNoop(c));
+});

scripts/sync-from-base-std/__tests__/eval-score.test.mjs

@@ -0,0 +1,63 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { addedLinesByFile, parseNumstat, scoreRun } from "../eval/run-eval.mjs";
+
+const EXPECT = {
+  must_touch: ["docs/ref/is-authorized.mdx"],
+  may_touch: { "docs/guide/seize.mdx": 3 },
+  must_not_touch: ["docs/guide/hold.mdx"],
+  unlisted_pages: "fail",
+  restatement: { owners: ["docs/ref/is-authorized.mdx"], keywords: ["malformed", "empty set", "inverted"], min_keywords: 2 },
+};
+
+test("parseNumstat and addedLinesByFile read git output", () => {
+  assert.deepEqual(parseNumstat("3\t1\tdocs/a.mdx\n-\t-\tdocs/b.png\n"), {
+    "docs/a.mdx": { added: 3, removed: 1 },
+    "docs/b.png": { added: 0, removed: 0 },
+  });
+  const diff = "diff --git a/docs/a.mdx b/docs/a.mdx\n--- a/docs/a.mdx\n+++ b/docs/a.mdx\n@@ -1 +1,2 @@\n line\n+new\n";
+  assert.deepEqual(addedLinesByFile(diff), { "docs/a.mdx": ["new"] });
+});
+
+test("scoreRun passes a run that edits only what it should", () => {
+  const r = scoreRun(EXPECT, { "docs/ref/is-authorized.mdx": { added: 20, removed: 5 }, "docs/guide/seize.mdx": { added: 1, removed: 0 } });
+  assert.equal(r.pass, true, r.violations.join("; "));
+});
+
+test("scoreRun flags missing, forbidden, oversized, unlisted, and restated edits", () => {
+  const r = scoreRun(
+    EXPECT,
+    {
+      "docs/guide/hold.mdx": { added: 2, removed: 0 },
+      "docs/guide/seize.mdx": { added: 9, removed: 0 },
+      "docs/other.mdx": { added: 1, removed: 0 },
+    },
+    { "docs/guide/seize.mdx": ["A malformed ID returns false; an unknown ID is an empty set."] },
+  );
+  assert.equal(r.pass, false);
+  const text = r.violations.join("\n");
+  assert.match(text, /missing required edit: docs\/ref\/is-authorized\.mdx/);
+  assert.match(text, /should leave alone: docs\/guide\/hold\.mdx/);
+  assert.match(text, /edit too large: docs\/guide\/seize\.mdx adds 9 lines \(budget 3\)/);
+  assert.match(text, /unlisted page: docs\/other\.mdx/);
+  assert.match(text, /restates the full rules outside their owner pages: docs\/guide\/seize\.mdx/);
+});
+
+test("scoreRun: must_mention fails a required edit that does not state the change", () => {
+  const expect = {
+    must_touch: ["a.mdx"],
+    must_mention: { "a.mdx": "address\\(this\\)" },
+  };
+  const numstat = { "a.mdx": { added: 1, removed: 1 } };
+  const cosmetic = scoreRun(expect, numstat, { "a.mdx": ["Reverts when `to` is zero."] });
+  assert.equal(cosmetic.pass, false);
+  assert.match(cosmetic.violations[0], /does not state the change/);
+  const real = scoreRun(expect, numstat, { "a.mdx": ["Reverts when `to == address(this)`."] });
+  assert.equal(real.pass, true);
+});
+
+test("scoreRun: unlisted_pages warn reports without failing", () => {
+  const r = scoreRun({ must_touch: [], unlisted_pages: "warn" }, { "x.mdx": { added: 2, removed: 0 } });
+  assert.equal(r.pass, true);
+  assert.equal(r.warnings.length, 1);
+});

scripts/sync-from-base-std/__tests__/stream-retry.test.mjs

@@ -0,0 +1,12 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { isTransientStreamError } from "../llm/client.mjs";
+
+test("isTransientStreamError: mid-stream overload is retried, real failures are not", () => {
+  assert.equal(
+    isTransientStreamError(new Error('{"type":"error","error":{"type":"overloaded_error","message":"The model stopped sending data. Retry this request."}}')),
+    true,
+  );
+  assert.equal(isTransientStreamError(new Error("LLM gateway returned no text content")), false);
+  assert.equal(isTransientStreamError(new Error('{"type":"error","error":{"type":"invalid_request_error"}}')), false);
+});

scripts/sync-from-base-std/__tests__/symbol-route-gate.test.mjs

@@ -0,0 +1,148 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  decideCall,
+  isCommentOnlyChange,
+  natspecDocumentedSymbols,
+  newestChangelogFork,
+  splitDiffByFile,
+  symbolRouteGate,
+} from "../release-utils.mjs";
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const fixture = (name) => JSON.parse(fs.readFileSync(path.join(HERE, "..", "fixtures", name), "utf8"));
+
+test("isCommentOnlyChange: base-std#234 (NatSpec-only) is a clarification", () => {
+  const payload = fixture("code-change-3820cf0-isauthorized-natspec.json");
+  assert.equal(isCommentOnlyChange(splitDiffByFile(payload.diff)), true);
+});
+
+test("isCommentOnlyChange: a code line makes it a real change", () => {
+  const diff = [
+    "diff --git a/src/I.sol b/src/I.sol",
+    "--- a/src/I.sol",
+    "+++ b/src/I.sol",
+    "@@ -1,2 +1,3 @@",
+    "-    /// @notice old",
+    "+    /// @notice new",
+    "+    function foo() external;",
+  ].join("\n");
+  assert.equal(isCommentOnlyChange(splitDiffByFile(diff)), false);
+});
+
+test("isCommentOnlyChange: base-std#232 (NatSpec + mocks + changelog) is a real change", () => {
+  const payload = fixture("code-change-1505323-token-self-recipient.json");
+  const byFile = splitDiffByFile(payload.diff);
+  assert.equal(isCommentOnlyChange(byFile), false);
+  // The interface edits alone are pure NatSpec; the mock change is what marks it.
+  const srcAndMocks = new Map([...byFile].filter(([f]) => f.startsWith("src/") || f.startsWith("test/lib/mocks/")));
+  assert.equal(isCommentOnlyChange(srcAndMocks), false);
+  const srcOnly = new Map([...byFile].filter(([f]) => f.startsWith("src/")));
+  assert.equal(isCommentOnlyChange(srcOnly), true);
+});
+
+test("isCommentOnlyChange: block comments count, tests are ignored, non-Solidity is not", () => {
+  const sol = [
+    "diff --git a/src/I.sol b/src/I.sol",
+    "--- a/src/I.sol",
+    "+++ b/src/I.sol",
+    "@@ -1 +1,3 @@",
+    "+/**",
+    "+ * text",
+    "+ */",
+    "diff --git a/test/I.t.sol b/test/I.t.sol",
+    "--- a/test/I.t.sol",
+    "+++ b/test/I.t.sol",
+    "@@ -1 +1 @@",
+    "+    assertTrue(x);",
+  ].join("\n");
+  assert.equal(isCommentOnlyChange(splitDiffByFile(sol)), true);
+  const md = ["diff --git a/docs/a.md b/docs/a.md", "--- a/docs/a.md", "+++ b/docs/a.md", "@@ -1 +1 @@", "+// text"].join("\n");
+  assert.equal(isCommentOnlyChange(splitDiffByFile(md)), false);
+  assert.equal(isCommentOnlyChange(new Map()), false);
+});
+
+const ENTRY_DIR = "docs/base-chain/specs/reference/b20/changelog";
+
+test("newestChangelogFork picks the highest-numbered fork", () => {
+  const pages = [
+    `${ENTRY_DIR}/02-cobalt-policyregistry-composite-policy.mdx`,
+    `${ENTRY_DIR}/03-denim-policyregistry-not-policy.mdx`,
+    "docs/upgrades/denim/overview.mdx",
+  ];
+  assert.equal(newestChangelogFork(pages, ENTRY_DIR), "denim");
+  assert.equal(newestChangelogFork(pages, ""), "");
+});
+
+test("symbolRouteGate: comment-only changes route only reference pages", () => {
+  const opts = { commentOnly: true, newestFork: "denim" };
+  assert.equal(symbolRouteGate("docs/x/reference/interfaces/i-policy-registry/is-authorized.mdx", { ...opts, role: "function-reference" }), null);
+  assert.equal(symbolRouteGate("docs/x/reference/interfaces/i-policy-registry/index.mdx", { ...opts, role: "interface-index" }), null);
+  for (const role of ["guide", "shared-reference", "changelog-entry"]) {
+    assert.match(symbolRouteGate("docs/build-on-base/a.mdx", { ...opts, role }), /comment-only/);
+  }
+});
+
+test("symbolRouteGate: historical changelog entries and upgrade pages are not routed", () => {
+  const opts = { commentOnly: false, newestFork: "denim" };
+  assert.match(
+    symbolRouteGate(`${ENTRY_DIR}/02-cobalt-policyregistry-composite-policy.mdx`, { ...opts, role: "changelog-entry" }),
+    /earlier hardfork \(cobalt/,
+  );
+  assert.equal(symbolRouteGate(`${ENTRY_DIR}/03-denim-policyregistry-not-policy.mdx`, { ...opts, role: "changelog-entry" }), null);
+  assert.match(symbolRouteGate("docs/upgrades/beryl/b20.mdx", { ...opts, role: "guide" }), /historical upgrade page \(beryl/);
+  assert.equal(symbolRouteGate("docs/upgrades/denim/overview.mdx", { ...opts, role: "guide" }), null);
+  assert.equal(symbolRouteGate("docs/build-on-base/a.mdx", { ...opts, role: "guide" }), null);
+  // No fork data → no fork gating.
+  assert.equal(symbolRouteGate("docs/upgrades/beryl/b20.mdx", { role: "guide", newestFork: "" }), null);
+});
+
+test("natspecDocumentedSymbols: a @dev hunk that stops above the declaration still names it", async () => {
+  const payload = fixture("code-change-1505323-token-self-recipient.json");
+  const section = splitDiffByFile(payload.diff).get("src/interfaces/IB20.sol");
+  // Minimal post-image: the mint hunk's lines, then @param lines, then the declaration.
+  const post = [];
+  const at = (n, text) => { while (post.length < n - 1) post.push(""); post.push(text); };
+  at(414, "    ///");
+  at(415, "    /// @dev Reverts with `ContractPaused(MINT)` when `MINT` is paused.");
+  at(416, "    /// @dev Reverts with `AccessControlUnauthorizedAccount` when the caller does not hold `MINT_ROLE`.");
+  at(417, "    /// @dev Reverts with `InvalidReceiver` when `to == address(0)` or `to == address(this)`.");
+  at(418, "    /// @dev Reverts with `PolicyForbids(MINT_RECEIVER_POLICY, ...)` when `to` is not authorized.");
+  at(419, "    /// @dev Reverts with `SupplyCapExceeded` when `totalSupply + amount > supplyCap`.");
+  at(420, "    ///");
+  at(421, "    /// @param to     Recipient.");
+  at(422, "    /// @param amount Amount.");
+  at(423, "    function mint(address to, uint256 amount) external;");
+  const got = natspecDocumentedSymbols(section, post.join("\n"));
+  assert.ok(got.includes("mint"), `got ${got}`);
+});
+
+test("natspecDocumentedSymbols: errors, events, constants; code lines stop the walk", () => {
+  const section = [
+    "@@ -1,6 +1,6 @@",
+    "-    /// @notice old",
+    "+    /// @notice new",
+    "     error InvalidReceiver(address receiver);",
+    " ",
+    "-    /// old",
+    "+    /// new",
+    "     bytes32 public constant SEIZE_ROLE = keccak256(\"SEIZE_ROLE\");",
+  ].join("\n");
+  const post = [
+    "    /// @notice new",
+    "    error InvalidReceiver(address receiver);",
+    "",
+    "    /// new",
+    "    bytes32 public constant SEIZE_ROLE = keccak256(\"SEIZE_ROLE\");",
+  ].join("\n");
+  assert.deepEqual(natspecDocumentedSymbols(section, post).sort(), ["InvalidReceiver", "SEIZE_ROLE"]);
+});
+
+test("decideCall: function-reference is called when its member's NatSpec changed", () => {
+  const content = '---\ntitle: "IB20.mint"\n---\nbody';
+  assert.match(decideCall({ role: "function-reference", content, diffSlice: "", symbols: [] }), /own symbol mint/);
+  assert.equal(decideCall({ role: "function-reference", content, diffSlice: "", symbols: [], documented: ["mint"] }), null);
+});

scripts/sync-from-base-std/eval/cases/1505323-token-self-recipient.json

@@ -0,0 +1,48 @@
+{
+  "description": "base-std#232 (1505323): a real behavior change. Transfers, mints, batch mints, and seizes now revert with InvalidReceiver when the recipient is the token's own address. In Base Std the interface edits are NatSpec only; the mocks and a changelog entry carry the change. Pinned to a docs commit where humans had already added the changelog entry and updated both seize guides, but the five function reference pages still list only address(0) (errors.mdx already covers it as a B20-prefix address). This case guards recall: the sync must still edit the owning reference pages for a real change. announce-a-distribution and issue-units list batchMint's revert reasons, so a one-line addition there is correct.",
+  "payload": "../../fixtures/code-change-1505323-token-self-recipient.json",
+  "docs_base_sha": "1169505620155cc1186cf3ce4ed56320de137c0b",
+  "expect": {
+    "must_touch": [
+      "docs/specifications/b20/reference/interfaces/ib20/transfer.mdx",
+      "docs/specifications/b20/reference/interfaces/ib20/transfer-from.mdx",
+      "docs/specifications/b20/reference/interfaces/ib20/mint.mdx",
+      "docs/specifications/b20/reference/interfaces/ib20/seize-with-memo.mdx",
+      "docs/specifications/b20/reference/interfaces/ib20-asset/batch-mint.mdx"
+    ],
+    "must_mention": {
+      "docs/specifications/b20/reference/interfaces/ib20/transfer.mdx": "address\\(this\\)|own address|token itself|token's address",
+      "docs/specifications/b20/reference/interfaces/ib20/transfer-from.mdx": "address\\(this\\)|own address|token itself|token's address",
+      "docs/specifications/b20/reference/interfaces/ib20/mint.mdx": "address\\(this\\)|own address|token itself|token's address",
+      "docs/specifications/b20/reference/interfaces/ib20/seize-with-memo.mdx": "address\\(this\\)|own address|token itself|token's address",
+      "docs/specifications/b20/reference/interfaces/ib20-asset/batch-mint.mdx": "address\\(this\\)|own address|token itself|token's address"
+    },
+    "may_touch": {
+      "docs/specifications/b20/reference/interfaces/ib20/index.mdx": 6,
+      "docs/specifications/b20/reference/interfaces/ib20-asset/index.mdx": 4,
+      "docs/specifications/b20/reference/interfaces.mdx": 4,
+      "docs/specifications/b20/reference/events.mdx": 2,
+      "docs/base-chain/specs/reference/b20/changelog/03-denim-b20-token-receiver.mdx": 6,
+      "docs/build-on-base/issue-rwa/seize-and-cancel-units.mdx": 4,
+      "docs/build-on-base/issue-stablecoins/recover-funds.mdx": 4,
+      "docs/specifications/b20/changelog.mdx": 4,
+      "docs/docs.json": 4,
+      "docs/specifications/b20/reference/errors.mdx": 2,
+      "docs/build-on-base/issue-rwa/announce-a-distribution.mdx": 2,
+      "docs/build-on-base/issue-rwa/issue-units.mdx": 2
+    },
+    "must_not_touch": [
+      "docs/build-on-base/accept-payments/request-a-payment.mdx",
+      "docs/build-on-base/accept-payments/send-a-payout.mdx",
+      "docs/build-on-base/accept-payments/split-a-payment.mdx",
+      "docs/build-on-base/accept-payments/verify-a-payment.mdx",
+      "docs/build-on-base/issue-rwa/create-an-asset-token.mdx",
+      "docs/build-on-base/issue-rwa/pause-transfers.mdx",
+      "docs/build-on-base/issue-stablecoins/issue-your-stablecoin.mdx",
+      "docs/build-on-base/issue-stablecoins/restrict-who-can-hold.mdx",
+      "docs/base-chain/specs/reference/b20/changelog/02-cobalt-b20-seize.mdx",
+      "docs/base-chain/specs/reference/b20/changelog/02-cobalt-b20asset-multiplier.mdx"
+    ],
+    "unlisted_pages": "warn"
+  }
+}

scripts/sync-from-base-std/eval/cases/3820cf0-isauthorized-natspec.json

@@ -0,0 +1,37 @@
+{
+  "description": "base-std#234 (3820cf0): NatSpec-only clarification of IPolicyRegistry.isAuthorized for malformed, unknown, and inverted IDs. No code or behavior change. The owning reference page and its interface-index row should change; guides that only mention isAuthorized in passing, and historical upgrade or older-fork changelog pages, should not.",
+  "payload": "../../fixtures/code-change-3820cf0-isauthorized-natspec.json",
+  "docs_base_sha": "5ceae2479cdd2ac09267bdcfb10342b933406cbc",
+  "expect": {
+    "must_touch": [
+      "docs/specifications/b20/reference/interfaces/i-policy-registry/is-authorized.mdx",
+      "docs/specifications/b20/reference/interfaces/i-policy-registry/index.mdx"
+    ],
+    "may_touch": {
+      "docs/specifications/b20/concepts/policies.mdx": 20,
+      "docs/specifications/b20/reference/constants.mdx": 4,
+      "docs/base-chain/specs/reference/b20/changelog/03-denim-policyregistry-not-policy.mdx": 20,
+      "docs/build-on-base/issue-rwa/seize-and-cancel-units.mdx": 3,
+      "docs/build-on-base/issue-stablecoins/block-an-account.mdx": 3,
+      "docs/specifications/b20/introduction.mdx": 3
+    },
+    "must_not_touch": [
+      "docs/build-on-base/issue-rwa/restrict-transfer-initiators.mdx",
+      "docs/build-on-base/issue-stablecoins/restrict-who-can-hold.mdx",
+      "docs/build-on-base/integrate-defi/list-tokenized-stocks.mdx",
+      "docs/base-chain/specs/reference/b20/changelog/02-cobalt-policyregistry-composite-policy.mdx",
+      "docs/base-chain/specs/reference/b20/changelog/03-denim-b20-transfer-executor-enforcement.mdx",
+      "docs/upgrades/beryl/b20.mdx"
+    ],
+    "unlisted_pages": "fail",
+    "restatement": {
+      "owners": [
+        "docs/specifications/b20/reference/interfaces/i-policy-registry/is-authorized.mdx",
+        "docs/specifications/b20/concepts/policies.mdx",
+        "docs/base-chain/specs/reference/b20/changelog/03-denim-policyregistry-not-policy.mdx"
+      ],
+      "keywords": ["malformed", "empty set", "INTERSECT", "inverted"],
+      "min_keywords": 3
+    }
+  }
+}