Fail CI when a new ruff finding exceeds today's baseline

.github/workflows/ci.yml

@@ -22,6 +22,10 @@ jobs:
           python-version: "3.12"
       - run: pip install ruff lizard==1.24.0 vulture==2.16
       - run: ruff check src/ tests/
+      - name: Ruff proving test
+        run: bash scripts/test-check-ruff.sh
+      - name: Ruff baseline
+        run: python3 scripts/check-ruff.py src tests
       - name: Complexity proving test
         run: bash scripts/test-check-complexity.sh
       - name: PR-diff complexity

config/ruff-baseline.txt

@@ -0,0 +1,5 @@
+# Ruff findings under the committed [tool.ruff.lint] select (E4, E7, E9, F).
+# Each line is: <count> <path> <code> <message>
+# Observed with ruff 0.15.7: src/ and tests/ have no findings.
+# check-ruff.py fails when a count rises or a new key appears.
+# CI reads this file and must not rewrite it. Do not refresh it to hide a finding.

scripts/check-ruff.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""Fail when ruff reports a new finding.
+
+The committed baseline counts today's findings under the repo ruff config.
+A higher count, or a finding that is not listed, fails. Removed findings are
+allowed. This script never writes the baseline.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from collections import Counter
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def finding_key(item: dict, root: Path) -> str:
+    filename = str(item.get("filename", ""))
+    path = Path(filename)
+    if path.is_absolute():
+        try:
+            filename = path.resolve().relative_to(root.resolve()).as_posix()
+        except ValueError:
+            filename = path.as_posix()
+    code = str(item.get("code", ""))
+    message = str(item.get("message", ""))
+    return f"{filename} {code} {message}"
+
+
+def parse_baseline(text: str) -> dict[str, int]:
+    counts: dict[str, int] = {}
+    for raw in text.splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        count_text, key = line.split(" ", 1)
+        counts[key] = int(count_text)
+    return counts
+
+
+def current_counts(paths: list[str], cwd: Path) -> Counter[str]:
+    proc = subprocess.run(
+        ["ruff", "check", *paths, "--output-format", "json", "--no-cache"],
+        cwd=cwd,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if proc.returncode not in (0, 1):
+        sys.stderr.write(proc.stderr)
+        raise RuntimeError(f"ruff exited {proc.returncode}")
+    payload = proc.stdout.strip() or "[]"
+    data = json.loads(payload)
+    if not isinstance(data, list):
+        raise RuntimeError("ruff JSON was not a list")
+    counts: Counter[str] = Counter()
+    for item in data:
+        if isinstance(item, dict):
+            counts[finding_key(item, cwd)] += 1
+    return counts
+
+
+def new_findings(current: Counter[str], baseline: dict[str, int]) -> list[str]:
+    failures = []
+    for key, count in sorted(current.items()):
+        allowed = baseline.get(key, 0)
+        if count > allowed:
+            failures.append(f"NEW {key} count {allowed} -> {count}")
+    return failures
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--baseline", type=Path, default=ROOT / "config" / "ruff-baseline.txt")
+    parser.add_argument("paths", nargs="*")
+    args = parser.parse_args()
+    paths = args.paths or ["src", "tests"]
+    if not args.baseline.is_file():
+        print(f"missing ruff baseline: {args.baseline}", file=sys.stderr)
+        return 2
+    baseline = parse_baseline(args.baseline.read_text(encoding="utf-8"))
+    try:
+        current = current_counts(paths, ROOT)
+    except (RuntimeError, json.JSONDecodeError, ValueError) as exc:
+        print(exc, file=sys.stderr)
+        return 2
+    failures = new_findings(current, baseline)
+    if failures:
+        print("check-ruff: FAIL", file=sys.stderr)
+        for line in failures:
+            print(line, file=sys.stderr)
+        return 1
+    print(f"check-ruff: PASS ({len(current)} finding key(s))")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/test-check-ruff.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# Proving test: a baselined finding passes; a new finding fails. CI must not rewrite the baseline.
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+CHECK="${SCRIPT_DIR}/check-ruff.py"
+BASELINE_FILE="${ROOT}/config/ruff-baseline.txt"
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+
+OK="$TMP/ok.py"
+BAD="$TMP/bad.py"
+BASELINE="$TMP/baseline.txt"
+printf 'import os\n' > "$OK"
+
+printf '# empty ceiling\n' > "$BASELINE"
+if python3 "$CHECK" --baseline "$BASELINE" "$OK"; then
+  echo "expected FAIL for a finding against an empty baseline" >&2
+  exit 1
+fi
+
+printf '1 %s F401 `os` imported but unused\n' "$OK" > "$BASELINE"
+if ! python3 "$CHECK" --baseline "$BASELINE" "$OK"; then
+  echo "expected PASS for a baselined unused import" >&2
+  exit 1
+fi
+
+printf 'import os\ndef a():\n    import os\n    return 1\n' > "$OK"
+printf '1 %s F401 `os` imported but unused\n' "$OK" > "$BASELINE"
+printf '1 %s F811 Redefinition of unused `os` from line 1: `os` redefined here\n' "$OK" >> "$BASELINE"
+if python3 "$CHECK" --baseline "$BASELINE" "$OK"; then
+  echo "expected FAIL when a baselined finding's count rises" >&2
+  exit 1
+fi
+
+printf 'import os\n' > "$OK"
+printf 'import sys\n' > "$BAD"
+printf '1 %s F401 `os` imported but unused\n' "$OK" > "$BASELINE"
+if python3 "$CHECK" --baseline "$BASELINE" "$OK" "$BAD"; then
+  echo "expected FAIL for a finding that is not in the baseline" >&2
+  exit 1
+fi
+
+if grep -q 'config/ruff-baseline.txt' "$ROOT/.github/workflows/ci.yml"; then
+  echo "CI must not rewrite the ruff baseline" >&2
+  exit 1
+fi
+if ! grep -q 'scripts/check-ruff.py' "$ROOT/.github/workflows/ci.yml"; then
+  echo "CI does not run check-ruff.py" >&2
+  exit 1
+fi
+if grep -E 'write_text|open\([^)]*["'\'']w' "$CHECK" >/dev/null; then
+  echo "check-ruff.py must not write the baseline" >&2
+  exit 1
+fi
+
+before="$(sha256sum "$BASELINE_FILE")"
+python3 "$CHECK" src tests
+after="$(sha256sum "$BASELINE_FILE")"
+if [[ "$before" != "$after" ]]; then
+  echo "check-ruff.py rewrote the committed baseline" >&2
+  exit 1
+fi
+echo "test-check-ruff: PASS"