diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2364ba..9b31719 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,10 @@ jobs: python-version: "3.12" - run: pip install ruff lizard==1.24.0 vulture==2.16 - run: ruff check src/ tests/ + - name: Ruff proving test + run: bash scripts/test-check-ruff.sh + - name: Ruff baseline + run: python3 scripts/check-ruff.py src tests - name: Complexity proving test run: bash scripts/test-check-complexity.sh - name: PR-diff complexity diff --git a/config/ruff-baseline.txt b/config/ruff-baseline.txt new file mode 100644 index 0000000..7c48091 --- /dev/null +++ b/config/ruff-baseline.txt @@ -0,0 +1,5 @@ +# Ruff findings under the committed [tool.ruff.lint] select (E4, E7, E9, F). +# Each line is: +# Observed with ruff 0.15.7: src/ and tests/ have no findings. +# check-ruff.py fails when a count rises or a new key appears. +# CI reads this file and must not rewrite it. Do not refresh it to hide a finding. diff --git a/scripts/check-ruff.py b/scripts/check-ruff.py new file mode 100644 index 0000000..c9e547e --- /dev/null +++ b/scripts/check-ruff.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Fail when ruff reports a new finding. + +The committed baseline counts today's findings under the repo ruff config. +A higher count, or a finding that is not listed, fails. Removed findings are +allowed. This script never writes the baseline. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from collections import Counter +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def finding_key(item: dict, root: Path) -> str: + filename = str(item.get("filename", "")) + path = Path(filename) + if path.is_absolute(): + try: + filename = path.resolve().relative_to(root.resolve()).as_posix() + except ValueError: + filename = path.as_posix() + code = str(item.get("code", "")) + message = str(item.get("message", "")) + return f"{filename} {code} {message}" + + +def parse_baseline(text: str) -> dict[str, int]: + counts: dict[str, int] = {} + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + count_text, key = line.split(" ", 1) + counts[key] = int(count_text) + return counts + + +def current_counts(paths: list[str], cwd: Path) -> Counter[str]: + proc = subprocess.run( + ["ruff", "check", *paths, "--output-format", "json", "--no-cache"], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + if proc.returncode not in (0, 1): + sys.stderr.write(proc.stderr) + raise RuntimeError(f"ruff exited {proc.returncode}") + payload = proc.stdout.strip() or "[]" + data = json.loads(payload) + if not isinstance(data, list): + raise RuntimeError("ruff JSON was not a list") + counts: Counter[str] = Counter() + for item in data: + if isinstance(item, dict): + counts[finding_key(item, cwd)] += 1 + return counts + + +def new_findings(current: Counter[str], baseline: dict[str, int]) -> list[str]: + failures = [] + for key, count in sorted(current.items()): + allowed = baseline.get(key, 0) + if count > allowed: + failures.append(f"NEW {key} count {allowed} -> {count}") + return failures + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline", type=Path, default=ROOT / "config" / "ruff-baseline.txt") + parser.add_argument("paths", nargs="*") + args = parser.parse_args() + paths = args.paths or ["src", "tests"] + if not args.baseline.is_file(): + print(f"missing ruff baseline: {args.baseline}", file=sys.stderr) + return 2 + baseline = parse_baseline(args.baseline.read_text(encoding="utf-8")) + try: + current = current_counts(paths, ROOT) + except (RuntimeError, json.JSONDecodeError, ValueError) as exc: + print(exc, file=sys.stderr) + return 2 + failures = new_findings(current, baseline) + if failures: + print("check-ruff: FAIL", file=sys.stderr) + for line in failures: + print(line, file=sys.stderr) + return 1 + print(f"check-ruff: PASS ({len(current)} finding key(s))") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test-check-ruff.sh b/scripts/test-check-ruff.sh new file mode 100644 index 0000000..e27e589 --- /dev/null +++ b/scripts/test-check-ruff.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Proving test: a baselined finding passes; a new finding fails. CI must not rewrite the baseline. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +CHECK="${SCRIPT_DIR}/check-ruff.py" +BASELINE_FILE="${ROOT}/config/ruff-baseline.txt" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +OK="$TMP/ok.py" +BAD="$TMP/bad.py" +BASELINE="$TMP/baseline.txt" +printf 'import os\n' > "$OK" + +printf '# empty ceiling\n' > "$BASELINE" +if python3 "$CHECK" --baseline "$BASELINE" "$OK"; then + echo "expected FAIL for a finding against an empty baseline" >&2 + exit 1 +fi + +printf '1 %s F401 `os` imported but unused\n' "$OK" > "$BASELINE" +if ! python3 "$CHECK" --baseline "$BASELINE" "$OK"; then + echo "expected PASS for a baselined unused import" >&2 + exit 1 +fi + +printf 'import os\ndef a():\n import os\n return 1\n' > "$OK" +printf '1 %s F401 `os` imported but unused\n' "$OK" > "$BASELINE" +printf '1 %s F811 Redefinition of unused `os` from line 1: `os` redefined here\n' "$OK" >> "$BASELINE" +if python3 "$CHECK" --baseline "$BASELINE" "$OK"; then + echo "expected FAIL when a baselined finding's count rises" >&2 + exit 1 +fi + +printf 'import os\n' > "$OK" +printf 'import sys\n' > "$BAD" +printf '1 %s F401 `os` imported but unused\n' "$OK" > "$BASELINE" +if python3 "$CHECK" --baseline "$BASELINE" "$OK" "$BAD"; then + echo "expected FAIL for a finding that is not in the baseline" >&2 + exit 1 +fi + +if grep -q 'config/ruff-baseline.txt' "$ROOT/.github/workflows/ci.yml"; then + echo "CI must not rewrite the ruff baseline" >&2 + exit 1 +fi +if ! grep -q 'scripts/check-ruff.py' "$ROOT/.github/workflows/ci.yml"; then + echo "CI does not run check-ruff.py" >&2 + exit 1 +fi +if grep -E 'write_text|open\([^)]*["'\'']w' "$CHECK" >/dev/null; then + echo "check-ruff.py must not write the baseline" >&2 + exit 1 +fi + +before="$(sha256sum "$BASELINE_FILE")" +python3 "$CHECK" src tests +after="$(sha256sum "$BASELINE_FILE")" +if [[ "$before" != "$after" ]]; then + echo "check-ruff.py rewrote the committed baseline" >&2 + exit 1 +fi +echo "test-check-ruff: PASS"