Run mutmut on the harness module

.github/workflows/tests.yml

@@ -46,3 +46,16 @@ jobs:
         run: python scripts/run_harness.py --backend stub --profile observed --fast-ms 1 --strong-ms 1
       - name: Deployment safety
         run: python scripts/check_deployment_safety.py --skip-listen
+
+  mutation:
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install package
+        run: python -m pip install --upgrade pip && python -m pip install -e '.[dev]' 'mutmut==3.8.0' 'pytest>=8,<9'
+      - name: Mutation harness.py
+        run: bash scripts/check-mutmut-harness.sh

.gitignore

@@ -9,5 +9,6 @@ __pycache__/
 venv/
 .DS_Store
 eval-runs/
+mutants/
 .local-ollama/
 *.local.md

pyproject.toml

@@ -24,3 +24,12 @@ src = ["src"]
 
 [tool.ruff.lint]
 select = ["F401", "F811"]
+
+# mutmut on the harness module only. Survivors fail the CI job.
+# There is no survived ceiling to raise.
+[tool.mutmut]
+source_paths = ["src"]
+only_mutate = ["src/local_coding_slm/eval/harness.py"]
+pytest_add_cli_args_test_selection = ["tests/test_harness_mutation.py"]
+use_git_change_detection = false
+mutate_only_covered_lines = true

scripts/check-mutmut-harness.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""Fail unless mutmut killed every harness.py mutant.
+
+Reads mutants/mutmut-cicd-stats.json. There is no survived ceiling.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_STATS = ROOT / "mutants" / "mutmut-cicd-stats.json"
+_BAD_KEYS = ("survived", "no_tests", "timeout", "suspicious", "segfault", "skipped")
+
+
+def _as_int(stats: dict[str, object], key: str) -> int:
+    value = stats.get(key) or 0
+    return int(value)  # type: ignore[arg-type]
+
+
+def evaluate(stats: dict[str, object]) -> str | None:
+    if _as_int(stats, "check_was_interrupted_by_user") > 0:
+        return "interrupted"
+    if _as_int(stats, "total") <= 0:
+        return "no mutants"
+    if _as_int(stats, "killed") <= 0:
+        return "killed none"
+    for key in _BAD_KEYS:
+        count = _as_int(stats, key)
+        if count > 0:
+            return f"{key} {count}"
+    return None
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--stats", type=Path, default=DEFAULT_STATS)
+    args = parser.parse_args()
+    if not args.stats.is_file():
+        print(f"check-mutmut-harness: missing {args.stats}", file=sys.stderr)
+        return 2
+    stats = json.loads(args.stats.read_text(encoding="utf-8"))
+    error = evaluate(stats)
+    if error is not None:
+        print(f"check-mutmut-harness: FAIL {error}", file=sys.stderr)
+        return 1
+    print(
+        "check-mutmut-harness: PASS "
+        f"killed {_as_int(stats, 'killed')} of {_as_int(stats, 'total')}"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/check-mutmut-harness.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# mutmut on src/local_coding_slm/eval/harness.py only.
+# mutmut 3 exits 0 when mutants survive. Fail unless every mutant was killed.
+# Do not write a survived ceiling.
+set -euo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT"
+rm -rf mutants
+run_log="$(mktemp)"
+trap 'rm -f "$run_log"' EXIT
+mutmut run --max-children 2 | tee "$run_log"
+if ! grep -E '\([1-9][0-9]* files mutated' "$run_log" >/dev/null; then
+  echo "mutmut: harness.py was not mutated" >&2
+  exit 1
+fi
+mutmut export-cicd-stats
+if ! python3 scripts/check-mutmut-harness.py; then
+  mutmut results || true
+  exit 1
+fi

src/local_coding_slm/eval/policy.py

@@ -32,7 +32,7 @@ class AttemptPlan:
 
 def next_plan(case: EvalCase, attempts: list[AttemptRecord]) -> AttemptPlan | None:
     """Return the next local_* call, or None if the case is done."""
-    del case
+    case.id
     if attempts and attempts[-1].passed:
         return None
     if len(attempts) >= MAX_ATTEMPTS: