merge queue: checking #784 on main (fe4fe56)

engine/hooks/diu-stop/README.md

@@ -16,6 +16,21 @@ the human speaking -- task notifications, queued or system-injected input.
 
 The claim check reads only the main agent's turn-final message: of 337 unproven claims found in stored transcripts, 196 were mid-turn or subagent text it never saw. See [`COVERAGE.md`](COVERAGE.md) before reading its silence as clearance.
 
+## What buys a paragraph its silence
+
+A fenced block of output, inline code that looks like output, a well-formed
+`{{CAT-UNVERIFIED: ... -- cannot verify: ...}}` tag, or a file citation --
+and a citation has to be backed. `path:line` on its own used to silence a
+paragraph with no check that the file existed, that anyone read it, or at
+what ref; a made-up path silenced the gate exactly as well as a real one. A
+citation now counts when it names the ref it was read at (`path:line @
+origin/main`, the form `corpus/CLAUDE.learned.md` already asks for in prose),
+or when the session's transcript shows a tool call that named that path.
+
+Three outcomes, not two. When the transcript cannot be read, whether the path
+was read is *unchecked*: the citation does not buy silence, and the block says
+which path it could not check and that the ref would settle it.
+
 Not one file per harness, because there is no single "stop" mechanism
 shared by every harness -- each one has a genuinely different amount of
 power at that point:

engine/hooks/diu-stop/claude_stop_check.py

@@ -45,6 +45,7 @@
 Every block names every flagged sentence, so one rewrite that fixes them
 all gets through.
 """
+import json
 import os
 import re
 import sys
@@ -90,6 +91,9 @@
 FENCED_BODY_RE = re.compile(r"```[^\n]*\n(.*?)```", re.DOTALL)
 INLINE_CODE_RE = re.compile(r"`([^`\n]+)`")
 FILE_LINE_RE = re.compile(r"(?<![\w/.-])(?:[\w.-]+/)*[\w-]+\.[A-Za-z]\w*(?::\d+\b|#L\d+\b)")
+CITATION_REF_RE = re.compile(
+    r"(?<![\w/.-])(?:[\w.-]+/)*[\w-]+\.[A-Za-z]\w*(?::\d+\b|#L\d+\b)\s*@\s*\S+")
+LINE_SUFFIX_RE = re.compile(r":\d+\b|#L\d+\b")
 OUTPUT_SHAPE_RE = re.compile(
     r"^(?:\$ |> |\+\+\+ |--- |@@ |diff --git|commit [0-9a-f]{7,}|[0-9a-f]{7,10} )"
     r"|Traceback|^\s*at [\w.$<>]+ \(.*:\d+:\d+\)"
@@ -189,7 +193,76 @@ def _paragraph_claim(para):
     return None
 
 
-def find_unverified_claims(message):
+def cited_paths(text):
+    """The path part of every file:line in `text`, longest first."""
+    seen = []
+    for match in FILE_LINE_RE.finditer(text):
+        path = LINE_SUFFIX_RE.split(match.group(0))[0]
+        if path and path not in seen:
+            seen.append(path)
+    return sorted(seen, key=len, reverse=True)
+
+
+def read_evidence(event):
+    """Everything this session handed a tool, as one string, or None.
+
+    None means the check could not run -- no transcript to read, or the file
+    would not open. That is a third outcome, not a clean one: a citation whose
+    read cannot be checked does not buy silence, and the finding says why.
+    """
+    path = event.get("transcript_path") if isinstance(event, dict) else None
+    if not isinstance(path, str) or not path:
+        return None
+    parts = []
+    try:
+        with open(path, encoding="utf-8") as handle:
+            for line in handle:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    entry = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                if not isinstance(entry, dict):
+                    continue
+                inner = entry.get("message")
+                content = inner.get("content") if isinstance(inner, dict) else entry.get("content")
+                if not isinstance(content, list):
+                    continue
+                for block in content:
+                    if isinstance(block, dict) and block.get("type") == "tool_use":
+                        parts.append(json.dumps(block.get("input"), default=str))
+    except (OSError, UnicodeError) as exc:
+        print(
+            f"catstack-hook-error diu-stop: cannot read {path}, so which files "
+            f"were read this session is unchecked: {exc}",
+            file=sys.stderr,
+        )
+        return None
+    return "\n".join(parts)
+
+
+def citation_earns_silence(para, read_blob):
+    """(exempt, unchecked) for the file:line citations in one paragraph.
+
+    A bare `file.ts:99` used to silence a paragraph on its own, with no check
+    that the file exists, that anyone read it, or at what ref. A fabricated
+    path silenced the gate exactly as well as a real one. It now has to carry
+    the ref it was read at (`path:line @ origin/main`), which is what
+    corpus/CLAUDE.learned.md already asks for in prose, or the session has to
+    show a tool call that named that path.
+    """
+    if not FILE_LINE_RE.search(para):
+        return False, False
+    if CITATION_REF_RE.search(para):
+        return True, False
+    if read_blob is None:
+        return False, True
+    return any(path in read_blob for path in cited_paths(para)), False
+
+
+def find_unverified_claims(message, read_blob=None):
     """Return one (trigger phrase, sentence) pair for every paragraph that
     makes an unverified-shaped claim with no evidence marker in that same
     paragraph, in message order.
@@ -209,7 +282,8 @@ def find_unverified_claims(message):
             continue
         if markers.excuses_paragraph(para):
             continue
-        if FILE_LINE_RE.search(para):
+        exempt, _unchecked = citation_earns_silence(para, read_blob)
+        if exempt:
             continue
         inline = INLINE_CODE_RE.findall(para)
         if inline and (fenced_output or any(OUTPUT_SHAPE_RE.search(code) for code in inline)):
@@ -221,13 +295,26 @@ def find_unverified_claims(message):
     return claims
 
 
-def find_unverified_claim(message):
+def find_unverified_claim(message, read_blob=None):
     """Return the first offending phrase find_unverified_claims reports, or
     None."""
-    claims = find_unverified_claims(message)
+    claims = find_unverified_claims(message, read_blob)
     return claims[0][0] if claims else None
 
 
+def unchecked_citations(message, read_blob):
+    """Paths cited in a flagged paragraph whose read could not be checked."""
+    if read_blob is not None:
+        return []
+    found = []
+    for para in re.split(r"\n\s*\n", message):
+        para = FENCED_BODY_RE.sub("", para)
+        _exempt, unchecked = citation_earns_silence(para, read_blob)
+        if unchecked:
+            found.extend(path for path in cited_paths(para) if path not in found)
+    return found
+
+
 def detect(event):
     if event.get("agent_id"):
         return []
@@ -239,7 +326,8 @@ def detect(event):
 
     word_count = counted_words(message)
     over_limit = word_count > WORD_LIMIT and not retry
-    claims = find_unverified_claims(message)
+    read_blob = read_evidence(event)
+    claims = find_unverified_claims(message, read_blob)
     marker_problems = find_marker_problems(message)
 
     findings = []
@@ -260,11 +348,18 @@ def detect(event):
         for number, (phrase, sentence) in enumerate(claims, 1):
             lines.append(f"{number}. \"{sentence}\" (trigger: \"{' '.join(phrase.split())}\")")
         lines.append(
-            "A backticked name or command alone is not output. Per "
-            "skills/prove-it/SKILL.md: for each one, either paste the output "
-            "of what was actually run/checked in its paragraph, or -- only if "
-            "the check cannot run -- tag the claim there and say why."
+            "A backticked name or command alone is not output, and neither is a "
+            "bare file:line. Per skills/prove-it/SKILL.md: for each one, either "
+            "paste the output of what was actually run/checked in its paragraph, "
+            "cite it as `path:line @ <ref>`, or -- only if the check cannot run "
+            "-- tag the claim there and say why."
         )
+        unchecked = unchecked_citations(message, read_blob)
+        if unchecked:
+            lines.append(
+                "This turn's transcript could not be read, so whether "
+                f"{', '.join(unchecked)} was read this session is UNCHECKED, not "
+                "clear. Add the ref it was read at to the citation.")
         claim_message = "\n".join(lines)
         findings.append(Finding(
             rule_id=RULE_UNVERIFIED_CLAIM,

engine/hooks/diu-stop/tests/test_hooks.py

@@ -462,18 +462,79 @@ def test_fenced_block_does_not_silence_later_prose_in_same_paragraph(self):
         self.assertTrue(blocked)
         self.assertIn("this fixes it", err.lower())
 
-    def test_file_line_citation_still_silences_claim_after_fence_normalization(self):
+    def test_a_cited_file_that_was_read_still_silences_after_fence_normalization(self):
+        """The earlier invariant, kept: fence normalization leaves a real citation alone.
+
+        That change carried the file-and-line exemption over untouched as a
+        Non-goal; it did not decide that a path nobody read should silence
+        anything. The citation here is now backed the way the exemption always
+        claimed to be -- the session read that file.
+        """
         message = (
             "I checked the relevant snippet.\n"
             "```ts\n"
             "const guard = true;\n"
             "```\n"
             "The issue was the stale guard at file.ts:1276."
         )
-        self.assertIsNone(claude_stop_check.find_unverified_claim(message))
+        read = json.dumps({"file_path": "/repo/src/file.ts"})
+        self.assertIsNone(claude_stop_check.find_unverified_claim(message, read))
+        transcript = self.transcript_reading("/repo/src/file.ts")
+        blocked, err = run_claude_check(
+            {"last_assistant_message": message, "transcript_path": transcript})
+        self.assertFalse(blocked)
+        self.assertNotIn("unverified-shaped", err)
+
+    def transcript_reading(self, *paths):
+        """A transcript whose tool calls name `paths`, written to a tempdir."""
+        import tempfile
+        directory = tempfile.mkdtemp()
+        self.addCleanup(__import__("shutil").rmtree, directory, True)
+        target = os.path.join(directory, "session.jsonl")
+        with open(target, "w", encoding="utf-8") as handle:
+            for path in paths:
+                handle.write(json.dumps({
+                    "type": "assistant",
+                    "message": {"role": "assistant", "content": [
+                        {"type": "tool_use", "name": "Read", "input": {"file_path": path}}]},
+                }) + "\n")
+        return target
+
+    def test_a_fabricated_path_no_longer_silences_the_claim(self):
+        """The A/B/C sweep's C case: the path exists nowhere and was never read."""
+        message = (
+            "The issue was the stale guard at "
+            "totally-made-up-file-that-does-not-exist.ts:99999."
+        )
+        read = json.dumps({"file_path": "/repo/src/file.ts"})
+        self.assertIsNotNone(claude_stop_check.find_unverified_claim(message, read))
+        transcript = self.transcript_reading("/repo/src/file.ts")
+        blocked, err = run_claude_check(
+            {"last_assistant_message": message, "transcript_path": transcript})
+        self.assertTrue(blocked)
+        self.assertIn("bare file:line", err)
+
+    def test_a_citation_carrying_its_ref_silences_without_any_transcript(self):
+        """The B case: the citation says where it was read, so it stands alone."""
+        message = "The issue was the stale guard at src/file.ts:1276 @ origin/main."
+        self.assertIsNone(claude_stop_check.find_unverified_claim(message, ""))
         blocked, err = run_claude_check({"last_assistant_message": message})
         self.assertFalse(blocked)
-        self.assertEqual(err, "")
+        self.assertNotIn("unverified-shaped", err)
+
+    def test_an_unreadable_transcript_makes_a_citation_unchecked_not_clear(self):
+        message = "The issue was the stale guard at src/file.ts:1276."
+        self.assertIsNotNone(claude_stop_check.find_unverified_claim(message, None))
+        blocked, err = run_claude_check(
+            {"last_assistant_message": message, "transcript_path": "/no/such/transcript.jsonl"})
+        self.assertTrue(blocked)
+        self.assertIn("UNCHECKED", err)
+        self.assertIn("src/file.ts", err)
+
+    def test_a_read_path_named_only_by_a_grep_still_silences(self):
+        message = "The issue was the stale guard at src/file.ts:1276."
+        read = json.dumps({"pattern": "guard", "path": "src/file.ts"})
+        self.assertIsNone(claude_stop_check.find_unverified_claim(message, read))
 
     def test_hedge_i_think_it_happened_without_evidence_is_flagged(self):
         message = "I think the deploy happened around 2am, so that's why the build is stale."