diff --git a/engine/hooks/diu-stop/README.md b/engine/hooks/diu-stop/README.md index d34f3182..1c931080 100644 --- a/engine/hooks/diu-stop/README.md +++ b/engine/hooks/diu-stop/README.md @@ -16,6 +16,21 @@ the human speaking -- task notifications, queued or system-injected input. The claim check reads only the main agent's turn-final message: of 337 unproven claims found in stored transcripts, 196 were mid-turn or subagent text it never saw. See [`COVERAGE.md`](COVERAGE.md) before reading its silence as clearance. +## What buys a paragraph its silence + +A fenced block of output, inline code that looks like output, a well-formed +`{{CAT-UNVERIFIED: ... -- cannot verify: ...}}` tag, or a file citation -- +and a citation has to be backed. `path:line` on its own used to silence a +paragraph with no check that the file existed, that anyone read it, or at +what ref; a made-up path silenced the gate exactly as well as a real one. A +citation now counts when it names the ref it was read at (`path:line @ +origin/main`, the form `corpus/CLAUDE.learned.md` already asks for in prose), +or when the session's transcript shows a tool call that named that path. + +Three outcomes, not two. When the transcript cannot be read, whether the path +was read is *unchecked*: the citation does not buy silence, and the block says +which path it could not check and that the ref would settle it. + Not one file per harness, because there is no single "stop" mechanism shared by every harness -- each one has a genuinely different amount of power at that point: diff --git a/engine/hooks/diu-stop/claude_stop_check.py b/engine/hooks/diu-stop/claude_stop_check.py index 125ec520..44b2270a 100755 --- a/engine/hooks/diu-stop/claude_stop_check.py +++ b/engine/hooks/diu-stop/claude_stop_check.py @@ -45,6 +45,7 @@ Every block names every flagged sentence, so one rewrite that fixes them all gets through. """ +import json import os import re import sys @@ -90,6 +91,9 @@ FENCED_BODY_RE = re.compile(r"```[^\n]*\n(.*?)```", re.DOTALL) INLINE_CODE_RE = re.compile(r"`([^`\n]+)`") FILE_LINE_RE = re.compile(r"(? |\+\+\+ |--- |@@ |diff --git|commit [0-9a-f]{7,}|[0-9a-f]{7,10} )" r"|Traceback|^\s*at [\w.$<>]+ \(.*:\d+:\d+\)" @@ -189,7 +193,76 @@ def _paragraph_claim(para): return None -def find_unverified_claims(message): +def cited_paths(text): + """The path part of every file:line in `text`, longest first.""" + seen = [] + for match in FILE_LINE_RE.finditer(text): + path = LINE_SUFFIX_RE.split(match.group(0))[0] + if path and path not in seen: + seen.append(path) + return sorted(seen, key=len, reverse=True) + + +def read_evidence(event): + """Everything this session handed a tool, as one string, or None. + + None means the check could not run -- no transcript to read, or the file + would not open. That is a third outcome, not a clean one: a citation whose + read cannot be checked does not buy silence, and the finding says why. + """ + path = event.get("transcript_path") if isinstance(event, dict) else None + if not isinstance(path, str) or not path: + return None + parts = [] + try: + with open(path, encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(entry, dict): + continue + inner = entry.get("message") + content = inner.get("content") if isinstance(inner, dict) else entry.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + parts.append(json.dumps(block.get("input"), default=str)) + except (OSError, UnicodeError) as exc: + print( + f"catstack-hook-error diu-stop: cannot read {path}, so which files " + f"were read this session is unchecked: {exc}", + file=sys.stderr, + ) + return None + return "\n".join(parts) + + +def citation_earns_silence(para, read_blob): + """(exempt, unchecked) for the file:line citations in one paragraph. + + A bare `file.ts:99` used to silence a paragraph on its own, with no check + that the file exists, that anyone read it, or at what ref. A fabricated + path silenced the gate exactly as well as a real one. It now has to carry + the ref it was read at (`path:line @ origin/main`), which is what + corpus/CLAUDE.learned.md already asks for in prose, or the session has to + show a tool call that named that path. + """ + if not FILE_LINE_RE.search(para): + return False, False + if CITATION_REF_RE.search(para): + return True, False + if read_blob is None: + return False, True + return any(path in read_blob for path in cited_paths(para)), False + + +def find_unverified_claims(message, read_blob=None): """Return one (trigger phrase, sentence) pair for every paragraph that makes an unverified-shaped claim with no evidence marker in that same paragraph, in message order. @@ -209,7 +282,8 @@ def find_unverified_claims(message): continue if markers.excuses_paragraph(para): continue - if FILE_LINE_RE.search(para): + exempt, _unchecked = citation_earns_silence(para, read_blob) + if exempt: continue inline = INLINE_CODE_RE.findall(para) if inline and (fenced_output or any(OUTPUT_SHAPE_RE.search(code) for code in inline)): @@ -221,13 +295,26 @@ def find_unverified_claims(message): return claims -def find_unverified_claim(message): +def find_unverified_claim(message, read_blob=None): """Return the first offending phrase find_unverified_claims reports, or None.""" - claims = find_unverified_claims(message) + claims = find_unverified_claims(message, read_blob) return claims[0][0] if claims else None +def unchecked_citations(message, read_blob): + """Paths cited in a flagged paragraph whose read could not be checked.""" + if read_blob is not None: + return [] + found = [] + for para in re.split(r"\n\s*\n", message): + para = FENCED_BODY_RE.sub("", para) + _exempt, unchecked = citation_earns_silence(para, read_blob) + if unchecked: + found.extend(path for path in cited_paths(para) if path not in found) + return found + + def detect(event): if event.get("agent_id"): return [] @@ -239,7 +326,8 @@ def detect(event): word_count = counted_words(message) over_limit = word_count > WORD_LIMIT and not retry - claims = find_unverified_claims(message) + read_blob = read_evidence(event) + claims = find_unverified_claims(message, read_blob) marker_problems = find_marker_problems(message) findings = [] @@ -260,11 +348,18 @@ def detect(event): for number, (phrase, sentence) in enumerate(claims, 1): lines.append(f"{number}. \"{sentence}\" (trigger: \"{' '.join(phrase.split())}\")") lines.append( - "A backticked name or command alone is not output. Per " - "skills/prove-it/SKILL.md: for each one, either paste the output " - "of what was actually run/checked in its paragraph, or -- only if " - "the check cannot run -- tag the claim there and say why." + "A backticked name or command alone is not output, and neither is a " + "bare file:line. Per skills/prove-it/SKILL.md: for each one, either " + "paste the output of what was actually run/checked in its paragraph, " + "cite it as `path:line @ `, or -- only if the check cannot run " + "-- tag the claim there and say why." ) + unchecked = unchecked_citations(message, read_blob) + if unchecked: + lines.append( + "This turn's transcript could not be read, so whether " + f"{', '.join(unchecked)} was read this session is UNCHECKED, not " + "clear. Add the ref it was read at to the citation.") claim_message = "\n".join(lines) findings.append(Finding( rule_id=RULE_UNVERIFIED_CLAIM, diff --git a/engine/hooks/diu-stop/tests/test_hooks.py b/engine/hooks/diu-stop/tests/test_hooks.py index 8c72b35d..abc4f9e9 100644 --- a/engine/hooks/diu-stop/tests/test_hooks.py +++ b/engine/hooks/diu-stop/tests/test_hooks.py @@ -462,7 +462,14 @@ def test_fenced_block_does_not_silence_later_prose_in_same_paragraph(self): self.assertTrue(blocked) self.assertIn("this fixes it", err.lower()) - def test_file_line_citation_still_silences_claim_after_fence_normalization(self): + def test_a_cited_file_that_was_read_still_silences_after_fence_normalization(self): + """The earlier invariant, kept: fence normalization leaves a real citation alone. + + That change carried the file-and-line exemption over untouched as a + Non-goal; it did not decide that a path nobody read should silence + anything. The citation here is now backed the way the exemption always + claimed to be -- the session read that file. + """ message = ( "I checked the relevant snippet.\n" "```ts\n" @@ -470,10 +477,64 @@ def test_file_line_citation_still_silences_claim_after_fence_normalization(self) "```\n" "The issue was the stale guard at file.ts:1276." ) - self.assertIsNone(claude_stop_check.find_unverified_claim(message)) + read = json.dumps({"file_path": "/repo/src/file.ts"}) + self.assertIsNone(claude_stop_check.find_unverified_claim(message, read)) + transcript = self.transcript_reading("/repo/src/file.ts") + blocked, err = run_claude_check( + {"last_assistant_message": message, "transcript_path": transcript}) + self.assertFalse(blocked) + self.assertNotIn("unverified-shaped", err) + + def transcript_reading(self, *paths): + """A transcript whose tool calls name `paths`, written to a tempdir.""" + import tempfile + directory = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, directory, True) + target = os.path.join(directory, "session.jsonl") + with open(target, "w", encoding="utf-8") as handle: + for path in paths: + handle.write(json.dumps({ + "type": "assistant", + "message": {"role": "assistant", "content": [ + {"type": "tool_use", "name": "Read", "input": {"file_path": path}}]}, + }) + "\n") + return target + + def test_a_fabricated_path_no_longer_silences_the_claim(self): + """The A/B/C sweep's C case: the path exists nowhere and was never read.""" + message = ( + "The issue was the stale guard at " + "totally-made-up-file-that-does-not-exist.ts:99999." + ) + read = json.dumps({"file_path": "/repo/src/file.ts"}) + self.assertIsNotNone(claude_stop_check.find_unverified_claim(message, read)) + transcript = self.transcript_reading("/repo/src/file.ts") + blocked, err = run_claude_check( + {"last_assistant_message": message, "transcript_path": transcript}) + self.assertTrue(blocked) + self.assertIn("bare file:line", err) + + def test_a_citation_carrying_its_ref_silences_without_any_transcript(self): + """The B case: the citation says where it was read, so it stands alone.""" + message = "The issue was the stale guard at src/file.ts:1276 @ origin/main." + self.assertIsNone(claude_stop_check.find_unverified_claim(message, "")) blocked, err = run_claude_check({"last_assistant_message": message}) self.assertFalse(blocked) - self.assertEqual(err, "") + self.assertNotIn("unverified-shaped", err) + + def test_an_unreadable_transcript_makes_a_citation_unchecked_not_clear(self): + message = "The issue was the stale guard at src/file.ts:1276." + self.assertIsNotNone(claude_stop_check.find_unverified_claim(message, None)) + blocked, err = run_claude_check( + {"last_assistant_message": message, "transcript_path": "/no/such/transcript.jsonl"}) + self.assertTrue(blocked) + self.assertIn("UNCHECKED", err) + self.assertIn("src/file.ts", err) + + def test_a_read_path_named_only_by_a_grep_still_silences(self): + message = "The issue was the stale guard at src/file.ts:1276." + read = json.dumps({"pattern": "guard", "path": "src/file.ts"}) + self.assertIsNone(claude_stop_check.find_unverified_claim(message, read)) def test_hedge_i_think_it_happened_without_evidence_is_flagged(self): message = "I think the deploy happened around 2am, so that's why the build is stale."