tests: run every suite hermetic, with none of this machine's settings

engine/hooks/_flags/flags.py

@@ -122,6 +122,9 @@ def repo_root(start: str | None) -> str | None:
         current = parent
 
 
+SKIP_ENV_FILES_VAR = "CATSTACK_SKIP_ENV_FILES"
+
+
 def env_file_candidates(environ: dict, cwd: str | None, home: str | None = None) -> list[str]:
     candidates: list[str] = []
     explicit = environ.get(ENV_FILE_VAR)
@@ -132,7 +135,12 @@ def env_file_candidates(environ: dict, cwd: str | None, home: str | None = None)
         candidates.append(os.path.join(root, ".env"))
     home_dir = home or environ.get("HOME") or os.path.expanduser("~")
     candidates.append(os.path.join(home_dir, HOME_ENV_FILE.replace("~/", "", 1)))
-    return candidates
+    skip = {
+        os.path.realpath(os.path.expanduser(path))
+        for path in environ.get(SKIP_ENV_FILES_VAR, "").split(os.pathsep)
+        if path
+    }
+    return [path for path in candidates if os.path.realpath(path) not in skip]
 
 
 def _unquote(value: str) -> str:

engine/hooks/_flags/tests/test_flags.py

@@ -116,6 +116,14 @@ def test_repo_env_beats_home_and_is_found_by_walking_up(self):
         found = self.resolve()
         self.assertEqual((found.value, found.source), ("1", self.box.repo_env))
 
+    def test_skipped_env_files_are_never_read(self):
+        self.box.write(self.box.home_env, f"{KEY}=1\n")
+        self.box.write(self.box.repo_env, f"{KEY}=1\n")
+        skip = os.pathsep.join([self.box.home_env, self.box.repo_env])
+        env = self.box.environ({flags.SKIP_ENV_FILES_VAR: skip})
+        self.assertIsNone(flags.resolve_flag(KEY, env, self.box.cwd, self.box.home).value)
+        self.assertEqual([], flags.env_file_candidates(env, self.box.cwd, self.box.home))
+
     def test_home_env_is_the_last_resort(self):
         self.box.write(self.box.home_env, f"{KEY}=1\n")
         found = self.resolve()

engine/hooks/llm-judge/judge_test_base.py

@@ -23,6 +23,19 @@ def setUp(self):
         })
         self.judge_env.start()
         os.environ.pop(judge.CHILD_ENV, None)
+        self.codex_catalog = ["catalog-first", "catalog-second"]
+        codex_home = tempfile.TemporaryDirectory()
+        self.addCleanup(codex_home.cleanup)
+        self.codex_config = os.path.join(codex_home.name, "config.toml")
+        with open(self.codex_config, "w", encoding="utf-8") as handle:
+            handle.write('model = "catalog-first"\n')
+        for name, stub in (
+            ("codex_listed_models", lambda: list(self.codex_catalog)),
+            ("codex_config_path", lambda: self.codex_config),
+        ):
+            patcher = patch.object(judge, name, stub)
+            patcher.start()
+            self.addCleanup(patcher.stop)
 
     def tearDown(self):
         self.judge_env.stop()

engine/hooks/llm-judge/tests/test_judge.py

@@ -251,6 +251,14 @@ def test_default_codex_runner_uses_the_account_model_not_a_pinned_one(self):
         self.assertNotIn("-m", codex_argv)
         self.assertNotIn("--model", codex_argv)
 
+    def test_codex_runner_pins_the_catalog_model_when_the_config_names_one_it_lacks(self):
+        with open(self.codex_config, "w", encoding="utf-8") as handle:
+            handle.write('model = "retired-model"\n')
+        with patch.dict(os.environ):
+            os.environ.pop(judge.RUNNERS_ENV)
+            codex_argv = dict(judge.runners())["codex"]
+        self.assertEqual(codex_argv[codex_argv.index("-m") + 1], "catalog-first")
+
     def test_investigate_runner_argv_is_read_only_and_excludes_cursor(self):
         os.environ.pop(judge.RUNNERS_ENV)
         self.assertEqual(

engine/hooks/unverified-tag-ledger/tests/test_hooks.py

@@ -28,6 +28,10 @@
 REAL_PAYLOAD = os.path.join(FIXTURES, "claude-stop-payload.json")
 REAL_TRANSCRIPT = os.path.join(FIXTURES, "claude-transcript.jsonl")
 sys.path.insert(0, HOOK)
+REAL_SETTINGS_FILES = os.pathsep.join([
+    os.path.expanduser("~/.catstack.env"),
+    os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(HOOK))), ".env"),
+])
 
 REAL_TAG_1 = (
     "{{CAT-UNVERIFIED: that it widened scope past the one session I gave it "
@@ -51,13 +55,15 @@ class LedgerTests(unittest.TestCase):
     def setUp(self) -> None:
         self.tmp = tempfile.TemporaryDirectory()
         os.environ["CATSTACK_TAG_LEDGER_DIR"] = self.tmp.name
+        os.environ["CATSTACK_SKIP_ENV_FILES"] = REAL_SETTINGS_FILES
         for module in ("detect", "markers"):
             sys.modules.pop(module, None)
         import detect
         self.detect = detect
 
     def tearDown(self) -> None:
         os.environ.pop("CATSTACK_TAG_LEDGER_DIR", None)
+        os.environ.pop("CATSTACK_SKIP_ENV_FILES", None)
         self.tmp.cleanup()
 
     def transcript(self, *, tools: bool, name: str = "transcript.jsonl") -> str:

engine/hooks/unverified-tag-ledger/tests/test_hooks_sdk_mode.py

@@ -16,6 +16,10 @@
 HOOK = os.path.dirname(HERE)
 FIXTURES = os.path.join(HERE, "fixtures")
 sys.path.insert(0, HOOK)
+REAL_SETTINGS_FILES = os.pathsep.join([
+    os.path.expanduser("~/.catstack.env"),
+    os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(HOOK))), ".env"),
+])
 
 import claude_stop_check  # noqa: E402
 
@@ -48,6 +52,7 @@ def setUp(self) -> None:
             {
                 "CATSTACK_TAG_LEDGER_DIR": self.ledger_tmp.name,
                 "CATSTACK_HOOK_METRICS_DIR": self.metrics_tmp.name,
+                "CATSTACK_SKIP_ENV_FILES": REAL_SETTINGS_FILES,
             },
             clear=False,
         )

scripts/test/hermetic_env.sh

@@ -0,0 +1,18 @@
+_catstack_real_home="${HOME:-}"
+_catstack_repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+for _catstack_name in $(compgen -e); do
+  case "$_catstack_name" in
+    CATSTACK_* | GIT_* | CLAUDE_* | CODEX_* | CURSOR_* | XDG_*) unset "$_catstack_name" ;;
+  esac
+done
+
+CATSTACK_TEST_HOME="$(mktemp -d "${TMPDIR:-/tmp}/catstack-test-home.XXXXXX")"
+export CATSTACK_TEST_HOME
+export HOME="$CATSTACK_TEST_HOME"
+export GIT_CONFIG_GLOBAL="$CATSTACK_TEST_HOME/.gitconfig"
+: > "$GIT_CONFIG_GLOBAL"
+export GIT_CONFIG_NOSYSTEM=1
+export CATSTACK_SKIP_ENV_FILES="$_catstack_real_home/.catstack.env:$_catstack_repo/.env"
+
+unset _catstack_name _catstack_real_home _catstack_repo

scripts/test/run_all_tests.sh

@@ -13,9 +13,10 @@ if [[ -L $self ]]; then
 fi
 REPO_DIR="$(cd "$(dirname "$self")/../.." && pwd)"
 cd "$REPO_DIR"
+source "$REPO_DIR/scripts/test/hermetic_env.sh"
 
 JUDGE_STATE_DIR="$(python3 -c 'import tempfile; print(tempfile.mkdtemp(prefix="catstack-llm-judge-tests-"))')"
-trap 'python3 -c '\''import os, shutil; shutil.rmtree(os.environ["CATSTACK_LLM_JUDGE_STATE_DIR"])'\''' EXIT
+trap 'python3 -c '\''import os, shutil; shutil.rmtree(os.environ["CATSTACK_LLM_JUDGE_STATE_DIR"]); shutil.rmtree(os.environ["CATSTACK_TEST_HOME"])'\''' EXIT
 export CATSTACK_LLM_JUDGE_STATE_DIR="$JUDGE_STATE_DIR"
 export CATSTACK_LLM_JUDGE_RUNNERS="$(python3 - <<'PY'
 import json
@@ -88,7 +89,7 @@ ensure_node_deps
 status=0
 bash scripts/test/ensure_node_toolchain.sh || status=1
 suite_log="$(mktemp)"
-trap 'rm -f "$suite_log"; python3 -c '\''import os, shutil; shutil.rmtree(os.environ["CATSTACK_LLM_JUDGE_STATE_DIR"])'\''' EXIT
+trap 'rm -f "$suite_log"; python3 -c '\''import os, shutil; shutil.rmtree(os.environ["CATSTACK_LLM_JUDGE_STATE_DIR"]); shutil.rmtree(os.environ["CATSTACK_TEST_HOME"])'\''' EXIT
 while IFS= read -r dir; do
   echo "=== $dir ==="
   if ! python3 -m unittest discover -s "$dir" -v 2>&1 | tee "$suite_log"; then

tests/test_hermetic_test_env.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""The test runner hides this machine from every test.
+
+A test that reads the machine it runs on passes or fails by whose machine it
+is: `CATSTACK_UNVERIFIED_TAG_BEHAVIOR=do_not_emit` in one person's
+~/.catstack.env failed seven ledger tests that pass in CI. The runner sources
+scripts/test/hermetic_env.sh before any suite runs; these tests start that
+script from a deliberately polluted environment and check nothing gets
+through: no inherited CATSTACK_* or GIT_* variable, a throwaway HOME and git
+config, and a flag reader that will not open the real ~/.catstack.env or the
+checkout's own .env.
+
+Run: python3 -m unittest tests/test_hermetic_test_env.py -v
+"""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import tempfile
+import unittest
+
+REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+SETUP = os.path.join(REPO, "scripts", "test", "hermetic_env.sh")
+RUNNER = os.path.join(REPO, "scripts", "test", "run_all_tests.sh")
+
+PROBE = r"""
+import json, os, sys
+sys.path.insert(0, os.path.join(sys.argv[1], "engine", "hooks", "_flags"))
+import flags
+print(json.dumps({
+    "env": dict(os.environ),
+    "candidates": flags.env_file_candidates(dict(os.environ), sys.argv[1]),
+}))
+"""
+
+
+class HermeticEnvTest(unittest.TestCase):
+    def setUp(self) -> None:
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        self.real_home = os.path.join(tmp.name, "real-home")
+        os.makedirs(self.real_home)
+        with open(os.path.join(self.real_home, ".catstack.env"), "w", encoding="utf-8") as handle:
+            handle.write("CATSTACK_UNVERIFIED_TAG_BEHAVIOR=do_not_emit\n")
+        polluted = {
+            "PATH": os.environ.get("PATH", ""),
+            "HOME": self.real_home,
+            "CATSTACK_UNVERIFIED_TAG_BEHAVIOR": "do_not_emit",
+            "CATSTACK_ENV_FILE": os.path.join(self.real_home, ".catstack.env"),
+            "GIT_DIR": "/nowhere/.git",
+            "XDG_CONFIG_HOME": os.path.join(self.real_home, ".config"),
+        }
+        result = subprocess.run(
+            ["bash", "-c", 'source "$1" && python3 -c "$2" "$3"', "_", SETUP, PROBE, REPO],
+            env=polluted, capture_output=True, text=True, check=False,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.seen = json.loads(result.stdout)
+
+    def test_no_inherited_catstack_or_git_variable_survives(self) -> None:
+        env = self.seen["env"]
+        self.assertNotIn("CATSTACK_UNVERIFIED_TAG_BEHAVIOR", env)
+        self.assertNotIn("CATSTACK_ENV_FILE", env)
+        self.assertNotIn("GIT_DIR", env)
+        self.assertNotIn("XDG_CONFIG_HOME", env)
+
+    def test_home_and_git_config_are_throwaway(self) -> None:
+        env = self.seen["env"]
+        self.assertNotEqual(env["HOME"], self.real_home)
+        self.assertTrue(env["HOME"].startswith(tempfile.gettempdir()) or "catstack-test-home" in env["HOME"])
+        self.assertTrue(env["GIT_CONFIG_GLOBAL"].startswith(env["HOME"]))
+        self.assertEqual(env.get("GIT_CONFIG_NOSYSTEM"), "1")
+
+    def test_flag_reader_skips_the_real_settings_files(self) -> None:
+        candidates = [os.path.realpath(path) for path in self.seen["candidates"]]
+        self.assertNotIn(os.path.realpath(os.path.join(self.real_home, ".catstack.env")), candidates)
+        self.assertNotIn(os.path.realpath(os.path.join(REPO, ".env")), candidates)
+
+    def test_the_runner_sources_the_setup_before_any_suite(self) -> None:
+        with open(RUNNER, encoding="utf-8") as handle:
+            text = handle.read()
+        self.assertIn('source "$REPO_DIR/scripts/test/hermetic_env.sh"', text)
+        self.assertLess(text.index("hermetic_env.sh"), text.index("unittest discover"))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_run_all_tests.py

@@ -24,6 +24,7 @@
 # run_all_tests.sh shells out to this sibling before discovering suites, so a
 # fake repo that omits it tests a script that cannot run.
 TOOLCHAIN_SCRIPT = REPO_ROOT / "scripts" / "test" / "ensure_node_toolchain.sh"
+HERMETIC_SCRIPT = REPO_ROOT / "scripts" / "test" / "hermetic_env.sh"
 
 PASSING_SUITE = """import unittest
 
@@ -59,6 +60,7 @@ def _fake_repo(tmp: Path, *, package: dict | None, installed: list[str]) -> Path
     (tmp / "scripts" / "test").mkdir(parents=True)
     shutil.copy2(SCRIPT, tmp / "scripts" / "test" / "run_all_tests.sh")
     shutil.copy2(TOOLCHAIN_SCRIPT, tmp / "scripts" / "test" / "ensure_node_toolchain.sh")
+    shutil.copy2(HERMETIC_SCRIPT, tmp / "scripts" / "test" / "hermetic_env.sh")
     (tmp / "tests").mkdir()
     (tmp / "tests" / "test_trivial.py").write_text(PASSING_SUITE, encoding="utf-8")
     if package is not None:

tests/test_run_all_tests_empty_suite.py

@@ -19,6 +19,7 @@
 REPO_ROOT = Path(__file__).resolve().parents[1]
 SCRIPT = REPO_ROOT / "scripts" / "test" / "run_all_tests.sh"
 TOOLCHAIN_SCRIPT = REPO_ROOT / "scripts" / "test" / "ensure_node_toolchain.sh"
+HERMETIC_SCRIPT = REPO_ROOT / "scripts" / "test" / "hermetic_env.sh"
 
 PASSING_SUITE = """import unittest
 
@@ -40,6 +41,7 @@ def _fake_repo(root: Path, *, with_empty_suite: bool) -> Path:
     (root / "scripts" / "test").mkdir(parents=True)
     shutil.copy2(SCRIPT, root / "scripts" / "test" / "run_all_tests.sh")
     shutil.copy2(TOOLCHAIN_SCRIPT, root / "scripts" / "test" / "ensure_node_toolchain.sh")
+    shutil.copy2(HERMETIC_SCRIPT, root / "scripts" / "test" / "hermetic_env.sh")
     (root / "tests").mkdir()
     (root / "tests" / "test_trivial.py").write_text(PASSING_SUITE, encoding="utf-8")
     if with_empty_suite: