From fcd263a7a4c5d1434965912e4c926d6baf62fe7c Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Thu, 24 Sep 2026 12:55:26 +0800 Subject: [PATCH] llm-judge: let codex use the account's configured model The codex runner pinned gpt-5.3-codex-spark, which a ChatGPT-account Codex login refuses, so every codex judge call exited 1. Drop -m so codex runs the model from ~/.codex/config.toml, like the investigate runner already does. Co-Authored-By: Claude Opus 5.5 (1M context) Change-Id: I6265302008c925a00186cf22a7b7e5f74426494b --- engine/hooks/llm-judge/README.md | 5 ++++- engine/hooks/llm-judge/judge.py | 2 +- engine/hooks/llm-judge/tests/test_judge.py | 7 +++++++ engine/hooks/wrong-check-reflect/README.md | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/engine/hooks/llm-judge/README.md b/engine/hooks/llm-judge/README.md index fe93a2e75..e4be571a1 100644 --- a/engine/hooks/llm-judge/README.md +++ b/engine/hooks/llm-judge/README.md @@ -76,7 +76,10 @@ the dictionary's `on_hit` text. `ask(prompt)` tries these in order and stops at the first one that answers: -1. **codex**: `codex exec --skip-git-repo-check -m gpt-5.3-codex-spark --sandbox read-only -c notify=[] PROMPT` +1. **codex**: `codex exec --skip-git-repo-check --sandbox read-only -c notify=[] PROMPT` + (no `-m`: codex runs the model set in `~/.codex/config.toml`, so the judge + uses a model the account can already call; a ChatGPT-account login refuses + API-only models) 2. **claude**: `claude -p --model haiku --settings '{"disableAllHooks": true}' PROMPT` 3. **cursor**: `cursor-agent -p --output-format text PROMPT` diff --git a/engine/hooks/llm-judge/judge.py b/engine/hooks/llm-judge/judge.py index 1a50cc74e..3fb4b1018 100644 --- a/engine/hooks/llm-judge/judge.py +++ b/engine/hooks/llm-judge/judge.py @@ -32,7 +32,7 @@ RUNNERS_ENV = "CATSTACK_LLM_JUDGE_RUNNERS" STATE_ENV = "CATSTACK_LLM_JUDGE_STATE_DIR" DEFAULT_RUNNERS = ( - ("codex", ["codex", "exec", "--skip-git-repo-check", "-m", "gpt-5.3-codex-spark", "--sandbox", "read-only", "-c", "notify=[]", PROMPT_SLOT]), + ("codex", ["codex", "exec", "--skip-git-repo-check", "--sandbox", "read-only", "-c", "notify=[]", PROMPT_SLOT]), ("claude", ["claude", "-p", "--model", "haiku", "--settings", '{"disableAllHooks": true}', PROMPT_SLOT]), ("cursor", ["cursor-agent", "-p", "--output-format", "text", PROMPT_SLOT]), ) diff --git a/engine/hooks/llm-judge/tests/test_judge.py b/engine/hooks/llm-judge/tests/test_judge.py index 5d083d3d8..ae164766f 100644 --- a/engine/hooks/llm-judge/tests/test_judge.py +++ b/engine/hooks/llm-judge/tests/test_judge.py @@ -179,6 +179,13 @@ def test_default_runner_order_is_codex_then_claude_then_cursor(self): os.environ.pop(judge.RUNNERS_ENV) self.assertEqual([name for name, _ in judge.runners()], ["codex", "claude", "cursor"]) + def test_default_codex_runner_uses_the_account_model_not_a_pinned_one(self): + with patch.dict(os.environ): + os.environ.pop(judge.RUNNERS_ENV) + codex_argv = dict(judge.runners())["codex"] + self.assertNotIn("-m", codex_argv) + self.assertNotIn("--model", codex_argv) + def test_investigate_runner_argv_is_read_only_and_excludes_cursor(self): os.environ.pop(judge.RUNNERS_ENV) self.assertEqual( diff --git a/engine/hooks/wrong-check-reflect/README.md b/engine/hooks/wrong-check-reflect/README.md index 7d8e92b9d..5fa53e40a 100644 --- a/engine/hooks/wrong-check-reflect/README.md +++ b/engine/hooks/wrong-check-reflect/README.md @@ -56,7 +56,7 @@ window left the `/reflect` typed beside it outside. Inside a judge run (`CATSTAC refuses the job. The model call runs in a detached background process, so the reply is never -held up. Runners are tried in `llm-judge` order: `codex` (gpt-5.3-codex-spark), +held up. Runners are tried in `llm-judge` order: `codex` (the account's configured model), then `claude` (haiku, hooks off), then `cursor-agent`, first answer wins. The verdict reports one turn later. On the next prompt the `llm-judge` inbox