diff --git a/corpus/skills/cat-mode/scripts/update_fleet.sh b/corpus/skills/cat-mode/scripts/update_fleet.sh index 03cd24147..732359f43 100755 --- a/corpus/skills/cat-mode/scripts/update_fleet.sh +++ b/corpus/skills/cat-mode/scripts/update_fleet.sh @@ -1,6 +1,29 @@ #!/bin/bash set -uo pipefail +usage() { + cat <<'USAGE' +update_fleet.sh -- put every machine on one Invoker release and the current +catstack. + + update_fleet.sh [--version ] [--hosts ] [--skip-invoker] + [--skip-catstack] [--with-app] [--dry-run] + + --version release tag to install (default: newest daily-* release) + --hosts subset of remoteTargets ids (default: all of them) + --skip-invoker leave the Invoker CLI where it is + --skip-catstack leave the catstack checkout where it is + --with-app also replace /Applications/Invoker.app on the Mac. This + quits a running Invoker, the live owner on that machine. An + interrupted replace parks the live bundle; the run puts it + back, and so does the next run if it was killed outright. + --dry-run resolve versions and print the table; change nothing. + +Every host gets one row. A row that could not be checked says so; it never +reads as ok. Exit is non-zero if any row failed. +USAGE +} + REPO="${INVOKER_RELEASE_REPO:-Neko-Catpital-Labs/Invoker}" CONFIG="${INVOKER_CONFIG:-$HOME/.invoker/config.json}" APP_DIR="${INVOKER_APP_DIR:-/Applications}" @@ -17,22 +40,6 @@ FAILED=0 cleanup() { rm -rf "$WORK_DIR"; } trap cleanup EXIT -usage() { - cat <<'USAGE' -Put every machine on one Invoker release and the current catstack. - - update_fleet.sh [--version ] [--hosts ] [--skip-invoker] - [--skip-catstack] [--with-app] [--dry-run] - ---version release tag to install (default: newest daily-* release) ---hosts subset of remoteTargets ids (default: all of them) ---skip-invoker ---skip-catstack ---with-app also replace /Applications/Invoker.app on the Mac ---dry-run check every host and print the table; change nothing -USAGE -} - while [ "$#" -gt 0 ]; do case "$1" in --version) VERSION="${2:?--version needs a tag}"; shift 2 ;; @@ -156,18 +163,44 @@ local_invoker() { row ok local "invoker $before -> $after" "$link" } +app_version() { + defaults read "$APP_DIR/Invoker.app/Contents/Info.plist" CFBundleShortVersionString 2>/dev/null || echo none +} + +parked_app() { + local candidate + [ -d "$APP_DIR/Invoker.app" ] && return 0 + for candidate in "$APP_DIR"/Invoker.app.replacing.*; do + [ -d "$candidate" ] || continue + printf '%s' "$candidate" + return 0 + done + return 0 +} + restore_parked_app() { local backup="$1" - [ -d "$backup" ] || return 0 + [ -n "$backup" ] && [ -d "$backup" ] || return 2 rm -rf "$APP_DIR/Invoker.app" - mv "$backup" "$APP_DIR/Invoker.app" + mv "$backup" "$APP_DIR/Invoker.app" || return 1 + return 0 } local_app() { - local dmg mount app before after backup + local dmg mount app before after backup parked rc [ "$(uname -s)" = "Darwin" ] || { row skip local "app: not macOS" ""; return 0; } - before="$(defaults read "$APP_DIR/Invoker.app/Contents/Info.plist" CFBundleShortVersionString 2>/dev/null || echo none)" - if [ "$DRY_RUN" = 1 ]; then row ok local "app $before -> $RELEASE_VERSION (dry-run)" ""; return 0; fi + parked="$(parked_app)" + if [ "$DRY_RUN" = 1 ]; then + if [ -n "$parked" ]; then + row warn local "app: an interrupted run left $parked and no $APP_DIR/Invoker.app; a real run puts it back first (dry-run)" "$parked" + return 0 + fi + row ok local "app $(app_version) -> $RELEASE_VERSION (dry-run)" ""; return 0 + fi + if [ -n "$parked" ] && ! restore_parked_app "$parked"; then + row fail local "app: $APP_DIR/Invoker.app is missing and $parked could not be moved back" "$parked"; return 1 + fi + before="$(app_version)" case "$(uname -m)" in arm64) dmg="Invoker-$RELEASE_VERSION-arm64.dmg" ;; *) dmg="Invoker-$RELEASE_VERSION-x64.dmg" ;; @@ -188,25 +221,40 @@ local_app() { hdiutil detach "$mount" >/dev/null 2>&1 row fail local "app: could not move $APP_DIR/Invoker.app aside; nothing replaced" ""; return 1 fi + trap 'restore_parked_app "$APP_DIR/Invoker.app.replacing.$$"; exit 130' INT TERM HUP if ! cp -R "$app" "$APP_DIR/"; then hdiutil detach "$mount" >/dev/null 2>&1 rm -rf "$APP_DIR/Invoker.app" - if ! restore_parked_app "$backup"; then + restore_parked_app "$backup"; rc="$?" + trap - INT TERM HUP + if [ "$rc" = 1 ]; then row fail local "app: copy failed and the only bundle left is $backup" "$backup"; return 1 fi + if [ "$rc" = 2 ]; then + row fail local "app: copy failed and $APP_DIR has no Invoker.app to put back" ""; return 1 + fi row fail local "app $before unchanged: could not copy the new bundle into $APP_DIR" ""; return 1 fi hdiutil detach "$mount" >/dev/null 2>&1 - after="$(defaults read "$APP_DIR/Invoker.app/Contents/Info.plist" CFBundleShortVersionString 2>/dev/null || echo unknown)" + after="$(app_version)" if [ "$after" != "$RELEASE_VERSION" ]; then rm -rf "$APP_DIR/Invoker.app" - if ! restore_parked_app "$backup"; then + restore_parked_app "$backup"; rc="$?" + trap - INT TERM HUP + if [ "$rc" = 1 ]; then row fail local "app $before -> $after (wanted $RELEASE_VERSION); the only bundle left is $backup" "$backup"; return 1 fi + if [ "$rc" = 2 ]; then + row fail local "app $before -> $after (wanted $RELEASE_VERSION); $APP_DIR has no Invoker.app to put back" ""; return 1 + fi row fail local "app $before unchanged: copied bundle read $after (wanted $RELEASE_VERSION)" ""; return 1 fi rm -rf "$APP_DIR/Invoker.app.old" - if [ -d "$backup" ]; then mv "$backup" "$APP_DIR/Invoker.app.old"; fi + if [ -d "$backup" ] && ! mv "$backup" "$APP_DIR/Invoker.app.old"; then + trap - INT TERM HUP + row fail local "app $before -> $after, but the previous bundle is still parked at $backup" "$backup"; return 1 + fi + trap - INT TERM HUP row ok local "app $before -> $after" "relaunch it to restore the owner" } diff --git a/engine/hooks/llm-judge/inbox.py b/engine/hooks/llm-judge/inbox.py index 25778cae7..669ed910d 100644 --- a/engine/hooks/llm-judge/inbox.py +++ b/engine/hooks/llm-judge/inbox.py @@ -87,31 +87,46 @@ def _is_harness_row(data: dict) -> bool: return _is_tool_result_row(data) +def _reverse_lines(path: str): + size = os.path.getsize(path) + remainder = b"" + with open(path, "rb") as handle: + position = size + while position > 0: + step = min(1024 * 1024, position) + position -= step + handle.seek(position) + lines = (handle.read(step) + remainder).split(b"\n") + remainder = lines.pop(0) + yield from reversed(lines) + if remainder: + yield remainder + + def last_turn(transcript_path: str) -> tuple[str, str]: last_human = "" last_assistant = "" try: - with open(transcript_path, encoding="utf-8") as handle: - for line in handle: - line = line.strip() - if not line: - continue - try: - data = json.loads(line) - except ValueError: - continue - if not isinstance(data, dict): - continue - if _is_harness_row(data): - continue - text = _turn_text(data) - if not text.strip(): - continue - role = _turn_role(data) - if role == "user": - last_human = text - elif role == "assistant": - last_assistant = text + for raw in _reverse_lines(transcript_path): + line = raw.strip() + if not line: + continue + try: + data = json.loads(line) + except ValueError: + continue + if not isinstance(data, dict) or _is_harness_row(data): + continue + text = _turn_text(data) + if not text.strip(): + continue + role = _turn_role(data) + if role == "user" and not last_human: + last_human = text + elif role == "assistant" and not last_assistant: + last_assistant = text + if last_human and last_assistant: + break except OSError as exc: judge.log(f"inbox: could not read {transcript_path} for its last turn: {exc}") return last_human, last_assistant diff --git a/engine/hooks/llm-judge/tests/check_latency.py b/engine/hooks/llm-judge/tests/check_latency.py new file mode 100644 index 000000000..f54e85204 --- /dev/null +++ b/engine/hooks/llm-judge/tests/check_latency.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +import json +import os +import sys +import tempfile +import time + +LIB_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, LIB_DIR) + +import inbox # noqa: E402 + + +HISTORY_ROWS = 400_000 +INVOCATIONS = 4 +BUDGET_SECONDS = 5.0 + + +def main() -> int: + with tempfile.TemporaryDirectory() as work: + transcript = os.path.join(work, "session.jsonl") + with open(transcript, "w", encoding="utf-8") as handle: + for index in range(HISTORY_ROWS): + handle.write(json.dumps({ + "type": "assistant", + "message": {"role": "assistant", "content": f"historical reply {index}"}, + }) + "\n") + handle.write(json.dumps({ + "type": "user", + "message": {"role": "user", "content": "current request"}, + }) + "\n") + handle.write(json.dumps({ + "type": "assistant", + "message": {"role": "assistant", "content": "probably done"}, + }) + "\n") + + started = time.perf_counter() + for invocation in range(1, INVOCATIONS + 1): + invocation_started = time.perf_counter() + result = inbox.last_turn(transcript) + elapsed = time.perf_counter() - invocation_started + print(f"llm-judge.last_turn[{invocation}] elapsed={elapsed:.3f}s") + if result != ("current request", "probably done") or elapsed >= BUDGET_SECONDS: + return 1 + total = time.perf_counter() - started + print(f"cat-mode hook latency total elapsed={total:.3f}s budget={BUDGET_SECONDS:.1f}s") + return 0 if total < BUDGET_SECONDS else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_cat_mode.py b/tests/test_cat_mode.py index 5a71d86dc..8eca54c7b 100644 --- a/tests/test_cat_mode.py +++ b/tests/test_cat_mode.py @@ -703,6 +703,21 @@ def test_remote_install_does_not_let_install_sh_eat_the_script(self): source = handle.read() self.assertIn("./install.sh > /tmp/catstack-install.log 2>&1