diff --git a/corpus/skills/cat-mode/scripts/update_fleet.sh b/corpus/skills/cat-mode/scripts/update_fleet.sh index a380e69ac..732359f43 100755 --- a/corpus/skills/cat-mode/scripts/update_fleet.sh +++ b/corpus/skills/cat-mode/scripts/update_fleet.sh @@ -1,6 +1,29 @@ #!/bin/bash set -uo pipefail +usage() { + cat <<'USAGE' +update_fleet.sh -- put every machine on one Invoker release and the current +catstack. + + update_fleet.sh [--version ] [--hosts ] [--skip-invoker] + [--skip-catstack] [--with-app] [--dry-run] + + --version release tag to install (default: newest daily-* release) + --hosts subset of remoteTargets ids (default: all of them) + --skip-invoker leave the Invoker CLI where it is + --skip-catstack leave the catstack checkout where it is + --with-app also replace /Applications/Invoker.app on the Mac. This + quits a running Invoker, the live owner on that machine. An + interrupted replace parks the live bundle; the run puts it + back, and so does the next run if it was killed outright. + --dry-run resolve versions and print the table; change nothing. + +Every host gets one row. A row that could not be checked says so; it never +reads as ok. Exit is non-zero if any row failed. +USAGE +} + REPO="${INVOKER_RELEASE_REPO:-Neko-Catpital-Labs/Invoker}" CONFIG="${INVOKER_CONFIG:-$HOME/.invoker/config.json}" APP_DIR="${INVOKER_APP_DIR:-/Applications}" @@ -17,22 +40,6 @@ FAILED=0 cleanup() { rm -rf "$WORK_DIR"; } trap cleanup EXIT -usage() { - cat <<'USAGE' -Put every machine on one Invoker release and the current catstack. - - update_fleet.sh [--version ] [--hosts ] [--skip-invoker] - [--skip-catstack] [--with-app] [--dry-run] - ---version release tag to install (default: newest daily-* release) ---hosts subset of remoteTargets ids (default: all of them) ---skip-invoker ---skip-catstack ---with-app also replace /Applications/Invoker.app on the Mac ---dry-run check every host and print the table; change nothing -USAGE -} - while [ "$#" -gt 0 ]; do case "$1" in --version) VERSION="${2:?--version needs a tag}"; shift 2 ;; @@ -55,25 +62,28 @@ need() { } need curl need python3 -need gh -if [ -z "$VERSION" ]; then - VERSION="$(gh release list --repo "$REPO" --limit 20 2>/dev/null \ - | awk '$0 ~ /daily-/ {for (i=1;i<=NF;i++) if ($i ~ /^daily-[0-9]+$/) {print $i; exit}}')" +resolve_release() { + need gh if [ -z "$VERSION" ]; then - echo "fail could not resolve the newest daily-* release from $REPO" >&2 - exit 1 + VERSION="$(gh release list --repo "$REPO" --limit 20 2>/dev/null \ + | awk '$0 ~ /daily-/ {for (i=1;i<=NF;i++) if ($i ~ /^daily-[0-9]+$/) {print $i; exit}}')" + if [ -z "$VERSION" ]; then + echo "fail could not resolve the newest daily-* release from $REPO" >&2 + exit 1 + fi fi -fi -echo "release $VERSION (repo $REPO)" + echo "release $VERSION (repo $REPO)" -RELEASE_VERSION="$(gh release view "$VERSION" --repo "$REPO" --json assets \ - -q '[.assets[].name | capture("invoker-cli-(?[0-9][^-]*)-") .v] | first' 2>/dev/null)" -if [ -z "$RELEASE_VERSION" ]; then - echo "fail release $VERSION has no invoker-cli asset to read a version from" >&2 - exit 1 -fi -echo "version $RELEASE_VERSION" + RELEASE_VERSION="$(gh release view "$VERSION" --repo "$REPO" --json assets \ + -q '[.assets[].name | capture("invoker-cli-(?[0-9][^-]*)-") .v] | first' 2>/dev/null)" + if [ -z "$RELEASE_VERSION" ]; then + echo "fail release $VERSION has no invoker-cli asset to read a version from" >&2 + exit 1 + fi + echo "version $RELEASE_VERSION" +} +[ "$DO_INVOKER" = 1 ] && resolve_release targets() { python3 - "$CONFIG" "$HOSTS" <<'PY' @@ -85,14 +95,18 @@ except OSError as exc: sys.exit(f"cannot read {path}: {exc}") except ValueError as exc: sys.exit(f"{path} is not valid JSON: {exc}") -keep = {h for h in wanted.split(",") if h} if wanted else None -for tid, t in (cfg.get("remoteTargets") or {}).items(): +keep = [h for h in wanted.split(",") if h] if wanted else None +known = cfg.get("remoteTargets") or {} +for tid, t in known.items(): if keep and tid not in keep: continue host, user = t.get("host"), t.get("user") if not host or not user: sys.exit(f"remoteTarget {tid} has no host/user") print(f"{tid}\t{user}\t{host}") +for tid in keep or []: + if tid not in known: + print(f"{tid}\t-\t-") PY } @@ -149,18 +163,44 @@ local_invoker() { row ok local "invoker $before -> $after" "$link" } +app_version() { + defaults read "$APP_DIR/Invoker.app/Contents/Info.plist" CFBundleShortVersionString 2>/dev/null || echo none +} + +parked_app() { + local candidate + [ -d "$APP_DIR/Invoker.app" ] && return 0 + for candidate in "$APP_DIR"/Invoker.app.replacing.*; do + [ -d "$candidate" ] || continue + printf '%s' "$candidate" + return 0 + done + return 0 +} + restore_parked_app() { local backup="$1" - [ -d "$backup" ] || return 0 + [ -n "$backup" ] && [ -d "$backup" ] || return 2 rm -rf "$APP_DIR/Invoker.app" - mv "$backup" "$APP_DIR/Invoker.app" + mv "$backup" "$APP_DIR/Invoker.app" || return 1 + return 0 } local_app() { - local dmg mount app before after backup + local dmg mount app before after backup parked rc [ "$(uname -s)" = "Darwin" ] || { row skip local "app: not macOS" ""; return 0; } - before="$(defaults read "$APP_DIR/Invoker.app/Contents/Info.plist" CFBundleShortVersionString 2>/dev/null || echo none)" - if [ "$DRY_RUN" = 1 ]; then row ok local "app $before -> $RELEASE_VERSION (dry-run)" ""; return 0; fi + parked="$(parked_app)" + if [ "$DRY_RUN" = 1 ]; then + if [ -n "$parked" ]; then + row warn local "app: an interrupted run left $parked and no $APP_DIR/Invoker.app; a real run puts it back first (dry-run)" "$parked" + return 0 + fi + row ok local "app $(app_version) -> $RELEASE_VERSION (dry-run)" ""; return 0 + fi + if [ -n "$parked" ] && ! restore_parked_app "$parked"; then + row fail local "app: $APP_DIR/Invoker.app is missing and $parked could not be moved back" "$parked"; return 1 + fi + before="$(app_version)" case "$(uname -m)" in arm64) dmg="Invoker-$RELEASE_VERSION-arm64.dmg" ;; *) dmg="Invoker-$RELEASE_VERSION-x64.dmg" ;; @@ -181,25 +221,40 @@ local_app() { hdiutil detach "$mount" >/dev/null 2>&1 row fail local "app: could not move $APP_DIR/Invoker.app aside; nothing replaced" ""; return 1 fi + trap 'restore_parked_app "$APP_DIR/Invoker.app.replacing.$$"; exit 130' INT TERM HUP if ! cp -R "$app" "$APP_DIR/"; then hdiutil detach "$mount" >/dev/null 2>&1 rm -rf "$APP_DIR/Invoker.app" - if ! restore_parked_app "$backup"; then + restore_parked_app "$backup"; rc="$?" + trap - INT TERM HUP + if [ "$rc" = 1 ]; then row fail local "app: copy failed and the only bundle left is $backup" "$backup"; return 1 fi + if [ "$rc" = 2 ]; then + row fail local "app: copy failed and $APP_DIR has no Invoker.app to put back" ""; return 1 + fi row fail local "app $before unchanged: could not copy the new bundle into $APP_DIR" ""; return 1 fi hdiutil detach "$mount" >/dev/null 2>&1 - after="$(defaults read "$APP_DIR/Invoker.app/Contents/Info.plist" CFBundleShortVersionString 2>/dev/null || echo unknown)" + after="$(app_version)" if [ "$after" != "$RELEASE_VERSION" ]; then rm -rf "$APP_DIR/Invoker.app" - if ! restore_parked_app "$backup"; then + restore_parked_app "$backup"; rc="$?" + trap - INT TERM HUP + if [ "$rc" = 1 ]; then row fail local "app $before -> $after (wanted $RELEASE_VERSION); the only bundle left is $backup" "$backup"; return 1 fi + if [ "$rc" = 2 ]; then + row fail local "app $before -> $after (wanted $RELEASE_VERSION); $APP_DIR has no Invoker.app to put back" ""; return 1 + fi row fail local "app $before unchanged: copied bundle read $after (wanted $RELEASE_VERSION)" ""; return 1 fi rm -rf "$APP_DIR/Invoker.app.old" - if [ -d "$backup" ]; then mv "$backup" "$APP_DIR/Invoker.app.old"; fi + if [ -d "$backup" ] && ! mv "$backup" "$APP_DIR/Invoker.app.old"; then + trap - INT TERM HUP + row fail local "app $before -> $after, but the previous bundle is still parked at $backup" "$backup"; return 1 + fi + trap - INT TERM HUP row ok local "app $before -> $after" "relaunch it to restore the owner" } @@ -234,6 +289,7 @@ else fi rm -f "$ASSET" echo "VERSION=$("$DIR/invoker-cli" --version 2>/dev/null || echo none)" +echo "PATH_VERSION=$(invoker-cli --version 2>/dev/null || echo none)" PAYLOAD cat > "$WORK_DIR/remote_catstack.sh" <<'PAYLOAD' @@ -286,7 +342,7 @@ PAYLOAD } remote_invoker() { - local id="$1" dest="$2" asset tarball out before arch after + local id="$1" dest="$2" asset tarball out before arch after path_version before="$(ssh_to "$dest" 'invoker-cli --version 2>/dev/null || echo none' /dev/null || echo unreachable)" if [ -z "$before" ] || [ "$before" = "unreachable" ]; then row fail "$id" "ssh failed; version unchecked" ""; return 1 @@ -307,6 +363,10 @@ remote_invoker() { if [ "$after" != "$RELEASE_VERSION" ]; then row fail "$id" "invoker $before -> ${after:-unreadable} (wanted $RELEASE_VERSION): ${out##*$'\n'}" ""; return 1 fi + path_version="$(printf '%s' "$out" | sed -n 's/^PATH_VERSION=//p')" + if [ "$path_version" != "$RELEASE_VERSION" ]; then + row warn "$id" "invoker $before -> $after in ~/.local/bin, but not on the ssh PATH (it runs ${path_version:-nothing}); no passwordless sudo for /usr/bin" ""; return 0 + fi row ok "$id" "invoker $before -> $after" "" } @@ -359,6 +419,9 @@ write_payloads while IFS=$'\t' read -r id user host; do [ -n "$id" ] || continue + if [ "$host" = "-" ]; then + row fail "$id" "not in remoteTargets of $CONFIG; unchecked" ""; continue + fi [ "$DO_INVOKER" = 1 ] && remote_invoker "$id" "$user@$host" [ "$DO_CATSTACK" = 1 ] && catstack_on "$id" "$user@$host" done <<< "$TARGET_LIST" diff --git a/tests/test_cat_mode.py b/tests/test_cat_mode.py index b9dc260f6..03951e933 100644 --- a/tests/test_cat_mode.py +++ b/tests/test_cat_mode.py @@ -658,6 +658,21 @@ def test_remote_install_does_not_let_install_sh_eat_the_script(self): source = handle.read() self.assertIn("./install.sh > /tmp/catstack-install.log 2>&1 &2\nexit 1\n") + write_stub(bin_dir, "scp", "exit 1\n") + write_stub(bin_dir, "ssh", "exit 255\n") + config = os.path.join(tmp, "config.json") + with open(config, "w", encoding="utf-8") as handle: + json.dump({"remoteTargets": targets}, handle) + env = dict(os.environ) + env.update(PATH=bin_dir + os.pathsep + env["PATH"], HOME=home, INVOKER_CONFIG=config) + return subprocess.run( + ["bash", script_path] + args, capture_output=True, text=True, env=env + ) + + +class TestFleetFlagsDoWhatTheySay(unittest.TestCase): + """--skip-invoker has to keep a catstack-only run away from the Invoker + release lookup, and --hosts has to account for every id it was handed.""" + + SCRIPT = os.path.join( + REPO_ROOT, "corpus", "skills", "cat-mode", "scripts", "update_fleet.sh" + ) + TARGETS = {"hostA": {"host": "10.0.0.1", "user": "me"}} + + def setUp(self): + import shutil + import tempfile + + self.tmp = tempfile.mkdtemp(prefix="fleet-flags-") + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + + def test_skip_invoker_never_needs_a_release(self): + out = run_fleet(self.SCRIPT, self.tmp, ["--skip-invoker"], self.TARGETS) + + self.assertNotIn("daily-*", out.stderr) + self.assertNotIn("invoker-cli asset", out.stderr) + self.assertIn("STATUS", out.stdout, out.stderr) + self.assertRegex(out.stdout, r"fail\s+hostA\s+catstack: scp") + + def test_an_unknown_host_id_gets_a_fail_row(self): + out = run_fleet( + self.SCRIPT, self.tmp, ["--skip-invoker", "--hosts", "hostA,hostTypo"], + self.TARGETS, + ) + + self.assertRegex(out.stdout, r"fail\s+hostTypo\s+not in remoteTargets", out.stderr) + self.assertRegex(out.stdout, r"fail\s+hostA\s+") + self.assertEqual(out.returncode, 1, out.stdout + out.stderr) + + def test_only_unknown_host_ids_still_fail_each_by_name(self): + out = run_fleet( + self.SCRIPT, self.tmp, ["--skip-invoker", "--skip-catstack", "--hosts", "nope"], + self.TARGETS, + ) + + self.assertRegex(out.stdout, r"fail\s+nope\s+not in remoteTargets", out.stderr) + self.assertEqual(out.returncode, 1) + + +REMOTE_INVOKER_HARNESS = """set -uo pipefail +WORK_DIR="$TEST_WORK_DIR" +RELEASE_VERSION="9.9.9" +DRY_RUN=0 +row() {{ printf '%s\\t%s\\t%s\\n' "$1" "$2" "$3" >> "$TEST_ROWS"; return 0; }} +{functions} +fetch_asset() {{ printf '%s' "$TEST_TARBALL"; }} +ssh_to() {{ shift; local cmd="$*"; env -i HOME="$TEST_REMOTE_HOME" PATH="$TEST_SSH_PATH" bash -c "${{cmd//\\/tmp\\//$TEST_REMOTE_TMP/}}"; }} +scp() {{ local a; for a in "$@"; do case "$a" in -*|BatchMode=*|ConnectTimeout=*|*:/tmp/) ;; *) cp "$a" "$TEST_REMOTE_TMP/" ;; esac; done; }} +write_payloads +remote_invoker hostA me@hostA +echo "RC=$?" +""" + + +def run_remote_invoker(script_path, tmp, sudo_ok, local_bin_on_path=False): + """Run update_fleet.sh's remote_invoker step against a fake remote: `ssh_to` + runs the command locally under a clean non-interactive environment (no + .bashrc) with the fake remote HOME, the same PATH a real `ssh host cmd` + gets when nothing adds ~/.local/bin.""" + import subprocess + import tarfile + + bin_dir = os.path.join(tmp, "ssh-bin") + remote_home = os.path.join(tmp, "remote-home") + remote_tmp = os.path.join(tmp, "remote-tmp") + work_dir = os.path.join(tmp, "work") + for d in (bin_dir, remote_home, remote_tmp, work_dir): + os.makedirs(d, exist_ok=True) + write_stub(bin_dir, "sudo", "exit 0\n" if sudo_ok else "exit 1\n") + write_stub(bin_dir, "uname", "echo x86_64\n") + + pkg = os.path.join(tmp, "pkg", "invoker-cli-9.9.9-linux-x64") + os.makedirs(pkg, exist_ok=True) + write_stub(pkg, "invoker-cli", "echo 9.9.9\n") + tarball = os.path.join(work_dir, "invoker-cli-9.9.9-linux-x64.tar.gz") + with tarfile.open(tarball, "w:gz") as tar: + tar.add(pkg, arcname="invoker-cli-9.9.9-linux-x64") + + with open(script_path, encoding="utf-8") as handle: + source = handle.read() + match = CATSTACK_FUNCTIONS.search(source) + if match is None: + raise AssertionError("could not slice the payload section out of the script") + harness = os.path.join(tmp, "remote-invoker-harness.sh") + with open(harness, "w", encoding="utf-8") as handle: + handle.write(REMOTE_INVOKER_HARNESS.format(functions=match.group(0))) + + rows = os.path.join(tmp, "rows.tsv") + open(rows, "w").close() + env = dict(os.environ) + env.update( + TEST_WORK_DIR=work_dir, + TEST_ROWS=rows, + TEST_TARBALL=tarball, + TEST_REMOTE_HOME=remote_home, + TEST_REMOTE_TMP=remote_tmp, + TEST_SSH_PATH=( + os.path.join(remote_home, ".local", "bin") + ":" if local_bin_on_path else "" + ) + bin_dir + ":/usr/bin:/bin", + ) + out = subprocess.run(["bash", harness], capture_output=True, text=True, env=env) + with open(rows, encoding="utf-8") as handle: + row = handle.read().strip() + return row, out + + +class TestRemoteInstallOffTheSshPathIsNotOk(unittest.TestCase): + """Without passwordless sudo the CLI lands in ~/.local/bin, which a + non-interactive `ssh host cmd` may never put on PATH. The row must say so + instead of reading ok because the binary answered by its full path.""" + + SCRIPT = os.path.join( + REPO_ROOT, "corpus", "skills", "cat-mode", "scripts", "update_fleet.sh" + ) + + def setUp(self): + import shutil + import tempfile + + self.tmp = tempfile.mkdtemp(prefix="fleet-remote-") + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + + def test_no_sudo_install_off_the_ssh_path_is_a_warn_row(self): + row, out = run_remote_invoker(self.SCRIPT, self.tmp, sudo_ok=False) + + self.assertTrue(row.startswith("warn\t"), f"row was {row!r}\n{out.stdout}{out.stderr}") + self.assertIn("not on the ssh PATH", row) + + def test_an_install_the_ssh_path_reaches_is_ok(self): + row, out = run_remote_invoker( + self.SCRIPT, self.tmp, sudo_ok=False, local_bin_on_path=True + ) + + self.assertTrue(row.startswith("ok\t"), f"row was {row!r}\n{out.stdout}{out.stderr}") + self.assertIn("invoker none -> 9.9.9", row) + + REFERENCE_DIR = os.path.join(REPO_ROOT, "corpus", "skills", "cat-mode", "references")