From ce795dcf5ff6b6c8ba0e60eaff59f84250c36830 Mon Sep 17 00:00:00 2001 From: Exotic209093 <134711311+Exotic209093@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:31:39 +0100 Subject: [PATCH 1/3] fix(server): clear systemd start limits before repairing services --- apps/server/src/cloud/bootService.test.ts | 100 +++++++++++++++++++++- apps/server/src/cloud/bootService.ts | 7 ++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 65bf0818a970..70759882c142 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -150,11 +150,15 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( linger: string; enabled: boolean; active: boolean; + startLimitHit: boolean; + resetFailed: boolean; } = { failCommand: undefined, linger: "yes", enabled: true, active: true, + startLimitHit: false, + resetFailed: false, }; const runner = ProcessRunner.ProcessRunner.of({ run: Effect.fn("test.run_boot_service_command")(function* ( @@ -167,7 +171,17 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( if (!failed && command === "loginctl enable-linger --no-ask-password 501") control.linger = "yes"; if (!failed && command === "systemctl --user enable t3code.service") control.enabled = true; - if (!failed && command === "systemctl --user restart t3code.service") control.active = true; + if (!failed && command === "systemctl --user reset-failed t3code.service") { + control.startLimitHit = false; + control.resetFailed = true; + } + if ( + !failed && + command === "systemctl --user restart t3code.service" && + !control.startLimitHit + ) { + control.active = true; + } if ( control.stateAfterStop !== undefined && (command === "systemctl --user stop t3code.service" || @@ -190,7 +204,11 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( : "", stderr: "", code: ChildProcessSpawner.ExitCode( - failed || (input.args[1] === "is-active" && !control.active) ? 1 : 0, + failed || + (command === "systemctl --user restart t3code.service" && control.startLimitHit) || + (input.args[1] === "is-active" && !control.active) + ? 1 + : 0, ), timedOut: false, stdoutTruncated: false, @@ -576,6 +594,84 @@ it.layer(NodeServices.layer)("boot service install", (it) => { "systemctl --user stop t3code.service", "systemctl --user daemon-reload", "systemctl --user enable t3code.service", + "systemctl --user reset-failed t3code.service", + "systemctl --user restart t3code.service", + ]); + }), + ); + + it.effect("clears a systemd start-limit before restarting the owned service", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness(); + yield* service.install(); + commands.length = 0; + control.active = false; + control.startLimitHit = true; + control.resetFailed = false; + + expect(yield* service.restart).toBe(true); + expect(control.resetFailed).toBe(true); + expect(control.startLimitHit).toBe(false); + expect(control.active).toBe(true); + expect( + commands.filter( + (command) => command.startsWith("systemctl ") && !command.includes("show-environment"), + ), + ).toEqual([ + "systemctl --user stop t3code.service", + "systemctl --user daemon-reload", + "systemctl --user enable t3code.service", + "systemctl --user reset-failed t3code.service", + "systemctl --user restart t3code.service", + ]); + }), + ); + + it.effect("clears a parked systemd service when installing with start=true", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness(); + yield* service.install(); + commands.length = 0; + control.active = false; + control.startLimitHit = true; + control.resetFailed = false; + + yield* service.install({ start: true }); + + expect(control.resetFailed).toBe(true); + expect(control.startLimitHit).toBe(false); + expect(control.active).toBe(true); + expect( + commands.filter( + (command) => command.startsWith("systemctl ") && !command.includes("show-environment"), + ), + ).toEqual([ + "systemctl --user stop t3code.service", + "systemctl --user daemon-reload", + "systemctl --user enable t3code.service", + "systemctl --user reset-failed t3code.service", + "systemctl --user restart t3code.service", + ]); + }), + ); + + it.effect("reports a reset-failed command error and still attempts recovery", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness(); + yield* service.install(); + commands.length = 0; + control.failCommand = "systemctl --user reset-failed t3code.service"; + + const error = yield* service.restart.pipe(Effect.flip); + expect(error).toMatchObject({ + _tag: "BootServiceCommandError", + step: "clearing the failed service state", + }); + expect(commands).toEqual([ + "systemctl --user stop t3code.service", + "systemctl --user daemon-reload", + "systemctl --user enable t3code.service", + "systemctl --user reset-failed t3code.service", "systemctl --user restart t3code.service", ]); }), diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 1d7d50b13185..3a70474c15c2 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -272,6 +272,13 @@ function systemdManager(input: { command: "systemctl", args: ["--user", "enable", BOOT_SERVICE_UNIT_FILE], }, + // A unit parked at start-limit-hit cannot be restarted until systemd's + // failed state is cleared. Keep this scoped to our own service. + { + step: "clearing the failed service state", + command: "systemctl", + args: ["--user", "reset-failed", BOOT_SERVICE_UNIT_FILE], + }, // Start last. No administrative state write occurs after this succeeds. { step: "starting the service", From 2643c8027c396797268ed90c05901f2a9466f254 Mon Sep 17 00:00:00 2001 From: Exotic209093 <134711311+Exotic209093@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:53:36 +0100 Subject: [PATCH 2/3] fix(server): reset failed state only for existing services --- apps/server/src/cloud/bootService.test.ts | 24 +++++++++++++++++----- apps/server/src/cloud/bootService.ts | 25 +++++++++++++++-------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 70759882c142..ab2a9bb343e1 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -416,6 +416,20 @@ it.layer(NodeServices.layer)("boot service install", (it) => { }), ); + it.effect("does not reset an un-loaded systemd unit on a fresh install", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness(); + // systemd can reject reset-failed before a fresh unit has been loaded; + // a first install must not issue that command at all. + control.failCommand = "systemctl --user reset-failed t3code.service"; + + yield* service.install(); + + expect(control.active).toBe(true); + expect(commands).not.toContain("systemctl --user reset-failed t3code.service"); + }), + ); + it.effect("reports an unknown version for invalid service state", () => Effect.gen(function* () { const { service, fs, statePath } = yield* makeHarness(); @@ -592,9 +606,9 @@ it.layer(NodeServices.layer)("boot service install", (it) => { ), ).toEqual([ "systemctl --user stop t3code.service", + "systemctl --user reset-failed t3code.service", "systemctl --user daemon-reload", "systemctl --user enable t3code.service", - "systemctl --user reset-failed t3code.service", "systemctl --user restart t3code.service", ]); }), @@ -619,9 +633,9 @@ it.layer(NodeServices.layer)("boot service install", (it) => { ), ).toEqual([ "systemctl --user stop t3code.service", + "systemctl --user reset-failed t3code.service", "systemctl --user daemon-reload", "systemctl --user enable t3code.service", - "systemctl --user reset-failed t3code.service", "systemctl --user restart t3code.service", ]); }), @@ -647,9 +661,9 @@ it.layer(NodeServices.layer)("boot service install", (it) => { ), ).toEqual([ "systemctl --user stop t3code.service", + "systemctl --user reset-failed t3code.service", "systemctl --user daemon-reload", "systemctl --user enable t3code.service", - "systemctl --user reset-failed t3code.service", "systemctl --user restart t3code.service", ]); }), @@ -669,8 +683,6 @@ it.layer(NodeServices.layer)("boot service install", (it) => { }); expect(commands).toEqual([ "systemctl --user stop t3code.service", - "systemctl --user daemon-reload", - "systemctl --user enable t3code.service", "systemctl --user reset-failed t3code.service", "systemctl --user restart t3code.service", ]); @@ -706,6 +718,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { ), ).toEqual([ "systemctl --user stop t3code.service", + "systemctl --user reset-failed t3code.service", "systemctl --user daemon-reload", "systemctl --user restart t3code.service", ]); @@ -727,6 +740,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { ), ).toEqual([ "systemctl --user stop t3code.service", + "systemctl --user reset-failed t3code.service", "systemctl --user daemon-reload", "systemctl --user restart t3code.service", ]); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 3a70474c15c2..de10402140f7 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -228,6 +228,8 @@ export interface BootServiceManager { readonly render: (plan: BootServicePlan) => string; /** Before rewriting files, when a unit is already installed. */ readonly stop: ReadonlyArray; + /** Clear manager failure state before starting an existing unit. */ + readonly resetFailed: ReadonlyArray; /** After files are written. The last entry starts the service. */ readonly activate: ReadonlyArray; /** Best-effort recovery after a failed repair of an installed service. */ @@ -261,6 +263,13 @@ function systemdManager(input: { timeout: STOP_STEP_TIMEOUT, }, ], + resetFailed: [ + { + step: "clearing the failed service state", + command: "systemctl", + args: ["--user", "reset-failed", BOOT_SERVICE_UNIT_FILE], + }, + ], activate: [ { step: "reloading systemd user units", @@ -272,13 +281,6 @@ function systemdManager(input: { command: "systemctl", args: ["--user", "enable", BOOT_SERVICE_UNIT_FILE], }, - // A unit parked at start-limit-hit cannot be restarted until systemd's - // failed state is cleared. Keep this scoped to our own service. - { - step: "clearing the failed service state", - command: "systemctl", - args: ["--user", "reset-failed", BOOT_SERVICE_UNIT_FILE], - }, // Start last. No administrative state write occurs after this succeeds. { step: "starting the service", @@ -353,6 +355,7 @@ function launchdManager(input: { timeout: STOP_STEP_TIMEOUT, }, ], + resetFailed: [], activate: [ // A persisted `launchctl disable` override refuses bootstrap; clear it. { @@ -898,6 +901,9 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { yield* writeDurably(unitPath, manager.render(plan)); if (start) { + if (installed) { + yield* runSteps(manager.resetFailed); + } yield* runSteps(manager.activate); yield* fs.remove(restartPendingPath, { force: true }); } @@ -924,7 +930,10 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { return false; } yield* runSteps(manager.stop); - yield* runSteps(manager.activate).pipe( + yield* Effect.gen(function* () { + yield* runSteps(manager.resetFailed); + yield* runSteps(manager.activate); + }).pipe( // Same recovery as a failed repair: a service that was running should // not be left stopped because daemon-reload or enable failed. Effect.tapError(() => runSteps(manager.restart).pipe(Effect.ignore)), From b59ca8d25a9679be84528048a7ffc8df72502e2c Mon Sep 17 00:00:00 2001 From: Exotic209093 <134711311+Exotic209093@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:29:54 +0100 Subject: [PATCH 3/3] fix(server): reset only loaded systemd service units --- apps/server/src/cloud/bootService.test.ts | 67 ++++++++++++++++++++--- apps/server/src/cloud/bootService.ts | 5 +- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index ab2a9bb343e1..4d1c7aae9fe0 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -152,6 +152,8 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( active: boolean; startLimitHit: boolean; resetFailed: boolean; + unitLoaded: boolean; + unitLoadedAfterStop: boolean; } = { failCommand: undefined, linger: "yes", @@ -159,6 +161,8 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( active: true, startLimitHit: false, resetFailed: false, + unitLoaded: true, + unitLoadedAfterStop: true, }; const runner = ProcessRunner.ProcessRunner.of({ run: Effect.fn("test.run_boot_service_command")(function* ( @@ -168,10 +172,16 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( commands.push(command); timeouts.set(command, input.timeout); const failed = command === control.failCommand; + const resetFailedForUnloadedUnit = + command === "systemctl --user reset-failed t3code.service" && !control.unitLoaded; if (!failed && command === "loginctl enable-linger --no-ask-password 501") control.linger = "yes"; if (!failed && command === "systemctl --user enable t3code.service") control.enabled = true; - if (!failed && command === "systemctl --user reset-failed t3code.service") { + if ( + !failed && + command === "systemctl --user reset-failed [t]3code.service" && + control.unitLoaded + ) { control.startLimitHit = false; control.resetFailed = true; } @@ -181,6 +191,10 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( !control.startLimitHit ) { control.active = true; + control.unitLoaded = true; + } + if (command === "systemctl --user stop t3code.service") { + control.unitLoaded = control.unitLoadedAfterStop; } if ( control.stateAfterStop !== undefined && @@ -205,6 +219,7 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( stderr: "", code: ChildProcessSpawner.ExitCode( failed || + resetFailedForUnloadedUnit || (command === "systemctl --user restart t3code.service" && control.startLimitHit) || (input.args[1] === "is-active" && !control.active) ? 1 @@ -421,11 +436,45 @@ it.layer(NodeServices.layer)("boot service install", (it) => { const { service, commands, control } = yield* makeHarness(); // systemd can reject reset-failed before a fresh unit has been loaded; // a first install must not issue that command at all. - control.failCommand = "systemctl --user reset-failed t3code.service"; + control.failCommand = "systemctl --user reset-failed [t]3code.service"; yield* service.install(); expect(control.active).toBe(true); + expect(commands).not.toContain("systemctl --user reset-failed [t]3code.service"); + }), + ); + + it.effect("repairs an installed service when systemd unloads it after stop", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness(); + yield* service.install(); + commands.length = 0; + control.active = false; + control.unitLoadedAfterStop = false; + + yield* service.install(); + + expect(control.active).toBe(true); + expect(control.resetFailed).toBe(false); + expect(commands).toContain("systemctl --user reset-failed [t]3code.service"); + expect(commands).not.toContain("systemctl --user reset-failed t3code.service"); + }), + ); + + it.effect("restarts an installed service when systemd unloads it after stop", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness(); + yield* service.install(); + commands.length = 0; + control.active = false; + control.unitLoadedAfterStop = false; + + expect(yield* service.restart).toBe(true); + + expect(control.active).toBe(true); + expect(control.resetFailed).toBe(false); + expect(commands).toContain("systemctl --user reset-failed [t]3code.service"); expect(commands).not.toContain("systemctl --user reset-failed t3code.service"); }), ); @@ -606,7 +655,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { ), ).toEqual([ "systemctl --user stop t3code.service", - "systemctl --user reset-failed t3code.service", + "systemctl --user reset-failed [t]3code.service", "systemctl --user daemon-reload", "systemctl --user enable t3code.service", "systemctl --user restart t3code.service", @@ -633,7 +682,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { ), ).toEqual([ "systemctl --user stop t3code.service", - "systemctl --user reset-failed t3code.service", + "systemctl --user reset-failed [t]3code.service", "systemctl --user daemon-reload", "systemctl --user enable t3code.service", "systemctl --user restart t3code.service", @@ -661,7 +710,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { ), ).toEqual([ "systemctl --user stop t3code.service", - "systemctl --user reset-failed t3code.service", + "systemctl --user reset-failed [t]3code.service", "systemctl --user daemon-reload", "systemctl --user enable t3code.service", "systemctl --user restart t3code.service", @@ -674,7 +723,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { const { service, commands, control } = yield* makeHarness(); yield* service.install(); commands.length = 0; - control.failCommand = "systemctl --user reset-failed t3code.service"; + control.failCommand = "systemctl --user reset-failed [t]3code.service"; const error = yield* service.restart.pipe(Effect.flip); expect(error).toMatchObject({ @@ -683,7 +732,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { }); expect(commands).toEqual([ "systemctl --user stop t3code.service", - "systemctl --user reset-failed t3code.service", + "systemctl --user reset-failed [t]3code.service", "systemctl --user restart t3code.service", ]); }), @@ -718,7 +767,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { ), ).toEqual([ "systemctl --user stop t3code.service", - "systemctl --user reset-failed t3code.service", + "systemctl --user reset-failed [t]3code.service", "systemctl --user daemon-reload", "systemctl --user restart t3code.service", ]); @@ -740,7 +789,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { ), ).toEqual([ "systemctl --user stop t3code.service", - "systemctl --user reset-failed t3code.service", + "systemctl --user reset-failed [t]3code.service", "systemctl --user daemon-reload", "systemctl --user restart t3code.service", ]); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index de10402140f7..3a264c831908 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -33,6 +33,9 @@ import { const BOOT_SERVICE_NAME = "t3code"; const BOOT_SERVICE_UNIT_FILE = `${BOOT_SERVICE_NAME}.service`; +// Systemd expands this exact-match glob only over loaded units, so an unloaded +// unit has no failed state to reset. +const BOOT_SERVICE_RESET_FAILED_PATTERN = "[t]3code.service"; // `.service` suffix keeps the label distinct from the desktop app's bundle id // (com.t3tools.t3code), so launchd and TCC records never collide. const BOOT_SERVICE_LAUNCHD_LABEL = "com.t3tools.t3code.service"; @@ -267,7 +270,7 @@ function systemdManager(input: { { step: "clearing the failed service state", command: "systemctl", - args: ["--user", "reset-failed", BOOT_SERVICE_UNIT_FILE], + args: ["--user", "reset-failed", BOOT_SERVICE_RESET_FAILED_PATTERN], }, ], activate: [