From e9f35928c9fb0f6058525d9190247f9f31ea92d5 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 21 Aug 2026 03:08:15 -0400 Subject: [PATCH 1/9] test(onboard): reproduce final lifecycle handoff race Signed-off-by: Julie Yaunches --- .../onboard/docker-gpu-patch-finalize.test.ts | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 428611aaf38..a88f4c48608 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -97,6 +97,55 @@ describe("finalizeDockerGpuPatchBackup", () => { ); }); + it("waits for the deleting lifecycle record to clear before restarting the replacement (#9531)", () => { + const events: string[] = []; + const dockerStop = vi.fn(() => { + events.push("stop replacement"); + return { status: 0 }; + }); + const dockerRm = vi.fn(() => { + events.push("remove backup"); + return { status: 0 }; + }); + const dockerStart = vi.fn(() => { + events.push("start replacement"); + return { status: 0 }; + }); + const runOpenshell = vi + .fn() + .mockImplementationOnce(() => { + events.push("observe deleting"); + return { status: 0, stdout: "alpha 2026-08-21 05:53:16 Deleting\n" }; + }) + .mockImplementationOnce(() => { + events.push("observe absent"); + return { status: 0, stdout: "No sandboxes found.\n" }; + }); + + const outcome = finalizeDockerGpuPatchBackup( + { + result: deferredCreateResult(), + supervisorReady: true, + sandboxName: "alpha", + lifecycleReleaseTimeoutSecs: 60, + } as Parameters[0], + { dockerStop, dockerRm, dockerStart, runOpenshell, sleep: vi.fn() }, + ); + + expect(outcome).toMatchObject({ + backupRemoved: true, + lifecycleReleaseObserved: true, + replacementRestarted: true, + }); + expect(events).toEqual([ + "stop replacement", + "remove backup", + "observe deleting", + "observe absent", + "start replacement", + ]); + }); + it("rolls back to the backup container when supervisor reconnect failed", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); From 66436ff58990cc47fdbf44aa876b1629b9b4d692 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 21 Aug 2026 03:31:48 -0400 Subject: [PATCH 2/9] fix(onboard): wait for final lifecycle release Signed-off-by: Julie Yaunches --- docs/reference/commands.mdx | 2 +- .../onboard/docker-gpu-patch-finalize.test.ts | 49 +++++++++++++++-- src/lib/onboard/docker-gpu-patch-finalize.ts | 18 +++++++ ...ocker-gpu-sandbox-create-lifecycle.test.ts | 52 ++++++++++++++++++- src/lib/onboard/docker-gpu-sandbox-create.ts | 30 ++++++++--- .../docker-gpu-supervisor-reconnect.ts | 37 +++++++++++++ 6 files changed, 176 insertions(+), 12 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index f76b34a6717..4f73296f38b 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1070,7 +1070,7 @@ On Jetson/Tegra hosts, the compatibility path uses the NVIDIA runtime and adds e These include selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, GPU, and applicable local-inference checks. If one of those checks fails before backup removal, onboarding prints failure diagnostics and attempts to restore the pre-patch container. -To commit the replacement, NemoClaw stops it, removes the rollback backup, starts the replacement as the final container lifecycle event, and verifies OpenShell supervisor readiness again. +To commit the replacement, NemoClaw stops it, removes the rollback backup, waits for OpenShell to retire the previous lifecycle record, starts the replacement as the final container lifecycle event, and verifies OpenShell supervisor readiness again within the same handoff deadline. If that final handoff cannot be confirmed, onboarding exits with the container diagnostics and cleanup guidance instead of reporting success. If rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance. GPU-proof diagnostics are captured before rollback and can print that guidance before the final container state is known, so inspect the sandbox and its labeled Docker containers before running a deletion command. diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index a88f4c48608..eeb24691dee 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -68,14 +68,25 @@ describe("finalizeDockerGpuPatchBackup", () => { const dockerRm = vi.fn((_name: string) => ({ status: 0 })); const dockerStart = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( - { result: deferredCreateResult(), supervisorReady: true }, - { dockerStop, dockerRm, dockerStart }, + { + result: deferredCreateResult(), + supervisorReady: true, + sandboxName: "alpha", + lifecycleReleaseTimeoutSecs: 60, + }, + { + dockerStop, + dockerRm, + dockerStart, + runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })), + }, ); expect(outcome).toEqual({ backupRemoved: true, rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: true, + lifecycleReleaseObserved: true, }); expect(dockerStop).toHaveBeenCalledWith( "new-container-id", @@ -128,7 +139,7 @@ describe("finalizeDockerGpuPatchBackup", () => { supervisorReady: true, sandboxName: "alpha", lifecycleReleaseTimeoutSecs: 60, - } as Parameters[0], + }, { dockerStop, dockerRm, dockerStart, runOpenshell, sleep: vi.fn() }, ); @@ -146,6 +157,38 @@ describe("finalizeDockerGpuPatchBackup", () => { ]); }); + it("does not treat failed lifecycle probes as a release receipt (#9531)", () => { + const runOpenshell = vi + .fn() + .mockReturnValueOnce({ status: 0, stdout: "Error: gateway unavailable" }) + .mockReturnValueOnce({ status: 1, stderr: "gateway unavailable" }); + const dockerStart = vi.fn(() => ({ status: 0 })); + + const outcome = finalizeDockerGpuPatchBackup( + { + result: deferredCreateResult(), + supervisorReady: true, + sandboxName: "alpha", + lifecycleReleaseTimeoutSecs: 1, + }, + { + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerStart, + runOpenshell, + sleep: vi.fn(), + }, + ); + + expect(outcome).toMatchObject({ + backupRemoved: true, + lifecycleReleaseObserved: false, + replacementRestarted: true, + }); + expect(runOpenshell).toHaveBeenCalledTimes(2); + expect(dockerStart).toHaveBeenCalledOnce(); + }); + it("rolls back to the backup container when supervisor reconnect failed", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index 26b8b7c0ea1..0d0c52a60a4 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -30,6 +30,7 @@ import { rollbackToBackupContainer, } from "./docker-gpu-patch-rollback"; import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "./docker-gpu-patch-types"; +import { waitForOpenShellSandboxLifecycleRelease } from "./docker-gpu-supervisor-reconnect"; export { restoreDockerGpuPatchBackupAfterRecreateFailure as rollbackDockerGpuPatchOnRecreateFailure, @@ -39,6 +40,8 @@ export { export type DockerGpuPatchFinalizeOptions = { result: DockerGpuPatchResult; supervisorReady: boolean; + sandboxName?: string; + lifecycleReleaseTimeoutSecs?: number; }; export type DockerGpuPatchFinalizeOutcome = { @@ -46,6 +49,7 @@ export type DockerGpuPatchFinalizeOutcome = { rolledBack: boolean; replacementStoppedForCommit?: boolean; replacementRestarted?: boolean; + lifecycleReleaseObserved?: boolean; replacementStopConfirmed?: boolean; replacementRemovalConfirmed?: boolean; replacementPresence?: "absent" | "present" | "unknown"; @@ -81,12 +85,26 @@ export function finalizeDockerGpuPatchBackup( } const rmResult = resolved.dockerRm(options.result.backupContainerName, containerOpts); const backupRemoved = hasZeroDockerExitStatus(rmResult); + if (backupRemoved && options.sandboxName && options.lifecycleReleaseTimeoutSecs) { + console.log( + ` Waiting for OpenShell to retire the previous lifecycle record before restarting the replacement (up to ${options.lifecycleReleaseTimeoutSecs}s)...`, + ); + } + const lifecycleReleaseObserved = + backupRemoved && options.sandboxName && options.lifecycleReleaseTimeoutSecs + ? waitForOpenShellSandboxLifecycleRelease( + options.sandboxName, + options.lifecycleReleaseTimeoutSecs, + deps, + ) + : undefined; const startResult = resolved.dockerStart(options.result.newContainerId, containerOpts); return { backupRemoved, rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: hasZeroDockerExitStatus(startResult), + ...(lifecycleReleaseObserved === undefined ? {} : { lifecycleReleaseObserved }), }; } const rollback = rollbackToBackupContainer( diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index 1f1bcd465d5..0350f7f1282 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -91,7 +91,15 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { await patch.commitAfterReady(); expect(finalizeBackup).toHaveBeenCalledTimes(1); - expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: true }, deps); + expect(finalizeBackup).toHaveBeenCalledWith( + { + result, + supervisorReady: true, + sandboxName: "alpha", + lifecycleReleaseTimeoutSecs: 900, + }, + deps, + ); expect(waitForSupervisor).toHaveBeenCalledTimes(2); expect(capturePreRollbackDiagnostics).not.toHaveBeenCalled(); expect(onPatchFailureExit).not.toHaveBeenCalled(); @@ -124,7 +132,15 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.waitForSupervisorReconnectIfNeeded(); await expect(patch.commitAfterReady()).resolves.toBeUndefined(); - expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: true }, deps); + expect(finalizeBackup).toHaveBeenCalledWith( + { + result, + supervisorReady: true, + sandboxName: "alpha", + lifecycleReleaseTimeoutSecs: 900, + }, + deps, + ); expect(waitForSupervisor).toHaveBeenCalledTimes(1); expect(onPatchFailureExit).not.toHaveBeenCalled(); }); @@ -157,6 +173,38 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(onPatchFailureExit).toHaveBeenCalledOnce(); }); + it("rejects final handoff when OpenShell never releases the deleting lifecycle record (#9531)", async () => { + const deps = makeDeps(); + const result = deferredCreateResult(); + const waitForSupervisor = vi.fn(() => true); + const onPatchFailureExit = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => result), + waitForSupervisor, + finalizeBackup: vi.fn(() => ({ + backupRemoved: true, + rolledBack: false, + lifecycleReleaseObserved: false, + replacementRestarted: true, + })), + onPatchFailureExit, + }, + }); + + patch.maybeApplyDuringCreate(); + patch.waitForSupervisorReconnectIfNeeded(); + await expect(patch.commitAfterReady()).rejects.toThrow("final runtime handoff"); + + expect(waitForSupervisor).toHaveBeenCalledOnce(); + expect(onPatchFailureExit).toHaveBeenCalledOnce(); + }); + it("reports a failed post-Ready rollback instead of treating it as restored", async () => { const deps = makeDeps(); const result = deferredCreateResult(); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index 14d333e1269..4308f327ae9 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -468,23 +468,41 @@ export function createDockerGpuSandboxCreatePatch( throw failure; } } + const supervisorReconnectTimeoutSecs = getDockerGpuSupervisorReconnectTimeoutSecs( + options.timeoutSecs, + ); + const finalHandoffDeadlineMs = Date.now() + supervisorReconnectTimeoutSecs * 1000; const finalizeOutcome = result - ? finalizeBackup({ result, supervisorReady: true }, options.deps) + ? finalizeBackup( + { + result, + supervisorReady: true, + sandboxName: options.sandboxName, + lifecycleReleaseTimeoutSecs: supervisorReconnectTimeoutSecs, + }, + options.deps, + ) : null; cutoverFinalized = true; if (!finalizeOutcome) return; if (finalizeOutcome.backupRemoved && finalizeOutcome.replacementRestarted === undefined) { return; } - if (finalizeOutcome.backupRemoved && finalizeOutcome.replacementRestarted) { - const supervisorReconnectTimeoutSecs = getDockerGpuSupervisorReconnectTimeoutSecs( - options.timeoutSecs, + if ( + finalizeOutcome.backupRemoved && + finalizeOutcome.replacementRestarted && + finalizeOutcome.lifecycleReleaseObserved !== false + ) { + const remainingReconnectTimeoutSecs = Math.max( + 0, + Math.ceil((finalHandoffDeadlineMs - Date.now()) / 1000), ); console.log( - ` Waiting for OpenShell supervisor to confirm the final container handoff (up to ${supervisorReconnectTimeoutSecs}s)...`, + ` Waiting for OpenShell supervisor to confirm the final container handoff (up to ${remainingReconnectTimeoutSecs}s)...`, ); if ( - waitForSupervisor(options.sandboxName, supervisorReconnectTimeoutSecs, { + remainingReconnectTimeoutSecs > 0 && + waitForSupervisor(options.sandboxName, remainingReconnectTimeoutSecs, { runOpenshell: options.deps.runOpenshell, runCaptureOpenshell: options.deps.runCaptureOpenshell, sleep: options.deps.sleep, diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index 9db684642c1..f6664c608c7 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -23,6 +23,7 @@ * recovers to Ready is the runtime evidence required. */ +import { parseLiveSandboxEntries } from "../runtime-recovery"; import { hasZeroDockerExitStatus } from "./docker-command-result"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import { envInt } from "./env"; @@ -72,6 +73,42 @@ export type DockerGpuSupervisorReconnectDeps = { errorPhaseDebouncePolls?: number; }; +/** + * Wait for OpenShell to retire the previous lifecycle record before Docker + * restarts the exact replacement container. A successful list command that + * omits the exact sandbox name is the authority; Docker state alone cannot + * release the OpenShell lifecycle record. + */ +export function waitForOpenShellSandboxLifecycleRelease( + sandboxName: string, + timeoutSecs: number, + deps: Pick, +): boolean { + if (!deps.runOpenshell) return false; + const sleep = deps.sleep ?? defaultSleep; + const boundedTimeoutSecs = Math.max(1, Math.round(timeoutSecs)); + const deadline = Date.now() + boundedTimeoutSecs * 1000; + const maxAttempts = Math.max(1, Math.ceil(boundedTimeoutSecs / 2) + 1); + + for (let attempt = 1; attempt <= maxAttempts && Date.now() <= deadline; attempt += 1) { + const result = deps.runOpenshell(["sandbox", "list"], { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if (hasZeroDockerExitStatus(result)) { + const output = String(result.stdout ?? "").trim(); + const entries = parseLiveSandboxEntries(output); + const sandboxPresent = entries.some((entry) => entry.name === sandboxName); + const hasPhaseBearingEntry = entries.some((entry) => entry.phase !== null); + const explicitEmptyList = output === "No sandboxes found" || output === "No sandboxes found."; + if (explicitEmptyList || (hasPhaseBearingEntry && !sandboxPresent)) return true; + } + if (attempt < maxAttempts && Date.now() <= deadline) sleep(2); + } + return false; +} + function defaultSleep(seconds: number): void { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, seconds) * 1000); } From 7717276800f12e779b943151ed26fb689595c2ae Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 21 Aug 2026 03:45:13 -0400 Subject: [PATCH 3/9] docs(onboard): record lifecycle wait contract Signed-off-by: Julie Yaunches --- .../docker-gpu-supervisor-reconnect.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index f6664c608c7..cdcd56bba50 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -74,10 +74,21 @@ export type DockerGpuSupervisorReconnectDeps = { }; /** - * Wait for OpenShell to retire the previous lifecycle record before Docker - * restarts the exact replacement container. A successful list command that - * omits the exact sandbox name is the authority; Docker state alone cannot - * release the OpenShell lifecycle record. + * Workaround contract for the OpenShell lifecycle race in #9531: + * + * - Removing the rollback backup can strand the exact sandbox in `Deleting` + * while its replacement container is healthy. + * - `openshell sandbox list` owns lifecycle authority. Docker health cannot + * prove that OpenShell retired the previous record. + * - This layer waits after backup removal and before replacement restart so + * OpenShell processes the stale deletion before the new registration. + * - `waits for the deleting lifecycle record to clear before restarting the + * replacement (#9531)` protects the event order. `rejects final handoff when + * OpenShell never releases the deleting lifecycle record (#9531)` protects + * the composed failure path. + * + * Remove this wait only when OpenShell binds deletion to the removed container + * identity or provides an identity-bound lifecycle-release receipt. */ export function waitForOpenShellSandboxLifecycleRelease( sandboxName: string, From 61af3e0a668ae66183b3ecd2b9c6a55d0e918c69 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 21 Aug 2026 04:39:56 -0400 Subject: [PATCH 4/9] fix(onboard): recognize stopped replacement lifecycle Signed-off-by: Julie Yaunches --- .../onboard/docker-gpu-patch-finalize.test.ts | 32 +++++++++++++++++-- .../docker-gpu-supervisor-reconnect.ts | 15 ++++++++- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index eeb24691dee..6dfe8d27dc1 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -129,8 +129,8 @@ describe("finalizeDockerGpuPatchBackup", () => { return { status: 0, stdout: "alpha 2026-08-21 05:53:16 Deleting\n" }; }) .mockImplementationOnce(() => { - events.push("observe absent"); - return { status: 0, stdout: "No sandboxes found.\n" }; + events.push("observe stopped replacement"); + return { status: 0, stdout: "alpha 2026-08-21 05:53:18 Error\n" }; }); const outcome = finalizeDockerGpuPatchBackup( @@ -152,7 +152,7 @@ describe("finalizeDockerGpuPatchBackup", () => { "stop replacement", "remove backup", "observe deleting", - "observe absent", + "observe stopped replacement", "start replacement", ]); }); @@ -189,6 +189,32 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(dockerStart).toHaveBeenCalledOnce(); }); + it("does not treat an unrelated terminal lifecycle phase as the stopped replacement (#9531)", () => { + const runOpenshell = vi.fn(() => ({ + status: 0, + stdout: "alpha 2026-08-21 05:53:18 Failed\n", + })); + + const outcome = finalizeDockerGpuPatchBackup( + { + result: deferredCreateResult(), + supervisorReady: true, + sandboxName: "alpha", + lifecycleReleaseTimeoutSecs: 1, + }, + { + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerStart: vi.fn(() => ({ status: 0 })), + runOpenshell, + sleep: vi.fn(), + }, + ); + + expect(outcome.lifecycleReleaseObserved).toBe(false); + expect(runOpenshell).toHaveBeenCalledTimes(2); + }); + it("rolls back to the backup container when supervisor reconnect failed", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index cdcd56bba50..638d109154e 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -82,6 +82,10 @@ export type DockerGpuSupervisorReconnectDeps = { * prove that OpenShell retired the previous record. * - This layer waits after backup removal and before replacement restart so * OpenShell processes the stale deletion before the new registration. + * - The caller enters this wait only after the replacement reached Ready and + * was deliberately stopped. Its exact `Error` row therefore proves the + * stale `Deleting` record no longer owns the sandbox name; the final start + * can make the replacement authoritative again. * - `waits for the deleting lifecycle record to clear before restarting the * replacement (#9531)` protects the event order. `rejects final handoff when * OpenShell never releases the deleting lifecycle record (#9531)` protects @@ -111,9 +115,18 @@ export function waitForOpenShellSandboxLifecycleRelease( const output = String(result.stdout ?? "").trim(); const entries = parseLiveSandboxEntries(output); const sandboxPresent = entries.some((entry) => entry.name === sandboxName); + const stoppedReplacementOwnsLifecycle = entries.some( + (entry) => entry.name === sandboxName && entry.phase === "Error", + ); const hasPhaseBearingEntry = entries.some((entry) => entry.phase !== null); const explicitEmptyList = output === "No sandboxes found" || output === "No sandboxes found."; - if (explicitEmptyList || (hasPhaseBearingEntry && !sandboxPresent)) return true; + if ( + explicitEmptyList || + stoppedReplacementOwnsLifecycle || + (hasPhaseBearingEntry && !sandboxPresent) + ) { + return true; + } } if (attempt < maxAttempts && Date.now() <= deadline) sleep(2); } From 9565fd2ace214ef16b428478925e8416f31787f2 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 21 Aug 2026 09:13:30 -0400 Subject: [PATCH 5/9] fix(onboard): require final lifecycle release Signed-off-by: Julie Yaunches --- src/lib/actions/sandbox/process-recovery.ts | 2 + .../sandbox/supervisor-relaunch.test.ts | 27 ++++++--- .../actions/sandbox/supervisor-relaunch.ts | 17 +++++- .../onboard/docker-gpu-patch-finalize.test.ts | 60 +++++++++++++++---- src/lib/onboard/docker-gpu-patch-finalize.ts | 48 ++++++++++----- ...ocker-gpu-sandbox-create-lifecycle.test.ts | 1 + src/lib/onboard/docker-gpu-sandbox-create.ts | 2 +- .../docker-gpu-supervisor-reconnect.ts | 11 +++- ...ocess-recovery-supervisor-relaunch.test.ts | 14 ++++- 9 files changed, 140 insertions(+), 42 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index bb9986ccc50..bb27a887b86 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -10,6 +10,7 @@ import { captureSandboxSshConfig, getOpenshellBinary, isCommandTimeout, + runOpenshell, } from "../../adapters/openshell/runtime"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import { @@ -748,6 +749,7 @@ function recoverSandboxProcesses( const relaunch = relaunchManagedSupervisorSessionImpl(sandboxName, { quiet, deps: { + runOpenshell, confirmMissingSupervisor: (containerId) => isExactlyMissingManagedSupervisor( requestPinnedGatewaySupervisorAction(sandboxName, "probe", 210000, containerId), diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index de8fff68946..95289eb55bf 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -74,6 +74,7 @@ function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) { failedFiles: [], })), removeBackup: vi.fn(() => true), + runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })), recreate: vi.fn(() => patchResult()), finalize: vi.fn(({ supervisorReady }) => supervisorReady @@ -153,10 +154,15 @@ describe("relaunchManagedSupervisorSession", () => { }); expect(deps.restoreState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); - expect(deps.finalize).toHaveBeenCalledWith({ - result: expect.objectContaining({ newContainerId: "new-container-id" }), - supervisorReady: true, - }); + expect(deps.finalize).toHaveBeenCalledWith( + { + lifecycleReleaseTimeoutSecs: 900, + result: expect.objectContaining({ newContainerId: "new-container-id" }), + sandboxName: "alpha", + supervisorReady: true, + }, + expect.objectContaining({ runOpenshell: deps.runOpenshell, sleep: expect.any(Function) }), + ); }); it("retries only transport-level state backup failures after a container restart", () => { @@ -381,10 +387,15 @@ describe("relaunchManagedSupervisorSession", () => { }); expect(order).toEqual(["restore-state", "restart-restored-gateway", "commit-container"]); expect(deps.restartRestoredManagedGateway).toHaveBeenCalledWith("new-container-id"); - expect(deps.finalize).toHaveBeenCalledWith({ - result: expect.objectContaining({ newContainerId: "new-container-id" }), - supervisorReady: true, - }); + expect(deps.finalize).toHaveBeenCalledWith( + { + lifecycleReleaseTimeoutSecs: 900, + result: expect.objectContaining({ newContainerId: "new-container-id" }), + sandboxName: "alpha", + supervisorReady: true, + }, + expect.objectContaining({ runOpenshell: deps.runOpenshell, sleep: expect.any(Function) }), + ); }); it("rolls back when managed health fails after state restore", () => { diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index ada035db3e6..defe0e3ebad 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -13,6 +13,7 @@ import { type DockerGpuPatchFinalizeOutcome, finalizeDockerGpuPatchBackup, } from "../../onboard/docker-gpu-patch-finalize"; +import { getDockerGpuSupervisorReconnectTimeoutSecs } from "../../onboard/docker-gpu-supervisor-reconnect"; import { recreateOpenShellDockerSandboxWithStartupCommand } from "../../onboard/docker-startup-command-patch"; import { buildSandboxRuntimeEnvArgs } from "../../onboard/sandbox-create-launch"; import { resolveDirectSandboxContainer } from "../../sandbox/privileged-exec"; @@ -57,6 +58,7 @@ export type ManagedSupervisorRelaunchDeps = { removeBackup?: typeof sandboxState.removeSandboxStateBackup; recreate?: typeof recreateOpenShellDockerSandboxWithStartupCommand; finalize?: typeof finalizeDockerGpuPatchBackup; + runOpenshell?: NonNullable[1]>["runOpenshell"]; }; function inspectContainer(containerId: string): DockerContainerInspect { @@ -284,8 +286,21 @@ export function relaunchManagedSupervisorSession( // both succeed. return finalizeFailure(); } + const runLifecycleProbe = deps.runOpenshell; + if (!runLifecycleProbe) return finalizeFailure(); const outcome = { - ...finalize({ result, supervisorReady: true }), + ...finalize( + { + result, + supervisorReady: true, + sandboxName, + lifecycleReleaseTimeoutSecs: getDockerGpuSupervisorReconnectTimeoutSecs(1), + }, + { + runOpenshell: runLifecycleProbe, + sleep, + }, + ), stateRestored: true, stateBackupRemoved: removeSettledStateBackup(), }; diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 6dfe8d27dc1..1721ef51a44 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -183,10 +183,14 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome).toMatchObject({ backupRemoved: true, lifecycleReleaseObserved: false, - replacementRestarted: true, + replacementRestarted: false, }); expect(runOpenshell).toHaveBeenCalledTimes(2); - expect(dockerStart).toHaveBeenCalledOnce(); + expect(runOpenshell.mock.calls[0]?.[1]?.timeout).toBeGreaterThan(0); + expect(runOpenshell.mock.calls[0]?.[1]?.timeout).toBeLessThanOrEqual(1000); + expect(runOpenshell.mock.calls[1]?.[1]?.timeout).toBeGreaterThan(0); + expect(runOpenshell.mock.calls[1]?.[1]?.timeout).toBeLessThanOrEqual(1000); + expect(dockerStart).not.toHaveBeenCalled(); }); it("does not treat an unrelated terminal lifecycle phase as the stopped replacement (#9531)", () => { @@ -195,6 +199,7 @@ describe("finalizeDockerGpuPatchBackup", () => { stdout: "alpha 2026-08-21 05:53:18 Failed\n", })); + const dockerStart = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( { result: deferredCreateResult(), @@ -205,7 +210,7 @@ describe("finalizeDockerGpuPatchBackup", () => { { dockerStop: vi.fn(() => ({ status: 0 })), dockerRm: vi.fn(() => ({ status: 0 })), - dockerStart: vi.fn(() => ({ status: 0 })), + dockerStart, runOpenshell, sleep: vi.fn(), }, @@ -213,6 +218,7 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome.lifecycleReleaseObserved).toBe(false); expect(runOpenshell).toHaveBeenCalledTimes(2); + expect(dockerStart).not.toHaveBeenCalled(); }); it("rolls back to the backup container when supervisor reconnect failed", () => { @@ -304,7 +310,15 @@ describe("finalizeDockerGpuPatchBackup", () => { it("is a no-op when the backup was already removed by the patch helper", () => { const dockerRm = vi.fn((_name: string) => ({ status: 0 })); const result = { ...deferredCreateResult(), backupRemoved: true }; - const outcome = finalizeDockerGpuPatchBackup({ result, supervisorReady: true }, { dockerRm }); + const outcome = finalizeDockerGpuPatchBackup( + { + result, + supervisorReady: true, + sandboxName: "alpha", + lifecycleReleaseTimeoutSecs: 60, + }, + { dockerRm }, + ); expect(outcome).toEqual({ backupRemoved: true, rolledBack: false }); expect(dockerRm).not.toHaveBeenCalled(); }); @@ -317,19 +331,26 @@ describe("finalizeDockerGpuPatchBackup", () => { })); const dockerStart = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( - { result: deferredCreateResult(), supervisorReady: true }, + { + result: deferredCreateResult(), + supervisorReady: true, + sandboxName: "alpha", + lifecycleReleaseTimeoutSecs: 60, + }, { dockerStop, dockerRm, dockerStart }, ); expect(outcome).toEqual({ backupRemoved: false, rolledBack: false, replacementStoppedForCommit: true, - replacementRestarted: true, + replacementRestarted: false, + lifecycleReleaseObserved: false, }); expect(dockerRm).toHaveBeenCalledWith( "openshell-alpha-nemoclaw-gpu-backup-1780491860342", expect.objectContaining({ ignoreError: true }), ); + expect(dockerStart).not.toHaveBeenCalled(); }); it("fails closed when backup removal has no exit status", () => { @@ -337,15 +358,22 @@ describe("finalizeDockerGpuPatchBackup", () => { const dockerRm = vi.fn((_name: string) => ({ status: null, stderr: "timed out" })); const dockerStart = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( - { result: deferredCreateResult(), supervisorReady: true }, + { + result: deferredCreateResult(), + supervisorReady: true, + sandboxName: "alpha", + lifecycleReleaseTimeoutSecs: 60, + }, { dockerStop, dockerRm, dockerStart }, ); expect(outcome).toEqual({ backupRemoved: false, rolledBack: false, replacementStoppedForCommit: true, - replacementRestarted: true, + replacementRestarted: false, + lifecycleReleaseObserved: false, }); + expect(dockerStart).not.toHaveBeenCalled(); }); it("retains the backup when the replacement cannot be stopped for the final handoff", () => { @@ -354,7 +382,12 @@ describe("finalizeDockerGpuPatchBackup", () => { const dockerStart = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( - { result: deferredCreateResult(), supervisorReady: true }, + { + result: deferredCreateResult(), + supervisorReady: true, + sandboxName: "alpha", + lifecycleReleaseTimeoutSecs: 60, + }, { dockerStop, dockerRm, dockerStart }, ); @@ -369,11 +402,17 @@ describe("finalizeDockerGpuPatchBackup", () => { it("reports a failed replacement restart after the backup is removed", () => { const outcome = finalizeDockerGpuPatchBackup( - { result: deferredCreateResult(), supervisorReady: true }, + { + result: deferredCreateResult(), + supervisorReady: true, + sandboxName: "alpha", + lifecycleReleaseTimeoutSecs: 60, + }, { dockerStop: vi.fn(() => ({ status: 0 })), dockerRm: vi.fn(() => ({ status: 0 })), dockerStart: vi.fn(() => ({ status: 1 })), + runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })), }, ); @@ -382,6 +421,7 @@ describe("finalizeDockerGpuPatchBackup", () => { rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: false, + lifecycleReleaseObserved: true, }); }); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index 0d0c52a60a4..75c9d90d43a 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -37,12 +37,17 @@ export { rollbackToBackupContainer, } from "./docker-gpu-patch-rollback"; -export type DockerGpuPatchFinalizeOptions = { - result: DockerGpuPatchResult; - supervisorReady: boolean; - sandboxName?: string; - lifecycleReleaseTimeoutSecs?: number; -}; +export type DockerGpuPatchFinalizeOptions = + | { + result: DockerGpuPatchResult; + supervisorReady: false; + } + | { + result: DockerGpuPatchResult; + supervisorReady: true; + sandboxName: string; + lifecycleReleaseTimeoutSecs: number; + }; export type DockerGpuPatchFinalizeOutcome = { backupRemoved: boolean; @@ -85,26 +90,37 @@ export function finalizeDockerGpuPatchBackup( } const rmResult = resolved.dockerRm(options.result.backupContainerName, containerOpts); const backupRemoved = hasZeroDockerExitStatus(rmResult); - if (backupRemoved && options.sandboxName && options.lifecycleReleaseTimeoutSecs) { + const sandboxName = options.sandboxName; + const lifecycleReleaseTimeoutSecs = options.lifecycleReleaseTimeoutSecs; + const hasLifecycleContext = + sandboxName.length > 0 && + Number.isFinite(lifecycleReleaseTimeoutSecs) && + lifecycleReleaseTimeoutSecs > 0; + if (backupRemoved && hasLifecycleContext) { console.log( - ` Waiting for OpenShell to retire the previous lifecycle record before restarting the replacement (up to ${options.lifecycleReleaseTimeoutSecs}s)...`, + ` Waiting for OpenShell to retire the previous lifecycle record before restarting the replacement (up to ${lifecycleReleaseTimeoutSecs}s)...`, ); } const lifecycleReleaseObserved = - backupRemoved && options.sandboxName && options.lifecycleReleaseTimeoutSecs - ? waitForOpenShellSandboxLifecycleRelease( - options.sandboxName, - options.lifecycleReleaseTimeoutSecs, - deps, - ) - : undefined; + backupRemoved && hasLifecycleContext + ? waitForOpenShellSandboxLifecycleRelease(sandboxName, lifecycleReleaseTimeoutSecs, deps) + : false; + if (!lifecycleReleaseObserved) { + return { + backupRemoved, + rolledBack: false, + replacementStoppedForCommit: true, + replacementRestarted: false, + lifecycleReleaseObserved: false, + }; + } const startResult = resolved.dockerStart(options.result.newContainerId, containerOpts); return { backupRemoved, rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: hasZeroDockerExitStatus(startResult), - ...(lifecycleReleaseObserved === undefined ? {} : { lifecycleReleaseObserved }), + lifecycleReleaseObserved: true, }; } const rollback = rollbackToBackupContainer( diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index 0350f7f1282..36bc94980ca 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -50,6 +50,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { const finalizeBackup = vi.fn(() => ({ backupRemoved: true, rolledBack: false, + lifecycleReleaseObserved: true, replacementRestarted: true, })); const capturePreRollbackDiagnostics = vi.fn(() => null); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index 4308f327ae9..cc4983bb01a 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -491,7 +491,7 @@ export function createDockerGpuSandboxCreatePatch( if ( finalizeOutcome.backupRemoved && finalizeOutcome.replacementRestarted && - finalizeOutcome.lifecycleReleaseObserved !== false + finalizeOutcome.lifecycleReleaseObserved === true ) { const remainingReconnectTimeoutSecs = Math.max( 0, diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index 638d109154e..c6f0c5f970d 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -105,11 +105,13 @@ export function waitForOpenShellSandboxLifecycleRelease( const deadline = Date.now() + boundedTimeoutSecs * 1000; const maxAttempts = Math.max(1, Math.ceil(boundedTimeoutSecs / 2) + 1); - for (let attempt = 1; attempt <= maxAttempts && Date.now() <= deadline; attempt += 1) { + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + const remainingBeforeProbeMs = deadline - Date.now(); + if (remainingBeforeProbeMs <= 0) break; const result = deps.runOpenshell(["sandbox", "list"], { ignoreError: true, suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, remainingBeforeProbeMs), }); if (hasZeroDockerExitStatus(result)) { const output = String(result.stdout ?? "").trim(); @@ -128,7 +130,10 @@ export function waitForOpenShellSandboxLifecycleRelease( return true; } } - if (attempt < maxAttempts && Date.now() <= deadline) sleep(2); + const remainingBeforeSleepMs = deadline - Date.now(); + if (attempt < maxAttempts && remainingBeforeSleepMs > 0) { + sleep(Math.min(2, remainingBeforeSleepMs / 1000)); + } } return false; } diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index 59ae878a3ef..8deaaaaa72d 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -109,6 +109,7 @@ function composedRelaunchTransaction( }; }), removeBackup: vi.fn(() => true), + runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })), recreate: vi.fn(() => ({ applied: true as const, oldContainerId: "old-container-id", @@ -451,7 +452,12 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(order).toEqual(["restore-state", "post-restore-restart", "commit-container"]); expect(finalizeTransaction).toHaveBeenCalledOnce(); expect(finalizeTransaction).toHaveBeenCalledWith( - expect.objectContaining({ supervisorReady: true }), + expect.objectContaining({ + lifecycleReleaseTimeoutSecs: 900, + sandboxName: "recovered-box", + supervisorReady: true, + }), + expect.objectContaining({ runOpenshell: expect.any(Function), sleep: expect.any(Function) }), ); }); @@ -464,8 +470,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { const dockerRm = vi.fn(() => ({ status: 0 })); const dockerStart = vi.fn(() => ({ status: 0 })); const finalizeTransaction = vi.fn( - (options: Parameters[0]) => - finalizeDockerGpuPatchBackup(options, { dockerStop, dockerRm, dockerStart }), + ( + options: Parameters[0], + deps: Parameters[1], + ) => finalizeDockerGpuPatchBackup(options, { ...deps, dockerStop, dockerRm, dockerStart }), ); const { relaunchManagedSupervisorSessionImpl } = composedRelaunchTransaction( order, From 5e700a8ec0dda79f1c29302cd9f1e1b4f33e8fbb Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 21 Aug 2026 09:29:19 -0400 Subject: [PATCH 6/9] fix(onboard): keep lifecycle polling host-bound Signed-off-by: Julie Yaunches --- .../sandbox/supervisor-relaunch.test.ts | 16 +++++++- .../actions/sandbox/supervisor-relaunch.ts | 9 +++-- .../docker-gpu-supervisor-reconnect.test.ts | 40 +++++++++++++++++++ ...ocess-recovery-supervisor-relaunch.test.ts | 2 +- 4 files changed, 60 insertions(+), 7 deletions(-) diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 95289eb55bf..08ea2622049 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -161,7 +161,7 @@ describe("relaunchManagedSupervisorSession", () => { sandboxName: "alpha", supervisorReady: true, }, - expect.objectContaining({ runOpenshell: deps.runOpenshell, sleep: expect.any(Function) }), + { runOpenshell: deps.runOpenshell }, ); }); @@ -394,10 +394,22 @@ describe("relaunchManagedSupervisorSession", () => { sandboxName: "alpha", supervisorReady: true, }, - expect.objectContaining({ runOpenshell: deps.runOpenshell, sleep: expect.any(Function) }), + { runOpenshell: deps.runOpenshell }, ); }); + it("uses only an injected host sleep for lifecycle polling after recreation (#9531)", () => { + const sleep = vi.fn(); + const deps = baseDeps({ sleep }); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(true)).toMatchObject({ backupRemoved: true, rolledBack: false }); + expect(deps.finalize).toHaveBeenCalledWith(expect.objectContaining({ supervisorReady: true }), { + runOpenshell: deps.runOpenshell, + sleep, + }); + }); + it("rolls back when managed health fails after state restore", () => { const order: string[] = []; const deps = baseDeps({ diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index defe0e3ebad..cf71bf3bf68 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -288,6 +288,10 @@ export function relaunchManagedSupervisorSession( } const runLifecycleProbe = deps.runOpenshell; if (!runLifecycleProbe) return finalizeFailure(); + const lifecycleDeps = { + runOpenshell: runLifecycleProbe, + ...(deps.sleep ? { sleep: deps.sleep } : {}), + }; const outcome = { ...finalize( { @@ -296,10 +300,7 @@ export function relaunchManagedSupervisorSession( sandboxName, lifecycleReleaseTimeoutSecs: getDockerGpuSupervisorReconnectTimeoutSecs(1), }, - { - runOpenshell: runLifecycleProbe, - sleep, - }, + lifecycleDeps, ), stateRestored: true, stateBackupRemoved: removeSettledStateBackup(), diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts index 7f77a5de8fb..6e65b66ece3 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts @@ -6,9 +6,49 @@ import { describe, expect, it, vi } from "vitest"; import { getDockerGpuSupervisorReconnectErrorDebouncePolls, getDockerGpuSupervisorReconnectTimeoutSecs, + waitForOpenShellSandboxLifecycleRelease, waitForOpenShellSupervisorReconnect, } from "./docker-gpu-supervisor-reconnect"; +describe("Docker GPU final lifecycle release", () => { + it.each([ + ["an explicit empty list", "No sandboxes found.\n"], + ["the stopped replacement Error row", "alpha 2026-08-21 05:53:18 Error\n"], + ["another phase-bearing sandbox", "beta 2026-08-21 05:53:18 Ready\n"], + ])("accepts %s as a release receipt (#9531)", (_receipt, stdout) => { + const runOpenshell = vi.fn(() => ({ status: 0, stdout })); + + expect( + waitForOpenShellSandboxLifecycleRelease("alpha", 1, { + runOpenshell, + sleep: vi.fn(), + }), + ).toBe(true); + expect(runOpenshell).toHaveBeenCalledOnce(); + }); + + it.each([ + ["a header", "NAME CREATED PHASE\n"], + ["a gateway error", "Error: gateway unavailable\n"], + ["a phase-free row", "beta 2026-08-21 05:53:18\n"], + ["an unrecognized phase", "beta 2026-08-21 05:53:18 Retiring\n"], + ["the selected sandbox in Deleting", "alpha 2026-08-21 05:53:18 Deleting\n"], + ["the selected sandbox in Ready", "alpha 2026-08-21 05:53:18 Ready\n"], + ["the selected sandbox in Provisioning", "alpha 2026-08-21 05:53:18 Provisioning\n"], + ["the selected sandbox in Failed", "alpha 2026-08-21 05:53:18 Failed\n"], + ])("rejects %s as a release receipt (#9531)", (_case, stdout) => { + const runOpenshell = vi.fn(() => ({ status: 0, stdout })); + + expect( + waitForOpenShellSandboxLifecycleRelease("alpha", 1, { + runOpenshell, + sleep: vi.fn(), + }), + ).toBe(false); + expect(runOpenshell).toHaveBeenCalledTimes(2); + }); +}); + // The Docker GPU patch supervisor-reconnect wait must absorb a transient // Error phase reported while OpenShell's sandbox-list cache catches up to // the newly-recreated GPU container. The old-container teardown briefly diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index 8deaaaaa72d..d2a2346b560 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -457,7 +457,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { sandboxName: "recovered-box", supervisorReady: true, }), - expect.objectContaining({ runOpenshell: expect.any(Function), sleep: expect.any(Function) }), + { runOpenshell: expect.any(Function) }, ); }); From 5b1af40cf76446073be5a9d8b8ff8eb9f92908b8 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 21 Aug 2026 11:52:25 -0400 Subject: [PATCH 7/9] test(onboard): cover lifecycle probe failures Signed-off-by: Julie Yaunches --- .../docker-gpu-supervisor-reconnect.test.ts | 15 +++++++++++++++ test/process-recovery-supervisor-relaunch.test.ts | 9 +++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts index 6e65b66ece3..41a2c822c19 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts @@ -47,6 +47,21 @@ describe("Docker GPU final lifecycle release", () => { ).toBe(false); expect(runOpenshell).toHaveBeenCalledTimes(2); }); + + it.each([ + ["a failed probe", { status: 1, stderr: "gateway unavailable" }], + ["a probe without an exit status", { status: null, stderr: "timed out" }], + ])("rejects %s as a release receipt (#9531)", (_case, result) => { + const runOpenshell = vi.fn(() => result); + + expect( + waitForOpenShellSandboxLifecycleRelease("alpha", 1, { + runOpenshell, + sleep: vi.fn(), + }), + ).toBe(false); + expect(runOpenshell).toHaveBeenCalledTimes(2); + }); }); // The Docker GPU patch supervisor-reconnect wait must absorb a transient diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index d2a2346b560..4304a0cbf90 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -77,6 +77,7 @@ function composedRelaunchTransaction( .fn() .mockReturnValueOnce("old-container-id") .mockReturnValue("replacement-container-id"); + const runOpenshell = vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })); const relaunchManagedSupervisorSessionImpl = vi.fn( (sandboxName: string, options: Parameters[1]) => relaunchManagedSupervisorSession(sandboxName, { @@ -109,7 +110,7 @@ function composedRelaunchTransaction( }; }), removeBackup: vi.fn(() => true), - runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })), + runOpenshell, recreate: vi.fn(() => ({ applied: true as const, oldContainerId: "old-container-id", @@ -128,7 +129,7 @@ function composedRelaunchTransaction( }, }), ); - return { finalizeTransaction, relaunchManagedSupervisorSessionImpl }; + return { finalizeTransaction, relaunchManagedSupervisorSessionImpl, runOpenshell }; } function scriptedPinnedGatewayRecovery( @@ -402,7 +403,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { mockOpenClawSandbox("recovered-box"); setImmediateRecoveryPolling(); const order: string[] = []; - const { finalizeTransaction, relaunchManagedSupervisorSessionImpl } = + const { finalizeTransaction, relaunchManagedSupervisorSessionImpl, runOpenshell } = composedRelaunchTransaction(order); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -457,7 +458,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { sandboxName: "recovered-box", supervisorReady: true, }), - { runOpenshell: expect.any(Function) }, + { runOpenshell }, ); }); From 7254137464264c114b3928e7c164d98ffe295de1 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 21 Aug 2026 15:54:45 -0400 Subject: [PATCH 8/9] fix(onboard): require sandbox name release Signed-off-by: Julie Yaunches --- docs/reference/commands.mdx | 3 ++- src/lib/actions/sandbox/process-recovery.ts | 3 +++ .../onboard/docker-gpu-patch-finalize.test.ts | 11 ++++++++--- .../docker-gpu-supervisor-reconnect.test.ts | 2 +- .../onboard/docker-gpu-supervisor-reconnect.ts | 16 ++++------------ .../process-recovery-supervisor-relaunch.test.ts | 15 +++++++++++++++ 6 files changed, 33 insertions(+), 17 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 4f73296f38b..4dbd5d4ea49 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1070,7 +1070,8 @@ On Jetson/Tegra hosts, the compatibility path uses the NVIDIA runtime and adds e These include selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, GPU, and applicable local-inference checks. If one of those checks fails before backup removal, onboarding prints failure diagnostics and attempts to restore the pre-patch container. -To commit the replacement, NemoClaw stops it, removes the rollback backup, waits for OpenShell to retire the previous lifecycle record, starts the replacement as the final container lifecycle event, and verifies OpenShell supervisor readiness again within the same handoff deadline. +To commit the replacement, NemoClaw stops it, removes the rollback backup, and waits until a successful OpenShell sandbox list has no row with that sandbox name. +NemoClaw then starts the replacement as the final container lifecycle event and verifies OpenShell supervisor readiness again within the same handoff deadline. If that final handoff cannot be confirmed, onboarding exits with the container diagnostics and cleanup guidance instead of reporting success. If rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance. GPU-proof diagnostics are captured before rollback and can print that guidance before the final container state is known, so inspect the sandbox and its labeled Docker containers before running a deletion command. diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index bb27a887b86..6656674bbe4 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -499,6 +499,9 @@ function finalRelaunchContainerFailureDetail( if (completion.replacementStoppedForCommit === false) { return "Docker could not stop the replacement container for the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward"; } + if (completion.lifecycleReleaseObserved === false) { + return "OpenShell did not release the sandbox name before the final recovery handoff. NemoClaw did not restart the replacement container or start the primary dashboard/API host forward"; + } if (completion.replacementRestarted === false) { return "Docker could not start the replacement container to complete the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward"; } diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 1721ef51a44..5a9d1b78d51 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -108,7 +108,7 @@ describe("finalizeDockerGpuPatchBackup", () => { ); }); - it("waits for the deleting lifecycle record to clear before restarting the replacement (#9531)", () => { + it("waits for the sandbox name to disappear before restarting the replacement (#9531)", () => { const events: string[] = []; const dockerStop = vi.fn(() => { events.push("stop replacement"); @@ -129,8 +129,12 @@ describe("finalizeDockerGpuPatchBackup", () => { return { status: 0, stdout: "alpha 2026-08-21 05:53:16 Deleting\n" }; }) .mockImplementationOnce(() => { - events.push("observe stopped replacement"); + events.push("observe error"); return { status: 0, stdout: "alpha 2026-08-21 05:53:18 Error\n" }; + }) + .mockImplementationOnce(() => { + events.push("observe name absence"); + return { status: 0, stdout: "beta 2026-08-21 05:53:20 Ready\n" }; }); const outcome = finalizeDockerGpuPatchBackup( @@ -152,7 +156,8 @@ describe("finalizeDockerGpuPatchBackup", () => { "stop replacement", "remove backup", "observe deleting", - "observe stopped replacement", + "observe error", + "observe name absence", "start replacement", ]); }); diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts index 41a2c822c19..b8c1282c261 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts @@ -13,7 +13,6 @@ import { describe("Docker GPU final lifecycle release", () => { it.each([ ["an explicit empty list", "No sandboxes found.\n"], - ["the stopped replacement Error row", "alpha 2026-08-21 05:53:18 Error\n"], ["another phase-bearing sandbox", "beta 2026-08-21 05:53:18 Ready\n"], ])("accepts %s as a release receipt (#9531)", (_receipt, stdout) => { const runOpenshell = vi.fn(() => ({ status: 0, stdout })); @@ -35,6 +34,7 @@ describe("Docker GPU final lifecycle release", () => { ["the selected sandbox in Deleting", "alpha 2026-08-21 05:53:18 Deleting\n"], ["the selected sandbox in Ready", "alpha 2026-08-21 05:53:18 Ready\n"], ["the selected sandbox in Provisioning", "alpha 2026-08-21 05:53:18 Provisioning\n"], + ["the selected sandbox in Error", "alpha 2026-08-21 05:53:18 Error\n"], ["the selected sandbox in Failed", "alpha 2026-08-21 05:53:18 Failed\n"], ])("rejects %s as a release receipt (#9531)", (_case, stdout) => { const runOpenshell = vi.fn(() => ({ status: 0, stdout })); diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index c6f0c5f970d..911d352d245 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -83,10 +83,9 @@ export type DockerGpuSupervisorReconnectDeps = { * - This layer waits after backup removal and before replacement restart so * OpenShell processes the stale deletion before the new registration. * - The caller enters this wait only after the replacement reached Ready and - * was deliberately stopped. Its exact `Error` row therefore proves the - * stale `Deleting` record no longer owns the sandbox name; the final start - * can make the replacement authoritative again. - * - `waits for the deleting lifecycle record to clear before restarting the + * was deliberately stopped. A successful list must omit the sandbox name; + * a name-and-phase row cannot identify which container owns that lifecycle. + * - `waits for the sandbox name to disappear before restarting the * replacement (#9531)` protects the event order. `rejects final handoff when * OpenShell never releases the deleting lifecycle record (#9531)` protects * the composed failure path. @@ -117,16 +116,9 @@ export function waitForOpenShellSandboxLifecycleRelease( const output = String(result.stdout ?? "").trim(); const entries = parseLiveSandboxEntries(output); const sandboxPresent = entries.some((entry) => entry.name === sandboxName); - const stoppedReplacementOwnsLifecycle = entries.some( - (entry) => entry.name === sandboxName && entry.phase === "Error", - ); const hasPhaseBearingEntry = entries.some((entry) => entry.phase !== null); const explicitEmptyList = output === "No sandboxes found" || output === "No sandboxes found."; - if ( - explicitEmptyList || - stoppedReplacementOwnsLifecycle || - (hasPhaseBearingEntry && !sandboxPresent) - ) { + if (explicitEmptyList || (hasPhaseBearingEntry && !sandboxPresent)) { return true; } } diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index 4304a0cbf90..07b6697fc5b 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -578,6 +578,21 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { finalPinnedAction: () => ACCEPTED_MANAGED_PROBE, finalReadinessReady: true, }, + { + condition: "OpenShell does not release the sandbox name", + finalizeOutcome: () => ({ + backupRemoved: true, + lifecycleReleaseObserved: false, + replacementRestarted: false, + replacementStoppedForCommit: true, + rolledBack: false, + stateRestored: true, + }), + expectedDetail: "OpenShell did not release the sandbox name", + expectedReadinessCalls: 1, + finalPinnedAction: () => ACCEPTED_MANAGED_PROBE, + finalReadinessReady: true, + }, { condition: "Docker cannot start the replacement container", finalizeOutcome: () => ({ From 0e1cf95d699bd81fd29e747c9f51d9536fd11493 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 21 Aug 2026 17:06:40 -0400 Subject: [PATCH 9/9] test(onboard): model lifecycle release in messaging fixtures Signed-off-by: Julie Yaunches --- test/onboard-messaging.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/onboard-messaging.test.ts b/test/onboard-messaging.test.ts index 042340d91c6..f5fa6e03269 100644 --- a/test/onboard-messaging.test.ts +++ b/test/onboard-messaging.test.ts @@ -87,7 +87,7 @@ const { EventEmitter } = require("node:events"); const fs = require("node:fs"); const commands = []; runner.run = (command, opts = {}) => { - commands.push({ command: _n(command), env: opts.env || null }); + commands.push({ command: _n(command), env: opts.env || null }); if (_n(command).includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; // provider-get returns not-found so messaging providers are created fresh if (_n(command).includes("provider get")) return { status: 1 }; return _n(command).includes("sandbox get") && _n(command).includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; @@ -366,7 +366,7 @@ const commands = []; let registeredSandbox = null; runner.run = (command, opts = {}) => { const normalized = _n(command); - commands.push({ command: normalized, env: opts.env || null }); + commands.push({ command: normalized, env: opts.env || null }); if (normalized.includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; if (normalized.includes("provider get")) return { status: 1 }; return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; @@ -536,7 +536,7 @@ registry.registerSandbox({ name: "my-assistant", messaging: { schemaVersion: 1, registry.addExtraProvider("my-assistant-extra-telegram-bot-token-agent-a"); registry.addExtraProvider("my-assistant-extra-telegram-bot-token-agent-b"); runner.run = (command) => { const normalized = _n(command); - commands.push({ command: normalized }); + commands.push({ command: normalized }); if (normalized.includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; const providerGet = normalized.match(/provider get -g nemoclaw ([^ ]+)$/)?.[1]; if (providerGet === process.env.NEMOCLAW_TEST_FAIL_PROVIDER) return { status: 2, stderr: "transport unavailable" }; if (providerGet && revisions.has(providerGet)) return { status: 0, stdout: "Name: " + providerGet + "\nType: " + (providerGet === "compatible-endpoint" ? "openai" : "generic") + "\nCredential keys: " + credentialKeys[providerGet] + "\nConfig keys: " + (providerGet === "compatible-endpoint" ? "OPENAI_BASE_URL" : "") + "\n" }; const refresh = normalized.match(/provider update -g nemoclaw ([^ ]+)$/)?.[1]; @@ -706,7 +706,7 @@ registry.registerSandbox({ }); runner.run = (command, opts = {}) => { const normalized = _n(command); - commands.push({ command: normalized, env: opts.env || null }); + commands.push({ command: normalized, env: opts.env || null }); if (normalized.includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; if (normalized.includes("provider get -g nemoclaw my-assistant-telegram-bridge")) return { status: 0, stdout: "Name: my-assistant-telegram-bridge\nType: generic\nCredential keys: TELEGRAM_BOT_TOKEN\nConfig keys: \n" }; if (normalized.includes("provider get")) return { status: 1 }; return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; @@ -859,7 +859,7 @@ const commands = []; let dockerfileContent; const registerCalls = []; runner.run = (command, opts = {}) => { const normalized = _n(command); - commands.push({ command: normalized, env: opts.env || null }); + commands.push({ command: normalized, env: opts.env || null }); if (normalized.includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; if (normalized.includes("provider get")) return { status: 1 }; return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; @@ -1020,7 +1020,7 @@ const commands = []; let dockerfileContent; const registerCalls = []; runner.run = (command, opts = {}) => { const normalized = _n(command); - commands.push({ command: normalized, env: opts.env || null }); + commands.push({ command: normalized, env: opts.env || null }); if (normalized.includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; if (normalized.includes("provider get")) return { status: 1 }; return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; @@ -1348,7 +1348,7 @@ const { EventEmitter } = require("node:events"); const commands = []; runner.run = (command, opts = {}) => { - commands.push({ command: _n(command), env: opts.env || null }); + commands.push({ command: _n(command), env: opts.env || null }); if (_n(command).includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; // provider-get returns not-found so messaging providers are created fresh if (_n(command).includes("provider get")) return { status: 1 }; return _n(command).includes("sandbox get") && _n(command).includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; @@ -1485,7 +1485,7 @@ const { EventEmitter } = require("node:events"); const commands = []; runner.run = (command, opts = {}) => { - commands.push({ command: _n(command), env: opts.env || null }); + commands.push({ command: _n(command), env: opts.env || null }); if (_n(command).includes("sandbox list")) return { status: 0, stdout: "No sandboxes found." }; return _n(command).includes("sandbox get") && _n(command).includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; runner.runCapture = (command) => {