diff --git a/src/lib/onboard/docker-gpu-local-inference.test.ts b/src/lib/onboard/docker-gpu-local-inference.test.ts index 702a4a7e757..d37b2976422 100644 --- a/src/lib/onboard/docker-gpu-local-inference.test.ts +++ b/src/lib/onboard/docker-gpu-local-inference.test.ts @@ -425,6 +425,27 @@ describe("verifyGpuSandboxLocalInferenceAndCommitAfterReady", () => { expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce(); expect(runtimePatch.commitAfterReady).not.toHaveBeenCalled(); }); + + it("treats a failed commit as terminal without attempting rollback", async () => { + const runtimePatch = { + commitAfterReady: vi.fn(async () => { + throw new Error("durable commit acknowledgement failed"); + }), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + }; + await expect( + verifyGpuSandboxLocalInferenceAndCommitAfterReady( + GPU_CONFIG, + "ollama-local", + { + ...options(), + deps: { execInSandbox: execEmitting("HTTP_200"), sleep: vi.fn() }, + }, + runtimePatch, + ), + ).rejects.toThrow("durable commit acknowledgement failed"); + expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + }); }); describe("printDockerGpuSandboxInferenceVerificationFailure", () => { diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index b4140641999..d1801a8080a 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -511,7 +511,6 @@ export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady( ): Promise { try { verifyGpuSandboxLocalInferenceAfterReady(config, provider, options); - await runtimePatch.commitAfterReady(); } catch (error) { const failure = error instanceof Error ? error : new Error(String(error)); try { @@ -523,4 +522,5 @@ export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady( } throw failure; } + await runtimePatch.commitAfterReady(); } diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index 262cd25d109..da5dbf13bd9 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -118,7 +118,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.waitForSupervisorReconnectIfNeeded(); expect(onPatchFailureExit).not.toHaveBeenCalled(); - await patch.commitAfterReady(); + await expect(patch.commitAfterReady()).rejects.toThrow("rollback backup"); expect(onPatchFailureExit).toHaveBeenCalledOnce(); expect(onPatchFailureExit.mock.calls[0]?.[1]).toEqual( @@ -136,6 +136,33 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { ); }); + it("rejects an early commit after rolling back before supervisor reconnect", async () => { + const deps = makeDeps(); + const result = deferredCreateResult(); + const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); + const onPatchFailureExit = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => result), + finalizeBackup, + onPatchFailureExit, + }, + }); + + patch.maybeApplyDuringCreate(); + + await expect(patch.commitAfterReady()).rejects.toThrow( + "cannot commit before the recreated OpenShell supervisor reconnects", + ); + expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: false }, deps); + expect(onPatchFailureExit).toHaveBeenCalledOnce(); + }); + it("rolls back to the backup container and surfaces rolledBack=true diagnostics when supervisorReady=false", () => { const deps = makeDeps(); const result = deferredCreateResult(); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index ed5f8dcf74e..7add41f7f62 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -368,18 +368,15 @@ export function createDockerGpuSandboxCreatePatch( "Managed startup cannot commit before the recreated OpenShell supervisor reconnects.", ); const rollbackError = await rollbackAfterFailure(); - onPatchFailureExit( - options.sandboxName, - rollbackError - ? new Error(`${error.message} Rollback failed: ${rollbackError.message}`) - : error, - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }, - ); - return; + const failure = rollbackError + ? new Error(`${error.message} Rollback failed: ${rollbackError.message}`) + : error; + onPatchFailureExit(options.sandboxName, failure, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }); + throw failure; } if (cutoverFinalization) { if (cutoverFinalizationOutcome !== "commit") { @@ -417,7 +414,7 @@ export function createDockerGpuSandboxCreatePatch( rolledBack: rollbackError === null, }, }); - return; + throw failure; } } const finalizeOutcome = result @@ -425,16 +422,16 @@ export function createDockerGpuSandboxCreatePatch( : null; cutoverFinalized = true; if (!finalizeOutcome || finalizeOutcome.backupRemoved) return; - onPatchFailureExit( - options.sandboxName, - new Error("Managed startup passed Ready, but its rollback backup could not be removed."), - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - context: failureContext(), - }, + const failure = new Error( + "Managed startup passed Ready, but its rollback backup could not be removed.", ); + onPatchFailureExit(options.sandboxName, failure, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: failureContext(), + }); + throw failure; })(); cutoverFinalization = finalization; cutoverFinalizationOutcome = "commit"; diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts index ec0ad3cb306..8ccc7295e87 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts @@ -233,7 +233,7 @@ describe("Docker startup-command sandbox creation", () => { rollback, }); - await patch.commitAfterReady(); + await expect(patch.commitAfterReady()).rejects.toThrow("receipt validation failed"); expect(events).toEqual(["commit", "rollback", "exit"]); expect(onPatchFailureExit).toHaveBeenCalledWith( diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index cf80b4d91a4..7c02f2314c5 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -44,7 +44,11 @@ all three names, both launch-spec hashes, image identity, profile fingerprint, and sandbox ID and then enter the destructive cutover. Rollback publishes `rollback-authorized` before exact replacement deletion; commit publishes `shared-state-committed` before exact backup deletion. Cleanup is bound to full -runtime IDs. Its private state root now retains enumerable, versioned unfinished +runtime IDs. Commit or rollback is claimed synchronously before asynchronous +finalization begins. Repeated calls for the claimed outcome share its one pending +result, while the opposite outcome remains invalid even if acknowledgement of the +first finalization is lost. Its private state root now retains enumerable, +versioned unfinished records containing the provider and sandbox identities, plan and profile fingerprints, exact original and replacement IDs, rollback target, and phase. Exact commit and cleanup receipts are durable terminal records, so adapter @@ -83,7 +87,7 @@ ordinary startup handoff. OpenClaw, Hermes, and DCode images now package the root-owned hold, trampoline, runtime bundle, and complete inert capability union. Pull-request and publication workflows build the exact images and run the direct root-stdin and hold contract without advertising buildless support. -No production provider invokes the trampoline yet. Until the later provider -activation and protected E2E slices tracked by +No production provider invokes the trampoline yet. Until the provider +activation and protected all-agent E2E exit criteria tracked by [epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) pass, every production runtime provider keeps bootstrap unsupported. diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts new file mode 100644 index 00000000000..e9b40f6340b --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const adapterMocks = vi.hoisted(() => ({ + activate: vi.fn(), + finalize: vi.fn(), + prepare: vi.fn(), +})); + +vi.mock("./adapter", async (importOriginal) => ({ + ...(await importOriginal()), + activateManagedBootstrapSequence: adapterMocks.activate, + finalizeManagedBootstrapSequence: adapterMocks.finalize, + prepareManagedBootstrapSequence: adapterMocks.prepare, +})); + +import type { + ManagedBootstrapActivatedTransaction, + ManagedBootstrapAdapter, + ManagedBootstrapPreparedTransaction, +} from "./adapter"; +import { createDockerManagedBootstrapSurface } from "./docker-runtime"; +import { authority, IDENTITY, NEW_ID, OLD_ID } from "./docker-test-fixture"; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("Docker managed-bootstrap lifecycle composition", () => { + it("does not finalize rollback after a claimed commit loses acknowledgement", async () => { + const seed = authority("openclaw"); + const prepared = Object.freeze({}) as ManagedBootstrapPreparedTransaction; + const activated = Object.freeze({ + snapshot: { runtimeId: OLD_ID }, + replacement: { replacementRuntimeId: NEW_ID }, + }) as ManagedBootstrapActivatedTransaction; + adapterMocks.prepare.mockImplementation(async (_adapter, input) => { + await input.create.launch({ + heldWorkloadArgv: seed.handle.heldWorkloadArgv, + bootstrapIdentity: IDENTITY, + }); + return prepared; + }); + adapterMocks.activate.mockResolvedValue(activated); + adapterMocks.finalize.mockRejectedValue(new Error("commit acknowledgement lost")); + const onPatchFailure = vi.fn((error: unknown): never => { + throw error; + }); + const lifecycle = createDockerManagedBootstrapSurface().createLifecycle({ + providerId: "docker", + bootstrapIdentity: IDENTITY, + request: seed.request, + image: seed.plan.image, + agentIdentity: seed.plan.agentIdentity, + intendedWorkloadArgv: seed.plan.intendedWorkloadArgv, + expectedSupervisorArgv: seed.plan.expectedSupervisorArgv, + launchArgv: ["openshell", "sandbox", "create", "--name", "alpha"], + heldWorkloadArgv: seed.handle.heldWorkloadArgv, + authorityStore: { + recordPreparedAuthority: vi.fn(), + }, + adapterOverride: {} as ManagedBootstrapAdapter, + route: "none", + persistStartupCommand: false, + sandboxName: "alpha", + sandboxGpuConfig: { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }, + requiredLimits: [], + timeoutSecs: 30, + onPatchFailure, + network: { + inferenceProvider: "openai", + dockerDriverGateway: false, + gatewayPort: 0, + }, + dependencies: {}, + }); + + await expect( + lifecycle.runCreate(async () => ({ value: "launched", receipt: seed.handle.createReceipt })), + ).resolves.toBe("launched"); + const failure = (await Promise.resolve(lifecycle.patch.commitAfterReady()).catch( + (error: unknown) => error, + )) as Error & { managedBootstrapRollbackError?: Error }; + + expect(failure).toBeInstanceOf(Error); + expect(failure.message).toBe("commit acknowledgement lost"); + expect(failure.managedBootstrapRollbackError?.message).toBe( + "Managed bootstrap rollback is no longer legal after commit finalization began.", + ); + expect(adapterMocks.finalize).toHaveBeenCalledOnce(); + expect(adapterMocks.finalize).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ outcome: "commit", transaction: activated }), + ); + expect(onPatchFailure).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts index ada0c3457ac..0364cd0ab5a 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -31,6 +31,7 @@ import type { ManagedBootstrapRuntimeCreateLifecycleInput, ManagedBootstrapRuntimeOnboardRoutingInput, } from "./runtime-create"; +import { createManagedBootstrapTerminalFinalizer } from "./runtime-create"; type SupportedBootstrapSurface = Extract< RuntimeProviderBootstrapSurface, @@ -191,7 +192,12 @@ function createDockerLifecycle( }); throw new Error("Managed bootstrap did not return its OpenShell create receipt."); } - let finalized = false; + const finalizer = createManagedBootstrapTerminalFinalizer((outcome) => + finalizeManagedBootstrapSequence(adapter, { + outcome, + transaction: activated, + }).then(() => undefined), + ); patch.attachManagedBootstrapCutover({ selectedMode: mode, failureContext: { @@ -201,22 +207,8 @@ function createDockerLifecycle( backupContainerName: null, selectedMode: mode, }, - async rollback() { - if (finalized) return; - await finalizeManagedBootstrapSequence(adapter, { - outcome: "rollback", - transaction: activated, - }); - finalized = true; - }, - async commit() { - if (finalized) return; - await finalizeManagedBootstrapSequence(adapter, { - outcome: "commit", - transaction: activated, - }); - finalized = true; - }, + rollback: finalizer.rollback, + commit: finalizer.commit, }); return launched.value; }, diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.test.ts b/src/lib/onboard/managed-bootstrap/runtime-create.test.ts new file mode 100644 index 00000000000..ad591f19277 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/runtime-create.test.ts @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { createManagedBootstrapTerminalFinalizer } from "./runtime-create"; + +describe("managed bootstrap terminal finalizer", () => { + it("shares one in-flight outcome and rejects an opposite concurrent outcome", async () => { + let release = (): void => {}; + const finalize = vi.fn( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + const finalizer = createManagedBootstrapTerminalFinalizer(finalize); + + const firstCommit = finalizer.commit(); + const duplicateCommit = finalizer.commit(); + + expect(duplicateCommit).toBe(firstCommit); + await expect(finalizer.rollback()).rejects.toThrow( + "rollback is no longer legal after commit finalization began", + ); + release(); + await expect(Promise.all([firstCommit, duplicateCommit])).resolves.toEqual([ + undefined, + undefined, + ]); + expect(finalize).toHaveBeenCalledExactlyOnceWith("commit"); + }); + + it("retains the claimed outcome after a lost finalization acknowledgement", async () => { + const finalize = vi.fn(async () => { + throw new Error("commit acknowledgement lost"); + }); + const finalizer = createManagedBootstrapTerminalFinalizer(finalize); + + await expect(finalizer.commit()).rejects.toThrow("commit acknowledgement lost"); + await expect(finalizer.rollback()).rejects.toThrow( + "rollback is no longer legal after commit finalization began", + ); + expect(finalize).toHaveBeenCalledExactlyOnceWith("commit"); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts index 1fe762d1a4e..a98dcf71c90 100644 --- a/src/lib/onboard/managed-bootstrap/runtime-create.ts +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -91,6 +91,42 @@ export interface ManagedBootstrapRuntimeCreateLaunchResult { readonly receipt: ManagedBootstrapCreateReceipt; } +export type ManagedBootstrapTerminalOutcome = "commit" | "rollback"; + +export interface ManagedBootstrapTerminalFinalizer { + commit(): Promise; + rollback(): Promise; +} + +/** + * Claim one terminal outcome before driver finalization starts. Duplicate calls + * for that outcome share the in-flight promise; the opposite outcome fails + * closed even when finalization loses acknowledgement. + */ +export function createManagedBootstrapTerminalFinalizer( + finalize: (outcome: ManagedBootstrapTerminalOutcome) => Promise, +): ManagedBootstrapTerminalFinalizer { + let claimedOutcome: ManagedBootstrapTerminalOutcome | null = null; + let pending: Promise | null = null; + const run = (outcome: ManagedBootstrapTerminalOutcome): Promise => { + if (claimedOutcome === outcome && pending !== null) return pending; + if (claimedOutcome !== null) { + return Promise.reject( + new Error( + `Managed bootstrap ${outcome} is no longer legal after ${claimedOutcome} finalization began.`, + ), + ); + } + claimedOutcome = outcome; + pending = Promise.resolve().then(() => finalize(outcome)); + return pending; + }; + return Object.freeze({ + commit: () => run("commit"), + rollback: () => run("rollback"), + }); +} + export interface ManagedBootstrapRuntimeCreateLifecycle { readonly launchArgv: readonly string[]; readonly patch: ManagedBootstrapRuntimePatch; diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 09e703f6d4b..be6fdef53a2 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -62,7 +62,13 @@ export interface SandboxCreateLaunchInput { openshellShellCommand: OpenshellShellCommand; openshellArgv?: OpenshellArgv; buildEnv?(): Record; - /** Dormant until a complete runtime bundle and durable authority store are selected. */ + /** + * Intentional partial migration: remains unset until production selects a + * complete runtime bundle with supported bootstrap after epic #7744's durable + * lifecycle, recovery, and rollback gates plus exact-head/base protected + * all-agent amd64/arm64, GPU/local-inference, and regression matrix pass. + * https://github.com/NVIDIA/NemoClaw/issues/7744 + */ managedStartupRootApplyRequest?: ManagedStartupRootApplyRequest | null; } diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index b642c66cc93..8d718b95d13 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -169,8 +169,21 @@ describe("runtime provider central source boundary", () => { /(?:driverId|providerId)\s*(?:===|!==)\s*["'](?:docker|podman)["']/iu, ); expect(bootstrapProtocol.join("\n")).not.toMatch(/\b(?:docker|podman|openshell|mxc)\b/iu); - expect(activationSources.join("\n")).not.toMatch( - /(?:from\s+["'][^"']*managed-bootstrap\/(?:docker|docker-journal|docker-runtime)|require\([^)]*managed-bootstrap)/u, + const activationWithoutAllowedProtocolImports = activationSources + .map((source) => + source + .replace( + /import\s+(?:type\s+)?\{[^}]*\}\s+from\s+["'][^"']*managed-bootstrap\/(?:adapter|envelope)["'];?/gu, + "", + ) + .replace( + /import\s+type\s+\{[^}]*\}\s+from\s+["'][^"']*managed-bootstrap\/runtime-create["'];?/gu, + "", + ), + ) + .join("\n"); + expect(activationWithoutAllowedProtocolImports).not.toMatch( + /(?:from\s+["'][^"']*managed-bootstrap|require\([^)]*managed-bootstrap|import\([^)]*managed-bootstrap)/u, ); expect(dockerProvider).not.toMatch( /(?:from\s+["'][^"']*managed-bootstrap|require\([^)]*managed-bootstrap)/u,