Skip to content
21 changes: 21 additions & 0 deletions src/lib/onboard/docker-gpu-local-inference.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,27 @@ describe("verifyGpuSandboxLocalInferenceAndCommitAfterReady", () => {
expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce();
expect(runtimePatch.commitAfterReady).not.toHaveBeenCalled();
});

it("treats a failed commit as terminal without attempting rollback", async () => {
const runtimePatch = {
commitAfterReady: vi.fn(async () => {
throw new Error("durable commit acknowledgement failed");
}),
rollbackManagedStartupAfterCreateFailure: vi.fn(),
};
await expect(
verifyGpuSandboxLocalInferenceAndCommitAfterReady(
GPU_CONFIG,
"ollama-local",
{
...options(),
deps: { execInSandbox: execEmitting("HTTP_200"), sleep: vi.fn() },
},
runtimePatch,
),
).rejects.toThrow("durable commit acknowledgement failed");
expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled();
});
});

describe("printDockerGpuSandboxInferenceVerificationFailure", () => {
Expand Down
2 changes: 1 addition & 1 deletion src/lib/onboard/docker-gpu-local-inference.ts
Original file line number Diff line number Diff line change
Expand Up @@ -511,7 +511,6 @@ export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady(
): Promise<void> {
try {
verifyGpuSandboxLocalInferenceAfterReady(config, provider, options);
await runtimePatch.commitAfterReady();
} catch (error) {
const failure = error instanceof Error ? error : new Error(String(error));
try {
Expand All @@ -523,4 +522,5 @@ export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady(
}
throw failure;
}
await runtimePatch.commitAfterReady();
}
29 changes: 28 additions & 1 deletion src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => {
patch.waitForSupervisorReconnectIfNeeded();
expect(onPatchFailureExit).not.toHaveBeenCalled();

await patch.commitAfterReady();
await expect(patch.commitAfterReady()).rejects.toThrow("rollback backup");

expect(onPatchFailureExit).toHaveBeenCalledOnce();
expect(onPatchFailureExit.mock.calls[0]?.[1]).toEqual(
Expand All @@ -136,6 +136,33 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => {
);
});

it("rejects an early commit after rolling back before supervisor reconnect", async () => {
const deps = makeDeps();
const result = deferredCreateResult();
const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true }));
const onPatchFailureExit = vi.fn();
const patch = createDockerGpuSandboxCreatePatch({
route: "compatibility",
sandboxName: "alpha",
timeoutSecs: 60,
deps,
overrides: {
findContainerIds: vi.fn(() => ["existing-container"]),
recreatePatch: vi.fn(() => result),
finalizeBackup,
onPatchFailureExit,
},
});

patch.maybeApplyDuringCreate();

await expect(patch.commitAfterReady()).rejects.toThrow(
"cannot commit before the recreated OpenShell supervisor reconnects",
);
expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: false }, deps);
expect(onPatchFailureExit).toHaveBeenCalledOnce();
});

it("rolls back to the backup container and surfaces rolledBack=true diagnostics when supervisorReady=false", () => {
const deps = makeDeps();
const result = deferredCreateResult();
Expand Down
41 changes: 19 additions & 22 deletions src/lib/onboard/docker-gpu-sandbox-create.ts
Original file line number Diff line number Diff line change
Expand Up @@ -368,18 +368,15 @@ export function createDockerGpuSandboxCreatePatch(
"Managed startup cannot commit before the recreated OpenShell supervisor reconnects.",
);
const rollbackError = await rollbackAfterFailure();
onPatchFailureExit(
options.sandboxName,
rollbackError
? new Error(`${error.message} Rollback failed: ${rollbackError.message}`)
: error,
{
runCaptureOpenshell: options.deps.runCaptureOpenshell,
dockerCapture: options.deps.dockerCapture,
additionalSummaryLines: routeAdapter.additionalSummaryLines,
},
);
return;
const failure = rollbackError
? new Error(`${error.message} Rollback failed: ${rollbackError.message}`)
: error;
onPatchFailureExit(options.sandboxName, failure, {
runCaptureOpenshell: options.deps.runCaptureOpenshell,
dockerCapture: options.deps.dockerCapture,
additionalSummaryLines: routeAdapter.additionalSummaryLines,
});
throw failure;
}
if (cutoverFinalization) {
if (cutoverFinalizationOutcome !== "commit") {
Expand Down Expand Up @@ -417,24 +414,24 @@ export function createDockerGpuSandboxCreatePatch(
rolledBack: rollbackError === null,
},
});
return;
throw failure;
}
}
const finalizeOutcome = result
? finalizeBackup({ result, supervisorReady: true }, options.deps)
: null;
cutoverFinalized = true;
if (!finalizeOutcome || finalizeOutcome.backupRemoved) return;
onPatchFailureExit(
options.sandboxName,
new Error("Managed startup passed Ready, but its rollback backup could not be removed."),
{
runCaptureOpenshell: options.deps.runCaptureOpenshell,
dockerCapture: options.deps.dockerCapture,
additionalSummaryLines: routeAdapter.additionalSummaryLines,
context: failureContext(),
},
const failure = new Error(
"Managed startup passed Ready, but its rollback backup could not be removed.",
);
onPatchFailureExit(options.sandboxName, failure, {
runCaptureOpenshell: options.deps.runCaptureOpenshell,
dockerCapture: options.deps.dockerCapture,
additionalSummaryLines: routeAdapter.additionalSummaryLines,
context: failureContext(),
});
throw failure;
})();
cutoverFinalization = finalization;
cutoverFinalizationOutcome = "commit";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ describe("Docker startup-command sandbox creation", () => {
rollback,
});

await patch.commitAfterReady();
await expect(patch.commitAfterReady()).rejects.toThrow("receipt validation failed");

expect(events).toEqual(["commit", "rollback", "exit"]);
expect(onPatchFailureExit).toHaveBeenCalledWith(
Expand Down
10 changes: 7 additions & 3 deletions src/lib/onboard/managed-bootstrap/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@ all three names, both launch-spec hashes, image identity, profile fingerprint,
and sandbox ID and then enter the destructive cutover. Rollback publishes
`rollback-authorized` before exact replacement deletion; commit publishes
`shared-state-committed` before exact backup deletion. Cleanup is bound to full
runtime IDs. Its private state root now retains enumerable, versioned unfinished
runtime IDs. Commit or rollback is claimed synchronously before asynchronous
finalization begins. Repeated calls for the claimed outcome share its one pending
result, while the opposite outcome remains invalid even if acknowledgement of the
first finalization is lost. Its private state root now retains enumerable,
versioned unfinished
records containing the provider and sandbox identities, plan and profile
fingerprints, exact original and replacement IDs, rollback target, and phase.
Exact commit and cleanup receipts are durable terminal records, so adapter
Expand Down Expand Up @@ -83,7 +87,7 @@ ordinary startup handoff. OpenClaw, Hermes, and DCode images now package the
root-owned hold, trampoline, runtime bundle, and complete inert capability
union. Pull-request and publication workflows build the exact images and run
the direct root-stdin and hold contract without advertising buildless support.
No production provider invokes the trampoline yet. Until the later provider
activation and protected E2E slices tracked by
No production provider invokes the trampoline yet. Until the provider
activation and protected all-agent E2E exit criteria tracked by
[epic #7744](https://github.com/NVIDIA/NemoClaw/issues/7744) pass, every
production runtime provider keeps bootstrap unsupported.
106 changes: 106 additions & 0 deletions src/lib/onboard/managed-bootstrap/docker-runtime.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { beforeEach, describe, expect, it, vi } from "vitest";

const adapterMocks = vi.hoisted(() => ({
activate: vi.fn<typeof import("./adapter").activateManagedBootstrapSequence>(),
finalize: vi.fn<typeof import("./adapter").finalizeManagedBootstrapSequence>(),
prepare: vi.fn<typeof import("./adapter").prepareManagedBootstrapSequence>(),
}));

vi.mock("./adapter", async (importOriginal) => ({
...(await importOriginal<typeof import("./adapter")>()),
activateManagedBootstrapSequence: adapterMocks.activate,
finalizeManagedBootstrapSequence: adapterMocks.finalize,
prepareManagedBootstrapSequence: adapterMocks.prepare,
}));

import type {
ManagedBootstrapActivatedTransaction,
ManagedBootstrapAdapter,
ManagedBootstrapPreparedTransaction,
} from "./adapter";
import { createDockerManagedBootstrapSurface } from "./docker-runtime";
import { authority, IDENTITY, NEW_ID, OLD_ID } from "./docker-test-fixture";

beforeEach(() => {
vi.clearAllMocks();
});

describe("Docker managed-bootstrap lifecycle composition", () => {
it("does not finalize rollback after a claimed commit loses acknowledgement", async () => {
const seed = authority("openclaw");
const prepared = Object.freeze({}) as ManagedBootstrapPreparedTransaction;
const activated = Object.freeze({
snapshot: { runtimeId: OLD_ID },
replacement: { replacementRuntimeId: NEW_ID },
}) as ManagedBootstrapActivatedTransaction;
adapterMocks.prepare.mockImplementation(async (_adapter, input) => {
await input.create.launch({
heldWorkloadArgv: seed.handle.heldWorkloadArgv,
bootstrapIdentity: IDENTITY,
});
return prepared;
});
adapterMocks.activate.mockResolvedValue(activated);
adapterMocks.finalize.mockRejectedValue(new Error("commit acknowledgement lost"));
const onPatchFailure = vi.fn((error: unknown): never => {
throw error;
});
const lifecycle = createDockerManagedBootstrapSurface().createLifecycle({
providerId: "docker",
bootstrapIdentity: IDENTITY,
request: seed.request,
image: seed.plan.image,
agentIdentity: seed.plan.agentIdentity,
intendedWorkloadArgv: seed.plan.intendedWorkloadArgv,
expectedSupervisorArgv: seed.plan.expectedSupervisorArgv,
launchArgv: ["openshell", "sandbox", "create", "--name", "alpha"],
heldWorkloadArgv: seed.handle.heldWorkloadArgv,
authorityStore: {
recordPreparedAuthority: vi.fn(),
},
adapterOverride: {} as ManagedBootstrapAdapter,
route: "none",
persistStartupCommand: false,
sandboxName: "alpha",
sandboxGpuConfig: {
mode: "0",
hostGpuDetected: false,
hostGpuPlatform: null,
sandboxGpuEnabled: false,
sandboxGpuDevice: null,
errors: [],
},
requiredLimits: [],
timeoutSecs: 30,
onPatchFailure,
network: {
inferenceProvider: "openai",
dockerDriverGateway: false,
gatewayPort: 0,
},
dependencies: {},
});

await expect(
lifecycle.runCreate(async () => ({ value: "launched", receipt: seed.handle.createReceipt })),
).resolves.toBe("launched");
const failure = (await Promise.resolve(lifecycle.patch.commitAfterReady()).catch(
(error: unknown) => error,
)) as Error & { managedBootstrapRollbackError?: Error };

expect(failure).toBeInstanceOf(Error);
expect(failure.message).toBe("commit acknowledgement lost");
expect(failure.managedBootstrapRollbackError?.message).toBe(
"Managed bootstrap rollback is no longer legal after commit finalization began.",
);
expect(adapterMocks.finalize).toHaveBeenCalledOnce();
expect(adapterMocks.finalize).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ outcome: "commit", transaction: activated }),
);
expect(onPatchFailure).toHaveBeenCalledOnce();
});
});
26 changes: 9 additions & 17 deletions src/lib/onboard/managed-bootstrap/docker-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import type {
ManagedBootstrapRuntimeCreateLifecycleInput,
ManagedBootstrapRuntimeOnboardRoutingInput,
} from "./runtime-create";
import { createManagedBootstrapTerminalFinalizer } from "./runtime-create";

type SupportedBootstrapSurface = Extract<
RuntimeProviderBootstrapSurface,
Expand Down Expand Up @@ -191,7 +192,12 @@ function createDockerLifecycle(
});
throw new Error("Managed bootstrap did not return its OpenShell create receipt.");
}
let finalized = false;
const finalizer = createManagedBootstrapTerminalFinalizer((outcome) =>
finalizeManagedBootstrapSequence(adapter, {
outcome,
transaction: activated,
}).then(() => undefined),
);
patch.attachManagedBootstrapCutover({
selectedMode: mode,
failureContext: {
Expand All @@ -201,22 +207,8 @@ function createDockerLifecycle(
backupContainerName: null,
selectedMode: mode,
},
async rollback() {
if (finalized) return;
await finalizeManagedBootstrapSequence(adapter, {
outcome: "rollback",
transaction: activated,
});
finalized = true;
},
async commit() {
if (finalized) return;
await finalizeManagedBootstrapSequence(adapter, {
outcome: "commit",
transaction: activated,
});
finalized = true;
},
rollback: finalizer.rollback,
commit: finalizer.commit,
});
return launched.value;
},
Expand Down
46 changes: 46 additions & 0 deletions src/lib/onboard/managed-bootstrap/runtime-create.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";

import { createManagedBootstrapTerminalFinalizer } from "./runtime-create";

describe("managed bootstrap terminal finalizer", () => {
it("shares one in-flight outcome and rejects an opposite concurrent outcome", async () => {
let release = (): void => {};
const finalize = vi.fn(
() =>
new Promise<void>((resolve) => {
release = resolve;
}),
);
const finalizer = createManagedBootstrapTerminalFinalizer(finalize);

const firstCommit = finalizer.commit();
const duplicateCommit = finalizer.commit();

expect(duplicateCommit).toBe(firstCommit);
await expect(finalizer.rollback()).rejects.toThrow(
"rollback is no longer legal after commit finalization began",
);
release();
await expect(Promise.all([firstCommit, duplicateCommit])).resolves.toEqual([
undefined,
undefined,
]);
expect(finalize).toHaveBeenCalledExactlyOnceWith("commit");
});

it("retains the claimed outcome after a lost finalization acknowledgement", async () => {
const finalize = vi.fn(async () => {
throw new Error("commit acknowledgement lost");
});
const finalizer = createManagedBootstrapTerminalFinalizer(finalize);

await expect(finalizer.commit()).rejects.toThrow("commit acknowledgement lost");
await expect(finalizer.rollback()).rejects.toThrow(
"rollback is no longer legal after commit finalization began",
);
expect(finalize).toHaveBeenCalledExactlyOnceWith("commit");
});
});
Loading
Loading