fix(cli): restore full E2E lifecycle convergence

docs/inference/switch-providers.mdx

@@ -74,7 +74,10 @@ Run the rebuild before relying on the running agent.
 Use `--no-verify` only when OpenShell cannot verify the target provider at switch time and you have already confirmed its provider and credential.
 This flag does not bypass shared-gateway compatibility checks.
 When you explicitly supply a direct compatible endpoint at `http://host.openshell.internal:<port>`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network.
-It then sends one validation request from the target sandbox before persisting the route in NemoClaw state; the request allows up to 16 output tokens.
+It then sends a validation request from the target sandbox before persisting the route in NemoClaw state; the request allows up to 16 output tokens.
+When the switch changes the API family and that request returns HTTP `400` or `404`, NemoClaw retries up to two times after delays of one and two seconds.
+Each retry allows another 16 output tokens.
+Other failures are not retried.
 If that request fails, NemoClaw attempts to restore the previous OpenShell selection and remove a provider that this switch created.
 If the error reports that rollback could not complete, rerun onboarding before using the route or retrying the switch.
 Endpoint-shape and shared-gateway compatibility checks still apply.

docs/reference/commands.mdx

@@ -1045,7 +1045,9 @@ On Docker Desktop WSL, the compatibility path skips CDI and tries Docker `--gpus
 On Jetson/Tegra hosts, the compatibility path uses the NVIDIA runtime and adds eligible host group IDs for the supported GPU device nodes.
 These include selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices.
 After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, GPU, and applicable local-inference checks.
-If a later check fails, onboarding prints failure diagnostics and attempts to restore the pre-patch container.
+If one of those checks fails before backup removal, onboarding prints failure diagnostics and attempts to restore the pre-patch container.
+To commit the replacement, NemoClaw stops it, removes the rollback backup, starts the replacement as the final container lifecycle event, and verifies OpenShell supervisor readiness again.
+If that final handoff cannot be confirmed, onboarding exits with the container diagnostics and cleanup guidance instead of reporting success.
 If rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance.
 GPU-proof diagnostics are captured before rollback and can print that guidance before the final container state is known, so inspect the sandbox and its labeled Docker containers before running a deletion command.
 
@@ -4126,7 +4128,10 @@ If the in-sandbox config write or integrity hash update fails, the OpenShell rou
 Supported provider names are `nvidia-prod`, `nvidia-nim`, `nvidia-router`, `openai-api`, `anthropic-prod`, `compatible-anthropic-endpoint`, `gemini-api`, `compatible-endpoint`, `hermes-provider`, `ollama-local`, and `vllm-local`.
 Use `--no-verify` only when OpenShell cannot verify the provider at switch time but you have already confirmed the provider and credential.
 When you explicitly supply a direct compatible endpoint at `http://host.openshell.internal:<port>`, NemoClaw skips OpenShell's host-side provider probe because that hostname resolves only inside the sandbox network.
-Before it persists the route in the NemoClaw registry or agent config, the command sends one validation request from the target sandbox with a 16-token output limit.
+Before it persists the route in the NemoClaw registry or agent config, the command sends a validation request from the target sandbox with a 16-token output limit.
+When the switch changes the API family and that request returns HTTP `400` or `404`, NemoClaw retries up to two times after delays of one and two seconds.
+Each retry has the same 16-token output limit.
+Other failures are not retried.
 If that request fails, the command attempts to restore the previous OpenShell selection and remove a provider that this switch created.
 If the error reports that rollback could not complete, rerun onboarding before using the route or retrying the switch.
 Endpoint-shape and shared-gateway compatibility checks still apply.

scripts/checks/run-managed-image-openshell-e2e.ts

@@ -8,6 +8,7 @@ import os from "node:os";
 import path from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { resolveAgent } from "../../src/lib/agent/onboard.ts";
+import { parseOpenShellSandboxId } from "../../src/lib/adapters/openshell/sandbox-identity.ts";
 import { isValidName, NAME_ALLOWED_FORMAT } from "../../src/lib/name-validation.ts";
 import {
   type StopHostGatewayResult,
@@ -21,6 +22,7 @@ import {
   MANAGED_BOOTSTRAP_SCHEMA_VERSION,
   type ManagedBootstrapAdapter,
   type ManagedBootstrapAuthorityStore,
+  ManagedBootstrapOwnerCleanupRequiredError,
 } from "../../src/lib/onboard/managed-bootstrap/adapter.ts";
 import { createDockerManagedBootstrapAdapter } from "../../src/lib/onboard/managed-bootstrap/docker.ts";
 import { createDockerManagedBootstrapSurface } from "../../src/lib/onboard/managed-bootstrap/docker-runtime.ts";
@@ -36,6 +38,7 @@ import type {
   RuntimeProviderBundle,
 } from "../../src/lib/onboard/runtime-provider/contract.ts";
 import { createDockerRuntimeProviderBundle } from "../../src/lib/onboard/runtime-provider/docker.ts";
+import { parseLiveSandboxNames } from "../../src/lib/runtime-recovery.ts";
 import {
   OPENSHELL_SANDBOX_SUPERVISOR_ARGV,
   prepareSandboxCreateLaunch,
@@ -786,23 +789,50 @@ export function assertExactSandboxImage(
   return resolved.exactIds[0] ?? "";
 }
 
-export function assertFailedBootstrapContainerCleanup(
+export function assertFailedBootstrapOwnerCleanupRetention(
   input: Inputs,
   networkName: string,
+  expectedRuntimeId: string,
   env: NodeJS.ProcessEnv,
   runCommand: ManagedImageCommandRunner = commandResult,
 ): void {
   const resolved = exactHarnessContainerIds(input, networkName, env, true, runCommand);
-  if (resolved.candidateCount !== 0 || resolved.exactIds.length !== 0) {
+  if (
+    resolved.candidateCount !== 1 ||
+    resolved.exactIds.length !== 1 ||
+    resolved.exactIds[0] !== expectedRuntimeId
+  ) {
+    throw new Error(
+      `managed-bootstrap rollback did not retain its one exact owner-cleanup runtime: found ${resolved.candidateCount} labeled and ${resolved.exactIds.length} exact containers`,
+    );
+  }
+  const inspect = runCommand(["docker", "inspect", expectedRuntimeId], env);
+  if (inspect.status !== 0) {
     throw new Error(
-      `managed-bootstrap rollback retained a failed held sandbox: found ${resolved.candidateCount} labeled and ${resolved.exactIds.length} exact containers`,
+      `managed-bootstrap rollback could not inspect its retained owner-cleanup runtime: ${commandDetail(inspect)}`,
+    );
+  }
+  try {
+    const records = JSON.parse(String(inspect.stdout ?? "")) as Array<{
+      State?: { Paused?: boolean; Restarting?: boolean; Running?: boolean };
+    }>;
+    const state = records.length === 1 ? records[0]?.State : undefined;
+    if (state?.Running !== false || state.Paused !== false || state.Restarting !== false) {
+      throw new Error("retained runtime is not explicitly quiescent");
+    }
+  } catch (error) {
+    throw new Error(
+      `managed-bootstrap rollback did not prove a quiescent owner-cleanup runtime: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
     );
   }
 }
 
-function assertFailedSandboxAbsent(
+export function assertFailedSandboxOwnerCleanupRetention(
   onboard: OnboardModule,
   input: Inputs,
+  expectedSandboxId: string,
   env: NodeJS.ProcessEnv,
 ): void {
   const get = onboard.runOpenshell(["sandbox", "get", input.sandbox], {
@@ -816,12 +846,13 @@ function assertFailedSandboxAbsent(
     stdio: ["ignore", "pipe", "pipe"],
   });
   if (
-    get.status === 0 ||
+    get.status !== 0 ||
+    parseOpenShellSandboxId(String(get.stdout ?? "")) !== expectedSandboxId ||
     list.status !== 0 ||
-    `${list.stdout ?? ""}\n${list.stderr ?? ""}`.includes(input.sandbox)
+    !parseLiveSandboxNames(String(list.stdout ?? "")).has(input.sandbox)
   ) {
     throw new Error(
-      `managed-bootstrap rollback retained failed OpenShell sandbox state: get=${commandDetail(get)} list=${commandDetail(list)}`,
+      `managed-bootstrap rollback did not retain its exact OpenShell owner-cleanup state: get=${commandDetail(get)} list=${commandDetail(list)}`,
     );
   }
 }
@@ -1021,11 +1052,29 @@ async function run<T extends ManagedImageOpenShellE2eLocalInferenceEvidence = ne
         error instanceof Error &&
         error.message.includes("protected-e2e-injected-bootstrap-completion-failure")
       ) {
-        assertFailedBootstrapContainerCleanup(input, networkName, launch.sandboxEnv);
-        assertFailedSandboxAbsent(onboard, input, launch.sandboxEnv);
+        const rollbackError = (error as Error & { managedBootstrapRollbackError?: unknown })
+          .managedBootstrapRollbackError;
+        if (
+          !(rollbackError instanceof ManagedBootstrapOwnerCleanupRequiredError) ||
+          rollbackError.sandboxName !== input.sandbox
+        ) {
+          throw error;
+        }
+        assertFailedBootstrapOwnerCleanupRetention(
+          input,
+          networkName,
+          rollbackError.runtimeId,
+          launch.sandboxEnv,
+        );
+        assertFailedSandboxOwnerCleanupRetention(
+          onboard,
+          input,
+          rollbackError.sandboxId,
+          launch.sandboxEnv,
+        );
         failureInjectionQualified = true;
         process.stdout.write(
-          `Injected managed-bootstrap completion failure removed the failed exact ${input.agent} sandbox before harness cleanup.\n`,
+          `Injected managed-bootstrap completion failure retained one exact quiescent ${input.agent} sandbox for owner cleanup.\n`,
         );
       } else {
         throw error;
@@ -1238,7 +1287,7 @@ async function run<T extends ManagedImageOpenShellE2eLocalInferenceEvidence = ne
   }
   if (failureInjectionQualified) {
     process.stdout.write(
-      `Managed-bootstrap failure injection left no sandbox, container, network, or harness state orphan for ${input.agent}.\n`,
+      `Managed-bootstrap failure injection retained only its exact quiescent sandbox until harness owner cleanup and left no sandbox, container, network, or harness state orphan for ${input.agent}.\n`,
     );
   }
   return {

src/lib/actions/inference-set-compatible-provider.test.ts

@@ -668,6 +668,208 @@ describe("runInferenceSet compatible providers", () => {
     );
   });
 
+  it("waits for a changed API family to replace the previous sandbox route", async () => {
+    const captureOpenshell = createCompatibleProviderCapture({
+      name: "compatible-anthropic-endpoint",
+      type: "anthropic",
+      credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
+      configKey: "ANTHROPIC_BASE_URL",
+      initiallyPresent: false,
+    });
+    const probeSandboxRoute = vi
+      .fn()
+      .mockReturnValueOnce({
+        ok: false,
+        detail: "sandbox inference invocation probe returned HTTP 400",
+        httpStatus: 400,
+      })
+      .mockReturnValueOnce({ ok: true });
+    const deps = createDeps({
+      config: {
+        agents: { defaults: { model: { primary: "inference/old-model" } } },
+        models: { providers: { inference: { api: "openai-completions", models: [] } } },
+      },
+      entry: {
+        name: "alpha",
+        agent: "openclaw",
+        provider: "compatible-endpoint",
+        model: "old-model",
+      },
+      session: baseSession({
+        provider: "compatible-endpoint",
+        model: "old-model",
+        preferredInferenceApi: "openai-completions",
+      }),
+      captureOpenshell,
+      probeSandboxRoute,
+    });
+
+    await runInferenceSet(
+      {
+        provider: "compatible-anthropic-endpoint",
+        model: "mock-anthropic-model",
+        endpointUrl: "http://host.openshell.internal:18767/",
+        credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
+        inferenceApi: "anthropic-messages",
+      },
+      deps,
+    );
+
+    expect(probeSandboxRoute).toHaveBeenCalledTimes(2);
+    expect(deps.calls.sleep).toHaveBeenCalledWith(1_000);
+    expect(deps.calls.updateSandbox).toHaveBeenCalled();
+  });
+
+  it("restores the prior route after changed-family convergence retries are exhausted", async () => {
+    const captureOpenshell = createCompatibleProviderCapture({
+      name: "compatible-anthropic-endpoint",
+      type: "anthropic",
+      credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
+      configKey: "ANTHROPIC_BASE_URL",
+      initiallyPresent: false,
+    });
+    const probeSandboxRoute = vi
+      .fn()
+      .mockReturnValueOnce({
+        ok: false,
+        detail: "sandbox inference invocation probe returned HTTP 400",
+        httpStatus: 400,
+      })
+      .mockReturnValueOnce({
+        ok: false,
+        detail: "sandbox inference invocation probe returned HTTP 404",
+        httpStatus: 404,
+      })
+      .mockReturnValueOnce({
+        ok: false,
+        detail: "sandbox inference invocation probe returned HTTP 400",
+        httpStatus: 400,
+      });
+    const deps = createDeps({
+      config: {
+        agents: { defaults: { model: { primary: "inference/old-model" } } },
+        models: { providers: { inference: { api: "openai-completions", models: [] } } },
+      },
+      entry: {
+        name: "alpha",
+        agent: "openclaw",
+        provider: "nvidia-prod",
+        model: "old-model",
+      },
+      session: baseSession({ provider: "nvidia-prod", model: "old-model" }),
+      captureOpenshell,
+      probeSandboxRoute,
+    });
+
+    await expect(
+      runInferenceSet(
+        {
+          provider: "compatible-anthropic-endpoint",
+          model: "mock-anthropic-model",
+          endpointUrl: "http://host.openshell.internal:18767/",
+          credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY",
+          inferenceApi: "anthropic-messages",
+        },
+        deps,
+      ),
+    ).rejects.toThrow(
+      /Sandbox-side verification rejected.*previous OpenShell inference selection was restored/s,
+    );
+
+    expect(probeSandboxRoute).toHaveBeenCalledTimes(3);
+    expect(deps.calls.sleep.mock.calls).toEqual([[1_000], [2_000]]);
+    expect(
+      captureOpenshell.mock.calls
+        .filter(([args]) => args[0] === "inference" && args[1] === "set")
+        .map(([args]) => args),
+    ).toEqual([
+      [
+        "inference",
+        "set",
+        "-g",
+        "nemoclaw",
+        "--provider",
+        "compatible-anthropic-endpoint",
+        "--model",
+        "mock-anthropic-model",
+        "--no-verify",
+      ],
+      [
+        "inference",
+        "set",
+        "-g",
+        "nemoclaw",
+        "--provider",
+        "nvidia-prod",
+        "--model",
+        "old-model",
+        "--no-verify",
+      ],
+    ]);
+    expect(
+      captureOpenshell.mock.calls
+        .filter(([args]) => args[0] === "provider" && args[1] === "delete")
+        .map(([args]) => args),
+    ).toEqual([
+      [
+        "provider",
+        "delete",
+        "-g",
+        "nemoclaw",
+        "compatible-anthropic-endpoint",
+      ],
+    ]);
+    expect(deps.calls.updateSandbox).not.toHaveBeenCalled();
+    expect(deps.calls.writeSandboxConfig).not.toHaveBeenCalled();
+  });
+
+  it("does not retry a target rejection when the API family did not change", async () => {
+    const captureOpenshell = createCompatibleProviderCapture({
+      name: "compatible-endpoint",
+      type: "openai",
+      credentialEnv: "COMPATIBLE_API_KEY",
+      configKey: "OPENAI_BASE_URL",
+      initiallyPresent: false,
+    });
+    const probeSandboxRoute = vi.fn(() => ({
+      ok: false as const,
+      detail: "sandbox inference invocation probe returned HTTP 400",
+      httpStatus: 400,
+    }));
+    const deps = createDeps({
+      config: {
+        agents: { defaults: { model: { primary: "inference/old-model" } } },
+        models: { providers: { inference: { api: "openai-completions", models: [] } } },
+      },
+      entry: {
+        name: "alpha",
+        agent: "openclaw",
+        provider: "nvidia-prod",
+        model: "old-model",
+      },
+      session: baseSession({ provider: "nvidia-prod", model: "old-model" }),
+      captureOpenshell,
+      probeSandboxRoute,
+    });
+
+    await expect(
+      runInferenceSet(
+        {
+          provider: "compatible-endpoint",
+          model: "mock-model",
+          endpointUrl: "http://host.openshell.internal:18767/",
+          credentialEnv: "COMPATIBLE_API_KEY",
+          inferenceApi: "openai-completions",
+        },
+        deps,
+      ),
+    ).rejects.toThrow(/Sandbox-side verification rejected/);
+
+    expect(probeSandboxRoute).toHaveBeenCalledOnce();
+    expect(deps.calls.sleep).not.toHaveBeenCalled();
+    expect(deps.calls.updateSandbox).not.toHaveBeenCalled();
+  });
+
   it.each([
     [
       "returns a rejection",