fix(sandbox): serialize Docker recreation handoff

src/lib/actions/sandbox/rebuild-flow-helpers.ts

@@ -28,6 +28,7 @@ import {
   recoverNamedGatewayRuntime,
 } from "../../gateway-runtime-action";
 import { resolveSandboxGatewayName } from "../../onboard/gateway-binding";
+import { removeStaleRebuildDockerOrphan } from "../../onboard/openshell-docker-sandbox-containers";
 import {
   captureSandboxListWithGatewayRecovery,
   printSandboxListFailureWithRecoveryContext,
@@ -214,6 +215,14 @@ export async function resolveRebuildLiveState(
     // provisioning, so rebuild recovers from registry metadata instead of
     // treating the preserved local entry as corrupt. Keep until OpenShell exposes
     // an atomic recreate-from-registry recovery API.
+    try {
+      removeStaleRebuildDockerOrphan(sandboxName, sb.openshellDriver, log);
+    } catch (error) {
+      bail(
+        `Stale-recovery Docker orphan cleanup failed: ${error instanceof Error ? error.message : String(error)}.`,
+      );
+      return null;
+    }
     console.log("");
     console.log(
       `  ${YW}⚠${R} Sandbox '${sandboxName}' is registered locally but absent from the live OpenShell gateway.`,

src/lib/actions/sandbox/rebuild-gateway-drift.test.ts

@@ -7,6 +7,7 @@ import * as gatewayDrift from "../../adapters/openshell/gateway-drift";
 import * as openshellRuntime from "../../adapters/openshell/runtime";
 import * as gatewayRuntime from "../../gateway-runtime-action";
 import * as dockerDriverRecovery from "../../onboard/docker-driver-sandbox-recovery";
+import * as openshellDockerContainers from "../../onboard/openshell-docker-sandbox-containers";
 import * as registry from "../../state/registry";
 import * as registryPersistence from "../../state/registry/persistence";
 import { type RebuildSandboxEntry, resolveRebuildLiveState } from "./rebuild-flow-helpers";
@@ -15,6 +16,10 @@ import {
   runRebuildGatewayIntentPreflight,
 } from "./rebuild-preflight-guards";
 
+const removeStaleRebuildDockerOrphan = openshellDockerContainers.removeStaleRebuildDockerOrphan;
+type QueryDockerContainers = typeof openshellDockerContainers.queryOpenShellDockerSandboxContainers;
+type ForceRemoveDockerContainer = (containerId: string) => { status?: number | null };
+
 const driftIssue: gatewayDrift.OpenShellStateRpcIssue = {
   kind: "image_drift",
   drift: {
@@ -57,6 +62,8 @@ describe("rebuild gateway drift preflight", () => {
   let recoverNamedGatewayRuntimeSpy: MockInstance;
   let getNamedGatewayLifecycleStateSpy: MockInstance;
   let recoverDockerDriverSandboxSpy: MockInstance;
+  let queryDockerContainersSpy: ReturnType<typeof vi.fn<QueryDockerContainers>>;
+  let forceRemoveDockerContainerSpy: ReturnType<typeof vi.fn<ForceRemoveDockerContainer>>;
   let errorSpy: MockInstance;
   let logSpy: MockInstance;
 
@@ -84,6 +91,19 @@ describe("rebuild gateway drift preflight", () => {
     recoverDockerDriverSandboxSpy = vi
       .spyOn(dockerDriverRecovery, "recoverDockerDriverSandbox")
       .mockReturnValue({ recovered: false, via: null });
+    queryDockerContainersSpy = vi
+      .fn<QueryDockerContainers>()
+      .mockReturnValue({ ok: true, ids: [] });
+    forceRemoveDockerContainerSpy = vi
+      .fn<ForceRemoveDockerContainer>()
+      .mockReturnValue({ status: 0 });
+    vi.spyOn(openshellDockerContainers, "removeStaleRebuildDockerOrphan").mockImplementation(
+      (sandboxName, openshellDriver, log) =>
+        removeStaleRebuildDockerOrphan(sandboxName, openshellDriver, log, {
+          queryContainers: queryDockerContainersSpy,
+          forceRemove: forceRemoveDockerContainerSpy,
+        }),
+    );
     vi.spyOn(registry, "getSandbox").mockReturnValue(makeSandboxEntry() as never);
     vi.spyOn(registryPersistence, "load").mockReturnValue({
       sandboxes: { alpha: makeSandboxEntry() },
@@ -187,6 +207,100 @@ describe("rebuild gateway drift preflight", () => {
     expect(behaviorLog.mock.calls.flat().join("\n")).toContain("Stale-sandbox recovery");
   });
 
+  it("removes one exactly labeled Docker orphan before a registry-only rebuild (#8720)", async () => {
+    const entry = { ...makeSandboxEntry(), openshellDriver: "docker" };
+    vi.mocked(registry.getSandbox).mockReturnValue(entry as never);
+    captureOpenshellSpy
+      .mockReturnValueOnce({ status: 0, output: "" })
+      .mockReturnValueOnce({ status: 1, output: "Error: sandbox not found" });
+    queryDockerContainersSpy
+      .mockReturnValueOnce({ ok: true, ids: ["orphan-container-id"] })
+      .mockReturnValueOnce({ ok: true, ids: [] });
+
+    await expect(resolveRebuildLiveState("alpha", entry, vi.fn(), bail)).resolves.toMatchObject({
+      staleRecovery: true,
+    });
+
+    expect(forceRemoveDockerContainerSpy).toHaveBeenCalledWith("orphan-container-id");
+    expect(queryDockerContainersSpy).toHaveBeenCalledTimes(2);
+  });
+
+  it("preserves legacy stale recovery when Docker inspection is unavailable (#8720)", async () => {
+    const entry = makeSandboxEntry();
+    vi.mocked(registry.getSandbox).mockReturnValue(entry as never);
+    captureOpenshellSpy
+      .mockReturnValueOnce({ status: 0, output: "" })
+      .mockReturnValueOnce({ status: 1, output: "Error: sandbox not found" });
+    queryDockerContainersSpy.mockReturnValue({ ok: false, ids: [], error: "docker unavailable" });
+
+    await expect(resolveRebuildLiveState("alpha", entry, vi.fn(), bail)).resolves.toMatchObject({
+      staleRecovery: true,
+    });
+
+    expect(forceRemoveDockerContainerSpy).not.toHaveBeenCalled();
+    expect(registryPersistence.load).toHaveBeenCalledOnce();
+  });
+
+  it("refuses ambiguous labeled Docker orphan cleanup without removing either container (#8720)", async () => {
+    const entry = { ...makeSandboxEntry(), openshellDriver: "docker" };
+    vi.mocked(registry.getSandbox).mockReturnValue(entry as never);
+    captureOpenshellSpy
+      .mockReturnValueOnce({ status: 0, output: "" })
+      .mockReturnValueOnce({ status: 1, output: "Error: sandbox not found" });
+    queryDockerContainersSpy.mockReturnValue({ ok: true, ids: ["first-id", "second-id"] });
+
+    await expect(resolveRebuildLiveState("alpha", entry, vi.fn(), bail)).rejects.toThrow(
+      "refusing ambiguous orphan cleanup",
+    );
+
+    expect(forceRemoveDockerContainerSpy).not.toHaveBeenCalled();
+    expect(registryPersistence.load).not.toHaveBeenCalled();
+  });
+
+  it.each([
+    {
+      failure: "query failure",
+      queryResults: [{ ok: false, ids: [], error: "docker unavailable" }],
+      removeResult: { status: 0 },
+      removeCalls: 0,
+    },
+    {
+      failure: "removal failure",
+      queryResults: [{ ok: true, ids: ["orphan-container-id"] }],
+      removeResult: { status: 1 },
+      removeCalls: 1,
+    },
+    {
+      failure: "confirmation failure",
+      queryResults: [
+        { ok: true, ids: ["orphan-container-id"] },
+        { ok: true, ids: ["orphan-container-id"] },
+      ],
+      removeResult: { status: 0 },
+      removeCalls: 1,
+    },
+  ])("fails closed before registry recovery on Docker orphan $failure (#8720)", async ({
+    queryResults,
+    removeResult,
+    removeCalls,
+  }) => {
+    const entry = { ...makeSandboxEntry(), openshellDriver: "docker" };
+    vi.mocked(registry.getSandbox).mockReturnValue(entry as never);
+    captureOpenshellSpy
+      .mockReturnValueOnce({ status: 0, output: "" })
+      .mockReturnValueOnce({ status: 1, output: "Error: sandbox not found" });
+    queryDockerContainersSpy.mockReturnValueOnce(queryResults[0] as never);
+    queryDockerContainersSpy.mockReturnValueOnce((queryResults[1] ?? queryResults[0]) as never);
+    forceRemoveDockerContainerSpy.mockReturnValue(removeResult);
+
+    await expect(resolveRebuildLiveState("alpha", entry, vi.fn(), bail)).rejects.toThrow(
+      "Stale-recovery Docker orphan cleanup failed",
+    );
+
+    expect(forceRemoveDockerContainerSpy).toHaveBeenCalledTimes(removeCalls);
+    expect(registryPersistence.load).not.toHaveBeenCalled();
+  });
+
   it.each([
     { gatewayName: "nemoclaw", gatewayPort: 8080 },
     { gatewayName: "nemoclaw-12345", gatewayPort: 12345 },

src/lib/onboard/openshell-docker-sandbox-containers.ts

@@ -10,6 +10,7 @@ export const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name";
 export const OPENSHELL_SANDBOX_ID_LABEL = "openshell.ai/sandbox-id";
 
 const DOCKER_SANDBOX_QUERY_TIMEOUT_MS = 30_000;
+const STALE_DOCKER_ORPHAN_TIMEOUT_MS = 30_000;
 
 type DockerSandboxContainerQueryDeps = Pick<DockerGpuPatchDeps, "dockerCapture" | "dockerRun">;
 
@@ -81,6 +82,56 @@ export function queryOpenShellDockerSandboxContainers(
   return { ok: true, ids };
 }
 
+type StaleDockerOrphanCleanupDeps = {
+  queryContainers?: typeof queryOpenShellDockerSandboxContainers;
+  forceRemove?: (containerId: string) => { status?: number | null };
+};
+
+/**
+ * Remove one exact Docker-owned orphan when a registry row outlives its OpenShell sandbox.
+ * Docker labels are the remaining authority in this invalid state; see the focused #8720 tests.
+ * Remove this workaround once OpenShell upgrades clean up or expose these orphans natively.
+ */
+export function removeStaleRebuildDockerOrphan(
+  sandboxName: string,
+  openshellDriver: string | null | undefined,
+  log: (message: string) => void,
+  deps: StaleDockerOrphanCleanupDeps = {},
+): void {
+  if (openshellDriver && openshellDriver !== "docker") return;
+  const queryContainers = deps.queryContainers ?? queryOpenShellDockerSandboxContainers;
+  const initial = queryContainers(sandboxName);
+  if (!initial.ok) {
+    if (openshellDriver === "docker") {
+      throw new Error(`could not inspect the owned Docker orphan: ${initial.error}`);
+    }
+    return;
+  }
+  if (initial.ids.length === 0) return;
+  if (initial.ids.length !== 1) {
+    throw new Error(
+      `found ${String(initial.ids.length)} exactly labeled Docker containers; refusing ambiguous orphan cleanup`,
+    );
+  }
+
+  const containerId = initial.ids[0];
+  const removal = deps.forceRemove
+    ? deps.forceRemove(containerId)
+    : dockerRun(["rm", "-f", containerId], {
+        ignoreError: true,
+        suppressOutput: true,
+        timeout: STALE_DOCKER_ORPHAN_TIMEOUT_MS,
+      });
+  if (Number(removal.status ?? 1) !== 0) {
+    throw new Error(`could not remove exactly labeled Docker orphan '${containerId}'`);
+  }
+  const confirmed = queryContainers(sandboxName);
+  if (!confirmed.ok || confirmed.ids.length !== 0) {
+    throw new Error("could not confirm exact Docker orphan removal");
+  }
+  log(`Removed exactly labeled Docker orphan '${containerId}' before stale rebuild`);
+}
+
 export type OpenShellDockerDeviceRequest = {
   Driver: string;
   Count: number;

src/lib/onboard/sandbox-create-step.test.ts

@@ -226,6 +226,55 @@ describe("runSandboxCreateStep", () => {
     expect(patch.maybeApplyDuringCreate).toHaveBeenCalledTimes(1);
   });
 
+  it("waits for the create ownership handoff before restart-safe recreation (#8720)", async () => {
+    vi.useFakeTimers();
+    const child = new FakeChild();
+    const patch = makePatch();
+    let ready = false;
+    let resolved = false;
+    const deps = makeDeps(
+      makeLaunch({ sandboxEnv: dockerEnv }),
+      patch,
+      { status: 0, output: "" },
+      {
+        streamCreate: ((command, args, sandboxEnv, options) =>
+          streamSandboxCreate(command, args, sandboxEnv, {
+            ...options,
+            ...makePollingOptions(child),
+          })) as SandboxCreateStepDeps["streamCreate"],
+        isSandboxReady: vi.fn(() => ready),
+      },
+    );
+
+    const create = runSandboxCreateStep(
+      makeContext({
+        prebuild: {
+          buildCtx: "/tmp/ctx",
+          buildId: "b1",
+          dockerDriverGateway: true,
+          origin: "generated",
+        },
+      }),
+      deps,
+    ).then((result) => {
+      resolved = true;
+      return result;
+    });
+
+    child.stdout.emit("data", Buffer.from("Created sandbox: alpha\n"));
+    ready = true;
+    await vi.advanceTimersByTimeAsync(6);
+
+    expect(resolved).toBe(false);
+    expect(child.kill).toHaveBeenCalledWith("SIGTERM");
+    expect(patch.maybeApplyDuringCreate).not.toHaveBeenCalled();
+
+    child.emit("close", 143);
+    await expect(create).resolves.toMatchObject({
+      createResult: { status: 0, forcedReady: true },
+    });
+  });
+
   it("threads the terminal-agent early-ready gate into stream options", async () => {
     const terminalDeps = makeDeps(
       makeLaunch(),

src/lib/onboard/sandbox-create-step.ts

@@ -98,6 +98,8 @@ export async function runSandboxCreateStep(
     context.agent,
     context.prebuild.dockerDriverGateway,
   );
+  const deferRestartSafeCutover =
+    startupCommandPatch.persistStartupCommand && !context.useDockerGpuPatch;
   const dockerGpuCreatePatch = deps.createDockerGpuPatch({
     route: context.useDockerGpuPatch ? "compatibility" : "native",
     persistStartupCommand: startupCommandPatch.persistStartupCommand,
@@ -123,13 +125,16 @@ export async function runSandboxCreateStep(
         const list = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true });
         return deps.isSandboxReady(list, context.sandboxName);
       },
-      onPoll: () => dockerGpuCreatePatch.maybeApplyDuringCreate(),
+      onPoll: () => {
+        if (!deferRestartSafeCutover) dockerGpuCreatePatch.maybeApplyDuringCreate();
+      },
       readyCheckOutputPatterns: getReadyCheckOutputPatternsForAgent(
         deps.isTerminalAgent(context.agent),
         sandboxEnv,
       ),
       failureCheck: dockerGpuCreatePatch.createFailureMessage,
       traceEvent: deps.addTraceEvent,
+      waitForReadyTermination: deferRestartSafeCutover,
     },
   );
   return { createResult, prebuild, effectiveDashboardPort, dockerGpuCreatePatch };

src/lib/sandbox/create-stream.test.ts

@@ -171,6 +171,34 @@ describe("sandbox-create-stream", () => {
     expect(child.unref).toHaveBeenCalled();
   });
 
+  it("aborts when the Ready ownership handoff does not terminate (#8720)", async () => {
+    vi.useFakeTimers();
+
+    const child = new FakeChild();
+    const promise = streamSandboxCreate("echo create", dockerEnv, {
+      spawnImpl: () => child,
+      readyCheck: () => true,
+      waitForReadyTermination: true,
+      pollIntervalMs: 5,
+      heartbeatIntervalMs: 1_000,
+      silentPhaseMs: 10_000,
+      logLine: vi.fn(),
+    });
+
+    child.stdout.emit("data", Buffer.from("Created sandbox: demo\n"));
+    await vi.advanceTimersByTimeAsync(6);
+    expect(child.kill).toHaveBeenCalledWith("SIGTERM");
+
+    await vi.advanceTimersByTimeAsync(5_001);
+    await expect(promise).resolves.toMatchObject({
+      status: 1,
+      output: expect.stringContaining("did not exit after Ready; aborting cutover"),
+    });
+    expect((await promise).forcedReady).toBeUndefined();
+    expect(child.kill).toHaveBeenCalledWith("SIGKILL");
+    expect(child.unref).not.toHaveBeenCalled();
+  });
+
   it("traces ready-check errors and keeps polling without forcing ready", async () => {
     vi.useFakeTimers();