fix(cli): let strict backup-all skip stranded orphan sandboxes

src/lib/actions/maintenance.test.ts

@@ -8,12 +8,14 @@ const mocks = vi.hoisted(() => ({
   backupSandboxState: vi.fn(),
   captureSandboxListWithGatewayPreflightOrExit: vi.fn(),
   parseReadySandboxNames: vi.fn(),
+  parseLiveSandboxNames: vi.fn(),
   dockerListImagesFormat: vi.fn().mockReturnValue(""),
   dockerRmi: vi.fn(),
   prompt: vi.fn(),
   startStoppedSandboxContainerForBackup: vi.fn(),
   backupStartedSandboxState: vi.fn(),
   returnSandboxContainerToStopped: vi.fn(),
+  isSandboxContainerDefinitivelyAbsent: vi.fn(),
 }));
 
 vi.mock("../state/registry", () => ({
@@ -30,6 +32,15 @@ vi.mock("../openshell-sandbox-list", () => ({
 }));
 vi.mock("../runtime-recovery", () => ({
   parseReadySandboxNames: mocks.parseReadySandboxNames,
+  parseLiveSandboxNames: mocks.parseLiveSandboxNames,
+}));
+// GATEWAY_PORT is baked from NEMOCLAW_GATEWAY_PORT at module load. Pin it so
+// the #6520 orphan-classification tests (which run the real gateway-binding
+// resolvers against literal ports) don't invert on a shell that exports a
+// non-default gateway port.
+vi.mock("../core/ports", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("../core/ports")>()),
+  GATEWAY_PORT: 8080,
 }));
 vi.mock("../adapters/docker", () => ({
   dockerListImagesFormat: mocks.dockerListImagesFormat,
@@ -45,6 +56,7 @@ vi.mock("./sandbox/stopped-sandbox-backup", () => ({
   startStoppedSandboxContainerForBackup: mocks.startStoppedSandboxContainerForBackup,
   backupStartedSandboxState: mocks.backupStartedSandboxState,
   returnSandboxContainerToStopped: mocks.returnSandboxContainerToStopped,
+  isSandboxContainerDefinitivelyAbsent: mocks.isSandboxContainerDefinitivelyAbsent,
 }));
 vi.mock("../domain/lifecycle/options", () => ({
   normalizeGarbageCollectImagesOptions: (o: unknown) => o || {},
@@ -70,6 +82,11 @@ describe("backupAll", () => {
       output: "sb-good\nsb-bad\n",
     });
     mocks.parseReadySandboxNames.mockReturnValue(new Set(["sb-good", "sb-bad"]));
+    // Defaults keep every pre-#6520 case on its original path: no sandbox is
+    // gateway-observed (so orphan classification is decided by the absence
+    // gate alone) and no container is ever definitively absent.
+    mocks.parseLiveSandboxNames.mockReturnValue(new Set());
+    mocks.isSandboxContainerDefinitivelyAbsent.mockReturnValue(false);
     mocks.startStoppedSandboxContainerForBackup.mockReturnValue(null);
     mocks.returnSandboxContainerToStopped.mockReturnValue(true);
   });
@@ -174,10 +191,17 @@ describe("backupAll", () => {
 
     await backupAll();
 
-    expect(mocks.captureSandboxListWithGatewayPreflightOrExit).toHaveBeenCalledWith({
-      action: "backing up registered sandboxes",
-      command: "nemoclaw backup-all",
-    });
+    // The listing must be pinned to the selected gateway (#6114/#6520):
+    // OpenShell's mutable current selection may be a sibling gateway, and an
+    // unpinned list would let the orphan classifier make a fail-open
+    // stranded call from another gateway's sandboxes.
+    expect(mocks.captureSandboxListWithGatewayPreflightOrExit).toHaveBeenCalledWith(
+      {
+        action: "backing up registered sandboxes",
+        command: "nemoclaw backup-all",
+      },
+      { gatewayName: "nemoclaw" },
+    );
     expect(mocks.backupSandboxState).toHaveBeenCalledWith("sb-good");
     logSpy.mockRestore();
   });
@@ -684,6 +708,184 @@ describe("backupAll", () => {
     errorSpy.mockRestore();
     exitSpy.mockRestore();
   });
+
+  it("skips a stranded orphan sandbox without failing strict backup (#6520)", async () => {
+    // Uninstall + reinstall strands a sandbox: gateway registration and
+    // container removed, sandboxes.json preserved. There is nothing left to
+    // back up, so strict backup-all must warn and move on instead of aborting
+    // before the installer's recovery phase can surface the orphan.
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "sb-good" }, { name: "sb-stranded" }],
+      defaultSandbox: null,
+    });
+    mocks.parseReadySandboxNames.mockReturnValue(new Set(["sb-good"]));
+    mocks.parseLiveSandboxNames.mockReturnValue(new Set(["sb-good"]));
+    mocks.isSandboxContainerDefinitivelyAbsent.mockImplementation(
+      (name: string) => name === "sb-stranded",
+    );
+    mocks.backupSandboxState.mockReturnValue({
+      success: true,
+      backedUpDirs: ["workspace"],
+      failedDirs: [],
+      backedUpFiles: [],
+      failedFiles: [],
+      manifest: { backupPath: "/backups/sb-good/timestamp" },
+    });
+    process.env.NEMOCLAW_REQUIRE_ALL_SANDBOX_BACKUPS = "1";
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
+    const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+
+    await backupAll();
+
+    expect(exitSpy).not.toHaveBeenCalled();
+    expect(mocks.backupSandboxState).toHaveBeenCalledWith("sb-good");
+    expect(mocks.backupStartedSandboxState).not.toHaveBeenCalled();
+    // The exemption requires a confirming second pinned listing after the loop.
+    expect(mocks.captureSandboxListWithGatewayPreflightOrExit).toHaveBeenCalledTimes(2);
+    expect(mocks.captureSandboxListWithGatewayPreflightOrExit).toHaveBeenNthCalledWith(
+      2,
+      {
+        action: "confirming stranded sandboxes remain absent from the selected gateway",
+        command: "nemoclaw backup-all",
+      },
+      { gatewayName: "nemoclaw" },
+    );
+    const logOutput = logSpy.mock.calls.flat().join("\n");
+    expect(logOutput).toContain(
+      "1 recorded sandbox(es) were not found on their recorded gateway: sb-stranded.",
+    );
+    expect(logOutput).toContain("destroy` to clear a stranded record");
+    expect(logOutput).toContain("onboard` to rebuild it");
+    expect(logOutput).toContain("1 backed up, 0 failed, 0 skipped");
+    expect(logOutput).not.toContain("Skipping 'sb-stranded'");
+  });
+
+  it("keeps the strict abort for an absent sandbox bound to a different gateway (#6520)", async () => {
+    // A sandbox persisted against a sibling gateway may be healthy there;
+    // this gateway's backup-all must never claim it is stranded, even when
+    // its container is absent on this host.
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "sb-other", gatewayPort: 9999 }],
+      defaultSandbox: null,
+    });
+    mocks.parseReadySandboxNames.mockReturnValue(new Set());
+    mocks.parseLiveSandboxNames.mockReturnValue(new Set());
+    mocks.isSandboxContainerDefinitivelyAbsent.mockReturnValue(true);
+    process.env.NEMOCLAW_REQUIRE_ALL_SANDBOX_BACKUPS = "1";
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
+    const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined);
+    const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+
+    await expect(backupAll()).rejects.toThrow("exit:1");
+
+    expect(exitSpy).toHaveBeenCalledWith(1);
+    const logOutput = logSpy.mock.calls.flat().join("\n");
+    expect(logOutput).toContain("Skipping 'sb-other' (not running");
+    expect(logOutput).not.toContain("were not found on their recorded gateway");
+    expect(errorSpy.mock.calls.flat().join("\n")).toContain(
+      "requires every registered sandbox to be backed up",
+    );
+  });
+
+  it("keeps the strict abort when an unobserved sandbox still has a container (#6520)", async () => {
+    // Orphan classification alone is race-prone: a sandbox mid-reconnect (or
+    // one whose gateway row is drifting) is unobserved on the gateway yet its
+    // container still exists. Only definitive container absence may downgrade
+    // the strict abort to a stranded-orphan warning.
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "sb-reconnecting" }],
+      defaultSandbox: null,
+    });
+    mocks.parseReadySandboxNames.mockReturnValue(new Set());
+    mocks.parseLiveSandboxNames.mockReturnValue(new Set());
+    mocks.isSandboxContainerDefinitivelyAbsent.mockReturnValue(false);
+    process.env.NEMOCLAW_REQUIRE_ALL_SANDBOX_BACKUPS = "1";
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
+    vi.spyOn(console, "error").mockImplementation(() => undefined);
+    const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+
+    await expect(backupAll()).rejects.toThrow("exit:1");
+
+    expect(exitSpy).toHaveBeenCalledWith(1);
+    expect(mocks.isSandboxContainerDefinitivelyAbsent).toHaveBeenCalledWith("sb-reconnecting");
+    const logOutput = logSpy.mock.calls.flat().join("\n");
+    expect(logOutput).toContain("Skipping 'sb-reconnecting' (not running");
+    expect(logOutput).not.toContain("were not found on their recorded gateway");
+  });
+
+  it("reverts a stranded candidate to a strict skip when the confirming listing observes it again (#6520)", async () => {
+    // The pre-loop listing can be minutes stale by the time the loop ends. A
+    // candidate the confirming second listing observes has reconnected — the
+    // exemption must not apply and strict mode must keep failing closed.
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "sb-flapping" }],
+      defaultSandbox: null,
+    });
+    mocks.parseReadySandboxNames.mockReturnValue(new Set());
+    mocks.captureSandboxListWithGatewayPreflightOrExit
+      .mockResolvedValueOnce({ status: 0, output: "" })
+      .mockResolvedValueOnce({
+        status: 0,
+        output: "sb-flapping  openshell  2026-07-21 10:00:00  Ready\n",
+      });
+    mocks.parseLiveSandboxNames.mockImplementation((output: string) =>
+      output.includes("sb-flapping") ? new Set(["sb-flapping"]) : new Set(),
+    );
+    mocks.isSandboxContainerDefinitivelyAbsent.mockReturnValue(true);
+    process.env.NEMOCLAW_REQUIRE_ALL_SANDBOX_BACKUPS = "1";
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
+    vi.spyOn(console, "error").mockImplementation(() => undefined);
+    const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+
+    await expect(backupAll()).rejects.toThrow("exit:1");
+
+    expect(exitSpy).toHaveBeenCalledWith(1);
+    expect(mocks.captureSandboxListWithGatewayPreflightOrExit).toHaveBeenCalledTimes(2);
+    const logOutput = logSpy.mock.calls.flat().join("\n");
+    expect(logOutput).toContain("Skipping 'sb-flapping' (not running");
+    expect(logOutput).toContain("0 backed up, 0 failed, 1 skipped");
+    expect(logOutput).not.toContain("were not found on their recorded gateway");
+  });
+
+  it("reverts a stranded candidate to a strict skip when its container reappears (#6520)", async () => {
+    mocks.listSandboxes.mockReturnValue({
+      sandboxes: [{ name: "sb-flapping" }],
+      defaultSandbox: null,
+    });
+    mocks.parseReadySandboxNames.mockReturnValue(new Set());
+    mocks.captureSandboxListWithGatewayPreflightOrExit.mockResolvedValue({
+      status: 0,
+      output: "",
+    });
+    mocks.parseLiveSandboxNames.mockReturnValue(new Set());
+    mocks.isSandboxContainerDefinitivelyAbsent.mockReturnValueOnce(true).mockReturnValueOnce(false);
+    process.env.NEMOCLAW_REQUIRE_ALL_SANDBOX_BACKUPS = "1";
+    const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined);
+    vi.spyOn(console, "error").mockImplementation(() => undefined);
+    const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`exit:${code}`);
+    }) as never);
+
+    await expect(backupAll()).rejects.toThrow("exit:1");
+
+    expect(exitSpy).toHaveBeenCalledWith(1);
+    expect(mocks.captureSandboxListWithGatewayPreflightOrExit).toHaveBeenCalledTimes(2);
+    expect(mocks.isSandboxContainerDefinitivelyAbsent).toHaveBeenCalledTimes(2);
+    expect(mocks.isSandboxContainerDefinitivelyAbsent).toHaveBeenNthCalledWith(1, "sb-flapping");
+    expect(mocks.isSandboxContainerDefinitivelyAbsent).toHaveBeenNthCalledWith(2, "sb-flapping");
+    const logOutput = logSpy.mock.calls.flat().join("\n");
+    expect(logOutput).toContain("Skipping 'sb-flapping' (not running");
+    expect(logOutput).toContain("0 backed up, 0 failed, 1 skipped");
+    expect(logOutput).not.toContain("were not found on their recorded gateway");
+  });
 });
 
 describe("shouldSkipUnreachableSandboxBackup", () => {