Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 2 additions & 16 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,6 @@ const {
dockerInspect,
dockerRemoveVolumesByPrefix,
dockerRm,
dockerRmi,
dockerStop,
} = docker;
const gatewayDrift: typeof import("./adapters/openshell/gateway-drift") = require("./adapters/openshell/gateway-drift");
Expand Down Expand Up @@ -2530,20 +2529,6 @@ async function createSandboxWithBaseImageResolution(
// biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail.
if (recreateRuntime.beginDelete() === "source") { runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); runOpenshell(["sandbox", "delete", "-g", recreateRuntime.journaledGatewayName ?? GATEWAY_NAME, sandboxName], { ignoreError: true }); if (!waitForSandboxRecreateDeleteAbsence(sandboxName, recreateRuntime.journaledGatewayName ?? GATEWAY_NAME, note)) throw new Error(`Cannot continue sandbox '${sandboxName}' recreation: OpenShell did not confirm explicit source absence after delete.`); }
recreateRuntime.confirmDeleted();
const replacementReusesPreviousImage =
replacementWorkload.source.kind === "managed-image" &&
replacementWorkload.source.reference === previousEntry?.imageTag;
if (
previousEntry?.imageTag &&
previousEntry.workload?.shared !== true &&
!replacementReusesPreviousImage
) {
// biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail.
const rmiResult = dockerRmi(previousEntry.imageTag, { ignoreError: true, suppressOutput: true });
if (rmiResult.status !== 0) {
console.warn(` Warning: failed to remove old sandbox image '${previousEntry.imageTag}'.`);
}
}
sandboxLifecycle.removeSandboxUnlessSessionReservation(previousEntry, sandboxName);
}
const preparedSandboxWorkload = await managedWorkloadRuntime.ensurePreparedWorkload();
Expand Down Expand Up @@ -2595,7 +2580,7 @@ async function createSandboxWithBaseImageResolution(
createArgv,
sandboxEnv,
sandboxStartupCommand,
lifecycleRegistrationFields: recreateRuntime.registrationFields,
lifecycleGeneration: recreateRuntime.targetGeneration,
prebuild,
restoreBackupPath,
terminalAgent: agentDefs.isTerminalAgent(agent),
Expand Down Expand Up @@ -2725,6 +2710,7 @@ async function createSandboxWithBaseImageResolution(
hermesDashboardState: finalHermesDashboardState,
dashboardPort: actualDashboardPort,
...lifecycleRegistrationFields,
...recreateRuntime.registrationFields,
gatewayName: GATEWAY_NAME,
gatewayPort: GATEWAY_PORT,
}),
Expand Down
55 changes: 55 additions & 0 deletions src/lib/onboard/machine/handlers/sandbox-recreate-journal.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,61 @@ it("journals not-ready repair on the selected non-default gateway (#6492)", asyn
expect(session.checkpoint?.sandboxRecreate).toBeNull();
});

it.each([
"replacement-unproven",
"shared-image",
"authority-unproven",
"no-owned-image",
"image-reused",
] as const)("reports the bounded %s image-retirement skip after journaled recreation", async (reason) => {
const session = createSession({ sandboxName: "saved", agent: "openclaw" });
const journal = bindJournaledRecreate(session);
const sourceEntry: SandboxEntry = {
name: "saved",
provider: "provider",
model: "model",
endpointUrl: null,
preferredInferenceApi: "openai-completions",
webSearchEnabled: false,
toolDisclosure: "progressive",
fromDockerfile: null,
hermesAuthMethod: null,
imageTag: "openshell/sandbox-from:old",
workload: {
schemaVersion: 1,
kind: "legacy-dockerfile",
reference: "openshell/sandbox-from:old",
shared: false,
},
};
const retireReplacedSandboxWorkload = vi.fn(() => ({
status: "skipped" as const,
reason,
}));
const { deps, calls } = createDeps(
{
getSandboxReuseState: () => "not_ready",
getSandboxRecreateObservation: journal.observe,
getSandboxRegistryEntry: () => sourceEntry,
createSandbox: journal.completeCreate,
retireReplacedSandboxWorkload,
},
session,
);

await handleSandboxState({
...baseOptions(deps, session),
resume: true,
sandboxName: "saved",
});

const diagnostics = calls.note.mock.calls
.map(([message]) => message)
.filter((message) => message.startsWith(" Obsolete sandbox image retirement skipped:"));
expect(diagnostics).toEqual([` Obsolete sandbox image retirement skipped: ${reason}`]);
expect(retireReplacedSandboxWorkload).toHaveBeenCalledOnce();
});

it("continues an outer rebuild journal after the outer rebuild deletes the source sandbox", async () => {
const session = createSession({ sandboxName: "saved", agent: "openclaw" });
session.steps.sandbox.status = "complete";
Expand Down
15 changes: 15 additions & 0 deletions src/lib/onboard/machine/handlers/sandbox.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,19 @@ import {
type SandboxResumeDecision,
} from "./sandbox-resume";

type SandboxRecreateWorkloadSkipReason = Extract<
ReplacedSandboxWorkloadCleanupResult,
{ readonly status: "skipped" }
>["reason"];

const SANDBOX_RECREATE_WORKLOAD_SKIP_DIAGNOSTIC = {
"replacement-unproven": " Obsolete sandbox image retirement skipped: replacement-unproven",
"shared-image": " Obsolete sandbox image retirement skipped: shared-image",
"authority-unproven": " Obsolete sandbox image retirement skipped: authority-unproven",
"no-owned-image": " Obsolete sandbox image retirement skipped: no-owned-image",
"image-reused": " Obsolete sandbox image retirement skipped: image-reused",
} as const satisfies Record<SandboxRecreateWorkloadSkipReason, string>;

function isAdvisoryPeerRouteDifference(
result: Exclude<GatewayRouteCompatibilityResult, { ok: true }>,
sandboxName: string,
Expand Down Expand Up @@ -1623,6 +1636,8 @@ class SandboxStateFlow<
this.deps.note(
` Warning: failed to remove obsolete ${retired.engineDisplayName} image ${retired.reference}; run '${this.deps.cliName()} gc' to clean up.`,
);
} else if (retired.status === "skipped") {
this.deps.note(SANDBOX_RECREATE_WORKLOAD_SKIP_DIAGNOSTIC[retired.reason]);
}
}

Expand Down
32 changes: 32 additions & 0 deletions src/lib/onboard/runtime-provider/replaced-workload.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ const TARGET_IDENTITY = "target-identity";
function entry(imageTag: string, generation: string): SandboxEntry {
return {
name: "alpha",
openshellDriver: "docker",
imageTag,
workload: {
schemaVersion: 1,
Expand Down Expand Up @@ -152,6 +153,37 @@ describe("same-name replacement workload cleanup", () => {
expect(removeImage).not.toHaveBeenCalled();
});

it.each([
["provider identity", ({ openshellDriver: _provider, ...source }: SandboxEntry) => source],
["workload receipt", ({ workload: _workload, ...source }: SandboxEntry) => source],
[
"matching workload receipt",
(source: SandboxEntry) => ({
...source,
workload: {
schemaVersion: 1 as const,
kind: "legacy-dockerfile" as const,
reference: "openshell/sandbox-from:foreign",
shared: false as const,
},
}),
],
] as const)("does not remove the source image without its durable %s", (_field, mutate) => {
const removeImage = vi.fn(() => ({ status: 0 }));

expect(
retireReplacedSandboxWorkload(
"alpha",
"target",
TARGET_IDENTITY,
mutate(entry(SOURCE_IMAGE, "source")),
entry(REPLACEMENT_IMAGE, "target"),
{ runtimeProviders: providers(removeImage) },
),
).toEqual({ status: "skipped", reason: "authority-unproven" });
expect(removeImage).not.toHaveBeenCalled();
});

it("skips image cleanup only for expected provider-selection failures", () => {
const removeImage = vi.fn(() => ({ status: 0 }));

Expand Down
18 changes: 7 additions & 11 deletions src/lib/onboard/sandbox-gpu-create-flow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -680,21 +680,17 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => {

it("configures the portable lifecycle after sandbox creation succeeds (#8441)", async () => {
const input = createInput();
input.lifecycleRegistrationFields = {
lifecycleGeneration: "current-generation",
lifecycleLiveIdentityFingerprint: "current-fingerprint",
};
input.lifecycleGeneration = "current-generation";
const deps = createDeps();
deps.installPortableDemoLifecycle = vi.fn(
() => input.lifecycleRegistrationFields?.lifecycleGeneration ?? null,
(_sandboxName, _startupCommand, _env, options) => options.registryGeneration ?? null,
);

await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({
lifecycleRegistrationFields: {
lifecycleGeneration: "current-generation",
lifecycleLiveIdentityFingerprint: "current-fingerprint",
},
route: "native",
const result = await runSandboxGpuCreateFlow(input, deps);

expect(result.route).toBe("native");
expect(result.lifecycleRegistrationFields).toEqual({
lifecycleGeneration: "current-generation",
Comment thread
coderabbitai[bot] marked this conversation as resolved.
});

expect(deps.installPortableDemoLifecycle).toHaveBeenCalledWith(
Expand Down
13 changes: 4 additions & 9 deletions src/lib/onboard/sandbox-gpu-create-flow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,7 @@ function exitForManagedBootstrapRecovery(error: ManagedBootstrapRecoveryBlockedE
type RunOpenshell = NonNullable<DockerGpuPatchDeps["runOpenshell"]>;
type RunCaptureOpenshell = NonNullable<DockerGpuPatchDeps["runCaptureOpenshell"]>;
type Sleep = NonNullable<DockerGpuPatchDeps["sleep"]>;
type LifecycleRegistrationFields = Pick<
SandboxEntry,
"lifecycleGeneration" | "lifecycleLiveIdentityFingerprint"
>;
type LifecycleRegistrationFields = Pick<SandboxEntry, "lifecycleGeneration">;

export interface SandboxGpuCreateFlowInput {
sandboxName: string;
Expand All @@ -84,7 +81,7 @@ export interface SandboxGpuCreateFlowInput {
createArgv: string[];
sandboxEnv: NodeJS.ProcessEnv;
sandboxStartupCommand: string[];
lifecycleRegistrationFields?: LifecycleRegistrationFields;
lifecycleGeneration?: SandboxEntry["lifecycleGeneration"];
prebuild: SandboxPrebuildResult;
restoreBackupPath: string | null;
terminalAgent: boolean;
Expand Down Expand Up @@ -264,9 +261,7 @@ export async function runSandboxGpuCreateFlow(
input.sandboxStartupCommand,
process.env,
{
...(input.lifecycleRegistrationFields?.lifecycleGeneration
? { registryGeneration: input.lifecycleRegistrationFields.lifecycleGeneration }
: {}),
...(input.lifecycleGeneration ? { registryGeneration: input.lifecycleGeneration } : {}),
},
) ?? null;
} catch (error) {
Expand All @@ -281,7 +276,7 @@ export async function runSandboxGpuCreateFlow(
registryImageRef,
lifecycleRegistrationFields: {
...(portableLifecycleGeneration ? { lifecycleGeneration: portableLifecycleGeneration } : {}),
...input.lifecycleRegistrationFields,
...(input.lifecycleGeneration ? { lifecycleGeneration: input.lifecycleGeneration } : {}),
},
};
}
13 changes: 13 additions & 0 deletions src/lib/onboard/sandbox-recreate-transaction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,19 @@ export function retireReplacedSandboxWorkload(
if (source.workload?.shared === true) {
return { status: "skipped", reason: "shared-image" };
}
if (!source.imageTag) {
return { status: "skipped", reason: "no-owned-image" };
}
if (
typeof source.openshellDriver !== "string" ||
source.openshellDriver.trim().length === 0 ||
source.workload?.schemaVersion !== 1 ||
source.workload.kind !== "legacy-dockerfile" ||
source.workload.shared !== false ||
source.workload.reference !== source.imageTag
) {
return { status: "skipped", reason: "authority-unproven" };
}

const cleanupSource = providerCleanupSource(source);
const providers = deps.runtimeProviders ?? CURRENT_RUNTIME_PROVIDER_BUNDLES;
Expand Down
46 changes: 45 additions & 1 deletion test/e2e/live/rebuild-hermes-image-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,20 @@ import {
} from "../../../src/lib/domain/sandbox/image-tag";

export interface RebuildHermesRegistryImageState {
openshellDriver: "docker";
imageTag: string;
fromDockerfile: null;
workload: {
schemaVersion: 1;
kind: "legacy-dockerfile";
reference: string;
shared: false;
};
}

export interface RebuildHermesReplacementLifecycleReceipt {
lifecycleGeneration: string;
lifecycleLiveIdentityFingerprint: string;
}

export async function cleanupTrackedRebuildHermesImage(
Expand All @@ -31,6 +43,28 @@ export function requireRebuildHermesInitialImageTag(value: unknown, sandboxName:
return imageTag;
}

export function requireRebuildHermesReplacementLifecycleReceipt(
value: Record<string, unknown>,
): RebuildHermesReplacementLifecycleReceipt {
const lifecycleGeneration = value.lifecycleGeneration;
const lifecycleLiveIdentityFingerprint = value.lifecycleLiveIdentityFingerprint;
if (
typeof lifecycleGeneration !== "string" ||
!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu.test(
lifecycleGeneration,
)
) {
throw new Error("rebuilt Hermes registry is missing its journaled lifecycle generation");
}
if (
typeof lifecycleLiveIdentityFingerprint !== "string" ||
!/^[0-9a-f]{64}$/u.test(lifecycleLiveIdentityFingerprint)
) {
throw new Error("rebuilt Hermes registry is missing its live lifecycle identity fingerprint");
}
return { lifecycleGeneration, lifecycleLiveIdentityFingerprint };
}

export function rebuildHermesRegistryImageState(
createOutput: string,
): RebuildHermesRegistryImageState {
Expand All @@ -42,5 +76,15 @@ export function rebuildHermesRegistryImageState(
`old Hermes sandbox create must report an exact ${prefix}<build-id> image tag; got ${imageTag ?? "<missing>"}`,
);
}
return { imageTag, fromDockerfile: null };
return {
openshellDriver: "docker",
imageTag,
fromDockerfile: null,
workload: {
schemaVersion: 1,
kind: "legacy-dockerfile",
reference: imageTag,
shared: false,
},
};
}
5 changes: 5 additions & 0 deletions test/e2e/live/rebuild-hermes.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ import {
type RebuildHermesRegistryImageState,
rebuildHermesRegistryImageState,
requireRebuildHermesInitialImageTag,
requireRebuildHermesReplacementLifecycleReceipt,
} from "./rebuild-hermes-image-state.ts";
import {
REBUILD_HERMES_OLD_BASE_FIXTURE,
Expand Down Expand Up @@ -1268,6 +1269,10 @@ test(STALE_BASE_REBUILD
resultText(oldImageInspect),
).toBe(true);
expect(resultText(oldImageInspect)).toMatch(/No such (?:image|object)(?::|\s)/iu);
await artifacts.writeJson(
"phase-6-replacement-registry-lifecycle-receipt.json",
requireRebuildHermesReplacementLifecycleReceipt(rebuiltRegistry),
);

progress.phase("validate upgraded state inference and backup hygiene");
const restoredMarker = await host.command(
Expand Down
Loading
Loading