From 8bf16d217c989640d523ace346957716c4136492 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 05:44:44 -0700 Subject: [PATCH 01/24] fix(onboard): claim bootstrap terminal outcomes Reconstruct the net #8077 terminal-outcome slice on current main. Signed-off-by: Aaron Erickson (cherry picked from commit ce6f313e9f7705aa9374c0184710ca995dafdfa8) --- .../docker-gpu-local-inference.test.ts | 21 ++++ src/lib/onboard/docker-gpu-local-inference.ts | 2 +- ...ocker-gpu-sandbox-create-lifecycle.test.ts | 29 ++++- src/lib/onboard/docker-gpu-sandbox-create.ts | 41 ++++--- ...ker-startup-command-sandbox-create.test.ts | 2 +- src/lib/onboard/managed-bootstrap/README.md | 9 +- .../managed-bootstrap/docker-runtime.test.ts | 106 ++++++++++++++++++ .../managed-bootstrap/docker-runtime.ts | 26 ++--- .../managed-bootstrap/runtime-create.test.ts | 46 ++++++++ .../managed-bootstrap/runtime-create.ts | 36 ++++++ src/lib/onboard/sandbox-create-launch.ts | 8 +- 11 files changed, 280 insertions(+), 46 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/docker-runtime.test.ts create mode 100644 src/lib/onboard/managed-bootstrap/runtime-create.test.ts diff --git a/src/lib/onboard/docker-gpu-local-inference.test.ts b/src/lib/onboard/docker-gpu-local-inference.test.ts index 3f0f6180ec4..7e0bd73b992 100644 --- a/src/lib/onboard/docker-gpu-local-inference.test.ts +++ b/src/lib/onboard/docker-gpu-local-inference.test.ts @@ -425,6 +425,27 @@ describe("verifyGpuSandboxLocalInferenceAndCommitAfterReady", () => { expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).toHaveBeenCalledOnce(); expect(runtimePatch.commitAfterReady).not.toHaveBeenCalled(); }); + + it("treats a failed commit as terminal without attempting rollback", async () => { + const runtimePatch = { + commitAfterReady: vi.fn(async () => { + throw new Error("durable commit acknowledgement failed"); + }), + rollbackManagedStartupAfterCreateFailure: vi.fn(), + }; + await expect( + verifyGpuSandboxLocalInferenceAndCommitAfterReady( + GPU_CONFIG, + "ollama-local", + { + ...options(), + deps: { execInSandbox: execEmitting("HTTP_200"), sleep: vi.fn() }, + }, + runtimePatch, + ), + ).rejects.toThrow("durable commit acknowledgement failed"); + expect(runtimePatch.rollbackManagedStartupAfterCreateFailure).not.toHaveBeenCalled(); + }); }); describe("printDockerGpuSandboxInferenceVerificationFailure", () => { diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index 496ee830448..f0999099539 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -512,7 +512,6 @@ export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady( ): Promise { try { verifyGpuSandboxLocalInferenceAfterReady(config, provider, options); - await runtimePatch.commitAfterReady(); } catch (error) { const failure = error instanceof Error ? error : new Error(String(error)); try { @@ -524,4 +523,5 @@ export async function verifyGpuSandboxLocalInferenceAndCommitAfterReady( } throw failure; } + await runtimePatch.commitAfterReady(); } diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index 6ddea611aa9..cf614528c94 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -159,7 +159,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.waitForSupervisorReconnectIfNeeded(); expect(onPatchFailureExit).not.toHaveBeenCalled(); - await patch.commitAfterReady(); + await expect(patch.commitAfterReady()).rejects.toThrow("rollback backup"); expect(onPatchFailureExit).toHaveBeenCalledOnce(); expect(onPatchFailureExit.mock.calls[0]?.[1]).toEqual( @@ -177,6 +177,33 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { ); }); + it("rejects an early commit after rolling back before supervisor reconnect", async () => { + const deps = makeDeps(); + const result = deferredCreateResult(); + const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); + const onPatchFailureExit = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => result), + finalizeBackup, + onPatchFailureExit, + }, + }); + + patch.maybeApplyDuringCreate(); + + await expect(patch.commitAfterReady()).rejects.toThrow( + "cannot commit before the recreated OpenShell supervisor reconnects", + ); + expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: false }, deps); + expect(onPatchFailureExit).toHaveBeenCalledOnce(); + }); + it("rolls back to the backup container and surfaces rolledBack=true diagnostics when supervisorReady=false", () => { const deps = makeDeps(); const result = deferredCreateResult(); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index 52c15d0b5be..cd7e9aa8af6 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -380,18 +380,15 @@ export function createDockerGpuSandboxCreatePatch( "Managed startup cannot commit before the recreated OpenShell supervisor reconnects.", ); const rollbackError = await rollbackAfterFailure(); - onPatchFailureExit( - options.sandboxName, - rollbackError - ? new Error(`${error.message} Rollback failed: ${rollbackError.message}`) - : error, - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - }, - ); - return; + const failure = rollbackError + ? new Error(`${error.message} Rollback failed: ${rollbackError.message}`) + : error; + onPatchFailureExit(options.sandboxName, failure, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }); + throw failure; } if (cutoverFinalization) { if (cutoverFinalizationOutcome !== "commit") { @@ -429,7 +426,7 @@ export function createDockerGpuSandboxCreatePatch( rolledBack: rollbackError === null, }, }); - return; + throw failure; } } const finalizeOutcome = result @@ -437,16 +434,16 @@ export function createDockerGpuSandboxCreatePatch( : null; cutoverFinalized = true; if (!finalizeOutcome || finalizeOutcome.backupRemoved) return; - onPatchFailureExit( - options.sandboxName, - new Error("Managed startup passed Ready, but its rollback backup could not be removed."), - { - runCaptureOpenshell: options.deps.runCaptureOpenshell, - dockerCapture: options.deps.dockerCapture, - additionalSummaryLines: routeAdapter.additionalSummaryLines, - context: failureContext(), - }, + const failure = new Error( + "Managed startup passed Ready, but its rollback backup could not be removed.", ); + onPatchFailureExit(options.sandboxName, failure, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: failureContext(), + }); + throw failure; })(); cutoverFinalization = finalization; cutoverFinalizationOutcome = "commit"; diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts index ec0ad3cb306..8ccc7295e87 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts @@ -233,7 +233,7 @@ describe("Docker startup-command sandbox creation", () => { rollback, }); - await patch.commitAfterReady(); + await expect(patch.commitAfterReady()).rejects.toThrow("receipt validation failed"); expect(events).toEqual(["commit", "rollback", "exit"]); expect(onPatchFailureExit).toHaveBeenCalledWith( diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index 186d8347a4c..b737ab6cb37 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -85,9 +85,12 @@ and sandbox ID and then enter the destructive cutover. Post-cutover rollback publishes `rollback-authorized` before exact replacement deletion; pre-cutover staged cleanup removes only the exact prepared replacement without that journal transition. Commit publishes `shared-state-committed` before exact backup -deletion. Cleanup is bound to full runtime IDs. Its private state root retains -versioned, identity-addressed transaction records containing the provider and -sandbox identities, plan and profile +deletion. Cleanup is bound to full runtime IDs. Commit or rollback is claimed +synchronously before asynchronous finalization begins. Repeated calls for the +claimed outcome share its one pending result, while the opposite outcome remains +invalid even if acknowledgement of the first finalization is lost. Its private +state root retains versioned, identity-addressed transaction records containing +the provider and sandbox identities, plan and profile fingerprints, exact original and replacement IDs, rollback target, and phase. Exact commit and cleanup receipts are durable terminal records, so adapter recreation does not depend on process-local transaction sets or tombstone maps. diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts new file mode 100644 index 00000000000..e9b40f6340b --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const adapterMocks = vi.hoisted(() => ({ + activate: vi.fn(), + finalize: vi.fn(), + prepare: vi.fn(), +})); + +vi.mock("./adapter", async (importOriginal) => ({ + ...(await importOriginal()), + activateManagedBootstrapSequence: adapterMocks.activate, + finalizeManagedBootstrapSequence: adapterMocks.finalize, + prepareManagedBootstrapSequence: adapterMocks.prepare, +})); + +import type { + ManagedBootstrapActivatedTransaction, + ManagedBootstrapAdapter, + ManagedBootstrapPreparedTransaction, +} from "./adapter"; +import { createDockerManagedBootstrapSurface } from "./docker-runtime"; +import { authority, IDENTITY, NEW_ID, OLD_ID } from "./docker-test-fixture"; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("Docker managed-bootstrap lifecycle composition", () => { + it("does not finalize rollback after a claimed commit loses acknowledgement", async () => { + const seed = authority("openclaw"); + const prepared = Object.freeze({}) as ManagedBootstrapPreparedTransaction; + const activated = Object.freeze({ + snapshot: { runtimeId: OLD_ID }, + replacement: { replacementRuntimeId: NEW_ID }, + }) as ManagedBootstrapActivatedTransaction; + adapterMocks.prepare.mockImplementation(async (_adapter, input) => { + await input.create.launch({ + heldWorkloadArgv: seed.handle.heldWorkloadArgv, + bootstrapIdentity: IDENTITY, + }); + return prepared; + }); + adapterMocks.activate.mockResolvedValue(activated); + adapterMocks.finalize.mockRejectedValue(new Error("commit acknowledgement lost")); + const onPatchFailure = vi.fn((error: unknown): never => { + throw error; + }); + const lifecycle = createDockerManagedBootstrapSurface().createLifecycle({ + providerId: "docker", + bootstrapIdentity: IDENTITY, + request: seed.request, + image: seed.plan.image, + agentIdentity: seed.plan.agentIdentity, + intendedWorkloadArgv: seed.plan.intendedWorkloadArgv, + expectedSupervisorArgv: seed.plan.expectedSupervisorArgv, + launchArgv: ["openshell", "sandbox", "create", "--name", "alpha"], + heldWorkloadArgv: seed.handle.heldWorkloadArgv, + authorityStore: { + recordPreparedAuthority: vi.fn(), + }, + adapterOverride: {} as ManagedBootstrapAdapter, + route: "none", + persistStartupCommand: false, + sandboxName: "alpha", + sandboxGpuConfig: { + mode: "0", + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }, + requiredLimits: [], + timeoutSecs: 30, + onPatchFailure, + network: { + inferenceProvider: "openai", + dockerDriverGateway: false, + gatewayPort: 0, + }, + dependencies: {}, + }); + + await expect( + lifecycle.runCreate(async () => ({ value: "launched", receipt: seed.handle.createReceipt })), + ).resolves.toBe("launched"); + const failure = (await Promise.resolve(lifecycle.patch.commitAfterReady()).catch( + (error: unknown) => error, + )) as Error & { managedBootstrapRollbackError?: Error }; + + expect(failure).toBeInstanceOf(Error); + expect(failure.message).toBe("commit acknowledgement lost"); + expect(failure.managedBootstrapRollbackError?.message).toBe( + "Managed bootstrap rollback is no longer legal after commit finalization began.", + ); + expect(adapterMocks.finalize).toHaveBeenCalledOnce(); + expect(adapterMocks.finalize).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ outcome: "commit", transaction: activated }), + ); + expect(onPatchFailure).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts index dba07739d18..a05a982ec90 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -31,6 +31,7 @@ import type { ManagedBootstrapRuntimeCreateLifecycleInput, ManagedBootstrapRuntimeOnboardRoutingInput, } from "./runtime-create"; +import { createManagedBootstrapTerminalFinalizer } from "./runtime-create"; type SupportedBootstrapSurface = Extract< RuntimeProviderBootstrapSurface, @@ -191,7 +192,12 @@ function createDockerLifecycle( }); throw new Error("Managed bootstrap did not return its OpenShell create receipt."); } - let finalized = false; + const finalizer = createManagedBootstrapTerminalFinalizer((outcome) => + finalizeManagedBootstrapSequence(adapter, { + outcome, + transaction: activated, + }).then(() => undefined), + ); patch.attachManagedBootstrapCutover({ selectedMode: mode, failureContext: { @@ -201,22 +207,8 @@ function createDockerLifecycle( backupContainerName: null, selectedMode: mode, }, - async rollback() { - if (finalized) return; - await finalizeManagedBootstrapSequence(adapter, { - outcome: "rollback", - transaction: activated, - }); - finalized = true; - }, - async commit() { - if (finalized) return; - await finalizeManagedBootstrapSequence(adapter, { - outcome: "commit", - transaction: activated, - }); - finalized = true; - }, + rollback: finalizer.rollback, + commit: finalizer.commit, }); return launched.value; }, diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.test.ts b/src/lib/onboard/managed-bootstrap/runtime-create.test.ts new file mode 100644 index 00000000000..ad591f19277 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/runtime-create.test.ts @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { createManagedBootstrapTerminalFinalizer } from "./runtime-create"; + +describe("managed bootstrap terminal finalizer", () => { + it("shares one in-flight outcome and rejects an opposite concurrent outcome", async () => { + let release = (): void => {}; + const finalize = vi.fn( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + const finalizer = createManagedBootstrapTerminalFinalizer(finalize); + + const firstCommit = finalizer.commit(); + const duplicateCommit = finalizer.commit(); + + expect(duplicateCommit).toBe(firstCommit); + await expect(finalizer.rollback()).rejects.toThrow( + "rollback is no longer legal after commit finalization began", + ); + release(); + await expect(Promise.all([firstCommit, duplicateCommit])).resolves.toEqual([ + undefined, + undefined, + ]); + expect(finalize).toHaveBeenCalledExactlyOnceWith("commit"); + }); + + it("retains the claimed outcome after a lost finalization acknowledgement", async () => { + const finalize = vi.fn(async () => { + throw new Error("commit acknowledgement lost"); + }); + const finalizer = createManagedBootstrapTerminalFinalizer(finalize); + + await expect(finalizer.commit()).rejects.toThrow("commit acknowledgement lost"); + await expect(finalizer.rollback()).rejects.toThrow( + "rollback is no longer legal after commit finalization began", + ); + expect(finalize).toHaveBeenCalledExactlyOnceWith("commit"); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts index 6ffcc028966..73efeac0cbb 100644 --- a/src/lib/onboard/managed-bootstrap/runtime-create.ts +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -90,6 +90,42 @@ export interface ManagedBootstrapRuntimeCreateLaunchResult { readonly receipt: ManagedBootstrapCreateReceipt; } +export type ManagedBootstrapTerminalOutcome = "commit" | "rollback"; + +export interface ManagedBootstrapTerminalFinalizer { + commit(): Promise; + rollback(): Promise; +} + +/** + * Claim one terminal outcome before driver finalization starts. Duplicate calls + * for that outcome share the in-flight promise; the opposite outcome fails + * closed even when finalization loses acknowledgement. + */ +export function createManagedBootstrapTerminalFinalizer( + finalize: (outcome: ManagedBootstrapTerminalOutcome) => Promise, +): ManagedBootstrapTerminalFinalizer { + let claimedOutcome: ManagedBootstrapTerminalOutcome | null = null; + let pending: Promise | null = null; + const run = (outcome: ManagedBootstrapTerminalOutcome): Promise => { + if (claimedOutcome === outcome && pending !== null) return pending; + if (claimedOutcome !== null) { + return Promise.reject( + new Error( + `Managed bootstrap ${outcome} is no longer legal after ${claimedOutcome} finalization began.`, + ), + ); + } + claimedOutcome = outcome; + pending = Promise.resolve().then(() => finalize(outcome)); + return pending; + }; + return Object.freeze({ + commit: () => run("commit"), + rollback: () => run("rollback"), + }); +} + export interface ManagedBootstrapRuntimeCreateLifecycle { readonly launchArgv: readonly string[]; readonly patch: ManagedBootstrapRuntimePatch; diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index f6f7b91f7e3..fe5079b9b94 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -63,7 +63,13 @@ export interface SandboxCreateLaunchInput { openshellShellCommand: OpenshellShellCommand; openshellArgv?: OpenshellArgv; buildEnv?(): Record; - /** Dormant until a complete runtime bundle and durable authority store are selected. */ + /** + * Intentional partial migration: remains unset until production selects a + * complete runtime bundle with supported bootstrap after epic #7744's durable + * lifecycle, recovery, and rollback gates plus exact-head/base protected + * all-agent amd64/arm64, GPU/local-inference, and regression matrix pass. + * https://github.com/NVIDIA/NemoClaw/issues/7744 + */ managedStartupRootApplyRequest?: ManagedStartupRootApplyRequest | null; } From 897408f74dcc055e7e9d263985931cf664a1c439 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 05:50:32 -0700 Subject: [PATCH 02/24] fix(onboard): preserve shared-state commit authority Reconstruct the net #8078 shared-state authority slice on current main. Signed-off-by: Aaron Erickson (cherry picked from commit 31236f767aa79c9e110be55bc1bf56b5396b227a) --- src/lib/onboard/managed-bootstrap/README.md | 7 + ...er-shared-state-rollback-authority.test.ts | 279 ++++++++++++++++++ .../managed-bootstrap/docker-shared-state.ts | 27 +- ...d-startup-shared-state-transaction.test.ts | 89 +++++- .../shared-state-transaction.ts | 49 ++- 5 files changed, 423 insertions(+), 28 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts diff --git a/src/lib/onboard/managed-bootstrap/README.md b/src/lib/onboard/managed-bootstrap/README.md index b737ab6cb37..437164f324c 100644 --- a/src/lib/onboard/managed-bootstrap/README.md +++ b/src/lib/onboard/managed-bootstrap/README.md @@ -105,6 +105,13 @@ commit atomically moves its pending manifest and backups into a durable receipt namespace, compacts that state to an exact commit receipt, and rejects rollback after a restart. The provider may retire that receipt only after it proves the external rollback backup is gone, leaving the next bootstrap attempt unblocked. +The parser accepts the exact canonical schema-v1 manifest written before +`bootstrapIdentity` was added only for the legacy null-identity path. It rejects +additional fields, missing historical fields, and legacy state presented as +identity-bound authority. Before rollback, the Docker adapter stops the +replacement and copies its writable-layer commit receipt to a protected host +path for verification. The immutable helper cannot obtain that receipt through +`--volumes-from`, which exposes volumes but not the replacement writable layer. Direct identity lookup reconstructs one known transaction record, while managed create-lifecycle startup uses unfinished-record enumeration to ask the selected provider to reconcile every identity-addressed record before a new sandbox diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts new file mode 100644 index 00000000000..34eca762bcb --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts @@ -0,0 +1,279 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types"; +import { + MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY, + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY, + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY, +} from "../managed-startup/shared-state-transaction"; +import { + type DockerManagedBootstrapSharedStateTransaction, + finalizeDockerManagedStartupSharedState, +} from "./docker-shared-state"; + +const CONTAINER_ID = "c".repeat(64); +const TRANSACTION: DockerManagedBootstrapSharedStateTransaction = { + agent: "openclaw", + bootstrapIdentity: "b".repeat(64), + containerId: CONTAINER_ID, + image: `sha256:${"a".repeat(64)}`, + profileFingerprint: "d".repeat(64), +}; + +interface SharedStateFixture { + readonly commands: readonly (readonly string[])[]; + readonly deps: DockerGpuPatchDeps; + readonly events: readonly string[]; + readonly state: () => "committed" | "none" | "pending"; +} + +interface SharedStateFixtureOptions { + readonly stateAfterCommitFailure?: "committed" | "none" | "pending"; + readonly stateAfterStop?: "committed" | "none" | "pending"; +} + +const copiedReceiptPaths: string[] = []; + +afterEach(() => { + vi.restoreAllMocks(); + for (const receiptPath of copiedReceiptPaths.splice(0)) { + fs.rmSync(path.dirname(receiptPath), { force: true, recursive: true }); + } +}); + +function fixture( + initialState: "committed" | "none" | "pending", + options: SharedStateFixtureOptions = {}, +): SharedStateFixture { + let state = initialState; + const commands: string[][] = []; + const events: string[] = []; + const copyPresentReceipt = (destination: string) => { + fs.mkdirSync(destination, { recursive: true }); + copiedReceiptPaths.push(destination); + return { status: 0 }; + }; + const copyMissingReceipt = (sourcePath: string) => ({ + status: 1, + stderr: `Error response from daemon: Could not find the file ${sourcePath} in container ${CONTAINER_ID}`, + }); + const dockerRun = vi.fn((args: readonly string[]) => { + commands.push([...args]); + switch (args[0]) { + case "cp": { + const source = String(args[2] ?? ""); + const destination = String(args[3] ?? ""); + const sourcePath = source.slice(`${CONTAINER_ID}:`.length); + const present = + (sourcePath === MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY && + state === "committed") || + (sourcePath === MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY && state === "pending"); + events.push(`copy:${path.basename(sourcePath)}:${present ? "present" : "absent"}`); + return present ? copyPresentReceipt(destination) : copyMissingReceipt(sourcePath); + } + case "run": { + const action = args.includes("--shared-state-transaction-status") + ? "status" + : args.includes("--rollback-shared-state-transaction") + ? "rollback" + : "unexpected"; + switch (action) { + case "status": + events.push(`status:${state}`); + return { status: 0, stdout: `${state}\n` }; + case "rollback": + events.push("rollback"); + state = "none"; + return { status: 0 }; + default: + throw new Error(`Unexpected Docker command: ${args.join(" ")}`); + } + } + case "exec": + switch (args.includes("--commit-shared-state-transaction")) { + case true: + events.push("commit:failed"); + state = options.stateAfterCommitFailure ?? state; + return { status: 1, stderr: "commit helper failed" }; + default: + throw new Error("Unexpected Docker commit command"); + } + default: + throw new Error(`Unexpected Docker command: ${args.join(" ")}`); + } + }); + return { + commands, + deps: { + dockerRm: vi.fn(() => ({ status: 0 })), + dockerRun, + dockerStop: vi.fn(() => { + events.push("stop"); + state = options.stateAfterStop ?? state; + return { status: 0 }; + }), + }, + events, + state: () => state, + }; +} + +describe("Docker managed-bootstrap shared-state rollback authority", () => { + it("copies and verifies writable-layer commit authority before rollback", () => { + const fake = fixture("committed"); + + expect(() => + finalizeDockerManagedStartupSharedState( + { transaction: TRANSACTION, supervisorReady: false }, + fake.deps, + ), + ).toThrow(/durably committed and cannot be rolled back/u); + + expect(fake.events).toEqual([ + "stop", + `copy:${path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY)}:present`, + "status:committed", + ]); + const statusCommand = fake.commands.find((args) => + args.includes("--shared-state-transaction-status"), + ); + expect(statusCommand).toContainEqual( + expect.stringMatching( + new RegExp( + `^type=bind,src=.+,dst=${MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY},readonly$`, + "u", + ), + ), + ); + expect(fake.commands.some((args) => args.includes("--rollback-shared-state-transaction"))).toBe( + false, + ); + }); + + it("proves pending authority after quiescence before starting the rollback helper", () => { + const fake = fixture("pending"); + + expect( + finalizeDockerManagedStartupSharedState( + { transaction: TRANSACTION, supervisorReady: false }, + fake.deps, + ), + ).toEqual({ supervisorReady: false, failure: null }); + + expect(fake.events[0]).toBe("stop"); + expect(fake.events.indexOf("status:pending")).toBeLessThan(fake.events.indexOf("rollback")); + expect(fake.state()).toBe("none"); + const rollbackCommand = fake.commands.find((args) => + args.includes("--rollback-shared-state-transaction"), + ); + expect(rollbackCommand).toContainEqual( + expect.stringMatching( + new RegExp( + `^type=bind,src=.+,dst=${MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY},readonly$`, + "u", + ), + ), + ); + }); + + it("removes the exact failed container without rollback when both receipts are absent", () => { + const fake = fixture("none"); + + expect( + finalizeDockerManagedStartupSharedState( + { transaction: TRANSACTION, supervisorReady: false }, + fake.deps, + ), + ).toEqual({ supervisorReady: false, failure: null }); + + expect(fake.events).toEqual([ + "stop", + `copy:${path.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY)}:absent`, + `copy:${path.basename(MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY)}:absent`, + ]); + expect(fake.commands.some((args) => args.includes("--rollback-shared-state-transaction"))).toBe( + false, + ); + expect(fake.deps.dockerRm).toHaveBeenCalledTimes(1); + expect(fake.deps.dockerRm).toHaveBeenCalledWith(CONTAINER_ID, expect.any(Object)); + expect(fake.state()).toBe("none"); + }); + + it("reuses one preserved pending receipt when commit validation fails", () => { + const fake = fixture("pending", { stateAfterCommitFailure: "none" }); + + const outcome = finalizeDockerManagedStartupSharedState( + { + retainContainerAfterRollback: true, + transaction: TRANSACTION, + supervisorReady: true, + }, + fake.deps, + ); + + expect(outcome.supervisorReady).toBe(false); + expect(outcome.failure).toEqual( + expect.objectContaining({ + message: expect.stringContaining("commit helper failed"), + }), + ); + const pendingSource = CONTAINER_ID + ":" + MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY; + const pendingCopies = fake.commands.filter( + (args) => args[0] === "cp" && args[2] === pendingSource, + ); + expect( + fake.events.filter( + (event) => + event === + "copy:" + path.basename(MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY) + ":present", + ), + ).toHaveLength(1); + const preservedReceiptPath = String(pendingCopies[0]?.[3] ?? ""); + const rollbackCommand = fake.commands.find((args) => + args.includes("--rollback-shared-state-transaction"), + ); + expect(rollbackCommand).toContain( + "type=bind,src=" + + preservedReceiptPath + + ",dst=" + + MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY + + ",readonly", + ); + expect(fake.deps.dockerRm).not.toHaveBeenCalled(); + }); + + it("rejects rollback when a failed commit becomes durable during quiescence", () => { + const fake = fixture("pending", { + stateAfterCommitFailure: "none", + stateAfterStop: "committed", + }); + + expect(() => + finalizeDockerManagedStartupSharedState( + { transaction: TRANSACTION, supervisorReady: true }, + fake.deps, + ), + ).toThrow(/durably committed and cannot be rolled back/u); + + expect( + fake.events.filter( + (event) => + event === + "copy:" + path.basename(MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY) + ":present", + ), + ).toHaveLength(1); + expect(fake.events).toContain("commit:failed"); + expect(fake.events).toContain("status:committed"); + expect(fake.events).not.toContain("rollback"); + expect(fake.commands.some((args) => args.includes("--rollback-shared-state-transaction"))).toBe( + false, + ); + expect(fake.deps.dockerRm).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts index a92d2e335ed..2fcd8595802 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -461,10 +461,21 @@ function copyManagedStartupReceipt( function rollbackManagedStartupSharedState( transaction: DockerManagedBootstrapSharedStateTransaction, - receiptPath: string, deps: DockerGpuPatchDeps, -): void { + preservedReceiptPath?: string, +): boolean { const dockerRun = deps.dockerRun ?? defaultDockerRun; + const status = probeDockerManagedStartupSharedState( + { transaction, profileFingerprint: transaction.profileFingerprint }, + deps, + ); + if (status === "committed") { + throw new Error("Managed-startup shared state is durably committed and cannot be rolled back."); + } + const receiptPath = + preservedReceiptPath ?? + (status === "pending" ? copyManagedStartupReceipt(transaction, deps) : null); + if (!receiptPath) return false; let restored = false; try { // The immutable image owns the canonical receipt parser and exact @@ -519,6 +530,7 @@ function rollbackManagedStartupSharedState( cleanupReceiptBestEffort(receiptPath); } } + return true; } function removeFailedUnbackedContainer( @@ -616,7 +628,7 @@ export function finalizeDockerManagedStartupSharedState( { cause: stopError }, ); } - rollbackManagedStartupSharedState(transaction, receiptPath, deps); + rollbackManagedStartupSharedState(transaction, deps, receiptPath); if (!input.patchResult && !input.retainContainerAfterRollback) { removeFailedUnbackedContainer(transaction, deps); } @@ -624,14 +636,7 @@ export function finalizeDockerManagedStartupSharedState( } quiesceManagedStartupContainer(transaction, deps); - const receiptPath = copyManagedStartupReceipt(transaction, deps, true); - if (!receiptPath) { - if (!input.patchResult && !input.retainContainerAfterRollback) { - removeFailedUnbackedContainer(transaction, deps); - } - return { supervisorReady: false, failure: null }; - } - rollbackManagedStartupSharedState(transaction, receiptPath, deps); + rollbackManagedStartupSharedState(transaction, deps); if (!input.patchResult && !input.retainContainerAfterRollback) { removeFailedUnbackedContainer(transaction, deps); } diff --git a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts index 6ec1563e73e..90f283e1a09 100644 --- a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts +++ b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts @@ -78,6 +78,20 @@ describe("managed startup shared-state transaction", () => { ); } + function rewriteManifest( + rewrite: (manifest: Record) => Record = (manifest) => + manifest, + ): void { + const manifestFile = path.join(transactionDirectory, "manifest.json"); + const manifest = JSON.parse(fs.readFileSync(manifestFile, "utf8")) as Record; + expect(manifest.bootstrapIdentity).toBeNull(); + delete manifest.bootstrapIdentity; + const rewritten = rewrite(manifest); + fs.chmodSync(manifestFile, 0o600); + fs.writeFileSync(manifestFile, `${JSON.stringify(rewritten, null, 2)}\n`); + fs.chmodSync(manifestFile, 0o400); + } + it.each([ "openclaw", "hermes", @@ -257,6 +271,70 @@ describe("managed startup shared-state transaction", () => { expect(commitManagedStartupSharedStateTransaction("openclaw", options)).toBe(false); }); + it.each([ + ["commits", "commit"], + ["rolls back", "rollback"], + ] as const)("%s an exact historical schema-v1 manifest without bootstrap identity", (_description, action) => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + const config = path.join(root, "openclaw.json"); + fs.writeFileSync(config, "before\n"); + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options); + rewriteManifest(); + fs.writeFileSync(config, "after\n"); + + const result = + action === "commit" + ? commitManagedStartupSharedStateTransaction("openclaw", options) + : rollbackManagedStartupSharedStateTransaction("openclaw", options); + + expect(result).toBe(true); + expect(fs.readFileSync(config, "utf8")).toBe(action === "commit" ? "after\n" : "before\n"); + expect(fs.existsSync(transactionDirectory)).toBe(false); + expect(fs.existsSync(commitReceiptDirectory())).toBe(false); + }); + + it.each([ + ["an extra field", (manifest: Record) => ({ ...manifest, extra: true })], + [ + "a missing historical field", + (manifest: Record) => { + delete manifest.directories; + return manifest; + }, + ], + ] as const)("rejects a schema-v1 legacy manifest with %s", (_case, rewrite) => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "before\n"); + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options); + rewriteManifest(rewrite); + + expect(() => commitManagedStartupSharedStateTransaction("openclaw", options)).toThrow( + /unexpected fields/u, + ); + expect(fs.existsSync(transactionDirectory)).toBe(true); + }); + + it("does not treat a legacy manifest as authority for an identity-bound bootstrap", () => { + const root = agentRoot("openclaw"); + fs.mkdirSync(root); + fs.writeFileSync(path.join(root, "openclaw.json"), "before\n"); + const boundOptions = { ...options, bootstrapIdentity: "b".repeat(64) }; + beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), boundOptions); + const manifestFile = path.join(transactionDirectory, "manifest.json"); + const manifest = JSON.parse(fs.readFileSync(manifestFile, "utf8")) as Record; + delete manifest.bootstrapIdentity; + fs.chmodSync(manifestFile, 0o600); + fs.writeFileSync(manifestFile, `${JSON.stringify(manifest, null, 2)}\n`); + fs.chmodSync(manifestFile, 0o400); + + expect(() => rollbackManagedStartupSharedStateTransaction("openclaw", boundOptions)).toThrow( + /different bootstrap attempt/u, + ); + expect(fs.existsSync(transactionDirectory)).toBe(true); + }); + it("fsyncs every transaction namespace before exposing a pending receipt", () => { const root = agentRoot("openclaw"); fs.mkdirSync(root); @@ -403,10 +481,13 @@ describe("managed startup shared-state transaction", () => { throw new Error("injected post-rename cleanup interruption"); })() : originalRmSync(target, removeOptions)) as typeof fs.rmSync); - expect(() => commitManagedStartupSharedStateTransaction("openclaw", boundOptions)).toThrow( - /injected post-rename cleanup interruption/u, - ); - rm.mockRestore(); + try { + expect(() => commitManagedStartupSharedStateTransaction("openclaw", boundOptions)).toThrow( + /injected post-rename cleanup interruption/u, + ); + } finally { + rm.mockRestore(); + } expect(fs.existsSync(transactionDirectory)).toBe(false); const committedDirectory = commitReceiptDirectory(); diff --git a/src/lib/onboard/managed-startup/shared-state-transaction.ts b/src/lib/onboard/managed-startup/shared-state-transaction.ts index 62894abeb5c..fd83d10c0c8 100644 --- a/src/lib/onboard/managed-startup/shared-state-transaction.ts +++ b/src/lib/onboard/managed-startup/shared-state-transaction.ts @@ -538,6 +538,20 @@ function canonicalManifest(manifest: TransactionManifest): string { return `${JSON.stringify(manifest, null, 2)}\n`; } +function canonicalLegacyManifest(manifest: TransactionManifest): string { + return `${JSON.stringify( + { + schemaVersion: manifest.schemaVersion, + agent: manifest.agent, + profileFingerprint: manifest.profileFingerprint, + files: manifest.files, + directories: manifest.directories, + }, + null, + 2, + )}\n`; +} + function canonicalCommitReceipt(receipt: CommitReceipt): string { return `${JSON.stringify(receipt, null, 2)}\n`; } @@ -597,23 +611,29 @@ function parseManifest(text: string): TransactionManifest { fail("transaction manifest must be an object"); } const record = parsed as Record; - requireExactKeys(record, [ - "agent", - "bootstrapIdentity", - "directories", - "files", - "profileFingerprint", - "schemaVersion", - ]); + const hasBootstrapIdentity = Object.hasOwn(record, "bootstrapIdentity"); + requireExactKeys( + record, + hasBootstrapIdentity + ? [ + "agent", + "bootstrapIdentity", + "directories", + "files", + "profileFingerprint", + "schemaVersion", + ] + : ["agent", "directories", "files", "profileFingerprint", "schemaVersion"], + ); + const bootstrapIdentity = hasBootstrapIdentity ? record.bootstrapIdentity : null; if ( record.schemaVersion !== TRANSACTION_SCHEMA_VERSION || !["openclaw", "hermes", "langchain-deepagents-code"].includes(String(record.agent)) || typeof record.profileFingerprint !== "string" || !/^[a-f0-9]{64}$/u.test(record.profileFingerprint) || !( - record.bootstrapIdentity === null || - (typeof record.bootstrapIdentity === "string" && - /^[a-f0-9]{64}$/u.test(record.bootstrapIdentity)) + bootstrapIdentity === null || + (typeof bootstrapIdentity === "string" && /^[a-f0-9]{64}$/u.test(bootstrapIdentity)) ) || !Array.isArray(record.files) || !Array.isArray(record.directories) || @@ -709,11 +729,14 @@ function parseManifest(text: string): TransactionManifest { schemaVersion: TRANSACTION_SCHEMA_VERSION, agent: record.agent as ManagedStartupAgent, profileFingerprint: record.profileFingerprint, - bootstrapIdentity: record.bootstrapIdentity as string | null, + bootstrapIdentity, files, directories, }; - if (canonicalManifest(manifest) !== text) { + const canonical = hasBootstrapIdentity + ? canonicalManifest(manifest) + : canonicalLegacyManifest(manifest); + if (canonical !== text) { fail("transaction manifest is not canonical"); } return manifest; From 08c0d3bbc58157a165e0492352bef44586b3db08 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 06:00:45 -0700 Subject: [PATCH 03/24] fix(onboard): preserve durable journal compatibility Reconstruct the net #8080 journal-compatibility slice on current main. Signed-off-by: Aaron Erickson (cherry picked from commit c52370db1119ec1b8f3365a0ce4c22beebdd28e4) --- .../onboard/managed-bootstrap/adapter.test.ts | 43 ++++++++++++++++ src/lib/onboard/managed-bootstrap/adapter.ts | 16 ++++++ .../managed-bootstrap/docker-journal.test.ts | 51 +++++++++++++------ .../managed-bootstrap/docker-journal.ts | 26 ---------- .../managed-bootstrap/docker-runtime.test.ts | 2 +- .../docker-shared-state.test.ts | 7 +-- .../managed-bootstrap/docker-test-fixture.ts | 4 +- .../onboard/managed-bootstrap/docker.test.ts | 25 +++++++++ src/lib/onboard/managed-bootstrap/docker.ts | 24 +++++---- src/lib/onboard/managed-bootstrap/index.ts | 2 + .../managed-bootstrap-test-fixture.ts | 6 +++ test/runtime-provider-source-shape.test.ts | 1 + 12 files changed, 150 insertions(+), 57 deletions(-) create mode 100644 src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts diff --git a/src/lib/onboard/managed-bootstrap/adapter.test.ts b/src/lib/onboard/managed-bootstrap/adapter.test.ts index c076af031cc..fa21399c97a 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.test.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.test.ts @@ -20,6 +20,7 @@ import { type ManagedBootstrapAuthorityStore, type ManagedBootstrapCompletionReceipt, type ManagedBootstrapCreateReceipt, + type ManagedBootstrapDurablePreparationReceipt, type ManagedBootstrapFinalizationReceipt, type ManagedBootstrapHeldWorkloadHandle, type ManagedBootstrapObservedSnapshot, @@ -29,7 +30,10 @@ import { prepareManagedBootstrapSequence, recoverManagedBootstrapTransactions, renderManagedBootstrapHeldCommand, + sameManagedBootstrapCompletionReceipt, + sameManagedBootstrapDurablePreparationReceipt, } from "./adapter"; +import { reverseKeys } from "./managed-bootstrap-test-fixture"; const IDENTITY = "1".repeat(64); const CONFIG_ID = `sha256:${"2".repeat(64)}`; @@ -356,6 +360,45 @@ async function captureFailure(promise: Promise) { } describe("managed bootstrap adapter contract", () => { + it("compares provider-neutral durable receipts by canonical value", () => { + const handle = handleFor(requestFor("hermes")); + const preparation: ManagedBootstrapDurablePreparationReceipt = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + authorityFingerprint: "a".repeat(64), + recordId: "mxc-durable-authority", + recordedAt: "2026-07-29T12:00:30.000Z", + }; + const reorderedPreparation = reverseKeys({ + ...preparation, + sandbox: reverseKeys({ ...preparation.sandbox }), + }); + expect(sameManagedBootstrapDurablePreparationReceipt(preparation, reorderedPreparation)).toBe( + true, + ); + expect( + sameManagedBootstrapDurablePreparationReceipt(preparation, { + ...reorderedPreparation, + recordId: "changed-authority", + }), + ).toBe(false); + + const completion = completionFor(requestFor("hermes"), handle); + const reorderedCompletion = reverseKeys({ + ...completion, + image: reverseKeys({ ...completion.image }), + sandbox: reverseKeys({ ...completion.sandbox }), + }); + expect(sameManagedBootstrapCompletionReceipt(completion, reorderedCompletion)).toBe(true); + expect( + sameManagedBootstrapCompletionReceipt(completion, { + ...reorderedCompletion, + transactionPending: false, + }), + ).toBe(false); + }); + it.each( MANAGED_STARTUP_AGENTS, )("prepares, durably records, and only then activates %s through a provider-neutral adapter", async (agent) => { diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index 19d0651881f..637a2f993e8 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -905,6 +905,22 @@ function canonicalJson(value: unknown): string { .join(",")}}`; } +/** Compare durable provider receipts by canonical value, independent of object key order. */ +export function sameManagedBootstrapDurablePreparationReceipt( + left: ManagedBootstrapDurablePreparationReceipt, + right: ManagedBootstrapDurablePreparationReceipt, +): boolean { + return canonicalJson(left) === canonicalJson(right); +} + +/** Compare completion receipts by canonical value, independent of object key order. */ +export function sameManagedBootstrapCompletionReceipt( + left: ManagedBootstrapCompletionReceipt, + right: ManagedBootstrapCompletionReceipt, +): boolean { + return canonicalJson(left) === canonicalJson(right); +} + export function assertManagedBootstrapIdentity(value: string): void { if (!SHA256_RE.test(value)) { protocolFail("identity must be 32 random bytes encoded as lowercase hex"); diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts index 457e9b4e154..087e188cb58 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.test.ts @@ -7,6 +7,10 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { + sameManagedBootstrapCompletionReceipt, + sameManagedBootstrapDurablePreparationReceipt, +} from "./adapter"; import { createFileDockerManagedBootstrapJournalStore, DOCKER_MANAGED_BOOTSTRAP_FINALIZATION_SCHEMA_VERSION, @@ -20,10 +24,10 @@ import { normalizeDockerManagedBootstrapJournal, parseDockerManagedBootstrapFinalizationRecord, parseDockerManagedBootstrapJournal, - sameDockerManagedBootstrapReceipt, serializeDockerManagedBootstrapFinalizationRecord, serializeDockerManagedBootstrapJournal, } from "./docker-journal"; +import { reverseKeys } from "./managed-bootstrap-test-fixture"; const roots: string[] = []; const IDENTITY = "1".repeat(64); @@ -366,9 +370,9 @@ describe("Docker managed bootstrap journal", () => { }, schemaVersion: preparation.schemaVersion, } satisfies typeof preparation; - expect( - sameDockerManagedBootstrapReceipt("preparation", preparation, reorderedPreparation), - ).toBe(true); + expect(sameManagedBootstrapDurablePreparationReceipt(preparation, reorderedPreparation)).toBe( + true, + ); const completion = finalization.commitReceipt; const reorderedCompletion = { @@ -391,9 +395,7 @@ describe("Docker managed bootstrap journal", () => { }, schemaVersion: completion.schemaVersion, } satisfies typeof completion; - expect(sameDockerManagedBootstrapReceipt("completion", completion, reorderedCompletion)).toBe( - true, - ); + expect(sameManagedBootstrapCompletionReceipt(completion, reorderedCompletion)).toBe(true); }); it("reloads exact terminal receipts from a new journal store", () => { @@ -440,15 +442,11 @@ describe("Docker managed bootstrap journal", () => { const store = createFileDockerManagedBootstrapJournalStore(root); store.create(journal); const directory = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY); - fs.writeFileSync( - path.join(directory, `.${IDENTITY}.json${suffix}.1234.deadbeef.tmp`), - "partial", - { - mode: 0o600, - }, - ); + const target = path.join(directory, `.${IDENTITY}.json${suffix}.1234.deadbeef.tmp`); + fs.writeFileSync(target, "partial", { mode: 0o600 }); expect(loadUnfinished(store)).toEqual([journal]); + expect(fs.existsSync(target)).toBe(true); }); it("rejects an unsupported journal-directory entry during enumeration", () => { @@ -466,6 +464,24 @@ describe("Docker managed bootstrap journal", () => { ); }); + it.each([ + `.${IDENTITY}.json.commit.123.a0.tmp`, + `.${IDENTITY}.json.decision.pid.a0.tmp`, + `.${IDENTITY}.json.finalized.123.A0.tmp`, + `${IDENTITY}.json.decision.123.a0.tmp`, + `.${IDENTITY}.json.decision.123.a0.tmp.extra`, + ])("rejects and retains near-miss atomic entry %s", (name) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); + roots.push(root); + const store = createFileDockerManagedBootstrapJournalStore(root); + expect(loadUnfinished(store)).toEqual([]); + const target = path.join(root, DOCKER_MANAGED_BOOTSTRAP_JOURNAL_DIRECTORY, name); + fs.writeFileSync(target, "near miss\n", { mode: 0o600 }); + + expect(() => store.listUnfinishedIdentities()).toThrow("unsupported entry"); + expect(fs.existsSync(target)).toBe(true); + }); + it("reloads the exact completion receipt from a new journal store", () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-journal-")); roots.push(root); @@ -476,7 +492,12 @@ describe("Docker managed bootstrap journal", () => { expect(completed.commitReceipt).toEqual(finalization.commitReceipt); const restarted = createFileDockerManagedBootstrapJournalStore(root); - expect(restarted.recordCompletion(IDENTITY, finalization.commitReceipt)).toEqual(completed); + const reorderedReceipt = reverseKeys({ + ...finalization.commitReceipt, + image: reverseKeys({ ...finalization.commitReceipt.image }), + sandbox: reverseKeys({ ...finalization.commitReceipt.sandbox }), + }); + expect(restarted.recordCompletion(IDENTITY, reorderedReceipt)).toEqual(completed); expect(loadUnfinished(restarted)).toEqual([completed]); expect(() => restarted.recordCompletion(IDENTITY, { diff --git a/src/lib/onboard/managed-bootstrap/docker-journal.ts b/src/lib/onboard/managed-bootstrap/docker-journal.ts index 47f1022e884..d0e1f4eaee0 100644 --- a/src/lib/onboard/managed-bootstrap/docker-journal.ts +++ b/src/lib/onboard/managed-bootstrap/docker-journal.ts @@ -740,32 +740,6 @@ function exactCompletionReceipt(value: unknown): ManagedBootstrapCompletionRecei }); } -export function sameDockerManagedBootstrapReceipt( - kind: "preparation", - left: ManagedBootstrapDurablePreparationReceipt, - right: ManagedBootstrapDurablePreparationReceipt, -): boolean; -export function sameDockerManagedBootstrapReceipt( - kind: "completion", - left: ManagedBootstrapCompletionReceipt, - right: ManagedBootstrapCompletionReceipt, -): boolean; -export function sameDockerManagedBootstrapReceipt( - kind: "preparation" | "completion", - left: ManagedBootstrapDurablePreparationReceipt | ManagedBootstrapCompletionReceipt, - right: ManagedBootstrapDurablePreparationReceipt | ManagedBootstrapCompletionReceipt, -): boolean { - if (kind === "preparation") { - return ( - JSON.stringify(exactPreparationReceipt(left)) === - JSON.stringify(exactPreparationReceipt(right)) - ); - } - return ( - JSON.stringify(exactCompletionReceipt(left)) === JSON.stringify(exactCompletionReceipt(right)) - ); -} - function exactCleanupReceipt(value: unknown): ManagedBootstrapFinalizationReceipt { if (typeof value !== "object" || value === null || Array.isArray(value)) { fail("cleanup receipt must be an object"); diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts index e9b40f6340b..954de3a49c0 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts @@ -78,7 +78,7 @@ describe("Docker managed-bootstrap lifecycle composition", () => { onPatchFailure, network: { inferenceProvider: "openai", - dockerDriverGateway: false, + gatewayUsesContainerBridge: false, gatewayPort: 0, }, dependencies: {}, diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts index ae965865eb7..02db650a396 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.test.ts @@ -112,9 +112,10 @@ describe("Docker managed-bootstrap shared-state helper environment", () => { expect(outcome).toEqual({ supervisorReady: false, failure: null }); const helpers = nodeHelperCalls(fake.deps); - expect(helpers).toHaveLength(1); - expect(helpers[0]).toContain("--rollback-shared-state-transaction"); - expectCleanRunNodeHelper(helpers[0]!); + expect(helpers).toHaveLength(2); + expect(helpers.some((args) => args.includes("--shared-state-transaction-status"))).toBe(true); + expect(helpers.some((args) => args.includes("--rollback-shared-state-transaction"))).toBe(true); + helpers.forEach(expectCleanRunNodeHelper); }); it("clears arbitrary container environment before the durable receipt-clear helper", () => { diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index 7c0f52b99c7..6e52d0c6a4d 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -18,6 +18,7 @@ import { type ManagedBootstrapObservedSnapshot, type ManagedBootstrapPreparedReplacementHandle, type ManagedBootstrapReplacementHandle, + sameManagedBootstrapCompletionReceipt, } from "./adapter"; import type { DockerManagedBootstrapDeps } from "./docker"; import { @@ -26,7 +27,6 @@ import { DockerManagedBootstrapJournalAcknowledgementLostError, type DockerManagedBootstrapJournalPhase, type DockerManagedBootstrapJournalStore, - sameDockerManagedBootstrapReceipt, serializeDockerManagedBootstrapFinalizationRecord, } from "./docker-journal"; import { normalizeDockerManagedBootstrapLaunchSpec } from "./docker-spec"; @@ -277,7 +277,7 @@ export function fixture(options: DockerFixtureOptions = {}) { } if ( journal.commitReceipt !== null && - !sameDockerManagedBootstrapReceipt("completion", journal.commitReceipt, receipt) + !sameManagedBootstrapCompletionReceipt(journal.commitReceipt, receipt) ) { throw new Error("completion changed"); } diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index cf1e1e63fdd..a72236b3b41 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -377,6 +377,31 @@ describe("Docker managed bootstrap adapter", () => { expect(fake.replacement).toBeNull(); }); + it("rejects a divergent snapshot image before creating durable recovery state", async () => { + const fake = fixture(); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request: rootRequest, snapshot } = authority(); + + await expect( + adapter.prepareBootstrapReplacement({ + handle, + snapshot: { + ...snapshot, + image: { + ...snapshot.image, + repository: "registry.example/nemoclaw/divergent", + }, + }, + request: rootRequest, + replacementOptions: { values: {} }, + }), + ).rejects.toThrow("replacement snapshot image does not match its plan"); + expect(fake.replacement).toBeNull(); + expect(fake.journal).toBeNull(); + expect(fake.events).not.toContain("create:replacement"); + expect(fake.events).not.toContain("journal:staged"); + }); + it.each( SUPPORTED_AGENTS, )("prepares, activates, and exactly rolls back the %s agent without a central switch", async (agent) => { diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 8dc146f1df7..817263d729c 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -65,6 +65,8 @@ import { type ManagedBootstrapReplacementOptions, type ManagedBootstrapSandboxIdentity, renderManagedBootstrapHeldCommand, + sameManagedBootstrapCompletionReceipt, + sameManagedBootstrapDurablePreparationReceipt, } from "./adapter"; import { createFileDockerManagedBootstrapJournalStore, @@ -77,7 +79,6 @@ import { type DockerManagedBootstrapJournalStore, DockerManagedBootstrapLegacyRecordRequiresAgentError, parseDockerManagedBootstrapJournal, - sameDockerManagedBootstrapReceipt, serializeDockerManagedBootstrapFinalizationRecord, serializeDockerManagedBootstrapJournal, } from "./docker-journal"; @@ -1629,8 +1630,7 @@ function assertDockerBootstrapTransactionAuthority( (durablePreparation !== undefined && durablePreparation !== null && (transaction.preparationReceipt === null || - !sameDockerManagedBootstrapReceipt( - "preparation", + !sameManagedBootstrapDurablePreparationReceipt( transaction.preparationReceipt, durablePreparation, ))) || @@ -2042,11 +2042,7 @@ export function createDockerManagedBootstrapAdapter( journal.phase === "shared-state-committed" && finalization.commitReceipt !== null && journal.commitReceipt !== null && - sameDockerManagedBootstrapReceipt( - "completion", - finalization.commitReceipt, - journal.commitReceipt, - )) || + sameManagedBootstrapCompletionReceipt(finalization.commitReceipt, journal.commitReceipt)) || (finalization.phase === "rolled-back" && (journal.phase === "staged" || journal.phase === "rollback-authorized" || @@ -2800,7 +2796,7 @@ export function createDockerManagedBootstrapAdapter( if ( finalized.phase !== "committed" || !finalized.commitReceipt || - !sameDockerManagedBootstrapReceipt("completion", finalized.commitReceipt, completion) + !sameManagedBootstrapCompletionReceipt(finalized.commitReceipt, completion) ) { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: handle.bootstrapIdentity, @@ -2880,7 +2876,7 @@ export function createDockerManagedBootstrapAdapter( ); if ( journal.commitReceipt === null || - !sameDockerManagedBootstrapReceipt("completion", journal.commitReceipt, completion) + !sameManagedBootstrapCompletionReceipt(journal.commitReceipt, completion) ) { throw new ManagedBootstrapCommitStateIndeterminateError({ bootstrapIdentity: journal.bootstrapIdentity, @@ -3155,6 +3151,14 @@ export function createDockerManagedBootstrapAdapter( ) { throw new Error("Managed bootstrap Docker replacement identities do not match."); } + if ( + snapshot.image.repository !== handle.plan.image.repository || + snapshot.image.manifestDigest !== handle.plan.image.manifestDigest + ) { + throw new Error( + "Managed bootstrap Docker replacement snapshot image does not match its plan.", + ); + } const parsed = parseDockerManagedBootstrapLaunchSpec(snapshot.specCanonicalJson); const normalizedOriginal = normalizeDockerManagedBootstrapLaunchSpec(parsed.inspect); if (normalizedOriginal.hash !== snapshot.specHash) { diff --git a/src/lib/onboard/managed-bootstrap/index.ts b/src/lib/onboard/managed-bootstrap/index.ts index 1c6378642e4..1e1f2c67fd9 100644 --- a/src/lib/onboard/managed-bootstrap/index.ts +++ b/src/lib/onboard/managed-bootstrap/index.ts @@ -17,6 +17,8 @@ export { type ManagedBootstrapRecoveryReport, prepareManagedBootstrapSequence, recoverManagedBootstrapTransactions, + sameManagedBootstrapCompletionReceipt, + sameManagedBootstrapDurablePreparationReceipt, } from "./adapter"; export { MANAGED_BOOTSTRAP_COMPLETION_FILE, diff --git a/src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts b/src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts new file mode 100644 index 00000000000..c4eb9cfb957 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export function reverseKeys(value: T): T { + return Object.fromEntries(Object.entries(value).reverse()) as T; +} diff --git a/test/runtime-provider-source-shape.test.ts b/test/runtime-provider-source-shape.test.ts index 7bcfd06ec4d..abd9e01f2da 100644 --- a/test/runtime-provider-source-shape.test.ts +++ b/test/runtime-provider-source-shape.test.ts @@ -143,6 +143,7 @@ describe("runtime provider central source boundary", () => { "src/lib/onboard/managed-bootstrap/envelope.ts", "src/lib/onboard/managed-bootstrap/image-runtime.ts", "src/lib/onboard/managed-bootstrap/index.ts", + "src/lib/onboard/managed-bootstrap/managed-bootstrap-test-fixture.ts", "src/lib/onboard/managed-bootstrap/runtime-create.ts", ]); }); From b3973cebb50d1841dda57b2b883dcfd1795beff6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 07:05:15 -0700 Subject: [PATCH 04/24] fix(onboard): retain failed terminal authority Signed-off-by: Aaron Erickson --- .../onboard/docker-gpu-sandbox-create-lifecycle.test.ts | 4 ++++ src/lib/onboard/docker-gpu-sandbox-create.ts | 5 +++++ .../docker-shared-state-rollback-authority.test.ts | 2 +- src/lib/onboard/managed-bootstrap/docker-shared-state.ts | 8 ++++---- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index cf614528c94..0206ce9a80d 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -159,6 +159,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.waitForSupervisorReconnectIfNeeded(); expect(onPatchFailureExit).not.toHaveBeenCalled(); + await expect(patch.commitAfterReady()).rejects.toThrow("rollback backup"); await expect(patch.commitAfterReady()).rejects.toThrow("rollback backup"); expect(onPatchFailureExit).toHaveBeenCalledOnce(); @@ -197,6 +198,9 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); + await expect(patch.commitAfterReady()).rejects.toThrow( + "cannot commit before the recreated OpenShell supervisor reconnects", + ); await expect(patch.commitAfterReady()).rejects.toThrow( "cannot commit before the recreated OpenShell supervisor reconnects", ); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index cd7e9aa8af6..b1e566785fc 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -147,6 +147,7 @@ export function createDockerGpuSandboxCreatePatch( let cutoverFinalized = false; let cutoverFinalization: Promise | null = null; let cutoverFinalizationOutcome: "commit" | "rollback" | null = null; + let cutoverFinalizationFailure: Error | null = null; const findContainerIds = options.overrides?.findContainerIds ?? findOpenShellDockerSandboxContainerIds; @@ -374,6 +375,7 @@ export function createDockerGpuSandboxCreatePatch( }, async commitAfterReady() { + if (cutoverFinalizationFailure) throw cutoverFinalizationFailure; if (cutoverFinalized || (!managedBootstrapCutover && !result)) return; if (needsSupervisorWait) { const error = new Error( @@ -383,6 +385,7 @@ export function createDockerGpuSandboxCreatePatch( const failure = rollbackError ? new Error(`${error.message} Rollback failed: ${rollbackError.message}`) : error; + cutoverFinalizationFailure = failure; onPatchFailureExit(options.sandboxName, failure, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, @@ -417,6 +420,7 @@ export function createDockerGpuSandboxCreatePatch( failure as Error & { managedBootstrapRollbackError?: unknown } ).managedBootstrapRollbackError = rollbackError; } + cutoverFinalizationFailure = failure; onPatchFailureExit(options.sandboxName, failure, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, @@ -437,6 +441,7 @@ export function createDockerGpuSandboxCreatePatch( const failure = new Error( "Managed startup passed Ready, but its rollback backup could not be removed.", ); + cutoverFinalizationFailure = failure; onPatchFailureExit(options.sandboxName, failure, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts index 34eca762bcb..2e88d215f1b 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state-rollback-authority.test.ts @@ -180,6 +180,7 @@ describe("Docker managed-bootstrap shared-state rollback authority", () => { ), ), ); + expect(fake.deps.dockerRm).not.toHaveBeenCalled(); }); it("removes the exact failed container without rollback when both receipts are absent", () => { @@ -210,7 +211,6 @@ describe("Docker managed-bootstrap shared-state rollback authority", () => { const outcome = finalizeDockerManagedStartupSharedState( { - retainContainerAfterRollback: true, transaction: TRANSACTION, supervisorReady: true, }, diff --git a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts index 2fcd8595802..6d616a9dc07 100644 --- a/src/lib/onboard/managed-bootstrap/docker-shared-state.ts +++ b/src/lib/onboard/managed-bootstrap/docker-shared-state.ts @@ -628,16 +628,16 @@ export function finalizeDockerManagedStartupSharedState( { cause: stopError }, ); } - rollbackManagedStartupSharedState(transaction, deps, receiptPath); - if (!input.patchResult && !input.retainContainerAfterRollback) { + const restoredSharedState = rollbackManagedStartupSharedState(transaction, deps, receiptPath); + if (!restoredSharedState && !input.patchResult && !input.retainContainerAfterRollback) { removeFailedUnbackedContainer(transaction, deps); } return { supervisorReady: false, failure }; } quiesceManagedStartupContainer(transaction, deps); - rollbackManagedStartupSharedState(transaction, deps); - if (!input.patchResult && !input.retainContainerAfterRollback) { + const restoredSharedState = rollbackManagedStartupSharedState(transaction, deps); + if (!restoredSharedState && !input.patchResult && !input.retainContainerAfterRollback) { removeFailedUnbackedContainer(transaction, deps); } return { supervisorReady: false, failure: null }; From 26bb959db30c95b031ea5f016758894b880e823b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 1 Aug 2026 07:58:57 -0700 Subject: [PATCH 05/24] ci(images): harden managed publication evidence Signed-off-by: Aaron Erickson (cherry picked from commit c3fea512b467c1a52c945435671dee43f07fb99f) --- .github/workflows/base-image.yaml | 36 +++++----- .../support/base-image-publication.test.ts | 65 ++----------------- ...managed-image-publication-workflow.test.ts | 58 +---------------- tools/e2e/base-image-publication.mts | 48 +------------- 4 files changed, 25 insertions(+), 182 deletions(-) diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 715f3cda49e..10daba7bc52 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -541,16 +541,14 @@ jobs: echo "ERROR: published manifest has unexpected platforms: $actual_platforms" >&2 exit 1 fi - manifest_inspect="$(docker buildx imagetools inspect "$first_tag")" - mapfile -t manifest_digests < <( - printf '%s\n' "$manifest_inspect" \ - | sed -nE 's/^Digest:[[:space:]]*(sha256:[0-9a-f]{64})$/\1/p' - ) - if [ "${#manifest_digests[@]}" -ne 1 ]; then + digest="$( + docker buildx imagetools inspect "$first_tag" \ + --format '{{.Manifest.Digest}}' + )" + if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then echo "ERROR: expected one published Hermes base digest." >&2 exit 1 fi - digest="${manifest_digests[0]}" reference="$IMAGE@$digest" docker buildx imagetools inspect "$reference" >/dev/null @@ -723,16 +721,14 @@ jobs: echo "ERROR: published manifest has unexpected platforms: $actual_platforms" >&2 exit 1 fi - manifest_inspect="$(docker buildx imagetools inspect "$first_tag")" - mapfile -t manifest_digests < <( - printf '%s\n' "$manifest_inspect" \ - | sed -nE 's/^Digest:[[:space:]]*(sha256:[0-9a-f]{64})$/\1/p' - ) - if [ "${#manifest_digests[@]}" -ne 1 ]; then + digest="$( + docker buildx imagetools inspect "$first_tag" \ + --format '{{.Manifest.Digest}}' + )" + if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then echo "ERROR: expected one published Deep Agents Code base digest." >&2 exit 1 fi - digest="${manifest_digests[0]}" reference="$IMAGE@$digest" docker buildx imagetools inspect "$reference" >/dev/null @@ -907,16 +903,14 @@ jobs: echo "ERROR: published manifest has unexpected platforms: $actual_platforms" >&2 exit 1 fi - manifest_inspect="$(docker buildx imagetools inspect "$first_tag")" - mapfile -t manifest_digests < <( - printf '%s\n' "$manifest_inspect" \ - | sed -nE 's/^Digest:[[:space:]]*(sha256:[0-9a-f]{64})$/\1/p' - ) - if [ "${#manifest_digests[@]}" -ne 1 ]; then + digest="$( + docker buildx imagetools inspect "$first_tag" \ + --format '{{.Manifest.Digest}}' + )" + if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then echo "ERROR: expected one published OpenClaw base digest." >&2 exit 1 fi - digest="${manifest_digests[0]}" reference="$IMAGE@$digest" docker buildx imagetools inspect "$reference" >/dev/null diff --git a/test/e2e/support/base-image-publication.test.ts b/test/e2e/support/base-image-publication.test.ts index eb82b9cc9d1..d1e56249dbb 100644 --- a/test/e2e/support/base-image-publication.test.ts +++ b/test/e2e/support/base-image-publication.test.ts @@ -233,68 +233,13 @@ describe("base-image publication evidence", () => { expect(() => parseBaseImagePushPaths(source)).toThrow(expected); }); - it("expands only reviewed glob families against first-parent Git history (#7744)", () => { - const calls: string[][] = []; - const expanded = expandBaseImagePushPaths( - EXPECTED_SHA, - ["Dockerfile", "agents/**", "src/lib/messaging/**"], - (args) => { - calls.push(args); - const pathspec = required(args.at(-1), "glob expansion call is missing a pathspec"); - return required( - new Map([ - [ - ":(glob)agents/**", - "agents/hermes/Dockerfile\nagents/openclaw/manifest.yaml\nagents/hermes/Dockerfile", - ], - [ - ":(glob)src/lib/messaging/**", - "src/lib/messaging/channels/slack.ts\nsrc/lib/messaging/types.ts", - ], - ]).get(pathspec), - `unexpected glob expansion: ${args.join(" ")}`, - ); - }, - ); - - expect(expanded).toEqual([ + it("passes only reviewed glob families as bounded Git pathspecs (#7744)", () => { + const expanded = expandBaseImagePushPaths(EXPECTED_SHA, [ "Dockerfile", - "agents/hermes/Dockerfile", - "agents/openclaw/manifest.yaml", - "src/lib/messaging/channels/slack.ts", - "src/lib/messaging/types.ts", + "agents/**", + "src/lib/messaging/**", ]); - expect(calls).toEqual([ - [ - "log", - "--first-parent", - "--diff-merges=first-parent", - "--format=", - "--name-only", - EXPECTED_SHA, - "--", - ":(glob)agents/**", - ], - [ - "log", - "--first-parent", - "--diff-merges=first-parent", - "--format=", - "--name-only", - EXPECTED_SHA, - "--", - ":(glob)src/lib/messaging/**", - ], - ]); - }); - - it("fails closed when a reviewed glob is empty or Git returns an out-of-family path (#7744)", () => { - expect(() => expandBaseImagePushPaths(EXPECTED_SHA, ["agents/**"], () => "")).toThrow( - /did not match Git history/u, - ); - expect(() => - expandBaseImagePushPaths(EXPECTED_SHA, ["agents/**"], () => "scripts/escaped.sh"), - ).toThrow(/outside reviewed/u); + expect(expanded).toEqual([":(glob)agents/**", ":(glob)src/lib/messaging/**", "Dockerfile"]); }); it("binds the applicable commit to the checked-out first-parent chain (#7372)", () => { diff --git a/test/managed-image-publication-workflow.test.ts b/test/managed-image-publication-workflow.test.ts index 802b6d785dd..f87b81e28f3 100644 --- a/test/managed-image-publication-workflow.test.ts +++ b/test/managed-image-publication-workflow.test.ts @@ -158,41 +158,6 @@ function publicationBoundaryErrors(baseWorkflow: Workflow, managedWorkflow: Work const base = step(publisher, "Validate exact base image contract"); const validate = step(publisher, "Validate exact managed image before promotion"); const workflowSource = JSON.stringify(managedWorkflow); - const publisherSource = JSON.stringify(publisher); - const validationMarkers = [ - 'mktemp -d "$RUNNER_TEMP/anonymous-docker-XXXXXX"', - 'DOCKER_CONFIG="$anonymous_config" docker pull --platform "$PLATFORM" "$reference"', - "bootstrap the GHCR package", - "/opt/nemoclaw-blueprint/blueprint.yaml", - "/usr/local/share/nemoclaw/node-tar-inventory.json", - "/usr/local/share/nemoclaw/corporate-ca.pem", - 'entry.status !== "fixed"', - '--entrypoint "$REQUIRED_BINARY"', - "io.nvidia.nemoclaw.managed-image.contract", - "io.nvidia.nemoclaw.managed-image.startup-profile", - "io.nvidia.nemoclaw.managed-image.capabilities", - "io.nvidia.nemoclaw.managed-image.cohort", - "^ghrun-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$", - "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION", - "@openclaw/diagnostics-otel", - "@openclaw/brave-plugin", - "@openclaw/discord", - "@tencent-weixin/openclaw-weixin", - "@openclaw/slack", - "@openclaw/whatsapp", - "@openclaw/msteams", - "@openclaw/googlechat", - "/sandbox/.openclaw/npm/projects", - "lstatSync(manifestPath).isFile()", - "matches.length !== 1", - "microsoft-teams-apps", - "config.plugins?.entries?.[id]?.enabled !== false", - 'config["platforms"].get(name) != {"enabled": False}', - "run-managed-image-direct-e2e.ts", - '--agent "$AGENT"', - '--image "$reference"', - '--platform "$PLATFORM"', - ]; const forbiddenPerLanePromotionMarkers = [ 'aliases=("${IMAGE}:${GITHUB_SHA}")', "docker buildx imagetools create", @@ -230,12 +195,6 @@ function publicationBoundaryErrors(baseWorkflow: Workflow, managedWorkflow: Work base.run.includes(".run == {id: $runId, attempt: $runAttempt}") ? [] : ["managed image build must consume the same-run exact base digest contract"]), - ...validationMarkers - .filter((marker) => !validate.run?.includes(marker)) - .map((marker) => `exact managed image validation is missing ${marker}`), - ...forbiddenPerLanePromotionMarkers - .filter((marker) => publisherSource.includes(marker)) - .map((marker) => `per-agent lane must not publish mutable alias with ${marker}`), ...(buildIndex >= 0 && buildIndex < validateIndex ? [] : ["managed image validation must follow its immutable digest build"]), @@ -263,18 +222,6 @@ describe("complete managed-image publication workflow", () => { const channelGuardStart = validationRun.lastIndexOf("for (const id of [", channelGuardEnd); expect(channelGuardStart).toBeGreaterThan(-1); expect(validationRun.slice(channelGuardStart, channelGuardEnd)).toContain('"googlechat",'); - const weakenedWorkflow = structuredClone(managedWorkflow); - const weakenedValidation = step( - managedPublisher(weakenedWorkflow), - "Validate exact managed image before promotion", - ); - weakenedValidation.run = weakenedValidation.run?.replace( - " || !fs.lstatSync(manifestPath).isFile()", - "", - ); - expect(publicationBoundaryErrors(baseWorkflow, weakenedWorkflow)).toContain( - "exact managed image validation is missing lstatSync(manifestPath).isFile()", - ); expect(publisher).toMatchObject({ needs: ["build-and-push-hermes", "build-and-push-dcode", "build-and-push-openclaw"], permissions: { @@ -317,6 +264,7 @@ describe("complete managed-image publication workflow", () => { const manifest = step(basePublisher, "Create and verify multi-platform manifest"); expect(manifest.id).toBe("manifest"); expect(manifest.run).toContain('reference="$IMAGE@$digest"'); + expect(manifest.run).toContain("--format '{{.Manifest.Digest}}'"); expect(manifest.run).toContain(`agent: "${expectedPublisher.agent}"`); expect(manifest.run).toContain("platformDigests: {"); expect(step(basePublisher, "Upload managed base image contract").with?.name).toBe( @@ -1010,12 +958,10 @@ fi ...publicationAgents.flatMap((agent) => publicationPlatforms.map((platform) => { const artifactPlatform = platform.replaceAll("/", "-"); - const displayAgent = - agent === "langchain-deepagents-code" ? "langchain-deepagents-code" : agent; return { name: "managed-image-${{ github.run_id }}-${{ github.run_attempt }}-" + - `${displayAgent}-${artifactPlatform}`, + `${agent}-${artifactPlatform}`, path: `\${{ runner.temp }}/managed-image-contracts/${agent}/${artifactPlatform}/contract.json`, "if-no-files-found": "error", "retention-days": 90, diff --git a/tools/e2e/base-image-publication.mts b/tools/e2e/base-image-publication.mts index 6a2db258051..f057ffbbce2 100644 --- a/tools/e2e/base-image-publication.mts +++ b/tools/e2e/base-image-publication.mts @@ -247,52 +247,10 @@ function defaultGit(args: string[]): string { export function expandBaseImagePushPaths( expectedSha: string, paths: readonly string[], - runGit: (args: string[]) => string = defaultGit, ): string[] { sha(expectedSha, "expected SHA"); - const expanded = new Set(); - - for (const path of paths) { - const matcher = REVIEWED_PATH_GLOBS.get(path); - if (!matcher) { - expanded.add(path); - continue; - } - - const matches = runGit([ - "log", - "--first-parent", - "--diff-merges=first-parent", - "--format=", - "--name-only", - expectedSha, - "--", - `:(glob)${path}`, - ]) - .split(/\r?\n/u) - .filter((candidate) => candidate.length > 0); - if (matches.length === 0) { - throw new Error(`reviewed base-image push glob did not match Git history: ${path}`); - } - for (const candidate of matches) { - if ( - !SAFE_PATH_PATTERN.test(candidate) || - candidate.startsWith("/") || - candidate.includes("//") || - candidate - .split("/") - .some((segment) => segment === "" || segment === "." || segment === "..") - ) { - throw new Error(`reviewed base-image push glob expanded to an unsafe path: ${candidate}`); - } - if (!matcher.test(candidate)) { - throw new Error(`Git returned a path outside reviewed base-image push glob ${path}`); - } - expanded.add(candidate); - } - } - - return [...expanded].sort(); + return [...new Set(paths.map((path) => (REVIEWED_PATH_GLOBS.has(path) ? `:(glob)${path}` : path)))] + .sort(); } export function resolveFirstParentHistory( @@ -312,7 +270,7 @@ export function resolveFirstParentHistory( if (runGit(["rev-parse", "--is-shallow-repository"]) !== "false") { throw new Error("base-image publication gate requires a complete Git history"); } - const expandedPaths = expandBaseImagePushPaths(expectedSha, paths, runGit); + const expandedPaths = expandBaseImagePushPaths(expectedSha, paths); if (expandedPaths.length === 0) { throw new Error("base-image push paths did not resolve to any Git paths"); } From 6b82eac5e54a66b6dfa7706683e96b2007628823 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 1 Aug 2026 08:15:26 -0700 Subject: [PATCH 06/24] refactor(images): share managed base contract export Signed-off-by: Aaron Erickson (cherry picked from commit 2060e9ccfd41363a2bc8818b8f0553e0579e455a) --- .github/workflows/base-image.yaml | 204 +++++------------- scripts/export-managed-base-image-contract.sh | 76 +++++++ test/dcode-base-image-workflow.test.ts | 6 + test/helpers/vitest-watch-triggers.ts | 9 + test/managed-base-image-contract.test.ts | 91 ++++++++ ...managed-image-publication-workflow.test.ts | 7 +- test/vitest-watch-triggers.test.ts | 12 ++ 7 files changed, 247 insertions(+), 158 deletions(-) create mode 100755 scripts/export-managed-base-image-contract.sh create mode 100644 test/managed-base-image-contract.test.ts diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 10daba7bc52..dbc8029399a 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -441,6 +441,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Download platform digests uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -473,6 +478,7 @@ jobs: - name: Create and verify multi-platform manifest id: manifest env: + AGENT: hermes IMAGE: ${{ env.REGISTRY }}/nvidia/nemoclaw/hermes-sandbox-base TAGS: ${{ steps.meta.outputs.tags }} run: | @@ -552,58 +558,16 @@ jobs: reference="$IMAGE@$digest" docker buildx imagetools inspect "$reference" >/dev/null - contract_dir="$RUNNER_TEMP/managed-base-contract" - mkdir -p "$contract_dir" - jq -n \ - --arg amd64Digest "${platform_digests[linux/amd64]}" \ - --arg amd64Reference "$IMAGE@${platform_digests[linux/amd64]}" \ - --arg arm64Digest "${platform_digests[linux/arm64]}" \ - --arg arm64Reference "$IMAGE@${platform_digests[linux/arm64]}" \ - --arg digest "$digest" \ - --arg image "$IMAGE" \ - --arg reference "$reference" \ - --arg revision "$GITHUB_SHA" \ - --argjson runAttempt "$GITHUB_RUN_ATTEMPT" \ - --argjson runId "$GITHUB_RUN_ID" \ - '{ - contractVersion: 1, - agent: "hermes", - image: $image, - digest: $digest, - reference: $reference, - platforms: ["linux/amd64", "linux/arm64"], - platformDigests: { - "linux/amd64": $amd64Digest, - "linux/arm64": $arm64Digest - }, - platformReferences: { - "linux/amd64": $amd64Reference, - "linux/arm64": $arm64Reference - }, - sourceRevision: $revision, - run: { - id: $runId, - attempt: $runAttempt - } - }' > "$contract_dir/contract.json" - jq -e \ - '.contractVersion == 1 - and .agent == "hermes" - and (.sourceRevision | test("^[0-9a-f]{40}$")) - and (.digest | test("^sha256:[0-9a-f]{64}$")) - and .reference == (.image + "@" + .digest) - and .platforms == ["linux/amd64", "linux/arm64"] - and (.platformDigests | keys | sort) == .platforms - and (.platformReferences | keys | sort) == .platforms - and ([ - .platforms[] as $platform - | ( - (.platformDigests[$platform] | test("^sha256:[0-9a-f]{64}$")) - and .platformReferences[$platform] == - (.image + "@" + .platformDigests[$platform]) - ) - ] | all)' \ - "$contract_dir/contract.json" >/dev/null + scripts/export-managed-base-image-contract.sh \ + "$AGENT" \ + "$IMAGE" \ + "$digest" \ + "${platform_digests[linux/amd64]}" \ + "${platform_digests[linux/arm64]}" \ + "$GITHUB_SHA" \ + "$GITHUB_RUN_ID" \ + "$GITHUB_RUN_ATTEMPT" \ + "$RUNNER_TEMP/managed-base-contract/contract.json" printf 'digest=%s\n' "$digest" >> "$GITHUB_OUTPUT" - name: Upload managed base image contract @@ -621,6 +585,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Download platform digests uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -653,6 +622,7 @@ jobs: - name: Create and verify multi-platform manifest id: manifest env: + AGENT: langchain-deepagents-code IMAGE: ${{ env.REGISTRY }}/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base TAGS: ${{ steps.meta.outputs.tags }} run: | @@ -732,58 +702,16 @@ jobs: reference="$IMAGE@$digest" docker buildx imagetools inspect "$reference" >/dev/null - contract_dir="$RUNNER_TEMP/managed-base-contract" - mkdir -p "$contract_dir" - jq -n \ - --arg amd64Digest "${platform_digests[linux/amd64]}" \ - --arg amd64Reference "$IMAGE@${platform_digests[linux/amd64]}" \ - --arg arm64Digest "${platform_digests[linux/arm64]}" \ - --arg arm64Reference "$IMAGE@${platform_digests[linux/arm64]}" \ - --arg digest "$digest" \ - --arg image "$IMAGE" \ - --arg reference "$reference" \ - --arg revision "$GITHUB_SHA" \ - --argjson runAttempt "$GITHUB_RUN_ATTEMPT" \ - --argjson runId "$GITHUB_RUN_ID" \ - '{ - contractVersion: 1, - agent: "langchain-deepagents-code", - image: $image, - digest: $digest, - reference: $reference, - platforms: ["linux/amd64", "linux/arm64"], - platformDigests: { - "linux/amd64": $amd64Digest, - "linux/arm64": $arm64Digest - }, - platformReferences: { - "linux/amd64": $amd64Reference, - "linux/arm64": $arm64Reference - }, - sourceRevision: $revision, - run: { - id: $runId, - attempt: $runAttempt - } - }' > "$contract_dir/contract.json" - jq -e \ - '.contractVersion == 1 - and .agent == "langchain-deepagents-code" - and (.sourceRevision | test("^[0-9a-f]{40}$")) - and (.digest | test("^sha256:[0-9a-f]{64}$")) - and .reference == (.image + "@" + .digest) - and .platforms == ["linux/amd64", "linux/arm64"] - and (.platformDigests | keys | sort) == .platforms - and (.platformReferences | keys | sort) == .platforms - and ([ - .platforms[] as $platform - | ( - (.platformDigests[$platform] | test("^sha256:[0-9a-f]{64}$")) - and .platformReferences[$platform] == - (.image + "@" + .platformDigests[$platform]) - ) - ] | all)' \ - "$contract_dir/contract.json" >/dev/null + scripts/export-managed-base-image-contract.sh \ + "$AGENT" \ + "$IMAGE" \ + "$digest" \ + "${platform_digests[linux/amd64]}" \ + "${platform_digests[linux/arm64]}" \ + "$GITHUB_SHA" \ + "$GITHUB_RUN_ID" \ + "$GITHUB_RUN_ATTEMPT" \ + "$RUNNER_TEMP/managed-base-contract/contract.json" printf 'digest=%s\n' "$digest" >> "$GITHUB_OUTPUT" - name: Upload managed base image contract @@ -803,6 +731,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Download platform digests uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -835,6 +768,7 @@ jobs: - name: Create and verify multi-platform manifest id: manifest env: + AGENT: openclaw IMAGE: ${{ env.REGISTRY }}/nvidia/nemoclaw/sandbox-base TAGS: ${{ steps.meta.outputs.tags }} run: | @@ -914,58 +848,16 @@ jobs: reference="$IMAGE@$digest" docker buildx imagetools inspect "$reference" >/dev/null - contract_dir="$RUNNER_TEMP/managed-base-contract" - mkdir -p "$contract_dir" - jq -n \ - --arg amd64Digest "${platform_digests[linux/amd64]}" \ - --arg amd64Reference "$IMAGE@${platform_digests[linux/amd64]}" \ - --arg arm64Digest "${platform_digests[linux/arm64]}" \ - --arg arm64Reference "$IMAGE@${platform_digests[linux/arm64]}" \ - --arg digest "$digest" \ - --arg image "$IMAGE" \ - --arg reference "$reference" \ - --arg revision "$GITHUB_SHA" \ - --argjson runAttempt "$GITHUB_RUN_ATTEMPT" \ - --argjson runId "$GITHUB_RUN_ID" \ - '{ - contractVersion: 1, - agent: "openclaw", - image: $image, - digest: $digest, - reference: $reference, - platforms: ["linux/amd64", "linux/arm64"], - platformDigests: { - "linux/amd64": $amd64Digest, - "linux/arm64": $arm64Digest - }, - platformReferences: { - "linux/amd64": $amd64Reference, - "linux/arm64": $arm64Reference - }, - sourceRevision: $revision, - run: { - id: $runId, - attempt: $runAttempt - } - }' > "$contract_dir/contract.json" - jq -e \ - '.contractVersion == 1 - and .agent == "openclaw" - and (.sourceRevision | test("^[0-9a-f]{40}$")) - and (.digest | test("^sha256:[0-9a-f]{64}$")) - and .reference == (.image + "@" + .digest) - and .platforms == ["linux/amd64", "linux/arm64"] - and (.platformDigests | keys | sort) == .platforms - and (.platformReferences | keys | sort) == .platforms - and ([ - .platforms[] as $platform - | ( - (.platformDigests[$platform] | test("^sha256:[0-9a-f]{64}$")) - and .platformReferences[$platform] == - (.image + "@" + .platformDigests[$platform]) - ) - ] | all)' \ - "$contract_dir/contract.json" >/dev/null + scripts/export-managed-base-image-contract.sh \ + "$AGENT" \ + "$IMAGE" \ + "$digest" \ + "${platform_digests[linux/amd64]}" \ + "${platform_digests[linux/arm64]}" \ + "$GITHUB_SHA" \ + "$GITHUB_RUN_ID" \ + "$GITHUB_RUN_ATTEMPT" \ + "$RUNNER_TEMP/managed-base-contract/contract.json" printf 'digest=%s\n' "$digest" >> "$GITHUB_OUTPUT" - name: Upload managed base image contract diff --git a/scripts/export-managed-base-image-contract.sh b/scripts/export-managed-base-image-contract.sh new file mode 100755 index 00000000000..643d93f5394 --- /dev/null +++ b/scripts/export-managed-base-image-contract.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [ "$#" -ne 9 ]; then + echo "Usage: $0 AGENT IMAGE DIGEST AMD64_DIGEST ARM64_DIGEST SOURCE_REVISION RUN_ID RUN_ATTEMPT OUTPUT" >&2 + exit 2 +fi + +readonly agent="$1" +readonly image="$2" +readonly digest="$3" +readonly amd64_digest="$4" +readonly arm64_digest="$5" +readonly source_revision="$6" +readonly run_id="$7" +readonly run_attempt="$8" +readonly output="$9" +readonly reference="${image}@${digest}" + +mkdir -p "$(dirname -- "$output")" +jq -n \ + --arg agent "$agent" \ + --arg amd64Digest "$amd64_digest" \ + --arg amd64Reference "$image@$amd64_digest" \ + --arg arm64Digest "$arm64_digest" \ + --arg arm64Reference "$image@$arm64_digest" \ + --arg digest "$digest" \ + --arg image "$image" \ + --arg reference "$reference" \ + --arg revision "$source_revision" \ + --argjson runAttempt "$run_attempt" \ + --argjson runId "$run_id" \ + '{ + contractVersion: 1, + agent: $agent, + image: $image, + digest: $digest, + reference: $reference, + platforms: ["linux/amd64", "linux/arm64"], + platformDigests: { + "linux/amd64": $amd64Digest, + "linux/arm64": $arm64Digest + }, + platformReferences: { + "linux/amd64": $amd64Reference, + "linux/arm64": $arm64Reference + }, + sourceRevision: $revision, + run: { + id: $runId, + attempt: $runAttempt + } + }' >"$output" + +jq -e \ + --arg agent "$agent" \ + '.contractVersion == 1 + and .agent == $agent + and (.sourceRevision | test("^[0-9a-f]{40}$")) + and (.digest | test("^sha256:[0-9a-f]{64}$")) + and .reference == (.image + "@" + .digest) + and .platforms == ["linux/amd64", "linux/arm64"] + and (.platformDigests | keys | sort) == .platforms + and (.platformReferences | keys | sort) == .platforms + and ([ + .platforms[] as $platform + | ( + (.platformDigests[$platform] | test("^sha256:[0-9a-f]{64}$")) + and .platformReferences[$platform] == + (.image + "@" + .platformDigests[$platform]) + ) + ] | all)' \ + "$output" >/dev/null diff --git a/test/dcode-base-image-workflow.test.ts b/test/dcode-base-image-workflow.test.ts index 121f14c6146..eaa66f3f7b6 100644 --- a/test/dcode-base-image-workflow.test.ts +++ b/test/dcode-base-image-workflow.test.ts @@ -448,6 +448,7 @@ describe("base-image publication behavior", () => { expect(metadata?.with?.tags).toContain("type=raw,value=latest"); expect(metadata?.with?.tags).toContain("type=ref,event=tag"); expect(metadata?.with?.tags).toContain("type=sha,prefix=,format=short"); + expect(createManifest?.env?.AGENT).toBe("openclaw"); const openClawManifestScript = createManifest?.run ?? ""; expect(openClawManifestScript).toContain('"${#digest_files[@]}" -ne 2'); expect(openClawManifestScript).toContain("^(amd64|arm64)-([0-9a-f]{64})$"); @@ -463,6 +464,7 @@ describe("base-image publication behavior", () => { 'docker buildx imagetools create "${tag_args[@]}" "${sources[@]}"', ); expect(openClawManifestScript).toContain('"amd64,arm64"'); + expect(openClawManifestScript).toContain("scripts/export-managed-base-image-contract.sh"); for (const step of (manifestJob?.steps ?? []).filter((step) => step.uses)) { expect(step.uses, step.name).toMatch(FULL_SHA_ACTION); } @@ -472,6 +474,7 @@ describe("base-image publication behavior", () => { const publishers = publisherJobs(workflow); const imagePublishers = [ { + agent: "hermes", platformJobName: "build-hermes-platforms", manifestJobName: "build-and-push-hermes", manifestName: "Build and push Hermes base image", @@ -479,6 +482,7 @@ describe("base-image publication behavior", () => { image: "${{ env.REGISTRY }}/nvidia/nemoclaw/hermes-sandbox-base", }, { + agent: "langchain-deepagents-code", platformJobName: "build-dcode-platforms", manifestJobName: "build-and-push-dcode", manifestName: "Build and push Deep Agents Code base image", @@ -571,6 +575,7 @@ describe("base-image publication behavior", () => { expect(metadata?.with?.tags).toContain("type=raw,value=latest"); expect(metadata?.with?.tags).toContain("type=ref,event=tag"); expect(metadata?.with?.tags).toContain("type=sha,prefix=,format=short"); + expect(createManifest?.env?.AGENT).toBe(imagePublisher.agent); expect(createManifest?.env?.IMAGE).toBe(imagePublisher.image); const manifestScript = createManifest?.run ?? ""; expect(manifestScript).toContain('"${#digest_files[@]}" -ne 2'); @@ -585,6 +590,7 @@ describe("base-image publication behavior", () => { 'docker buildx imagetools create "${tag_args[@]}" "${sources[@]}"', ); expect(manifestScript).toContain('"amd64,arm64"'); + expect(manifestScript).toContain("scripts/export-managed-base-image-contract.sh"); for (const step of (manifestJob?.steps ?? []).filter((step) => step.uses)) { expect(step.uses, step.name).toMatch(FULL_SHA_ACTION); } diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 2ef92a8b845..b89004eb195 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -91,6 +91,15 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ pattern: /(?:^|\/)scripts\/setup-jetson\.sh$/, testsToRun: runTests("test/setup-jetson.test.ts"), }, + { + pattern: + /(?:^|\/)(?:\.github\/workflows\/base-image\.yaml|scripts\/export-managed-base-image-contract\.sh)$/, + testsToRun: runTests( + "test/managed-base-image-contract.test.ts", + "test/managed-image-publication-workflow.test.ts", + "test/dcode-base-image-workflow.test.ts", + ), + }, { pattern: /(?:^|\/)scripts\/e2e\/sanitize-trace-timing\.py$/, testsToRun: runTests( diff --git a/test/managed-base-image-contract.test.ts b/test/managed-base-image-contract.test.ts new file mode 100644 index 00000000000..d3cb78bbc8d --- /dev/null +++ b/test/managed-base-image-contract.test.ts @@ -0,0 +1,91 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const repoRoot = path.resolve(import.meta.dirname, ".."); +const exporter = path.join(repoRoot, "scripts", "export-managed-base-image-contract.sh"); +const sourceRevision = "c".repeat(40); +const digest = `sha256:${"d".repeat(64)}`; +const amd64Digest = `sha256:${"a".repeat(64)}`; +const arm64Digest = `sha256:${"b".repeat(64)}`; +const agents = [ + { + agent: "openclaw", + image: "ghcr.io/nvidia/nemoclaw/sandbox-base", + }, + { + agent: "hermes", + image: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base", + }, + { + agent: "langchain-deepagents-code", + image: "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base", + }, +] as const; + +function exportContract(output: string, agent: string, image: string, amd64 = amd64Digest) { + return spawnSync( + exporter, + [agent, image, digest, amd64, arm64Digest, sourceRevision, "42", "3", output], + { encoding: "utf8" }, + ); +} + +describe("managed base image contract exporter", () => { + it("binds both native platform digests for every managed agent (#7744)", () => { + const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-contract-")); + + try { + for (const { agent, image } of agents) { + const output = path.join(temporaryRoot, agent, "contract.json"); + const result = exportContract(output, agent, image); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(fs.readFileSync(output, "utf8"))).toEqual({ + contractVersion: 1, + agent, + image, + digest, + reference: `${image}@${digest}`, + platforms: ["linux/amd64", "linux/arm64"], + platformDigests: { + "linux/amd64": amd64Digest, + "linux/arm64": arm64Digest, + }, + platformReferences: { + "linux/amd64": `${image}@${amd64Digest}`, + "linux/arm64": `${image}@${arm64Digest}`, + }, + sourceRevision, + run: { id: 42, attempt: 3 }, + }); + } + } finally { + fs.rmSync(temporaryRoot, { recursive: true, force: true }); + } + }); + + it("rejects a platform digest that is not a full SHA-256 value (#7744)", () => { + const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-contract-")); + + try { + const output = path.join(temporaryRoot, "contract.json"); + const result = exportContract( + output, + "openclaw", + "ghcr.io/nvidia/nemoclaw/sandbox-base", + "sha256:bad", + ); + + expect(result.status).not.toBe(0); + } finally { + fs.rmSync(temporaryRoot, { recursive: true, force: true }); + } + }); +}); diff --git a/test/managed-image-publication-workflow.test.ts b/test/managed-image-publication-workflow.test.ts index f87b81e28f3..95e7dd6df33 100644 --- a/test/managed-image-publication-workflow.test.ts +++ b/test/managed-image-publication-workflow.test.ts @@ -263,10 +263,13 @@ describe("complete managed-image publication workflow", () => { ); const manifest = step(basePublisher, "Create and verify multi-platform manifest"); expect(manifest.id).toBe("manifest"); + expect(manifest.env?.AGENT).toBe(expectedPublisher.agent); expect(manifest.run).toContain('reference="$IMAGE@$digest"'); expect(manifest.run).toContain("--format '{{.Manifest.Digest}}'"); - expect(manifest.run).toContain(`agent: "${expectedPublisher.agent}"`); - expect(manifest.run).toContain("platformDigests: {"); + expect(manifest.run).toContain("scripts/export-managed-base-image-contract.sh"); + expect(manifest.run).toContain('"${platform_digests[linux/amd64]}"'); + expect(manifest.run).toContain('"${platform_digests[linux/arm64]}"'); + expect(step(basePublisher, "Checkout").with?.["persist-credentials"]).toBe(false); expect(step(basePublisher, "Upload managed base image contract").with?.name).toBe( expectedPublisher.artifact, ); diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index 9d2fcec9768..206f9cfa020 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -60,6 +60,8 @@ const OPAQUE_INPUTS = [ "agents/hermes/mcp-config-transaction.py", "test/e2e/lib/ci-compatible-inference.sh", "scripts/setup-jetson.sh", + ".github/workflows/base-image.yaml", + "scripts/export-managed-base-image-contract.sh", "scripts/e2e/sanitize-trace-timing.py", "test/e2e/manifests/openclaw-nvidia.yaml", "test/e2e/docs/parity-inventory.generated.json", @@ -123,6 +125,16 @@ describe("Vitest opaque-input watch triggers", () => { "test/e2e/support/hosted-inference.test.ts", ]); expect(triggeredBy("scripts/setup-jetson.sh")).toEqual(["test/setup-jetson.test.ts"]); + expect(triggeredBy(".github/workflows/base-image.yaml")).toEqual([ + "test/managed-base-image-contract.test.ts", + "test/managed-image-publication-workflow.test.ts", + "test/dcode-base-image-workflow.test.ts", + ]); + expect(triggeredBy("scripts/export-managed-base-image-contract.sh")).toEqual([ + "test/managed-base-image-contract.test.ts", + "test/managed-image-publication-workflow.test.ts", + "test/dcode-base-image-workflow.test.ts", + ]); expect(triggeredBy("scripts/e2e/sanitize-trace-timing.py")).toEqual([ "test/e2e/support/e2e-scorecard.test.ts", "test/e2e/support/sanitize-trace-timing.test.ts", From 71a858f0520e1adbee7330674ec97b896871e980 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 06:21:40 -0700 Subject: [PATCH 07/24] test(images): add protected multiarch build contract Signed-off-by: Aaron Erickson (cherry picked from commit 5976d9d0b45f08597d39c6fea4ef368deb7d5de5) --- .../checks/build-protected-managed-images.sh | 215 ++++++++++++++++++ .../protected-managed-image-contract.ts | 119 ++++++++++ test/protected-managed-image-contract.test.ts | 86 +++++++ 3 files changed, 420 insertions(+) create mode 100755 scripts/checks/build-protected-managed-images.sh create mode 100644 scripts/checks/protected-managed-image-contract.ts create mode 100644 test/protected-managed-image-contract.test.ts diff --git a/scripts/checks/build-protected-managed-images.sh b/scripts/checks/build-protected-managed-images.sh new file mode 100755 index 00000000000..127f2088d19 --- /dev/null +++ b/scripts/checks/build-protected-managed-images.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +usage() { + echo "usage: $0 --output --revision --cohort --platform --openclaw-base --hermes-base --dcode-base " >&2 + exit 2 +} + +output="" +revision="" +cohort="" +platform="" +openclaw_base="" +hermes_base="" +dcode_base="" +while (($# > 0)); do + case "$1" in + --output) + (($# >= 2)) || usage + output="$2" + shift 2 + ;; + --revision) + (($# >= 2)) || usage + revision="$2" + shift 2 + ;; + --cohort) + (($# >= 2)) || usage + cohort="$2" + shift 2 + ;; + --platform) + (($# >= 2)) || usage + platform="$2" + shift 2 + ;; + --openclaw-base) + (($# >= 2)) || usage + openclaw_base="$2" + shift 2 + ;; + --hermes-base) + (($# >= 2)) || usage + hermes_base="$2" + shift 2 + ;; + --dcode-base) + (($# >= 2)) || usage + dcode_base="$2" + shift 2 + ;; + *) + usage + ;; + esac +done + +[[ "$output" == /* && "$output" != *$'\n'* ]] || usage +[[ "$revision" =~ ^[a-f0-9]{40}$ ]] || usage +[[ "$cohort" =~ ^protected-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$ ]] || usage +[[ "$platform" == "linux/amd64" || "$platform" == "linux/arm64" ]] || usage +[[ "$openclaw_base" =~ ^ghcr[.]io/nvidia/nemoclaw/sandbox-base@sha256:[a-f0-9]{64}$ ]] || usage +[[ "$hermes_base" =~ ^ghcr[.]io/nvidia/nemoclaw/hermes-sandbox-base@sha256:[a-f0-9]{64}$ ]] || usage +[[ "$dcode_base" =~ ^ghcr[.]io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base@sha256:[a-f0-9]{64}$ ]] || usage + +for command in docker jq sha256sum; do + command -v "$command" >/dev/null 2>&1 || { + echo "ERROR: protected managed-image build requires $command" >&2 + exit 1 + } +done + +work_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/nemoclaw-protected-images.XXXXXX")" +trap 'rm -rf "$work_dir"' EXIT +contracts="$work_dir/contracts.jsonl" +: >"$contracts" + +build_agent() { + local agent="$1" + local dockerfile="$2" + local base_reference="$3" + local image_repository="localhost:5000/nemoclaw-managed-protected/${agent}" + local exact_base_raw="$work_dir/${agent}-base-exact.raw" + local metadata="$work_dir/${agent}-build-metadata.json" + local exact_image_raw="$work_dir/${agent}-image-exact.raw" + + local base_digest="${base_reference##*@}" + docker buildx imagetools inspect "$base_reference" --raw >"$exact_base_raw" + local actual_base + actual_base="sha256:$(sha256sum "$exact_base_raw" | awk '{print $1}')" + [[ "$actual_base" == "$base_digest" ]] || { + echo "ERROR: ${agent} exact base bytes do not match its descriptor" >&2 + exit 1 + } + + scripts/check-production-build-args.sh \ + -f "$dockerfile" \ + --build-arg "BASE_IMAGE=${base_reference}" \ + --build-arg "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1" \ + --build-arg "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root" + + docker buildx build \ + --file "$dockerfile" \ + --platform "$platform" \ + --push \ + --provenance=false \ + --sbom=false \ + --metadata-file "$metadata" \ + --tag "${image_repository}:${revision}" \ + --label "org.opencontainers.image.source=https://github.com/NVIDIA/NemoClaw" \ + --label "org.opencontainers.image.revision=${revision}" \ + --label "io.nvidia.nemoclaw.agent=${agent}" \ + --label "io.nvidia.nemoclaw.managed-image.contract=1" \ + --label "io.nvidia.nemoclaw.managed-image.platform=${platform}" \ + --label "io.nvidia.nemoclaw.managed-image.startup-profile=1" \ + --label "io.nvidia.nemoclaw.managed-image.capabilities=1" \ + --label "io.nvidia.nemoclaw.managed-image.cohort=${cohort}" \ + --build-arg "BASE_IMAGE=${base_reference}" \ + --build-arg "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1" \ + --build-arg "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root" \ + . + + local digest + digest="$(jq -er '."containerimage.digest"' "$metadata")" + [[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]] || { + echo "ERROR: ${agent} build did not return an immutable manifest digest" >&2 + exit 1 + } + local reference="${image_repository}@${digest}" + docker buildx imagetools inspect "$reference" --raw >"$exact_image_raw" + local actual_image + actual_image="sha256:$(sha256sum "$exact_image_raw" | awk '{print $1}')" + [[ "$actual_image" == "$digest" ]] || { + echo "ERROR: ${agent} isolated-registry bytes do not match the build digest" >&2 + exit 1 + } + docker pull --platform "$platform" "$reference" + local image_json + image_json="$(docker image inspect "$reference")" + local image_id + image_id="$(jq -er 'if length == 1 then .[0].Id else error("not one image") end' <<<"$image_json")" + [[ "$image_id" =~ ^sha256:[a-f0-9]{64}$ ]] || { + echo "ERROR: ${agent} exact manifest did not resolve to one local content ID" >&2 + exit 1 + } + jq -e \ + --arg agent "$agent" \ + --arg cohort "$cohort" \ + --arg image_id "$image_id" \ + --arg platform "$platform" \ + --arg revision "$revision" ' + length == 1 and + .[0].Id == $image_id and + ((.[0].Config.User // "") as $user | + $user == "" or $user == "root" or $user == "0") and + .[0].Config.Labels["io.nvidia.nemoclaw.agent"] == $agent and + .[0].Config.Labels["io.nvidia.nemoclaw.managed-image.contract"] == "1" and + .[0].Config.Labels["io.nvidia.nemoclaw.managed-image.platform"] == $platform and + .[0].Config.Labels["io.nvidia.nemoclaw.managed-image.startup-profile"] == "1" and + .[0].Config.Labels["io.nvidia.nemoclaw.managed-image.capabilities"] == "1" and + .[0].Config.Labels["io.nvidia.nemoclaw.managed-image.cohort"] == $cohort and + .[0].Config.Labels["org.opencontainers.image.revision"] == $revision + ' <<<"$image_json" >/dev/null || { + echo "ERROR: ${agent} exact protected image contract is invalid" >&2 + exit 1 + } + + jq -nc \ + --arg agent "$agent" \ + --arg reference "$reference" \ + --arg digest "$digest" \ + --arg localContentId "$image_id" \ + --arg baseReference "$base_reference" \ + --arg platform "$platform" \ + '{ + agent: $agent, + platform: $platform, + reference: $reference, + digest: $digest, + localContentId: $localContentId, + baseReference: $baseReference + }' >>"$contracts" +} + +build_agent \ + openclaw \ + Dockerfile \ + "$openclaw_base" +build_agent \ + hermes \ + agents/hermes/Dockerfile \ + "$hermes_base" +build_agent \ + langchain-deepagents-code \ + agents/langchain-deepagents-code/Dockerfile \ + "$dcode_base" + +mkdir -p "$(dirname "$output")" +jq -se \ + --arg platform "$platform" ' + if ( + length == 3 and + ([.[].agent] | sort) == ["hermes", "langchain-deepagents-code", "openclaw"] and + ([.[].platform] | unique) == [$platform] and + ([.[].reference] | unique | length) == 3 + ) + then . + else error("protected managed-image set is incomplete") + end +' "$contracts" >"${output}.tmp" +mv "${output}.tmp" "$output" diff --git a/scripts/checks/protected-managed-image-contract.ts b/scripts/checks/protected-managed-image-contract.ts new file mode 100644 index 00000000000..db92ccd0133 --- /dev/null +++ b/scripts/checks/protected-managed-image-contract.ts @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ManagedStartupAgent } from "../../src/lib/onboard/managed-startup/profile.ts"; + +export const PROTECTED_MANAGED_IMAGE_AGENTS = [ + "openclaw", + "hermes", + "langchain-deepagents-code", +] as const satisfies readonly ManagedStartupAgent[]; + +export const PROTECTED_MANAGED_IMAGE_PLATFORMS = ["linux/amd64", "linux/arm64"] as const; + +export type ProtectedManagedImagePlatform = (typeof PROTECTED_MANAGED_IMAGE_PLATFORMS)[number]; + +export type ProtectedManagedImageContract = { + readonly agent: ManagedStartupAgent; + readonly baseReference: string; + readonly digest: string; + readonly localContentId: string; + readonly platform: ProtectedManagedImagePlatform; + readonly reference: string; +}; + +const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/u; +const BASE_REPOSITORIES: Readonly> = Object.freeze({ + openclaw: "ghcr.io/nvidia/nemoclaw/sandbox-base", + hermes: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base", + "langchain-deepagents-code": "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base", +}); + +function record(value: unknown): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("protected managed-image contract entry must be an object"); + } + return value as Record; +} + +function exactKeys(value: Record): void { + const actual = Object.keys(value).sort(); + const expected = [ + "agent", + "baseReference", + "digest", + "localContentId", + "platform", + "reference", + ].sort(); + if (actual.some((key, index) => key !== expected[index]) || actual.length !== expected.length) { + throw new Error("protected managed-image contract entry has unexpected fields"); + } +} + +function parseEntry( + value: unknown, + expectedPlatform: ProtectedManagedImagePlatform, +): ProtectedManagedImageContract { + const entry = record(value); + exactKeys(entry); + if ( + typeof entry.agent !== "string" || + !PROTECTED_MANAGED_IMAGE_AGENTS.includes(entry.agent as ManagedStartupAgent) + ) { + throw new Error("protected managed-image contract entry has an invalid agent"); + } + if (entry.platform !== expectedPlatform) { + throw new Error("protected managed-image contract entry has the wrong platform"); + } + if ( + typeof entry.digest !== "string" || + !DIGEST_PATTERN.test(entry.digest) || + typeof entry.localContentId !== "string" || + !DIGEST_PATTERN.test(entry.localContentId) + ) { + throw new Error("protected managed-image contract entry has an invalid content identity"); + } + const expectedRepository = `localhost:5000/nemoclaw-managed-protected/${entry.agent}`; + if (entry.reference !== `${expectedRepository}@${entry.digest}`) { + throw new Error("protected managed-image contract entry is not the exact agent digest"); + } + const basePrefix = `${BASE_REPOSITORIES[entry.agent as ManagedStartupAgent]}@`; + if ( + typeof entry.baseReference !== "string" || + !entry.baseReference.startsWith(basePrefix) || + !DIGEST_PATTERN.test(entry.baseReference.slice(basePrefix.length)) + ) { + throw new Error("protected managed-image contract entry has an invalid base reference"); + } + return { + agent: entry.agent as ManagedStartupAgent, + baseReference: entry.baseReference, + digest: entry.digest, + localContentId: entry.localContentId, + platform: expectedPlatform, + reference: entry.reference, + }; +} + +export function parseProtectedManagedImageContracts( + value: unknown, + expectedPlatform: ProtectedManagedImagePlatform, +): ProtectedManagedImageContract[] { + if (!Array.isArray(value) || value.length !== PROTECTED_MANAGED_IMAGE_AGENTS.length) { + throw new Error("protected managed-image contract must contain exactly all shipped agents"); + } + const contracts = value.map((entry) => parseEntry(entry, expectedPlatform)); + const actualAgents = contracts.map(({ agent }) => agent).sort(); + const expectedAgents = [...PROTECTED_MANAGED_IMAGE_AGENTS].sort(); + if (actualAgents.some((agent, index) => agent !== expectedAgents[index])) { + throw new Error("protected managed-image contract must contain each shipped agent once"); + } + if ( + new Set(contracts.map(({ reference }) => reference)).size !== contracts.length || + new Set(contracts.map(({ localContentId }) => localContentId)).size !== contracts.length + ) { + throw new Error("protected managed-image contract must contain unique immutable images"); + } + return contracts; +} diff --git a/test/protected-managed-image-contract.test.ts b/test/protected-managed-image-contract.test.ts new file mode 100644 index 00000000000..52137411533 --- /dev/null +++ b/test/protected-managed-image-contract.test.ts @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + PROTECTED_MANAGED_IMAGE_AGENTS, + PROTECTED_MANAGED_IMAGE_PLATFORMS, + type ProtectedManagedImagePlatform, + parseProtectedManagedImageContracts, +} from "../scripts/checks/protected-managed-image-contract.ts"; + +const BASE_REPOSITORIES = { + openclaw: "sandbox-base", + hermes: "hermes-sandbox-base", + "langchain-deepagents-code": "langchain-deepagents-code-sandbox-base", +} as const; + +function contracts(platform: ProtectedManagedImagePlatform) { + return PROTECTED_MANAGED_IMAGE_AGENTS.map((agent, index) => { + const digit = String(index + 1); + const digest = `sha256:${digit.repeat(64)}`; + return { + agent, + baseReference: `ghcr.io/nvidia/nemoclaw/${BASE_REPOSITORIES[agent]}@sha256:${String(index + 4).repeat(64)}`, + digest, + localContentId: `sha256:${String(index + 7).repeat(64)}`, + platform, + reference: `localhost:5000/nemoclaw-managed-protected/${agent}@${digest}`, + }; + }); +} + +describe("protected managed-image build contract", () => { + it.each( + PROTECTED_MANAGED_IMAGE_PLATFORMS, + )("accepts one unique immutable image for every shipped agent on %s (#7744)", (platform) => { + const value = contracts(platform); + expect(parseProtectedManagedImageContracts(value, platform)).toEqual(value); + }); + + it("rejects an incomplete or duplicated all-agent cohort (#7744)", () => { + const value = contracts("linux/amd64"); + expect(() => parseProtectedManagedImageContracts(value.slice(0, 2), "linux/amd64")).toThrow( + "exactly all shipped agents", + ); + expect(() => + parseProtectedManagedImageContracts([value[0], value[0], value[2]], "linux/amd64"), + ).toThrow("each shipped agent once"); + }); + + it("rejects cross-platform or mutable image evidence (#7744)", () => { + const value = contracts("linux/amd64"); + expect(() => parseProtectedManagedImageContracts(value, "linux/arm64")).toThrow( + "wrong platform", + ); + expect(() => + parseProtectedManagedImageContracts( + [{ ...value[0], reference: value[0].reference.split("@")[0] }, value[1], value[2]], + "linux/amd64", + ), + ).toThrow("exact agent digest"); + }); + + it("rejects identity drift and unexpected receipt fields (#7744)", () => { + const value = contracts("linux/arm64"); + expect(() => + parseProtectedManagedImageContracts( + [{ ...value[0], digest: `sha256:${"f".repeat(64)}` }, value[1], value[2]], + "linux/arm64", + ), + ).toThrow("exact agent digest"); + expect(() => + parseProtectedManagedImageContracts( + [{ ...value[0], baseReference: value[1].baseReference }, value[1], value[2]], + "linux/arm64", + ), + ).toThrow("invalid base reference"); + expect(() => + parseProtectedManagedImageContracts( + [{ ...value[0], aliases: ["latest"] }, value[1], value[2]], + "linux/arm64", + ), + ).toThrow("unexpected fields"); + }); +}); From b17865596260e032e2c5d18b97a3aef28f4446bc Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sun, 2 Aug 2026 07:54:29 -0700 Subject: [PATCH 08/24] ci(images): bootstrap protected multiarch startup lane Signed-off-by: Aaron Erickson (cherry picked from commit c9a92ae77f44f619bede7ce84a1ed11b45a30ed2) --- .github/workflows/e2e.yaml | 330 ++++++++++++++++++ .../protected-managed-image-contract.ts | 188 ++++++++++ .../checks/run-managed-image-direct-e2e.ts | 3 +- .../managed-image-multiarch-startup.test.ts | 106 ++++++ test/pr-risk-plan.test.ts | 18 +- test/protected-managed-image-contract.test.ts | 87 +++++ tools/advisors/risk-plan.mts | 24 +- ...aged-image-multiarch-workflow-boundary.mts | 272 +++++++++++++++ tools/e2e/prepare-e2e-workflow-boundary.mts | 1 + ...upload-e2e-artifacts-workflow-boundary.mts | 7 + tools/e2e/workflow-boundary.mts | 2 + 11 files changed, 1035 insertions(+), 3 deletions(-) create mode 100644 test/e2e/live/managed-image-multiarch-startup.test.ts create mode 100644 tools/e2e/managed-image-multiarch-workflow-boundary.mts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 07840ded266..c38a2948ce1 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1702,6 +1702,335 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + # This explicit-only lane is intentionally dormant until the checked-out + # candidate carries ci/protected-managed-image-multiarch-activation-v1.json. + # The workflow and its narrow activation rule must first land on trusted main; + # only then can the follow-on candidate execute and prove its own exact head. + managed-image-multiarch-startup: + name: Protected managed-image startup (${{ matrix.platform }}) + needs: generate-matrix + if: ${{ contains(format(',{0},', inputs.jobs), ',managed-image-multiarch-startup,') || contains(format(',{0},', inputs.targets), ',managed-image-multiarch-startup,') }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 210 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-24.04 + shard: linux-amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + shard: linux-arm64 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }} + E2E_DEFAULT_ENABLED: "0" + E2E_JOB: "1" + E2E_TARGET_ID: "managed-image-multiarch-startup" + NEMOCLAW_E2E_SHARD: ${{ matrix.shard }} + NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: ${{ inputs.base_sha }} + NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: protected-${{ github.run_id }}-${{ github.run_attempt }} + NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/contracts.json + NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/evidence.json + NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM: ${{ matrix.platform }} + NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + NEMOCLAW_PROTECTED_REGISTRY_NAME: nemoclaw-managed-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }} + NEMOCLAW_RUN_LIVE_E2E: "1" + steps: + - name: Validate protected exact-head dispatch + env: + ACTOR: ${{ github.actor }} + BASE_SHA: ${{ inputs.base_sha }} + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + PLATFORM: ${{ matrix.platform }} + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + shell: bash + run: | + set -euo pipefail + [[ "$REPOSITORY" == "NVIDIA/NemoClaw" && "$REF" == "refs/heads/main" && "$EVENT_NAME" == "workflow_dispatch" ]] || { + echo "::error::Protected managed-image startup must run from trusted NVIDIA/NemoClaw main" >&2 + exit 1 + } + [[ "$ACTOR" == "github-actions[bot]" ]] || { + echo "::error::Protected managed-image startup requires the trusted controller actor" >&2 + exit 1 + } + [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { + echo "::error::Protected managed-image startup requires exact PR and base SHAs" >&2 + exit 1 + } + [[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA" ]] || { + echo "::error::Protected managed-image startup requires the exact trusted workflow SHA" >&2 + exit 1 + } + case "${PLATFORM}:${RUNNER_ARCH_KIND}" in + linux/amd64:X64 | linux/arm64:ARM64) ;; + *) + echo "::error::Protected managed-image startup requires a native ${PLATFORM} runner" >&2 + exit 1 + ;; + esac + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + + - *dockerhub-auth + + - name: Set up protected managed-image Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + with: + driver-opts: network=host + buildkitd-config-inline: | + [registry."localhost:5000"] + http = true + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + with: + build-cli: "false" + + - name: Validate candidate activation contract + env: + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + shell: bash + run: | + set -euo pipefail + activation="ci/protected-managed-image-multiarch-activation-v1.json" + [[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { + echo "::error::Protected managed-image checkout does not match the exact PR SHA" >&2 + exit 1 + } + [[ -f "$activation" && ! -L "$activation" ]] || { + echo "::error::Protected managed-image activation contract is absent" >&2 + exit 1 + } + jq -e ' + (keys | sort) == ["agents", "contractVersion", "jobId", "platforms"] and + .contractVersion == 1 and + .jobId == "managed-image-multiarch-startup" and + .agents == ["openclaw", "hermes", "langchain-deepagents-code"] and + .platforms == ["linux/amd64", "linux/arm64"] + ' "$activation" >/dev/null || { + echo "::error::Protected managed-image activation contract is invalid" >&2 + exit 1 + } + install -d -m 0700 "$E2E_ARTIFACT_DIR" + + - id: bases + name: Resolve exact platform base images + env: + PLATFORM: ${{ matrix.platform }} + shell: bash + run: | + set -euo pipefail + arch="${PLATFORM#linux/}" + work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-protected-bases.XXXXXX")" + trap 'rm -rf -- "$work_dir"' EXIT + + resolve_base() { + local output_name="$1" + local alias="$2" + local repository="$3" + local alias_raw="$work_dir/${output_name}-alias.raw" + local exact_raw="$work_dir/${output_name}-exact.raw" + docker buildx imagetools inspect "$alias" --raw > "$alias_raw" + local digest + digest="$( + jq -er --arg arch "$arch" ' + if ( + .mediaType == "application/vnd.oci.image.index.v1+json" or + .mediaType == "application/vnd.docker.distribution.manifest.list.v2+json" + ) then + [.manifests[] | select(.platform.os == "linux" and .platform.architecture == $arch)] + | if length == 1 then .[0].digest else error("not one exact platform descriptor") end + else + error("base alias is not a platform index") + end + ' "$alias_raw" + )" + [[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]] || { + echo "::error::${output_name} base alias returned an invalid digest" >&2 + exit 1 + } + local reference="${repository}@${digest}" + docker buildx imagetools inspect "$reference" --raw > "$exact_raw" + [[ "sha256:$(sha256sum "$exact_raw" | awk '{print $1}')" == "$digest" ]] || { + echo "::error::${output_name} exact base bytes do not match the selected digest" >&2 + exit 1 + } + printf '%s=%s\n' "$output_name" "$reference" >> "$GITHUB_OUTPUT" + } + + resolve_base openclaw \ + ghcr.io/nvidia/nemoclaw/sandbox-base:latest \ + ghcr.io/nvidia/nemoclaw/sandbox-base + resolve_base hermes \ + ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest \ + ghcr.io/nvidia/nemoclaw/hermes-sandbox-base + resolve_base dcode \ + ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:latest \ + ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base + + - name: Start isolated protected managed-image registry + shell: bash + run: | + set -euo pipefail + if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then + echo "::error::Protected registry name already exists" >&2 + exit 1 + fi + if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then + echo "::error::Refusing to reuse an existing localhost:5000 registry" >&2 + exit 1 + fi + docker run --detach \ + --name "$NEMOCLAW_PROTECTED_REGISTRY_NAME" \ + --label "io.nvidia.nemoclaw.e2e-owner=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT}" \ + --label "io.nvidia.nemoclaw.e2e-platform=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM}" \ + --publish 127.0.0.1:5000:5000 \ + docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373 + for _ in $(seq 1 30); do + if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null; then + exit 0 + fi + sleep 1 + done + docker logs "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >&2 + exit 1 + + - name: Build exact all-agent protected managed images + env: + BASE_DCODE: ${{ steps.bases.outputs.dcode }} + BASE_HERMES: ${{ steps.bases.outputs.hermes }} + BASE_OPENCLAW: ${{ steps.bases.outputs.openclaw }} + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + shell: bash + run: | + set -euo pipefail + scripts/checks/build-protected-managed-images.sh \ + --output "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" \ + --revision "$CHECKOUT_SHA" \ + --cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \ + --platform "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" \ + --openclaw-base "$BASE_OPENCLAW" \ + --hermes-base "$BASE_HERMES" \ + --dcode-base "$BASE_DCODE" + + - name: Run every exact managed-image contract directly + env: + BASE_SHA: ${{ inputs.base_sha }} + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + WORKFLOW_SHA: ${{ inputs.workflow_sha }} + shell: bash + run: | + set -euo pipefail + direct_runs="${RUNNER_TEMP}/protected-managed-image-direct-runs.jsonl" + : > "$direct_runs" + while IFS= read -r entry; do + agent="$(jq -er '.agent' <<< "$entry")" + digest="$(jq -er '.digest' <<< "$entry")" + platform="$(jq -er '.platform' <<< "$entry")" + reference="$(jq -er '.reference' <<< "$entry")" + [[ "$platform" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" ]] || { + echo "::error::Protected managed-image contract changed platform" >&2 + exit 1 + } + npx --no-install tsx scripts/checks/run-managed-image-direct-e2e.ts \ + --agent "$agent" \ + --image "$reference" \ + --platform "$platform" + jq -nc \ + --arg agent "$agent" \ + --arg digest "$digest" \ + --arg platform "$platform" \ + --arg reference "$reference" \ + '{agent: $agent, digest: $digest, platform: $platform, reference: $reference}' \ + >> "$direct_runs" + done < <(jq -c '.[]' "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT") + + contract_sha="sha256:$(sha256sum "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" | awk '{print $1}')" + jq -n \ + --arg baseSha "$BASE_SHA" \ + --arg cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \ + --arg contractSha256 "$contract_sha" \ + --arg headSha "$CHECKOUT_SHA" \ + --arg platform "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" \ + --arg workflowSha "$WORKFLOW_SHA" \ + --argjson runAttempt "$GITHUB_RUN_ATTEMPT" \ + --argjson runId "$GITHUB_RUN_ID" \ + --slurpfile contracts "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" \ + --slurpfile directRuns "$direct_runs" \ + '{ + kind: "nemoclaw-protected-managed-image-multiarch-v1", + headSha: $headSha, + baseSha: $baseSha, + workflowSha: $workflowSha, + platform: $platform, + cohort: $cohort, + contractSha256: $contractSha256, + contracts: $contracts[0], + directRuns: $directRuns, + run: {id: $runId, attempt: $runAttempt} + }' > "${NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE}.tmp" + mv \ + "${NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE}.tmp" \ + "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE" + + - name: Remove isolated protected managed-image registry + if: always() + shell: bash + run: | + set -euo pipefail + if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then + owner="$( + docker container inspect \ + --format '{{index .Config.Labels "io.nvidia.nemoclaw.e2e-owner"}}' \ + "$NEMOCLAW_PROTECTED_REGISTRY_NAME" + )" + [[ "$owner" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" ]] || { + echo "::error::Refusing to remove a registry not owned by this protected shard" >&2 + exit 1 + } + docker rm -f "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null + fi + if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then + echo "::error::Protected managed-image registry container remained after cleanup" >&2 + exit 1 + fi + if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then + echo "::error::Protected managed-image registry listener remained after cleanup" >&2 + exit 1 + fi + + - name: Validate protected managed-image evidence + shell: bash + run: >- + npx tsx tools/e2e/live-vitest-invocation.mts run + --test-path test/e2e/live/managed-image-multiarch-startup.test.ts + + - name: Upload protected managed-image evidence + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: e2e-managed-image-multiarch-startup-${{ matrix.shard }} + path: e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/ + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + agent-turn-latency: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',agent-turn-latency,') || contains(format(',{0},', inputs.targets), ',agent-turn-latency,') }} @@ -5739,6 +6068,7 @@ jobs: inference-routing, cloud-inference, gpu-e2e, + managed-image-multiarch-startup, agent-turn-latency, kimi-inference-compat, hermes-inference-switch, diff --git a/scripts/checks/protected-managed-image-contract.ts b/scripts/checks/protected-managed-image-contract.ts index db92ccd0133..9fe25a9228b 100644 --- a/scripts/checks/protected-managed-image-contract.ts +++ b/scripts/checks/protected-managed-image-contract.ts @@ -10,6 +10,9 @@ export const PROTECTED_MANAGED_IMAGE_AGENTS = [ ] as const satisfies readonly ManagedStartupAgent[]; export const PROTECTED_MANAGED_IMAGE_PLATFORMS = ["linux/amd64", "linux/arm64"] as const; +export const PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID = "managed-image-multiarch-startup" as const; +export const PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH = + "ci/protected-managed-image-multiarch-activation-v1.json" as const; export type ProtectedManagedImagePlatform = (typeof PROTECTED_MANAGED_IMAGE_PLATFORMS)[number]; @@ -22,7 +25,45 @@ export type ProtectedManagedImageContract = { readonly reference: string; }; +export type ProtectedManagedImageActivation = { + readonly agents: readonly ManagedStartupAgent[]; + readonly contractVersion: 1; + readonly jobId: typeof PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID; + readonly platforms: readonly ProtectedManagedImagePlatform[]; +}; + +export type ProtectedManagedImageDirectRun = Pick< + ProtectedManagedImageContract, + "agent" | "digest" | "platform" | "reference" +>; + +export type ProtectedManagedImageEvidence = { + readonly baseSha: string; + readonly cohort: string; + readonly contracts: readonly ProtectedManagedImageContract[]; + readonly contractSha256: string; + readonly directRuns: readonly ProtectedManagedImageDirectRun[]; + readonly headSha: string; + readonly kind: "nemoclaw-protected-managed-image-multiarch-v1"; + readonly platform: ProtectedManagedImagePlatform; + readonly run: { + readonly attempt: number; + readonly id: number; + }; + readonly workflowSha: string; +}; + +export type ProtectedManagedImageEvidenceIdentity = Pick< + ProtectedManagedImageEvidence, + "baseSha" | "cohort" | "headSha" | "platform" | "workflowSha" +> & { + readonly runAttempt: number; + readonly runId: number; +}; + const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/u; +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const COHORT_PATTERN = /^protected-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$/u; const BASE_REPOSITORIES: Readonly> = Object.freeze({ openclaw: "ghcr.io/nvidia/nemoclaw/sandbox-base", hermes: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base", @@ -51,6 +92,21 @@ function exactKeys(value: Record): void { } } +function requireExactKeys( + value: Record, + expected: readonly string[], + label: string, +): void { + const actual = Object.keys(value).sort(); + const sortedExpected = [...expected].sort(); + if ( + actual.length !== sortedExpected.length || + actual.some((key, index) => key !== sortedExpected[index]) + ) { + throw new Error(`${label} has unexpected fields`); + } +} + function parseEntry( value: unknown, expectedPlatform: ProtectedManagedImagePlatform, @@ -117,3 +173,135 @@ export function parseProtectedManagedImageContracts( } return contracts; } + +export function parseProtectedManagedImageActivation( + value: unknown, +): ProtectedManagedImageActivation { + const activation = record(value); + requireExactKeys( + activation, + ["agents", "contractVersion", "jobId", "platforms"], + "protected managed-image activation", + ); + if ( + activation.contractVersion !== 1 || + activation.jobId !== PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID || + JSON.stringify(activation.agents) !== JSON.stringify(PROTECTED_MANAGED_IMAGE_AGENTS) || + JSON.stringify(activation.platforms) !== JSON.stringify(PROTECTED_MANAGED_IMAGE_PLATFORMS) + ) { + throw new Error("protected managed-image activation contract is invalid"); + } + return { + agents: PROTECTED_MANAGED_IMAGE_AGENTS, + contractVersion: 1, + jobId: PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID, + platforms: PROTECTED_MANAGED_IMAGE_PLATFORMS, + }; +} + +export function parseProtectedManagedImageEvidence( + value: unknown, + expected: ProtectedManagedImageEvidenceIdentity, +): ProtectedManagedImageEvidence { + const evidence = record(value); + requireExactKeys( + evidence, + [ + "baseSha", + "cohort", + "contracts", + "contractSha256", + "directRuns", + "headSha", + "kind", + "platform", + "run", + "workflowSha", + ], + "protected managed-image evidence", + ); + if ( + evidence.kind !== "nemoclaw-protected-managed-image-multiarch-v1" || + evidence.headSha !== expected.headSha || + evidence.baseSha !== expected.baseSha || + evidence.workflowSha !== expected.workflowSha || + evidence.platform !== expected.platform || + evidence.cohort !== expected.cohort || + typeof evidence.headSha !== "string" || + typeof evidence.baseSha !== "string" || + typeof evidence.workflowSha !== "string" || + !SHA_PATTERN.test(evidence.headSha) || + !SHA_PATTERN.test(evidence.baseSha) || + !SHA_PATTERN.test(evidence.workflowSha) || + typeof evidence.cohort !== "string" || + !COHORT_PATTERN.test(evidence.cohort) || + typeof evidence.contractSha256 !== "string" || + !DIGEST_PATTERN.test(evidence.contractSha256) + ) { + throw new Error("protected managed-image evidence identity is invalid"); + } + + const run = record(evidence.run); + requireExactKeys(run, ["attempt", "id"], "protected managed-image evidence run"); + if ( + run.id !== expected.runId || + run.attempt !== expected.runAttempt || + !Number.isSafeInteger(run.id) || + !Number.isSafeInteger(run.attempt) || + Number(run.id) < 1 || + Number(run.attempt) < 1 + ) { + throw new Error("protected managed-image evidence run identity is invalid"); + } + + const contracts = parseProtectedManagedImageContracts(evidence.contracts, expected.platform); + if (!Array.isArray(evidence.directRuns) || evidence.directRuns.length !== contracts.length) { + throw new Error("protected managed-image evidence must directly run every contract"); + } + const contractByAgent = new Map(contracts.map((contract) => [contract.agent, contract])); + const seenAgents = new Set(); + const directRuns = evidence.directRuns.map((value): ProtectedManagedImageDirectRun => { + const directRun = record(value); + requireExactKeys( + directRun, + ["agent", "digest", "platform", "reference"], + "protected managed-image direct run", + ); + const contract = + typeof directRun.agent === "string" + ? contractByAgent.get(directRun.agent as ManagedStartupAgent) + : undefined; + if ( + !contract || + seenAgents.has(contract.agent) || + directRun.digest !== contract.digest || + directRun.platform !== contract.platform || + directRun.reference !== contract.reference + ) { + throw new Error("protected managed-image direct run does not match its exact contract"); + } + seenAgents.add(contract.agent); + return { + agent: contract.agent, + digest: contract.digest, + platform: contract.platform, + reference: contract.reference, + }; + }); + if (seenAgents.size !== PROTECTED_MANAGED_IMAGE_AGENTS.length) { + throw new Error("protected managed-image evidence must directly run every shipped agent"); + } + + return { + baseSha: evidence.baseSha, + cohort: evidence.cohort, + contracts, + contractSha256: evidence.contractSha256, + directRuns, + headSha: evidence.headSha, + kind: "nemoclaw-protected-managed-image-multiarch-v1", + platform: expected.platform, + run: { attempt: expected.runAttempt, id: expected.runId }, + workflowSha: evidence.workflowSha, + }; +} diff --git a/scripts/checks/run-managed-image-direct-e2e.ts b/scripts/checks/run-managed-image-direct-e2e.ts index c3c01ea7b45..c142cf72abe 100755 --- a/scripts/checks/run-managed-image-direct-e2e.ts +++ b/scripts/checks/run-managed-image-direct-e2e.ts @@ -29,6 +29,7 @@ import { MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, managedStartupE2eProfile, } from "./generate-managed-startup-profile-fixture.mts"; +import type { ProtectedManagedImagePlatform } from "./protected-managed-image-contract.ts"; const CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; const IMMUTABLE_IMAGE_RE = /^sha256:[a-f0-9]{64}$/u; @@ -47,7 +48,7 @@ const FIXED_ROOT_ENV = [ export interface ManagedImageDirectE2eInputs { readonly agent: ManagedStartupAgent; readonly image: string; - readonly platform: "linux/amd64" | "linux/arm64"; + readonly platform: ProtectedManagedImagePlatform; } interface CommandResult { diff --git a/test/e2e/live/managed-image-multiarch-startup.test.ts b/test/e2e/live/managed-image-multiarch-startup.test.ts new file mode 100644 index 00000000000..229c09f113a --- /dev/null +++ b/test/e2e/live/managed-image-multiarch-startup.test.ts @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import { + PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, + PROTECTED_MANAGED_IMAGE_PLATFORMS, + type ProtectedManagedImagePlatform, + parseProtectedManagedImageActivation, + parseProtectedManagedImageContracts, + parseProtectedManagedImageEvidence, +} from "../../../scripts/checks/protected-managed-image-contract.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; + +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const COHORT_PATTERN = /^protected-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$/u; + +function requiredEnvironment(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +function positiveIntegerEnvironment(name: string): number { + const value = requiredEnvironment(name); + if (!/^[1-9][0-9]*$/u.test(value)) throw new Error(`${name} must be a positive integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) throw new Error(`${name} must be a safe integer`); + return parsed; +} + +function regularArtifact(file: string, artifactDirectory: string): Buffer { + const relative = path.relative(fs.realpathSync(artifactDirectory), fs.realpathSync(file)); + if (!relative || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new Error(`${file} must be a child of the protected artifact directory`); + } + const status = fs.lstatSync(file); + if (!status.isFile() || status.isSymbolicLink() || status.size > 1024 * 1024) { + throw new Error(`${file} must be a bounded regular file`); + } + return fs.readFileSync(file); +} + +test("binds protected all-agent direct startup to the exact multiarch dispatch (#7744)", { + meta: { + e2ePhases: [ + "validate protected activation and dispatch identity", + "validate exact all-agent managed-image contracts", + "validate direct-start evidence binding", + ], + }, +}, ({ progress }) => { + progress.phase("validate protected activation and dispatch identity"); + const workspace = fs.realpathSync(requiredEnvironment("GITHUB_WORKSPACE")); + const artifactDirectory = requiredEnvironment("E2E_ARTIFACT_DIR"); + const contractFile = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT"); + const evidenceFile = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE"); + const platform = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM"); + const cohort = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT"); + const headSha = requiredEnvironment("NEMOCLAW_E2E_EXPECTED_SHA"); + const baseSha = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA"); + const workflowSha = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA"); + if ( + !(PROTECTED_MANAGED_IMAGE_PLATFORMS as readonly string[]).includes(platform) || + !COHORT_PATTERN.test(cohort) || + !SHA_PATTERN.test(headSha) || + !SHA_PATTERN.test(baseSha) || + !SHA_PATTERN.test(workflowSha) + ) { + throw new Error("protected managed-image dispatch identity is invalid"); + } + const expectedPlatform = platform as ProtectedManagedImagePlatform; + + const activationPath = path.join(workspace, PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH); + const activationStatus = fs.lstatSync(activationPath); + expect(activationStatus.isFile()).toBe(true); + expect(activationStatus.isSymbolicLink()).toBe(false); + parseProtectedManagedImageActivation(JSON.parse(fs.readFileSync(activationPath, "utf8"))); + + progress.phase("validate exact all-agent managed-image contracts"); + const contractBytes = regularArtifact(contractFile, artifactDirectory); + const evidenceBytes = regularArtifact(evidenceFile, artifactDirectory); + const contracts = parseProtectedManagedImageContracts( + JSON.parse(contractBytes.toString("utf8")), + expectedPlatform, + ); + + progress.phase("validate direct-start evidence binding"); + const evidence = parseProtectedManagedImageEvidence(JSON.parse(evidenceBytes.toString("utf8")), { + baseSha, + cohort, + headSha, + platform: expectedPlatform, + runAttempt: positiveIntegerEnvironment("GITHUB_RUN_ATTEMPT"), + runId: positiveIntegerEnvironment("GITHUB_RUN_ID"), + workflowSha, + }); + + expect(evidence.contractSha256).toBe( + `sha256:${createHash("sha256").update(contractBytes).digest("hex")}`, + ); + expect(evidence.contracts).toEqual(contracts); +}); diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index 5565b55bf31..07fad134f43 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -75,7 +75,7 @@ describe("deterministic PR risk plan", () => { const second = plan("src/lib/onboard.ts", "src/lib/state/registry.ts"); expect(first).toEqual(second); - expect(first.version).toBe(12); + expect(first.version).toBe(13); expect(first.headSha).toBe(HEAD_SHA); expect(first.planHash).toMatch(/^[a-f0-9]{64}$/u); expect(first.changedFiles).toEqual(["src/lib/onboard.ts", "src/lib/state/registry.ts"]); @@ -313,6 +313,22 @@ describe("deterministic PR risk plan", () => { ]); }); + it("keeps the protected managed-image lane dormant until its trusted activation marker (#7744)", () => { + const activation = "ci/protected-managed-image-multiarch-activation-v1.json"; + const result = plan(activation); + const preActivationRuntime = plan("scripts/checks/run-managed-image-direct-e2e.ts"); + + expect(result.families).toContainEqual( + expect.objectContaining({ + id: "managed-image-multiarch", + matchedFiles: [activation], + requiredJobs: ["managed-image-multiarch-startup"], + }), + ); + expect(riskPlanRequiredJobIds(result)).toEqual(["managed-image-multiarch-startup"]); + expect(preActivationRuntime.families).toEqual([]); + }); + it("runs snapshot commands for restored-gateway pairing runtime changes (#7431)", () => { const runtimeFiles = [ "src/lib/actions/sandbox/restore-gateway-pairing.ts", diff --git a/test/protected-managed-image-contract.test.ts b/test/protected-managed-image-contract.test.ts index 52137411533..5b7cb001708 100644 --- a/test/protected-managed-image-contract.test.ts +++ b/test/protected-managed-image-contract.test.ts @@ -5,9 +5,12 @@ import { describe, expect, it } from "vitest"; import { PROTECTED_MANAGED_IMAGE_AGENTS, + PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID, PROTECTED_MANAGED_IMAGE_PLATFORMS, type ProtectedManagedImagePlatform, + parseProtectedManagedImageActivation, parseProtectedManagedImageContracts, + parseProtectedManagedImageEvidence, } from "../scripts/checks/protected-managed-image-contract.ts"; const BASE_REPOSITORIES = { @@ -31,7 +34,59 @@ function contracts(platform: ProtectedManagedImagePlatform) { }); } +const HEAD_SHA = "a".repeat(40); +const BASE_SHA = "b".repeat(40); +const WORKFLOW_SHA = "c".repeat(40); +const COHORT = "protected-42-1"; + +function evidence(platform: ProtectedManagedImagePlatform) { + const built = contracts(platform); + return { + baseSha: BASE_SHA, + cohort: COHORT, + contracts: built, + contractSha256: `sha256:${"d".repeat(64)}`, + directRuns: built.map(({ agent, digest, reference }) => ({ + agent, + digest, + platform, + reference, + })), + headSha: HEAD_SHA, + kind: "nemoclaw-protected-managed-image-multiarch-v1", + platform, + run: { attempt: 1, id: 42 }, + workflowSha: WORKFLOW_SHA, + }; +} + +function evidenceIdentity(platform: ProtectedManagedImagePlatform) { + return { + baseSha: BASE_SHA, + cohort: COHORT, + headSha: HEAD_SHA, + platform, + runAttempt: 1, + runId: 42, + workflowSha: WORKFLOW_SHA, + }; +} + describe("protected managed-image build contract", () => { + it("accepts only the dormant all-agent multiarch activation contract (#7744)", () => { + const activation = { + agents: PROTECTED_MANAGED_IMAGE_AGENTS, + contractVersion: 1, + jobId: PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID, + platforms: PROTECTED_MANAGED_IMAGE_PLATFORMS, + }; + + expect(parseProtectedManagedImageActivation(activation)).toEqual(activation); + expect(() => + parseProtectedManagedImageActivation({ ...activation, jobId: "untrusted-job" }), + ).toThrow("activation contract is invalid"); + }); + it.each( PROTECTED_MANAGED_IMAGE_PLATFORMS, )("accepts one unique immutable image for every shipped agent on %s (#7744)", (platform) => { @@ -83,4 +138,36 @@ describe("protected managed-image build contract", () => { ), ).toThrow("unexpected fields"); }); + + it.each( + PROTECTED_MANAGED_IMAGE_PLATFORMS, + )("binds exact protected build and direct-start evidence on %s (#7744)", (platform) => { + const value = evidence(platform); + expect(parseProtectedManagedImageEvidence(value, evidenceIdentity(platform))).toEqual(value); + }); + + it("rejects stale identity and incomplete direct-start evidence (#7744)", () => { + const value = evidence("linux/arm64"); + expect(() => + parseProtectedManagedImageEvidence( + { ...value, headSha: "e".repeat(40) }, + evidenceIdentity("linux/arm64"), + ), + ).toThrow("evidence identity is invalid"); + expect(() => + parseProtectedManagedImageEvidence( + { ...value, directRuns: value.directRuns.slice(0, 2) }, + evidenceIdentity("linux/arm64"), + ), + ).toThrow("directly run every contract"); + expect(() => + parseProtectedManagedImageEvidence( + { + ...value, + directRuns: [value.directRuns[0], value.directRuns[0], value.directRuns[2]], + }, + evidenceIdentity("linux/arm64"), + ), + ).toThrow("does not match its exact contract"); + }); }); diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 1727be449c1..8655c85193f 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; -export const RISK_PLAN_VERSION = 12 as const; +export const RISK_PLAN_VERSION = 13 as const; export const PR_E2E_TYPED_TARGET_IDS = [ "ubuntu-repo-cloud-langchain-deepagents-code", @@ -49,6 +49,8 @@ const HERMES_MANAGED_POLICY_FILES = new Set([ "agents/hermes/start.sh", "src/lib/hermes-managed-route.ts", ]); +const MANAGED_IMAGE_MULTIARCH_ACTIVATION = + "ci/protected-managed-image-multiarch-activation-v1.json"; export type RiskTier = 0 | 1 | 2 | 3; export type RiskFamilyId = @@ -61,6 +63,7 @@ export type RiskFamilyId = | "openclaw-image" | "credentials-security" | "e2e-control-plane" + | "managed-image-multiarch" | "sandbox-boundary" | "focused-e2e"; @@ -382,6 +385,25 @@ export const RISK_RULES: readonly RiskRule[] = [ file.startsWith(".github/actions/prepare-e2e/") || file.startsWith(".github/actions/upload-e2e-artifacts/"), }, + { + id: "managed-image-multiarch", + summary: + "Protected managed-image qualification must build and directly start every shipped agent on each supported architecture from exact base and candidate digests.", + tier: 3, + requiredJobs: ["managed-image-multiarch-startup"], + invariants: [ + "OpenClaw, Hermes, and Deep Agents Code use platform-specific digest-pinned bases from one exact PR head and cohort", + "each built image is addressed by its isolated-registry digest and exercises the managed root-stdin and sandbox-hold startup boundary", + "amd64 and arm64 shards emit exact head, base, platform, cohort, base, image, and direct-start evidence before cleanup", + "the isolated registry is removed before a shard can publish passing risk evidence", + ], + // Bootstrap contract: this first trusted-controller slice recognizes only + // the activation marker. The follow-on candidate adds that marker and + // broadens the runtime paths after this job exists on trusted main, which + // lets the follow-on prove its own exact head without loading PR-authored + // workflow structure into the controller. + matches: (file) => file === MANAGED_IMAGE_MULTIARCH_ACTIVATION, + }, { id: "sandbox-boundary", summary: diff --git a/tools/e2e/managed-image-multiarch-workflow-boundary.mts b/tools/e2e/managed-image-multiarch-workflow-boundary.mts new file mode 100644 index 00000000000..a828feb39a8 --- /dev/null +++ b/tools/e2e/managed-image-multiarch-workflow-boundary.mts @@ -0,0 +1,272 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; + +type WorkflowRecord = Record; +type WorkflowStep = WorkflowRecord & { + env?: WorkflowRecord; + name?: string; + run?: string; + uses?: string; + with?: WorkflowRecord; +}; + +const JOB_ID = "managed-image-multiarch-startup"; +const SELECTOR = + "${{ contains(format(',{0},', inputs.jobs), ',managed-image-multiarch-startup,') || contains(format(',{0},', inputs.targets), ',managed-image-multiarch-startup,') }}"; +const ACTIVATION_PATH = "ci/protected-managed-image-multiarch-activation-v1.json"; +const DIRECT_TEST_PATH = "test/e2e/live/managed-image-multiarch-startup.test.ts"; +const REGISTRY_IMAGE = + "docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"; + +function record(value: unknown): WorkflowRecord { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as WorkflowRecord) + : {}; +} + +function workflowSteps(value: unknown): WorkflowStep[] { + return Array.isArray(value) ? (value as WorkflowStep[]) : []; +} + +function text(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +function requireStep( + errors: string[], + steps: readonly WorkflowStep[], + name: string, +): WorkflowStep | undefined { + const matching = steps.filter((step) => step.name === name); + if (matching.length !== 1) { + errors.push(`${JOB_ID} must define exactly one '${name}' step`); + } + return matching[0]; +} + +function requireValues( + errors: string[], + subject: string, + actual: WorkflowRecord, + expected: Readonly>, +): void { + for (const [key, value] of Object.entries(expected)) { + if (actual[key] !== value) errors.push(`${subject} must bind ${key} to ${String(value)}`); + } +} + +function requireFragments( + errors: string[], + step: WorkflowStep | undefined, + fragments: readonly string[], +): void { + if (!step) return; + const run = text(step.run); + for (const fragment of fragments) { + if (!run.includes(fragment)) { + errors.push(`${JOB_ID} step '${step.name}' must include ${fragment}`); + } + } +} + +function requireOrderedSteps( + errors: string[], + steps: readonly WorkflowStep[], + names: readonly string[], +): void { + const indexes = names.map((name) => steps.findIndex((step) => step.name === name)); + if (indexes.some((index) => index < 0)) return; + if (indexes.some((index, offset) => offset > 0 && index <= indexes[offset - 1])) { + errors.push( + `${JOB_ID} protected build, execution, cleanup, validation, and upload steps drifted`, + ); + } +} + +export function validateManagedImageMultiarchWorkflow(workflow: WorkflowRecord): string[] { + const errors: string[] = []; + const job = record(record(workflow.jobs)[JOB_ID]); + if (Object.keys(job).length === 0) { + return [`workflow missing ${JOB_ID} job`]; + } + + if (job.needs !== "generate-matrix") errors.push(`${JOB_ID} must depend on generate-matrix`); + if (job.if !== SELECTOR) errors.push(`${JOB_ID} must remain explicit-only and selector-bound`); + if (job["runs-on"] !== "${{ matrix.runner }}") { + errors.push(`${JOB_ID} must run on the native matrix runner`); + } + if (job["timeout-minutes"] !== 210) errors.push(`${JOB_ID} must keep the 210 minute timeout`); + if (record(job.permissions).contents !== "read") { + errors.push(`${JOB_ID} permissions must be contents: read`); + } + + const expectedStrategy = { + "fail-fast": false, + matrix: { + include: [ + { platform: "linux/amd64", runner: "ubuntu-24.04", shard: "linux-amd64" }, + { platform: "linux/arm64", runner: "ubuntu-24.04-arm", shard: "linux-arm64" }, + ], + }, + }; + if (!isDeepStrictEqual(job.strategy, expectedStrategy)) { + errors.push(`${JOB_ID} must preserve the native amd64 and arm64 runner matrix`); + } + + requireValues(errors, `${JOB_ID} env`, record(job.env), { + E2E_ARTIFACT_DIR: + "${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}", + E2E_DEFAULT_ENABLED: "0", + E2E_JOB: "1", + E2E_TARGET_ID: JOB_ID, + NEMOCLAW_E2E_SHARD: "${{ matrix.shard }}", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: "${{ inputs.base_sha }}", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: + "protected-${{ github.run_id }}-${{ github.run_attempt }}", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT: + "${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/contracts.json", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE: + "${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/evidence.json", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM: "${{ matrix.platform }}", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA: "${{ inputs.workflow_sha }}", + NEMOCLAW_PROTECTED_REGISTRY_NAME: + "nemoclaw-managed-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }}", + NEMOCLAW_RUN_LIVE_E2E: "1", + }); + + const steps = workflowSteps(job.steps); + const guard = requireStep(errors, steps, "Validate protected exact-head dispatch"); + requireValues(errors, `${JOB_ID} exact-head guard env`, record(guard?.env), { + ACTOR: "${{ github.actor }}", + BASE_SHA: "${{ inputs.base_sha }}", + CHECKOUT_SHA: "${{ inputs.checkout_sha }}", + EVENT_NAME: "${{ github.event_name }}", + EXPECTED_WORKFLOW_SHA: "${{ inputs.workflow_sha }}", + PLATFORM: "${{ matrix.platform }}", + REF: "${{ github.ref }}", + REPOSITORY: "${{ github.repository }}", + RUNNER_ARCH_KIND: "${{ runner.arch }}", + WORKFLOW_SHA: "${{ github.workflow_sha }}", + }); + requireFragments(errors, guard, [ + '"NVIDIA/NemoClaw"', + '"refs/heads/main"', + '"workflow_dispatch"', + '"github-actions[bot]"', + '[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]]', + '"$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA"', + "linux/amd64:X64 | linux/arm64:ARM64", + ]); + + const checkouts = steps.filter((step) => text(step.uses).startsWith("actions/checkout@")); + if (checkouts.length !== 1) errors.push(`${JOB_ID} must define exactly one candidate checkout`); + requireValues(errors, `${JOB_ID} candidate checkout`, record(checkouts[0]?.with), { + repository: "${{ inputs.checkout_repository || github.repository }}", + ref: "${{ inputs.checkout_sha || github.sha }}", + "fetch-depth": 0, + "persist-credentials": false, + }); + + const buildx = requireStep(errors, steps, "Set up protected managed-image Buildx"); + if (buildx?.uses !== "docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c") { + errors.push(`${JOB_ID} must pin the reviewed Buildx setup action`); + } + requireValues(errors, `${JOB_ID} Buildx setup`, record(buildx?.with), { + "driver-opts": "network=host", + "buildkitd-config-inline": '[registry."localhost:5000"]\n http = true\n', + }); + + const activation = requireStep(errors, steps, "Validate candidate activation contract"); + requireFragments(errors, activation, [ + `activation="${ACTIVATION_PATH}"`, + '[[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]]', + '[[ -f "$activation" && ! -L "$activation" ]]', + '(keys | sort) == ["agents", "contractVersion", "jobId", "platforms"]', + '.agents == ["openclaw", "hermes", "langchain-deepagents-code"]', + '.platforms == ["linux/amd64", "linux/arm64"]', + ]); + + const bases = requireStep(errors, steps, "Resolve exact platform base images"); + requireFragments(errors, bases, [ + 'arch="${PLATFORM#linux/}"', + 'docker buildx imagetools inspect "$alias" --raw', + '.platform.os == "linux" and .platform.architecture == $arch', + 'reference="${repository}@${digest}"', + '"sha256:$(sha256sum "$exact_raw" | awk \'{print $1}\')" == "$digest"', + "ghcr.io/nvidia/nemoclaw/sandbox-base:latest", + "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest", + "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:latest", + ]); + + const registry = requireStep(errors, steps, "Start isolated protected managed-image registry"); + requireFragments(errors, registry, [ + 'docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME"', + "http://127.0.0.1:5000/v2/", + "io.nvidia.nemoclaw.e2e-owner=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT}", + "--publish 127.0.0.1:5000:5000", + REGISTRY_IMAGE, + ]); + + const build = requireStep(errors, steps, "Build exact all-agent protected managed images"); + requireFragments(errors, build, [ + "scripts/checks/build-protected-managed-images.sh", + '--revision "$CHECKOUT_SHA"', + '--cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT"', + '--platform "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM"', + '--openclaw-base "$BASE_OPENCLAW"', + '--hermes-base "$BASE_HERMES"', + '--dcode-base "$BASE_DCODE"', + ]); + + const direct = requireStep(errors, steps, "Run every exact managed-image contract directly"); + requireFragments(errors, direct, [ + "scripts/checks/run-managed-image-direct-e2e.ts", + "done < <(jq -c '.[]' \"$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT\")", + '--agent "$agent"', + '--image "$reference"', + '--platform "$platform"', + 'kind: "nemoclaw-protected-managed-image-multiarch-v1"', + "headSha: $headSha", + "baseSha: $baseSha", + "workflowSha: $workflowSha", + "platform: $platform", + "cohort: $cohort", + "contractSha256: $contractSha256", + "contracts: $contracts[0]", + "directRuns: $directRuns", + "run: {id: $runId, attempt: $runAttempt}", + ]); + + const cleanup = requireStep(errors, steps, "Remove isolated protected managed-image registry"); + if (cleanup?.if !== "always()") errors.push(`${JOB_ID} registry cleanup must always run`); + requireFragments(errors, cleanup, [ + "io.nvidia.nemoclaw.e2e-owner", + '[[ "$owner" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" ]]', + 'docker rm -f "$NEMOCLAW_PROTECTED_REGISTRY_NAME"', + "http://127.0.0.1:5000/v2/", + ]); + + const evidence = requireStep(errors, steps, "Validate protected managed-image evidence"); + requireFragments(errors, evidence, [ + "tools/e2e/live-vitest-invocation.mts run", + `--test-path ${DIRECT_TEST_PATH}`, + ]); + requireStep(errors, steps, "Upload protected managed-image evidence"); + requireStep(errors, steps, "Clean up Docker auth"); + requireOrderedSteps(errors, steps, [ + "Validate protected exact-head dispatch", + "Validate candidate activation contract", + "Resolve exact platform base images", + "Start isolated protected managed-image registry", + "Build exact all-agent protected managed images", + "Run every exact managed-image contract directly", + "Remove isolated protected managed-image registry", + "Validate protected managed-image evidence", + "Upload protected managed-image evidence", + "Clean up Docker auth", + ]); + + return errors; +} diff --git a/tools/e2e/prepare-e2e-workflow-boundary.mts b/tools/e2e/prepare-e2e-workflow-boundary.mts index 1fce8f61247..d1c6d4e83ff 100644 --- a/tools/e2e/prepare-e2e-workflow-boundary.mts +++ b/tools/e2e/prepare-e2e-workflow-boundary.mts @@ -27,6 +27,7 @@ const RETIRED_SELECTOR_COMPATIBILITY_JOB = "retired-selector-compatibility"; const NO_BUILD_JOBS = new Set([ "generate-matrix", "bootstrap-install-smoke", + "managed-image-multiarch-startup", "ollama-auth-proxy", "security-posture", "shields-config", diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 978d3805158..481196cad25 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -143,6 +143,13 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ path: "e2e-artifacts/live/hermes-inference-switch/${{ matrix.mode }}/", }, ], + [ + "managed-image-multiarch-startup", + { + name: "e2e-managed-image-multiarch-startup-${{ matrix.shard }}", + path: "e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/", + }, + ], [ "network-policy", { diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 4849190ec74..02aa66417c1 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -26,6 +26,7 @@ import { type InferenceSwitchWorkflow, validateInferenceSwitchWorkflow, } from "./inference-switch-workflow-boundary.mts"; +import { validateManagedImageMultiarchWorkflow } from "./managed-image-multiarch-workflow-boundary.mts"; import { type OpenClawPluginRuntimeExdevWorkflow, validateOpenClawPluginRuntimeExdevWorkflow, @@ -4237,6 +4238,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { errors.push(...validateHermesDashboardWorkflow(workflow as unknown as HermesDashboardWorkflow)); errors.push(...validateHermesGpuStartupWorkflow(workflow)); errors.push(...validateInferenceSwitchWorkflow(workflow as unknown as InferenceSwitchWorkflow)); + errors.push(...validateManagedImageMultiarchWorkflow(workflow)); errors.push( ...validateOpenClawPluginRuntimeExdevWorkflow( workflow as unknown as OpenClawPluginRuntimeExdevWorkflow, From 12b3a1bbb5216dbf24bd8e839015385bd12b19af Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 07:23:14 -0700 Subject: [PATCH 09/24] test(e2e): keep multiarch qualification linear Signed-off-by: Aaron Erickson --- ...managed-image-multiarch-startup-helpers.ts | 85 +++++++++++++++++++ .../managed-image-multiarch-startup.test.ts | 77 ++++------------- test/e2e/mock-parity.json | 9 ++ 3 files changed, 110 insertions(+), 61 deletions(-) create mode 100644 test/e2e/live/managed-image-multiarch-startup-helpers.ts diff --git a/test/e2e/live/managed-image-multiarch-startup-helpers.ts b/test/e2e/live/managed-image-multiarch-startup-helpers.ts new file mode 100644 index 00000000000..a863c3dfbe3 --- /dev/null +++ b/test/e2e/live/managed-image-multiarch-startup-helpers.ts @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { + PROTECTED_MANAGED_IMAGE_PLATFORMS, + type ProtectedManagedImagePlatform, +} from "../../../scripts/checks/protected-managed-image-contract.ts"; + +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const COHORT_PATTERN = /^protected-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$/u; + +export interface ProtectedManagedImageDispatchEnvironment { + artifactDirectory: string; + baseSha: string; + cohort: string; + contractFile: string; + evidenceFile: string; + headSha: string; + platform: ProtectedManagedImagePlatform; + runAttempt: number; + runId: number; + workflowSha: string; + workspace: string; +} + +function requiredEnvironment(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +function positiveIntegerEnvironment(name: string): number { + const value = requiredEnvironment(name); + if (!/^[1-9][0-9]*$/u.test(value)) throw new Error(`${name} must be a positive integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) throw new Error(`${name} must be a safe integer`); + return parsed; +} + +export function protectedManagedImageDispatchEnvironment(): ProtectedManagedImageDispatchEnvironment { + const platform = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM"); + const cohort = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT"); + const headSha = requiredEnvironment("NEMOCLAW_E2E_EXPECTED_SHA"); + const baseSha = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA"); + const workflowSha = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA"); + + if ( + !(PROTECTED_MANAGED_IMAGE_PLATFORMS as readonly string[]).includes(platform) || + !COHORT_PATTERN.test(cohort) || + !SHA_PATTERN.test(headSha) || + !SHA_PATTERN.test(baseSha) || + !SHA_PATTERN.test(workflowSha) + ) { + throw new Error("protected managed-image dispatch identity is invalid"); + } + + return { + artifactDirectory: requiredEnvironment("E2E_ARTIFACT_DIR"), + baseSha, + cohort, + contractFile: requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT"), + evidenceFile: requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE"), + headSha, + platform: platform as ProtectedManagedImagePlatform, + runAttempt: positiveIntegerEnvironment("GITHUB_RUN_ATTEMPT"), + runId: positiveIntegerEnvironment("GITHUB_RUN_ID"), + workflowSha, + workspace: fs.realpathSync(requiredEnvironment("GITHUB_WORKSPACE")), + }; +} + +export function readRegularArtifact(file: string, artifactDirectory: string): Buffer { + const relative = path.relative(fs.realpathSync(artifactDirectory), fs.realpathSync(file)); + if (!relative || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new Error(`${file} must be a child of the protected artifact directory`); + } + const status = fs.lstatSync(file); + if (!status.isFile() || status.isSymbolicLink() || status.size > 1024 * 1024) { + throw new Error(`${file} must be a bounded regular file`); + } + return fs.readFileSync(file); +} diff --git a/test/e2e/live/managed-image-multiarch-startup.test.ts b/test/e2e/live/managed-image-multiarch-startup.test.ts index 229c09f113a..af4e7e33425 100644 --- a/test/e2e/live/managed-image-multiarch-startup.test.ts +++ b/test/e2e/live/managed-image-multiarch-startup.test.ts @@ -7,42 +7,15 @@ import path from "node:path"; import { PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, - PROTECTED_MANAGED_IMAGE_PLATFORMS, - type ProtectedManagedImagePlatform, parseProtectedManagedImageActivation, parseProtectedManagedImageContracts, parseProtectedManagedImageEvidence, } from "../../../scripts/checks/protected-managed-image-contract.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; - -const SHA_PATTERN = /^[a-f0-9]{40}$/u; -const COHORT_PATTERN = /^protected-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$/u; - -function requiredEnvironment(name: string): string { - const value = process.env[name]; - if (!value) throw new Error(`${name} is required`); - return value; -} - -function positiveIntegerEnvironment(name: string): number { - const value = requiredEnvironment(name); - if (!/^[1-9][0-9]*$/u.test(value)) throw new Error(`${name} must be a positive integer`); - const parsed = Number(value); - if (!Number.isSafeInteger(parsed)) throw new Error(`${name} must be a safe integer`); - return parsed; -} - -function regularArtifact(file: string, artifactDirectory: string): Buffer { - const relative = path.relative(fs.realpathSync(artifactDirectory), fs.realpathSync(file)); - if (!relative || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { - throw new Error(`${file} must be a child of the protected artifact directory`); - } - const status = fs.lstatSync(file); - if (!status.isFile() || status.isSymbolicLink() || status.size > 1024 * 1024) { - throw new Error(`${file} must be a bounded regular file`); - } - return fs.readFileSync(file); -} +import { + protectedManagedImageDispatchEnvironment, + readRegularArtifact, +} from "./managed-image-multiarch-startup-helpers.ts"; test("binds protected all-agent direct startup to the exact multiarch dispatch (#7744)", { meta: { @@ -54,49 +27,31 @@ test("binds protected all-agent direct startup to the exact multiarch dispatch ( }, }, ({ progress }) => { progress.phase("validate protected activation and dispatch identity"); - const workspace = fs.realpathSync(requiredEnvironment("GITHUB_WORKSPACE")); - const artifactDirectory = requiredEnvironment("E2E_ARTIFACT_DIR"); - const contractFile = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT"); - const evidenceFile = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE"); - const platform = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM"); - const cohort = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT"); - const headSha = requiredEnvironment("NEMOCLAW_E2E_EXPECTED_SHA"); - const baseSha = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA"); - const workflowSha = requiredEnvironment("NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA"); - if ( - !(PROTECTED_MANAGED_IMAGE_PLATFORMS as readonly string[]).includes(platform) || - !COHORT_PATTERN.test(cohort) || - !SHA_PATTERN.test(headSha) || - !SHA_PATTERN.test(baseSha) || - !SHA_PATTERN.test(workflowSha) - ) { - throw new Error("protected managed-image dispatch identity is invalid"); - } - const expectedPlatform = platform as ProtectedManagedImagePlatform; + const dispatch = protectedManagedImageDispatchEnvironment(); - const activationPath = path.join(workspace, PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH); + const activationPath = path.join(dispatch.workspace, PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH); const activationStatus = fs.lstatSync(activationPath); expect(activationStatus.isFile()).toBe(true); expect(activationStatus.isSymbolicLink()).toBe(false); parseProtectedManagedImageActivation(JSON.parse(fs.readFileSync(activationPath, "utf8"))); progress.phase("validate exact all-agent managed-image contracts"); - const contractBytes = regularArtifact(contractFile, artifactDirectory); - const evidenceBytes = regularArtifact(evidenceFile, artifactDirectory); + const contractBytes = readRegularArtifact(dispatch.contractFile, dispatch.artifactDirectory); + const evidenceBytes = readRegularArtifact(dispatch.evidenceFile, dispatch.artifactDirectory); const contracts = parseProtectedManagedImageContracts( JSON.parse(contractBytes.toString("utf8")), - expectedPlatform, + dispatch.platform, ); progress.phase("validate direct-start evidence binding"); const evidence = parseProtectedManagedImageEvidence(JSON.parse(evidenceBytes.toString("utf8")), { - baseSha, - cohort, - headSha, - platform: expectedPlatform, - runAttempt: positiveIntegerEnvironment("GITHUB_RUN_ATTEMPT"), - runId: positiveIntegerEnvironment("GITHUB_RUN_ID"), - workflowSha, + baseSha: dispatch.baseSha, + cohort: dispatch.cohort, + headSha: dispatch.headSha, + platform: dispatch.platform, + runAttempt: dispatch.runAttempt, + runId: dispatch.runId, + workflowSha: dispatch.workflowSha, }); expect(evidence.contractSha256).toBe( diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 8ba1529fcf7..9f912d2fc21 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -2,6 +2,15 @@ "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0", "version": 1, "entries": [ + { + "live": "test/e2e/live/managed-image-multiarch-startup.test.ts", + "fast": [ + "test/e2e/support/base-image-publication.test.ts", + "test/managed-image-publication-workflow.test.ts", + "test/pr-risk-plan.test.ts", + "test/protected-managed-image-contract.test.ts" + ] + }, { "live": "test/e2e/live/hermes-gpu-startup.test.ts", "fast": [ From b166aebdd43c107a8e674e8cedb62a0b810f95fa Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 07:33:05 -0700 Subject: [PATCH 10/24] fix(ci): gate dormant release qualifications Signed-off-by: Aaron Erickson --- .../SKILL.md | 2 + .../scripts/release-e2e-evidence.mts | 46 +++++++++++++++++-- .../references/release-train.md | 3 +- .github/workflows/e2e.yaml | 1 + .../e2e-cross-runtime-compatibility.test.ts | 4 +- test/maintainer-skills-policy.test.ts | 9 +++- test/release-e2e-evidence.test.ts | 22 ++++++++- 7 files changed, 79 insertions(+), 8 deletions(-) diff --git a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md index ff416fc4113..a059fc76bbc 100644 --- a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md @@ -143,6 +143,8 @@ The preflight derives every required execution and these dispatch groups from th - `parallelExplicit`: explicit-only selectors that require neither the Launchable E2E job nor runner confirmation; and - `conditional`: Jetson or another lane that must not queue until its authoritative runner inventory is confirmed online. +An explicit-only job may declare `RELEASE_E2E_ACTIVATION_PATH` in its workflow environment. The preflight includes that job and all of its expanded executions only when the exact relative path exists at the candidate SHA. When the path is absent, the job is a dormant lane: do not dispatch it and do not treat it as missing release evidence. + First feed applicable existing runs for the candidate SHA into the ledger. Dispatch only groups that still lack green evidence. When no applicable exact Brev evidence exists, load `nemoclaw-maintainer-e2e` and dispatch full mode for that SHA. diff --git a/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts b/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts index 7dec78c2596..0612394901d 100644 --- a/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts +++ b/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts @@ -101,6 +101,7 @@ const REPO_ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", const DEFAULT_WORKFLOW_PATH = path.join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); const SHA_PATTERN = /^[a-f0-9]{40}$/u; const SELECTOR_PATTERN = /^[A-Za-z0-9_-]+$/u; +const SAFE_REPO_PATH_PATTERN = /^(?!\/)(?!.*(?:^|\/)\.\.(?:\/|$))[^\\]+$/u; const MATRIX_EXPRESSION_PATTERN = /\$\{\{\s*matrix\.([A-Za-z0-9_-]+)\s*\}\}/gu; function record(value: unknown, label: string): JsonRecord { @@ -271,8 +272,39 @@ function requiresConfirmedJetsonRunner(job: JsonRecord): boolean { return typeof runsOn === "string" && runsOn.includes("inputs.allow_jetson_runner_queue"); } +function releaseActivationPath(job: JsonRecord, jobId: string): string | undefined { + const rawEnvironment = job.env; + if (rawEnvironment === undefined) return undefined; + const environment = record(rawEnvironment, `workflow.jobs.${jobId}.env`); + const activationPath = environment.RELEASE_E2E_ACTIVATION_PATH; + if (activationPath === undefined) return undefined; + if ( + typeof activationPath !== "string" || + activationPath.length === 0 || + !SAFE_REPO_PATH_PATTERN.test(activationPath) + ) { + throw new Error( + `${jobId}.env.RELEASE_E2E_ACTIVATION_PATH must be a nonempty relative repository path without backslashes or parent-directory segments`, + ); + } + return activationPath; +} + +function candidatePathExists(candidateSha: string, candidatePath: string): boolean { + try { + execFileSync("git", ["cat-file", "-e", `${candidateSha}:${candidatePath}`], { + cwd: REPO_ROOT, + stdio: "ignore", + }); + return true; + } catch { + return false; + } +} + export function buildReleaseE2ePreflight(input: { candidateSha: string; + candidatePathExists?: (candidateSha: string, candidatePath: string) => boolean; jetsonRunnerOnline?: RunnerStatus; plan?: E2eWorkflowPlan; workflowPath?: string; @@ -285,7 +317,15 @@ export function buildReleaseE2ePreflight(input: { const inventory = readFreeStandingJobsInventory(workflowPath); const plan = input.plan ?? buildE2eWorkflowPlan(); const explicitJobs = new Set(inventory.explicitOnlyJobs); - const launchableE2eJobs = inventory.explicitOnlyJobs.filter((jobId) => + const pathExists = input.candidatePathExists ?? candidatePathExists; + const releaseExplicitJobs = inventory.explicitOnlyJobs.filter((jobId) => { + const activationPath = releaseActivationPath( + record(jobs[jobId], `workflow.jobs.${jobId}`), + jobId, + ); + return activationPath === undefined || pathExists(input.candidateSha, activationPath); + }); + const launchableE2eJobs = releaseExplicitJobs.filter((jobId) => isLaunchableE2eJob(record(jobs[jobId], `workflow.jobs.${jobId}`)), ); if (launchableE2eJobs.length !== 1) { @@ -294,12 +334,12 @@ export function buildReleaseE2ePreflight(input: { ); } const launchableE2eJobId = launchableE2eJobs[0]!; - const conditionalJobs = inventory.explicitOnlyJobs.filter( + const conditionalJobs = releaseExplicitJobs.filter( (jobId) => jobId !== launchableE2eJobId && requiresConfirmedJetsonRunner(record(jobs[jobId], `workflow.jobs.${jobId}`)), ); - const parallelExplicitJobs = inventory.explicitOnlyJobs.filter( + const parallelExplicitJobs = releaseExplicitJobs.filter( (jobId) => jobId !== launchableE2eJobId && !conditionalJobs.includes(jobId), ); diff --git a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md index 73e50a5aade..c48579303b7 100644 --- a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md +++ b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md @@ -54,13 +54,14 @@ Before asking for the release confirmation phrase, build and show an evidence le - Preflight the candidate workflow, conditional runner readiness, and existing candidate evidence before dispatching new work. - Dispatch independent default-suite and unconditional explicit-only work concurrently. Dispatch a conditional hardware lane only after its authoritative runner inventory confirms that it is online; otherwise record its required itemized exception without queueing it. - Derive the denominator and dispatch selectors from the candidate workflow. Do not copy them into a second release test list. +- An explicit-only job that declares `RELEASE_E2E_ACTIVATION_PATH` enters the release denominator only when that exact relative path exists at the candidate SHA. Until then, treat it as a declared dormant lane: do not dispatch it and do not count it as missing evidence. - For every accepted run, require the workflow-produced trusted dispatch receipt to bind the candidate SHA, run ID, attempt, and actual selector inputs. Derive default-suite or selective coverage only from that receipt, never from a release manifest claim. - Run `nemoclaw-maintainer-e2e` in full mode if no applicable exact Brev Launchable evidence exists for the candidate SHA. - For full-mode exact Brev evidence, require one workflow run for the candidate SHA that includes the default-enabled suite and a successful `Exact staging Brev Launchable` job. - For that evidence, require the trusted dispatch receipt to bind the run and attempt to empty selectors and `include_staging_brev_launchable=true`. - Require its Launchable E2E receipt to identify the candidate SHA in the repository and provision records. - Require its cleanup receipt to identify the qualified workspace and report `ABSENT`. -- Every E2E test execution declared by the workflow must have at least one completed, successful execution for the candidate SHA. This includes tests that require explicit selection and every expanded matrix execution. +- Every release-eligible E2E test execution declared by the workflow must have at least one completed, successful execution for the candidate SHA. This includes tests that require explicit selection, including activation-gated jobs whose path exists at that SHA, and every expanded matrix execution. - Treat each expanded matrix execution as a separate ledger entry. Use its matrix `id`, or all distinguishing matrix dimensions when no single ID exists, in the test identifier so results for distinct expansions are never collapsed under the parent job. - Green evidence may accumulate across multiple workflow runs, selective runs, reruns, and attempts. A later failure does not erase an earlier successful execution for the same test and SHA. - Skipped, unexecuted, queued, in-progress, cancelled, and failing results are not green evidence. diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c38a2948ce1..7c67bd097a9 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1729,6 +1729,7 @@ jobs: E2E_DEFAULT_ENABLED: "0" E2E_JOB: "1" E2E_TARGET_ID: "managed-image-multiarch-startup" + RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-multiarch-activation-v1.json NEMOCLAW_E2E_SHARD: ${{ matrix.shard }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: ${{ inputs.base_sha }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: protected-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/test/e2e/support/e2e-cross-runtime-compatibility.test.ts b/test/e2e/support/e2e-cross-runtime-compatibility.test.ts index 2a9e563a53f..787a63f5024 100644 --- a/test/e2e/support/e2e-cross-runtime-compatibility.test.ts +++ b/test/e2e/support/e2e-cross-runtime-compatibility.test.ts @@ -29,7 +29,7 @@ describe("cross-runtime foundation compatibility", () => { ), ).toBe("6272aab16cf4b9555bdc4b3f4c0cdd24b5faa55118cbd61cbb4b30a3d418a63a"); expect(digestOutput(buildE2eWorkflowPlan())).toBe( - "45a00867e0c501ba5004a8cd9557d846371524eeb300a5ec979ce840500f74e9", + "36795de73b09280ad17f7a6296d5690572e23dabe40b374e754836066589d145", ); }); @@ -45,7 +45,7 @@ describe("cross-runtime foundation compatibility", () => { ]; expect(digestOutput(cases.map(buildRiskPlan))).toBe( - "b9f2f00f87b7a18caac504048c72eea224c10621690d2cf66549733487d3d342", + "311bd367e8d6ee469a9ec99aba13ab9b806ac679f7e71381c09d4fc4beafd4a2", ); }); }); diff --git a/test/maintainer-skills-policy.test.ts b/test/maintainer-skills-policy.test.ts index 9fa8498ebab..3bb1b898460 100644 --- a/test/maintainer-skills-policy.test.ts +++ b/test/maintainer-skills-policy.test.ts @@ -140,7 +140,14 @@ describe("maintainer skills follow canonical workflow policy", () => { expect(policy).toContain("Do not maintain a separate release-gating test list"); expect(policy).toContain("at least one completed, successful execution"); expect(policy).toContain("multiple workflow runs, selective runs, reruns, and attempts"); - expect(policy).toContain("explicit selection and every expanded matrix execution"); + expect(policy).toContain( + "explicit selection, including activation-gated jobs whose path exists at that SHA", + ); + expect(policy).toContain("and every expanded matrix execution"); + expect(policy).toContain("`RELEASE_E2E_ACTIVATION_PATH` enters the release denominator only"); + expect(policy).toContain("do not dispatch it and do not count it as missing evidence"); + expect(release).toContain("`RELEASE_E2E_ACTIVATION_PATH` in its workflow environment"); + expect(release).toContain("do not dispatch it and do not treat it as missing release evidence"); expect(policy).toContain("each expanded matrix execution as a separate ledger entry"); expect(policy).toContain("matrix `id`"); expect(policy).toContain("A later failure does not erase an earlier successful execution"); diff --git a/test/release-e2e-evidence.test.ts b/test/release-e2e-evidence.test.ts index 2f3a3e647d9..1431a359414 100644 --- a/test/release-e2e-evidence.test.ts +++ b/test/release-e2e-evidence.test.ts @@ -13,9 +13,15 @@ import { const candidateSha = "a".repeat(40); -function preflight(input: { jetsonRunnerOnline?: "true" | "unknown" } = {}) { +function preflight( + input: { + candidatePathExists?: (candidateSha: string, candidatePath: string) => boolean; + jetsonRunnerOnline?: "true" | "unknown"; + } = {}, +) { return buildReleaseE2ePreflight({ candidateSha, + candidatePathExists: input.candidatePathExists, jetsonRunnerOnline: input.jetsonRunnerOnline ?? "true", }); } @@ -105,6 +111,20 @@ describe("release E2E evidence", () => { expect(plan.exceptionsRequired).toEqual(["jetson-nvmap-gpu"]); }); + it("includes an activation-gated explicit lane only when its candidate marker exists", () => { + const plan = preflight({ + candidatePathExists: (_sha, candidatePath) => + candidatePath === "ci/protected-managed-image-multiarch-activation-v1.json", + }); + + expect(plan.dispatches.parallelExplicit.jobs.split(",")).toContain( + "managed-image-multiarch-startup", + ); + expect( + plan.executions.filter((execution) => execution.jobId === "managed-image-multiarch-startup"), + ).toHaveLength(2); + }); + it("keeps every static and dynamic matrix row as a distinct execution", () => { const plan = preflight(); const ids = plan.executions.map((execution) => execution.id); From 8c1501021e5f8670c69771736c2c042989880702 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 08:06:08 -0700 Subject: [PATCH 11/24] ci(images): activate protected multiarch qualification Signed-off-by: Aaron Erickson --- ...managed-image-multiarch-activation-v1.json | 6 +++ test/pr-e2e-gate-signal-shards.test.ts | 4 +- test/pr-e2e-gate.test.ts | 3 +- test/pr-risk-plan.test.ts | 38 ++++++++++++++---- test/protected-managed-image-contract.test.ts | 18 ++++++++- tools/advisors/risk-plan.mts | 40 ++++++++++++++++--- 6 files changed, 91 insertions(+), 18 deletions(-) create mode 100644 ci/protected-managed-image-multiarch-activation-v1.json diff --git a/ci/protected-managed-image-multiarch-activation-v1.json b/ci/protected-managed-image-multiarch-activation-v1.json new file mode 100644 index 00000000000..5960e13e340 --- /dev/null +++ b/ci/protected-managed-image-multiarch-activation-v1.json @@ -0,0 +1,6 @@ +{ + "agents": ["openclaw", "hermes", "langchain-deepagents-code"], + "contractVersion": 1, + "jobId": "managed-image-multiarch-startup", + "platforms": ["linux/amd64", "linux/arm64"] +} diff --git a/test/pr-e2e-gate-signal-shards.test.ts b/test/pr-e2e-gate-signal-shards.test.ts index c6b0666d0c3..edd01697f6a 100644 --- a/test/pr-e2e-gate-signal-shards.test.ts +++ b/test/pr-e2e-gate-signal-shards.test.ts @@ -47,8 +47,8 @@ describe("PR E2E signal shard policy", () => { }); const broadPlan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: BROAD_FILES }); const broadShards = expectedSignalShards(riskPlanRequiredJobIds(broadPlan)); - expect(Object.keys(broadShards)).toHaveLength(13); - expect(Object.values(broadShards).flat()).toHaveLength(15); + expect(Object.keys(broadShards)).toHaveLength(14); + expect(Object.values(broadShards).flat()).toHaveLength(17); expect(() => expectedSignalShards(["not-a-workflow-job"])).toThrow(/does not define/u); }); diff --git a/test/pr-e2e-gate.test.ts b/test/pr-e2e-gate.test.ts index 548d5929a8c..85d99c96f52 100644 --- a/test/pr-e2e-gate.test.ts +++ b/test/pr-e2e-gate.test.ts @@ -62,6 +62,7 @@ const BROAD_FILES = [ const BROAD_JOBS = [ "cloud-inference", "cloud-onboard", + "managed-image-multiarch-startup", "security-posture", "channels-add-remove", "channels-stop-start", @@ -1408,7 +1409,7 @@ describe("PR E2E controller", () => { expect(checkUpdates[1]?.body).toMatchObject({ status: "in_progress", output: { - title: "Running 13 E2E checks", + title: "Running 14 E2E checks", summary: expect.stringContaining("rebuild-openclaw"), }, }); diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index 07fad134f43..edfdb3a340f 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -20,6 +20,7 @@ const HERMES_SANDBOX_BOUNDARY_JOBS = [ "full-e2e", "hermes-e2e", "hermes-inference-switch", + "managed-image-multiarch-startup", "security-posture", ]; const HERMES_CLI_ADAPTER_JOBS = ["channels-stop-start", "mcp-bridge"]; @@ -219,6 +220,7 @@ describe("deterministic PR risk plan", () => { "full-e2e", "hermes-e2e", "hermes-inference-switch", + "managed-image-multiarch-startup", "security-posture", ]); }); @@ -313,20 +315,33 @@ describe("deterministic PR risk plan", () => { ]); }); - it("keeps the protected managed-image lane dormant until its trusted activation marker (#7744)", () => { + it("activates protected multiarch qualification for every managed-image build input (#7744)", () => { const activation = "ci/protected-managed-image-multiarch-activation-v1.json"; - const result = plan(activation); - const preActivationRuntime = plan("scripts/checks/run-managed-image-direct-e2e.ts"); + const managedImageInputs = [ + activation, + ".github/workflows/managed-images.yaml", + "Dockerfile", + "agents/hermes/Dockerfile", + "agents/langchain-deepagents-code/Dockerfile", + "scripts/checks/run-managed-image-direct-e2e.ts", + "src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.99.json", + "src/lib/onboard/managed-startup/image-runtime.ts", + ]; + const result = plan(...managedImageInputs); + const adjacentOnboardChange = plan("src/lib/onboard/provider-selection.ts"); expect(result.families).toContainEqual( expect.objectContaining({ id: "managed-image-multiarch", - matchedFiles: [activation], + matchedFiles: [...managedImageInputs].sort((left, right) => left.localeCompare(right)), requiredJobs: ["managed-image-multiarch-startup"], }), ); - expect(riskPlanRequiredJobIds(result)).toEqual(["managed-image-multiarch-startup"]); - expect(preActivationRuntime.families).toEqual([]); + expect(riskPlanRequiredJobIds(result)).toContain("managed-image-multiarch-startup"); + expect(riskPlanRequiredJobIds(plan(activation))).toEqual(["managed-image-multiarch-startup"]); + expect( + adjacentOnboardChange.families.some((family) => family.id === "managed-image-multiarch"), + ).toBe(false); }); it("runs snapshot commands for restored-gateway pairing runtime changes (#7431)", () => { @@ -437,7 +452,8 @@ describe("deterministic PR risk plan", () => { matchedFiles: ["agents/langchain-deepagents-code/patch-managed-deepagents-code.py"], }), ]); - expect(result.tier).toBe(2); + expect(result.tier).toBe(3); + expect(riskPlanRequiredJobIds(result)).toContain("managed-image-multiarch-startup"); expect(riskPlanRequiredTargetIds(docsAndTestsOnly)).toEqual([]); }); @@ -544,8 +560,13 @@ describe("deterministic PR risk plan", () => { expect(rootImage.families.map((family) => family.id)).toEqual([ "platform-install", "openclaw-image", + "managed-image-multiarch", + ]); + expect(riskPlanRequiredJobIds(rootImage)).toEqual([ + "cloud-onboard", + "full-e2e", + "managed-image-multiarch-startup", ]); - expect(riskPlanRequiredJobIds(rootImage)).toEqual(["cloud-onboard", "full-e2e"]); expect(adjacentImage.families.map((family) => family.id)).toEqual(["platform-install"]); expect(riskPlanRequiredJobIds(adjacentImage)).toEqual(["cloud-onboard"]); }); @@ -656,6 +677,7 @@ describe("deterministic PR risk plan", () => { expect(riskPlanRequiredJobIds(result)).toEqual([ "cloud-inference", "cloud-onboard", + "managed-image-multiarch-startup", "security-posture", "channels-add-remove", "channels-stop-start", diff --git a/test/protected-managed-image-contract.test.ts b/test/protected-managed-image-contract.test.ts index 5b7cb001708..b96b9a1ec5d 100644 --- a/test/protected-managed-image-contract.test.ts +++ b/test/protected-managed-image-contract.test.ts @@ -1,9 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { readFileSync } from "node:fs"; + import { describe, expect, it } from "vitest"; import { + PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, PROTECTED_MANAGED_IMAGE_AGENTS, PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID, PROTECTED_MANAGED_IMAGE_PLATFORMS, @@ -73,7 +76,7 @@ function evidenceIdentity(platform: ProtectedManagedImagePlatform) { } describe("protected managed-image build contract", () => { - it("accepts only the dormant all-agent multiarch activation contract (#7744)", () => { + it("accepts only the all-agent multiarch activation contract (#7744)", () => { const activation = { agents: PROTECTED_MANAGED_IMAGE_AGENTS, contractVersion: 1, @@ -87,6 +90,19 @@ describe("protected managed-image build contract", () => { ).toThrow("activation contract is invalid"); }); + it("ships the exact activation contract consumed by the trusted lane (#7744)", () => { + const activation = JSON.parse( + readFileSync(PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, "utf8"), + ) as unknown; + + expect(parseProtectedManagedImageActivation(activation)).toEqual({ + agents: PROTECTED_MANAGED_IMAGE_AGENTS, + contractVersion: 1, + jobId: PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID, + platforms: PROTECTED_MANAGED_IMAGE_PLATFORMS, + }); + }); + it.each( PROTECTED_MANAGED_IMAGE_PLATFORMS, )("accepts one unique immutable image for every shipped agent on %s (#7744)", (platform) => { diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 8655c85193f..8c615200ab3 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -51,6 +51,32 @@ const HERMES_MANAGED_POLICY_FILES = new Set([ ]); const MANAGED_IMAGE_MULTIARCH_ACTIVATION = "ci/protected-managed-image-multiarch-activation-v1.json"; +const MANAGED_IMAGE_MULTIARCH_INPUTS = new Set([ + MANAGED_IMAGE_MULTIARCH_ACTIVATION, + ".dockerignore", + ".github/workflows/managed-images.yaml", + "Dockerfile", + "ci/npm-audit-exceptions.json", + "src/lib/core/json-types.ts", + "src/lib/core/ports.ts", + "src/lib/onboard/managed-bootstrap/envelope.ts", + "src/lib/security/credential-hash.ts", + "src/lib/state/paths.ts", + "src/lib/state/state-root.ts", + "src/lib/tool-disclosure.ts", + "tsconfig.runtime-preloads.json", +]); +const MANAGED_IMAGE_MULTIARCH_CHILD_CREDENTIALS = + /^src\/lib\/actions\/sandbox\/openshell-child-visible-credentials[.]v[^/]+[.]json$/u; +const MANAGED_IMAGE_MULTIARCH_INPUT_PREFIXES = [ + "agents/", + "nemoclaw/", + "nemoclaw-blueprint/", + "scripts/", + "src/lib/messaging/", + "src/lib/onboard/managed-startup/", + "tools/mcp-tool-discovery-runtime/", +] as const; export type RiskTier = 0 | 1 | 2 | 3; export type RiskFamilyId = @@ -397,12 +423,14 @@ export const RISK_RULES: readonly RiskRule[] = [ "amd64 and arm64 shards emit exact head, base, platform, cohort, base, image, and direct-start evidence before cleanup", "the isolated registry is removed before a shard can publish passing risk evidence", ], - // Bootstrap contract: this first trusted-controller slice recognizes only - // the activation marker. The follow-on candidate adds that marker and - // broadens the runtime paths after this job exists on trusted main, which - // lets the follow-on prove its own exact head without loading PR-authored - // workflow structure into the controller. - matches: (file) => file === MANAGED_IMAGE_MULTIARCH_ACTIVATION, + // Keep this source boundary synchronized with the managed-image workflow's + // path filter. The preceding trusted-controller slice intentionally matched + // only the activation marker; after that lane lands, this candidate can + // select and prove its own exact head before broadening future qualification. + matches: (file) => + MANAGED_IMAGE_MULTIARCH_INPUTS.has(file) || + MANAGED_IMAGE_MULTIARCH_CHILD_CREDENTIALS.test(file) || + MANAGED_IMAGE_MULTIARCH_INPUT_PREFIXES.some((prefix) => file.startsWith(prefix)), }, { id: "sandbox-boundary", From 5e7991ca601f6f2b26f472a1f54bde8e44980f3f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 08:25:49 -0700 Subject: [PATCH 12/24] test(images): add protected runtime qualification harness Signed-off-by: Aaron Erickson --- ...anaged-image-protected-runtime-contract.ts | 95 ++ .../checks/run-managed-image-openshell-e2e.ts | 960 ++++++++++++++++++ 2 files changed, 1055 insertions(+) create mode 100644 scripts/checks/managed-image-protected-runtime-contract.ts create mode 100644 scripts/checks/run-managed-image-openshell-e2e.ts diff --git a/scripts/checks/managed-image-protected-runtime-contract.ts b/scripts/checks/managed-image-protected-runtime-contract.ts new file mode 100644 index 00000000000..79d9aa05c84 --- /dev/null +++ b/scripts/checks/managed-image-protected-runtime-contract.ts @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + ManagedStartupAgent, + ManagedStartupProfile, +} from "../../src/lib/onboard/managed-startup/profile.ts"; +import { PROTECTED_MANAGED_IMAGE_AGENTS } from "./protected-managed-image-contract.ts"; + +export { + PROTECTED_MANAGED_IMAGE_AGENTS, + type ProtectedManagedImageContract, + parseProtectedManagedImageContracts, +} from "./protected-managed-image-contract.ts"; + +export const MANAGED_IMAGE_LOCAL_INFERENCE_KINDS = ["ollama", "nim", "vllm"] as const; + +export type ManagedImageLocalInferenceKind = (typeof MANAGED_IMAGE_LOCAL_INFERENCE_KINDS)[number]; + +export type ManagedImageLocalInferenceRoute = { + readonly kind: ManagedImageLocalInferenceKind; + readonly providerName: "ollama-local" | "vllm-local"; + readonly credentialEnv: "NEMOCLAW_OLLAMA_PROXY_TOKEN" | "NEMOCLAW_VLLM_LOCAL_TOKEN"; + readonly defaultBaseUrl: string; +}; + +const LOCAL_INFERENCE_ROUTES: Readonly< + Record +> = Object.freeze({ + ollama: Object.freeze({ + kind: "ollama", + providerName: "ollama-local", + credentialEnv: "NEMOCLAW_OLLAMA_PROXY_TOKEN", + defaultBaseUrl: "http://host.openshell.internal:11435/v1", + }), + // Local NIM exposes the same OpenAI-compatible host route as local vLLM. + // Keep the source kinds distinct even though OpenShell intentionally binds + // both to vllm-local; this prevents a future engine-specific route change + // from being silently treated as equivalent. + nim: Object.freeze({ + kind: "nim", + providerName: "vllm-local", + credentialEnv: "NEMOCLAW_VLLM_LOCAL_TOKEN", + defaultBaseUrl: "http://host.openshell.internal:8000/v1", + }), + vllm: Object.freeze({ + kind: "vllm", + providerName: "vllm-local", + credentialEnv: "NEMOCLAW_VLLM_LOCAL_TOKEN", + defaultBaseUrl: "http://host.openshell.internal:8000/v1", + }), +}); + +export function isManagedImageLocalInferenceKind( + value: string, +): value is ManagedImageLocalInferenceKind { + return (MANAGED_IMAGE_LOCAL_INFERENCE_KINDS as readonly string[]).includes(value); +} + +export function resolveManagedImageLocalInferenceRoute( + kind: ManagedImageLocalInferenceKind, +): ManagedImageLocalInferenceRoute { + return LOCAL_INFERENCE_ROUTES[kind]; +} + +export function withManagedImageLocalInferenceProfile( + profile: ManagedStartupProfile, + route: ManagedImageLocalInferenceRoute, + model: string, +): ManagedStartupProfile { + const primaryModelRef = + profile.agent === "openclaw" ? `inference/${model}` : profile.inference.primaryModelRef; + return { + ...profile, + inference: { + ...profile.inference, + routeProvider: "inference", + upstreamProvider: route.providerName, + model, + primaryModelRef, + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions", + }, + } as ManagedStartupProfile; +} + +export function managedImageProtectedSandboxName( + agent: ManagedStartupAgent, + routeKind: ManagedImageLocalInferenceKind | "rollback", +): string { + const agentToken = + agent === "langchain-deepagents-code" ? "dcode" : agent.replace(/[^a-z0-9-]+/gu, "-"); + return `nemoclaw-managed-${agentToken}-${routeKind}`; +} diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts new file mode 100644 index 00000000000..2c81c9cc53b --- /dev/null +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -0,0 +1,960 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { resolveAgent } from "../../src/lib/agent/onboard.ts"; +import { + type InitialSandboxPolicy, + prepareInitialSandboxCreatePolicy, +} from "../../src/lib/onboard/initial-policy.ts"; +import { + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapAdapter, + type ManagedBootstrapAuthorityStore, +} from "../../src/lib/onboard/managed-bootstrap/adapter.ts"; +import { createDockerManagedBootstrapAdapter } from "../../src/lib/onboard/managed-bootstrap/docker.ts"; +import { createDockerManagedBootstrapSurface } from "../../src/lib/onboard/managed-bootstrap/docker-runtime.ts"; +import { + encodeManagedStartupProfile, + type ManagedStartupAgent, +} from "../../src/lib/onboard/managed-startup/profile.ts"; +import { createManagedStartupRootApplyRequest } from "../../src/lib/onboard/managed-startup/root-apply.ts"; +import type { + RuntimeProviderBootstrapSurface, + RuntimeProviderBundle, +} from "../../src/lib/onboard/runtime-provider/contract.ts"; +import { createDockerRuntimeProviderBundle } from "../../src/lib/onboard/runtime-provider/docker.ts"; +import { prepareSandboxCreateLaunch } from "../../src/lib/onboard/sandbox-create-launch.ts"; +import { + resolveDockerStartupCommandPatch, + runSandboxGpuCreateFlow, +} from "../../src/lib/onboard/sandbox-gpu-create-flow.ts"; +import { createDirectSandboxGpuVerifier } from "../../src/lib/onboard/sandbox-gpu-preflight.ts"; +import { + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + managedStartupE2eProfile, +} from "./generate-managed-startup-profile-fixture.mts"; +import { + isManagedImageLocalInferenceKind, + type ManagedImageLocalInferenceKind, + resolveManagedImageLocalInferenceRoute, + withManagedImageLocalInferenceProfile, +} from "./managed-image-protected-runtime-contract.ts"; + +const MANAGED_AGENTS = new Set([ + "openclaw", + "hermes", + "langchain-deepagents-code", +]); +const MODEL = "nvidia/nemotron-3-ultra-550b-a55b"; +const GATEWAY_PORT = 8080; +const IMMUTABLE_MANIFEST_REFERENCE_RE = /^([^\s@]+)@(sha256:[a-f0-9]{64})$/u; +const MANAGED_AGENT_BASE_POLICIES: Record = { + openclaw: ["nemoclaw-blueprint", "policies", "openclaw-sandbox.yaml"], + hermes: ["agents", "hermes", "policy-additions.yaml"], + "langchain-deepagents-code": ["agents", "langchain-deepagents-code", "policy-additions.yaml"], +}; + +function compactText(value = ""): string { + return String(value).replace(/\s+/gu, " ").trim(); +} + +function redactProtectedGpuProof(value: string): string { + return String(value) + .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/giu, "Bearer ") + .replace(/\b([A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD))=([^\s]*)/giu, "$1="); +} + +type Inputs = { + agent: ManagedStartupAgent; + image: string; + sandbox: string; + gpu?: true; + localProvider?: ManagedImageLocalInferenceKind; + model?: string; + failureInjection?: "bootstrap-completion"; +}; + +type OnboardModule = { + openshellArgv(args: string[]): string[]; + runOpenshell(args: string[], opts?: Record): ReturnType; + runCaptureOpenshell(args: string[], opts?: Record): string; + sleepSeconds(seconds: number): void; + startGatewayForRecovery(options: { gatewayName: string; gatewayPort: number }): Promise; +}; + +function requiredValue(argv: readonly string[], flag: string): string { + const index = argv.indexOf(flag); + const value = index >= 0 ? argv[index + 1] : undefined; + if (!value || value.startsWith("--")) throw new Error(`${flag} is required`); + return value; +} + +export function parseManagedImageOpenShellE2eInputs(argv: readonly string[]): Inputs { + const valueFlags = new Set(["--agent", "--image", "--sandbox", "--local-provider", "--model"]); + const booleanFlags = new Set(["--gpu", "--inject-bootstrap-completion-failure"]); + for (let index = 0; index < argv.length; index += 1) { + const value = argv[index] ?? ""; + if (booleanFlags.has(value)) continue; + if (!valueFlags.has(value)) throw new Error(`unsupported arguments: ${value}`); + const next = argv[index + 1]; + if (!next || next.startsWith("--")) throw new Error(`${value} is required`); + index += 1; + } + const agentValue = requiredValue(argv, "--agent"); + if (!MANAGED_AGENTS.has(agentValue as ManagedStartupAgent)) { + throw new Error("--agent must identify a shipped managed-image agent"); + } + const image = requiredValue(argv, "--image"); + if (!IMMUTABLE_MANIFEST_REFERENCE_RE.test(image)) { + throw new Error("--image must be an immutable repository@sha256 manifest reference"); + } + const sandbox = requiredValue(argv, "--sandbox"); + if (!/^[a-z0-9](?:[a-z0-9.-]{0,61}[a-z0-9])?$/u.test(sandbox)) { + throw new Error("--sandbox must be a valid RFC 1123 label"); + } + const gpu = argv.includes("--gpu"); + const localProviderValue = argv.includes("--local-provider") + ? requiredValue(argv, "--local-provider") + : null; + if (localProviderValue && !isManagedImageLocalInferenceKind(localProviderValue)) { + throw new Error("--local-provider must be one of: ollama, nim, vllm"); + } + const model = argv.includes("--model") ? requiredValue(argv, "--model") : null; + if (model && !/^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,255}$/u.test(model)) { + throw new Error("--model must be one bounded model identifier"); + } + const failureInjection = argv.includes("--inject-bootstrap-completion-failure"); + if (gpu && (!localProviderValue || !model)) { + throw new Error("--gpu requires --local-provider and --model"); + } + if (!gpu && (localProviderValue || model)) { + throw new Error("--local-provider and --model require --gpu"); + } + if (failureInjection && gpu) { + throw new Error("bootstrap failure injection cannot be combined with the GPU qualification"); + } + return { + agent: agentValue as ManagedStartupAgent, + image, + sandbox, + ...(gpu ? { gpu: true as const } : {}), + ...(localProviderValue + ? { localProvider: localProviderValue as ManagedImageLocalInferenceKind } + : {}), + ...(model ? { model } : {}), + ...(failureInjection ? { failureInjection: "bootstrap-completion" as const } : {}), + }; +} + +export function managedImageOpenShellBasePolicyPath(agent: ManagedStartupAgent): string { + return path.resolve(__dirname, "..", "..", ...MANAGED_AGENT_BASE_POLICIES[agent]); +} + +function commandResult(argv: readonly string[], env: NodeJS.ProcessEnv, timeout = 20_000) { + const [command, ...args] = argv; + if (!command) throw new Error("command argv must not be empty"); + return spawnSync(command, args, { + encoding: "utf8", + env, + killSignal: "SIGKILL", + stdio: ["ignore", "pipe", "pipe"], + timeout, + }); +} + +function commandDetail(result: ReturnType): string { + return `${result.error?.message ?? ""}\n${result.stdout ?? ""}\n${result.stderr ?? ""}` + .trim() + .slice(-8_000); +} + +function isDockerNotFound(result: ReturnType): boolean { + return ( + result.status !== 0 && + /(?:no such (?:container|network|object)|not found)/iu.test(commandDetail(result)) + ); +} + +function readGatewayPid(stateDir: string): number | null { + try { + const value = Number.parseInt( + fs.readFileSync(path.join(stateDir, "openshell-gateway.pid"), "utf8").trim(), + 10, + ); + return Number.isSafeInteger(value) && value > 1 ? value : null; + } catch { + return null; + } +} + +function stopProcess(pid: number | null): void { + if (!pid) return; + try { + process.kill(pid, "SIGTERM"); + } catch { + return; + } + for (let attempt = 0; attempt < 50; attempt += 1) { + try { + process.kill(pid, 0); + } catch { + return; + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100); + } + try { + process.kill(pid, "SIGKILL"); + } catch { + // The process exited between the liveness probe and the final signal. + } +} + +function createProtectedAuthorityStore(stateDir: string): ManagedBootstrapAuthorityStore { + const authorityDir = path.join(stateDir, "managed-bootstrap-authority"); + fs.mkdirSync(authorityDir, { mode: 0o700, recursive: true }); + return { + async recordPreparedAuthority(authority) { + const finalPath = path.join(authorityDir, `${authority.bootstrapIdentity}.json`); + const temporaryPath = `${finalPath}.tmp-${process.pid}`; + const serialized = `${JSON.stringify(authority)}\n`; + const file = fs.openSync(temporaryPath, "wx", 0o600); + try { + fs.writeFileSync(file, serialized, "utf8"); + fs.fsyncSync(file); + } finally { + fs.closeSync(file); + } + fs.renameSync(temporaryPath, finalPath); + const directory = fs.openSync(authorityDir, "r"); + try { + fs.fsyncSync(directory); + } finally { + fs.closeSync(directory); + } + if (fs.readFileSync(finalPath, "utf8") !== serialized) { + throw new Error("protected managed-bootstrap authority was not durably re-readable"); + } + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: authority.sandbox, + bootstrapIdentity: authority.bootstrapIdentity, + authorityFingerprint: authority.authorityFingerprint, + recordId: `protected-${authority.bootstrapIdentity}`, + recordedAt: new Date().toISOString(), + }; + }, + }; +} + +async function assertGatewayPortAvailable(): Promise { + await new Promise((resolve, reject) => { + const server = net.createServer(); + server.unref(); + server.once("error", () => { + reject( + new Error( + `refusing to disturb an existing listener on the managed-image E2E gateway port ${GATEWAY_PORT}`, + ), + ); + }); + server.listen(GATEWAY_PORT, "127.0.0.1", () => { + server.close((error) => { + if (error) reject(error); + else resolve(); + }); + }); + }); +} + +function managedConfigPath(agent: ManagedStartupAgent): string { + switch (agent) { + case "openclaw": + return "/sandbox/.openclaw/openclaw.json"; + case "hermes": + return "/sandbox/.hermes/config.yaml"; + case "langchain-deepagents-code": + return "/sandbox/.deepagents/config.toml"; + } +} + +export function managedImageOpenShellProbe( + agent: ManagedStartupAgent, + model: string = MODEL, +): string { + const healthProbe = + agent === "openclaw" + ? "/usr/bin/curl -fsS --max-time 5 http://127.0.0.1:18789/health >/dev/null" + : agent === "hermes" + ? "/usr/bin/curl -fsS --max-time 5 http://127.0.0.1:8642/health >/dev/null" + : "/usr/local/bin/dcode --version >/dev/null"; + return [ + "set -eu", + `test -x ${ + agent === "openclaw" + ? "/usr/local/bin/openclaw" + : agent === "hermes" + ? "/usr/local/bin/hermes" + : "/usr/local/bin/dcode" + }`, + `grep -F ${JSON.stringify(model)} ${JSON.stringify(managedConfigPath(agent))} >/dev/null`, + "test ! -L /run/nemoclaw/managed-startup-runtime.env", + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-runtime.env)" = "0:0:444"', + "test ! -L /run/nemoclaw/managed-startup-complete.json", + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-complete.json)" = "0:0:444"', + "test -s /usr/local/share/nemoclaw/corporate-ca.pem", + 'test "$(stat -c "%u:%g:%a" /usr/local/share/nemoclaw/corporate-ca.pem)" = "0:0:444"', + "test -s /run/nemoclaw/managed-startup-ca-bundle.pem", + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-ca-bundle.pem)" = "0:0:444"', + healthProbe, + ].join("\n"); +} + +export function managedImageOpenShellCommittedProbe(): string { + return [ + "set -eu", + "test ! -e /var/lib/nemoclaw/managed-startup-shared-state-transaction-v1", + ].join("\n"); +} + +async function waitForCommittedSandboxProbe( + onboard: OnboardModule, + input: Inputs, + env: NodeJS.ProcessEnv, + requireCommitted = true, +): Promise { + const healthProbe = managedImageOpenShellProbe(input.agent, input.model ?? MODEL); + const committedProbe = managedImageOpenShellCommittedProbe(); + const deadline = Date.now() + 240_000; + const runProbe = (probe: string, timeoutMs: number) => + commandResult( + onboard.openshellArgv([ + "sandbox", + "exec", + "--name", + input.sandbox, + "--", + "/bin/sh", + "-eu", + "-c", + probe, + ]), + env, + timeoutMs, + ); + let lastHealthDetail = ""; + while (Date.now() < deadline) { + const remainingMs = deadline - Date.now(); + const health = runProbe(healthProbe, Math.max(1, Math.min(15_000, remainingMs))); + if (health.status === 0) { + if (!requireCommitted) return; + const committed = runProbe( + committedProbe, + Math.max(1, Math.min(15_000, deadline - Date.now())), + ); + if (committed.status !== 0) { + throw new Error( + `managed bootstrap committed, but transaction cleanup was not observable through the exact sandbox: ${commandDetail(committed)}`, + ); + } + return; + } + lastHealthDetail = commandDetail(health); + const sleepMs = Math.min(2_000, Math.max(0, deadline - Date.now())); + if (sleepMs > 0) await new Promise((resolve) => setTimeout(resolve, sleepMs)); + } + throw new Error( + `OpenShell sandbox did not pass the exact-image managed-bootstrap probe within 240s: ${lastHealthDetail}`, + ); +} + +function localInferenceBaseUrl(input: Inputs): string { + if (!input.localProvider) throw new Error("local provider is required"); + const route = resolveManagedImageLocalInferenceRoute(input.localProvider); + const configured = String(process.env.NEMOCLAW_E2E_LOCAL_INFERENCE_BASE_URL ?? "").trim(); + const value = configured || route.defaultBaseUrl; + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw new Error("protected local inference base URL is invalid"); + } + if ( + parsed.protocol !== "http:" || + parsed.hostname !== "host.openshell.internal" || + !/^[1-9][0-9]{0,4}$/u.test(parsed.port) || + parsed.pathname.replace(/\/+$/u, "") !== "/v1" || + parsed.username || + parsed.password || + parsed.search || + parsed.hash + ) { + throw new Error("protected local inference must use http://host.openshell.internal:/v1"); + } + return value.replace(/\/+$/u, ""); +} + +function configureLocalInferenceRoute( + onboard: OnboardModule, + input: Inputs, + env: NodeJS.ProcessEnv, +): void { + if (!input.localProvider || !input.model) return; + const route = resolveManagedImageLocalInferenceRoute(input.localProvider); + const credential = String(env[route.credentialEnv] ?? "").trim(); + if (!credential || /[\0\r\n]/u.test(credential)) { + throw new Error(`${route.credentialEnv} is required for protected local inference`); + } + const commandEnv = { ...env, [route.credentialEnv]: credential }; + const create = onboard.runOpenshell( + [ + "provider", + "create", + "--name", + route.providerName, + "--type", + "openai", + "--credential", + route.credentialEnv, + "--config", + `OPENAI_BASE_URL=${localInferenceBaseUrl(input)}`, + ], + { ignoreError: true, env: commandEnv, stdio: ["ignore", "pipe", "pipe"] }, + ); + if (create.status !== 0) { + throw new Error(`protected local inference provider creation failed: ${commandDetail(create)}`); + } + const setRoute = onboard.runOpenshell( + [ + "inference", + "set", + "--no-verify", + "--provider", + route.providerName, + "--model", + input.model, + "--timeout", + "120", + ], + { ignoreError: true, env: commandEnv, stdio: ["ignore", "pipe", "pipe"] }, + ); + if (setRoute.status !== 0) { + throw new Error(`protected local inference route failed: ${commandDetail(setRoute)}`); + } +} + +function localInferenceProbe(input: Inputs): string { + if (!input.model) throw new Error("local inference model is required"); + const payload = JSON.stringify({ + model: input.model, + messages: [{ role: "user", content: "Reply with exactly one word: PONG" }], + reasoning_effort: "none", + max_tokens: 32, + }); + return [ + "set -eu", + "response=/tmp/nemoclaw-managed-image-inference.json", + `curl -fsS --max-time 180 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' --data ${JSON.stringify(payload)} > "$response"`, + "node - \"$response\" <<'NODE'", + 'const fs = require("node:fs");', + 'const body = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));', + "const choice = Array.isArray(body.choices) ? body.choices[0] : null;", + 'const text = choice && choice.message && typeof choice.message.content === "string"', + " ? choice.message.content", + ' : choice && typeof choice.text === "string" ? choice.text : "";', + 'if (!/pong/i.test(text)) throw new Error("local inference did not return PONG");', + "NODE", + 'rm -f "$response"', + ].join("\n"); +} + +function assertProtectedLocalInference( + onboard: OnboardModule, + input: Inputs, + env: NodeJS.ProcessEnv, +): void { + const result = commandResult( + onboard.openshellArgv([ + "sandbox", + "exec", + "--name", + input.sandbox, + "--", + "/bin/sh", + "-eu", + "-c", + localInferenceProbe(input), + ]), + env, + 210_000, + ); + if (result.status !== 0) { + throw new Error(`sandbox inference.local completion failed: ${commandDetail(result)}`); + } +} + +function failureInjectingAdapter(onboard: OnboardModule): ManagedBootstrapAdapter { + const adapter = createDockerManagedBootstrapAdapter({ + runCaptureOpenshell: onboard.runCaptureOpenshell, + runOpenshell: onboard.runOpenshell, + sleep: onboard.sleepSeconds, + }); + return { + ...adapter, + async awaitBootstrap(input) { + await adapter.awaitBootstrap(input); + throw new Error("protected-e2e-injected-bootstrap-completion-failure"); + }, + }; +} + +function parseImmutableManifestReference(image: string): { + repository: string; + manifestDigest: `sha256:${string}`; +} { + const match = IMMUTABLE_MANIFEST_REFERENCE_RE.exec(image); + if (!match?.[1] || !match[2]) { + throw new Error("--image must be an immutable repository@sha256 manifest reference"); + } + return { + repository: match[1], + manifestDigest: match[2] as `sha256:${string}`, + }; +} + +function resolveLocalImageContentId(image: string, env: NodeJS.ProcessEnv): string { + const inspect = commandResult(["docker", "image", "inspect", "--format", "{{.Id}}", image], env); + const contentId = String(inspect.stdout ?? "").trim(); + if (inspect.status !== 0 || !/^sha256:[a-f0-9]{64}$/u.test(contentId)) { + throw new Error( + `--image does not resolve to one immutable local image content ID: ${commandDetail(inspect)}`, + ); + } + return contentId; +} + +function exactHarnessContainerIds( + input: Inputs, + networkName: string, + env: NodeJS.ProcessEnv, +): { candidateCount: number; exactIds: string[] } { + const expectedContentId = resolveLocalImageContentId(input.image, env); + const list = commandResult( + [ + "docker", + "ps", + "-aq", + "--no-trunc", + "--filter", + "label=openshell.ai/managed-by=openshell", + "--filter", + `label=openshell.ai/sandbox-name=${input.sandbox}`, + ], + env, + ); + if (list.status !== 0) { + throw new Error(`could not resolve the OpenShell sandbox container: ${commandDetail(list)}`); + } + const candidates = String(list.stdout ?? "") + .trim() + .split(/\s+/u) + .filter(Boolean); + const exactIds: string[] = []; + for (const candidate of candidates) { + const inspect = commandResult(["docker", "inspect", candidate], env); + if (inspect.status !== 0) continue; + try { + const records = JSON.parse(String(inspect.stdout ?? "")) as Array<{ + Config?: { Labels?: Record }; + Image?: string; + NetworkSettings?: { Networks?: Record }; + }>; + const record = records.length === 1 ? records[0] : undefined; + if ( + record?.Config?.Labels?.["openshell.ai/managed-by"] === "openshell" && + record.Config.Labels["openshell.ai/sandbox-name"] === input.sandbox && + record.Image === expectedContentId && + Object.hasOwn(record.NetworkSettings?.Networks ?? {}, networkName) + ) { + exactIds.push(candidate); + } + } catch { + // An unparseable inspection result cannot establish cleanup ownership. + } + } + return { candidateCount: candidates.length, exactIds }; +} + +function assertExactSandboxImage( + input: Inputs, + networkName: string, + env: NodeJS.ProcessEnv, +): string { + const resolved = exactHarnessContainerIds(input, networkName, env); + if (resolved.candidateCount !== 1 || resolved.exactIds.length !== 1) { + throw new Error( + `OpenShell did not launch exactly one harness-owned PR image container: found ${resolved.candidateCount} labeled and ${resolved.exactIds.length} exact`, + ); + } + return resolved.exactIds[0] ?? ""; +} + +function assertFailedSandboxAbsent( + onboard: OnboardModule, + input: Inputs, + env: NodeJS.ProcessEnv, +): void { + const get = onboard.runOpenshell(["sandbox", "get", input.sandbox], { + ignoreError: true, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + const list = onboard.runOpenshell(["sandbox", "list"], { + ignoreError: true, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + if ( + get.status === 0 || + list.status !== 0 || + `${list.stdout ?? ""}\n${list.stderr ?? ""}`.includes(input.sandbox) + ) { + throw new Error( + `managed-bootstrap rollback retained failed OpenShell sandbox state: get=${commandDetail(get)} list=${commandDetail(list)}`, + ); + } +} + +async function run(input: Inputs): Promise { + const stateParent = process.env.RUNNER_TEMP || os.tmpdir(); + const stateDir = fs.mkdtempSync(path.join(stateParent, "nemoclaw-managed-openshell-")); + const networkName = `nemoclaw-managed-pr-${process.pid}-${Date.now().toString(36)}`; + process.env.NEMOCLAW_NON_INTERACTIVE = "1"; + process.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR = stateDir; + process.env.NEMOCLAW_GATEWAY_PORT = String(GATEWAY_PORT); + process.env.NEMOCLAW_DOCKER_GPU_SUPERVISOR_RECONNECT_TIMEOUT = "240"; + process.env.OPENSHELL_DOCKER_NETWORK_NAME = networkName; + process.env.XDG_CONFIG_HOME = path.join(stateDir, "xdg-config"); + process.env.XDG_DATA_HOME = path.join(stateDir, "xdg-data"); + process.env.XDG_STATE_HOME = path.join(stateDir, "xdg-state"); + process.env.PATH = `${path.join(os.homedir(), ".local", "bin")}:${process.env.PATH ?? ""}`; + + let onboard: OnboardModule | null = null; + let ownedContainerId: string | null = null; + let initialSandboxPolicy: InitialSandboxPolicy | null = null; + let failureInjectionQualified = false; + try { + await assertGatewayPortAvailable(); + const image = parseImmutableManifestReference(input.image); + resolveLocalImageContentId(input.image, process.env); + + const onboardImport = (await import("../../src/lib/onboard.ts")) as unknown as + | OnboardModule + | { default: OnboardModule }; + onboard = "default" in onboardImport ? onboardImport.default : onboardImport; + await onboard.startGatewayForRecovery({ + gatewayName: "nemoclaw", + gatewayPort: GATEWAY_PORT, + }); + configureLocalInferenceRoute(onboard, input, process.env); + + const baseProfile = managedStartupE2eProfile(input.agent, false, true, true); + const protectedProfile = + input.localProvider && input.model + ? withManagedImageLocalInferenceProfile( + baseProfile, + resolveManagedImageLocalInferenceRoute(input.localProvider), + input.model, + ) + : baseProfile; + const profile = encodeManagedStartupProfile(protectedProfile); + const rootApplyRequest = createManagedStartupRootApplyRequest({ + agent: input.agent, + encodedProfile: profile, + corporateCaB64: Buffer.from(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, "utf8").toString("base64"), + }); + initialSandboxPolicy = prepareInitialSandboxCreatePolicy( + managedImageOpenShellBasePolicyPath(input.agent), + [], + { + agentName: input.agent, + directGpu: input.gpu === true, + hostGpuAvailable: input.gpu === true, + additionalPresets: input.localProvider ? ["local-inference"] : [], + }, + ); + const createArgs = [ + "--from", + input.image, + "--name", + input.sandbox, + "--policy", + initialSandboxPolicy.policyPath, + ...(input.gpu ? ["--gpu"] : []), + ]; + const launch = prepareSandboxCreateLaunch({ + agent: resolveAgent({ agentFlag: input.agent }), + sandboxName: input.sandbox, + chatUiUrl: "", + createArgs, + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: { config: null, enabled: false }, + manageDashboard: false, + openshellShellCommand: (args: string[]) => args.map((arg) => JSON.stringify(arg)).join(" "), + openshellArgv: onboard.openshellArgv, + managedStartupRootApplyRequest: rootApplyRequest, + }); + const prebuild = { createArgs: [...createArgs], imageRef: null, imageId: null }; + if ( + prebuild.imageId !== null || + prebuild.imageRef !== null || + prebuild.createArgs.join("\0") !== createArgs.join("\0") || + launch.createArgv.filter((value) => value === "--from").length !== 1 || + launch.createArgv[launch.createArgv.indexOf("--from") + 1] !== input.image || + launch.createArgv.filter((value) => value === "--policy").length !== 1 || + launch.createArgv[launch.createArgv.indexOf("--policy") + 1] !== + initialSandboxPolicy.policyPath + ) { + throw new Error("managed-image launch renderer altered the exact PR image identity"); + } + const startupPlan = resolveDockerStartupCommandPatch( + { name: input.agent } as Parameters[0], + true, + ); + if ( + !launch.managedStartupRootApplyRequest || + !launch.managedBootstrapIdentity || + !launch.intendedSandboxStartupCommand + ) { + throw new Error("managed-image launch did not retain its identity-bound bootstrap contract"); + } + + const gpuEnabled = input.gpu === true; + const gpuConfig = { + mode: gpuEnabled ? ("1" as const) : ("0" as const), + hostGpuDetected: gpuEnabled, + hostGpuPlatform: gpuEnabled ? ("linux" as const) : null, + sandboxGpuEnabled: gpuEnabled, + sandboxGpuDevice: null, + errors: [], + }; + const verifyDirectSandboxGpu = gpuEnabled + ? createDirectSandboxGpuVerifier({ + runOpenshell: onboard.runOpenshell, + compactText, + redact: redactProtectedGpuProof, + }) + : () => ({ + status: "unverified" as const, + cudaVerified: false, + label: "disabled", + detail: null, + at: new Date().toISOString(), + }); + const runtimeProvider = { + ...createDockerRuntimeProviderBundle(), + bootstrap: createDockerManagedBootstrapSurface("docker"), + } as RuntimeProviderBundle & { + readonly bootstrap: Extract; + }; + let flow: Awaited>; + try { + flow = await runSandboxGpuCreateFlow( + { + sandboxName: input.sandbox, + provider: input.localProvider + ? resolveManagedImageLocalInferenceRoute(input.localProvider).providerName + : "nvidia", + sandboxGpuConfig: gpuConfig, + gpuRoutePlan: gpuEnabled ? "native-only" : "none", + initialGpuRoute: gpuEnabled ? "native" : "none", + compatibilityPolicyPath: null, + dockerDriverGateway: true, + gatewayPort: GATEWAY_PORT, + sandboxReadyTimeoutSecs: 240, + createArgv: launch.createArgv, + sandboxEnv: launch.sandboxEnv, + sandboxStartupCommand: launch.sandboxStartupCommand, + prebuild, + restoreBackupPath: null, + terminalAgent: input.agent === "langchain-deepagents-code", + managedBootstrap: { + bootstrapIdentity: launch.managedBootstrapIdentity, + runtimeProvider, + authorityStore: createProtectedAuthorityStore(stateDir), + request: launch.managedStartupRootApplyRequest, + image, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + intendedWorkloadArgv: launch.intendedSandboxStartupCommand, + expectedSupervisorArgv: ["/opt/openshell/bin/openshell-sandbox"], + }, + ...startupPlan, + }, + { + runOpenshell: onboard.runOpenshell, + runCaptureOpenshell: onboard.runCaptureOpenshell, + sleep: onboard.sleepSeconds, + openshellArgv: onboard.openshellArgv, + verifyDirectSandboxGpu, + ...(input.failureInjection + ? { createManagedBootstrapAdapter: () => failureInjectingAdapter(onboard!) } + : {}), + }, + ); + } catch (error) { + if ( + input.failureInjection === "bootstrap-completion" && + error instanceof Error && + error.message.includes("protected-e2e-injected-bootstrap-completion-failure") + ) { + const resolved = exactHarnessContainerIds(input, networkName, launch.sandboxEnv); + if (resolved.candidateCount !== 0 || resolved.exactIds.length !== 0) { + throw new Error( + `managed-bootstrap rollback retained a failed held sandbox: found ${resolved.candidateCount} labeled and ${resolved.exactIds.length} exact containers`, + ); + } + assertFailedSandboxAbsent(onboard, input, launch.sandboxEnv); + failureInjectionQualified = true; + process.stdout.write( + `Injected managed-bootstrap completion failure removed the failed exact ${input.agent} sandbox before harness cleanup.\n`, + ); + return; + } + throw error; + } + const expectedRoute = gpuEnabled ? "native" : "none"; + if (flow.route !== expectedRoute || flow.createResult.status !== 0) { + throw new Error( + `production managed-bootstrap flow did not complete the exact PR image create: route=${flow.route} status=${flow.createResult.status}`, + ); + } + + await waitForCommittedSandboxProbe(onboard, input, launch.sandboxEnv, !gpuEnabled); + ownedContainerId = assertExactSandboxImage(input, networkName, launch.sandboxEnv); + if (gpuEnabled) { + assertProtectedLocalInference(onboard, input, launch.sandboxEnv); + await flow.runtimePatch.commitAfterReady(); + await waitForCommittedSandboxProbe(onboard, input, launch.sandboxEnv); + } + process.stdout.write( + `OpenShell launched exact ${input.agent} PR image ${input.image} through the production managed-bootstrap sequence${gpuEnabled ? ` with real NVIDIA GPU access and ${input.localProvider} inference.local completion` : ""}.\n`, + ); + } finally { + const cleanupErrors: string[] = []; + if (onboard) { + commandResult( + onboard.openshellArgv(["sandbox", "delete", input.sandbox]), + process.env, + 15_000, + ); + } + stopProcess(readGatewayPid(stateDir)); + if (onboard) { + commandResult(onboard.openshellArgv(["gateway", "remove", "nemoclaw"]), process.env, 15_000); + } + try { + const resolved = exactHarnessContainerIds(input, networkName, process.env); + const cleanupContainerId = + resolved.exactIds.length === 1 ? (resolved.exactIds[0] ?? null) : null; + if (cleanupContainerId) { + const remove = commandResult( + ["docker", "rm", "-f", cleanupContainerId], + process.env, + 15_000, + ); + const verify = commandResult( + ["docker", "container", "inspect", cleanupContainerId], + process.env, + 15_000, + ); + if (verify.status === 0 || !isDockerNotFound(verify)) { + cleanupErrors.push( + `exact harness container ${cleanupContainerId} was not removed: ${commandDetail(remove)} ${commandDetail(verify)}`.trim(), + ); + } + } else if (resolved.exactIds.length > 1) { + cleanupErrors.push( + `refusing ambiguous exact harness container cleanup: ${resolved.exactIds.length} matches`, + ); + } else if (ownedContainerId) { + const verify = commandResult( + ["docker", "container", "inspect", ownedContainerId], + process.env, + 15_000, + ); + if (verify.status === 0 || !isDockerNotFound(verify)) { + cleanupErrors.push( + `could not prove exact harness container ${ownedContainerId} was removed: ${commandDetail(verify)}`, + ); + } + } + } catch (error) { + cleanupErrors.push(error instanceof Error ? error.message : String(error)); + } + const removeNetwork = commandResult( + ["docker", "network", "rm", networkName], + process.env, + 15_000, + ); + const verifyNetwork = commandResult( + ["docker", "network", "inspect", networkName], + process.env, + 15_000, + ); + if (verifyNetwork.status === 0 || !isDockerNotFound(verifyNetwork)) { + cleanupErrors.push( + `harness network ${networkName} was not removed: ${commandDetail(removeNetwork)} ${commandDetail(verifyNetwork)}`.trim(), + ); + } + const remainingSandboxContainers = commandResult( + [ + "docker", + "ps", + "-aq", + "--filter", + "label=openshell.ai/managed-by=openshell", + "--filter", + `label=openshell.ai/sandbox-name=${input.sandbox}`, + ], + process.env, + 15_000, + ); + if ( + remainingSandboxContainers.status !== 0 || + String(remainingSandboxContainers.stdout ?? "").trim() !== "" + ) { + cleanupErrors.push( + `managed-image sandbox/container orphan remained after cleanup: ${commandDetail(remainingSandboxContainers)}`, + ); + } + try { + initialSandboxPolicy?.cleanup?.(); + } catch (error) { + cleanupErrors.push(error instanceof Error ? error.message : String(error)); + } + fs.rmSync(stateDir, { recursive: true, force: true }); + if (cleanupErrors.length > 0) { + throw new Error(`managed-image OpenShell cleanup failed: ${cleanupErrors.join("; ")}`); + } + if (failureInjectionQualified) { + process.stdout.write( + `Managed-bootstrap failure injection left no sandbox, container, network, or harness state orphan for ${input.agent}.\n`, + ); + } + } +} + +if (require.main === module) { + run(parseManagedImageOpenShellE2eInputs(process.argv.slice(2))).catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} + +export { run as runManagedImageOpenShellE2e }; From 1741458afa527867afac07bf96c575de5b0e6278 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 08:46:31 -0700 Subject: [PATCH 13/24] ci(images): add protected runtime qualification lane Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 267 ++++++++++++ ...managed-image-protected-runtime-helpers.ts | 408 ++++++++++++++++++ .../managed-image-protected-runtime.test.ts | 33 ++ test/e2e/mock-parity.json | 10 + ...d-image-protected-runtime-workflow.test.ts | 74 ++++ ...d-image-protected-runtime-contract.test.ts | 136 ++++++ test/pr-risk-plan.test.ts | 25 +- tools/advisors/risk-plan.mts | 22 +- ...ge-protected-runtime-workflow-boundary.mts | 274 ++++++++++++ tools/e2e/prepare-e2e-workflow-boundary.mts | 1 + ...upload-e2e-artifacts-workflow-boundary.mts | 7 + tools/e2e/workflow-boundary.mts | 2 + 12 files changed, 1257 insertions(+), 2 deletions(-) create mode 100644 test/e2e/live/managed-image-protected-runtime-helpers.ts create mode 100644 test/e2e/live/managed-image-protected-runtime.test.ts create mode 100644 test/e2e/support/managed-image-protected-runtime-workflow.test.ts create mode 100644 test/managed-image-protected-runtime-contract.test.ts create mode 100644 tools/e2e/managed-image-protected-runtime-workflow-boundary.mts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 7c67bd097a9..2e91f836023 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2032,6 +2032,272 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + # This explicit-only lane remains dormant until its trusted workflow and + # validation boundary land on main. A follow-on candidate activates it with + # ci/protected-managed-image-runtime-activation-v1.json so the trusted + # controller can qualify that candidate's exact head without executing + # PR-controlled workflow code. + managed-image-protected-runtime: + name: Protected managed-image GPU and local inference + needs: generate-matrix + if: ${{ contains(format(',{0},', inputs.jobs), ',managed-image-protected-runtime,') || contains(format(',{0},', inputs.targets), ',managed-image-protected-runtime,') }} + runs-on: linux-amd64-gpu-rtxpro6000-latest-1 + timeout-minutes: 300 + permissions: + contents: read + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime + E2E_DEFAULT_ENABLED: "0" + E2E_JOB: "1" + E2E_TARGET_ID: "managed-image-protected-runtime" + RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-runtime-activation-v1.json + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_E2E_SHARD: linux-amd64-gpu + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: ${{ inputs.base_sha }} + NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: protected-${{ github.run_id }}-${{ github.run_attempt }} + NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime/contracts.json + NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM: linux/amd64 + NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + NEMOCLAW_PROTECTED_REGISTRY_NAME: nemoclaw-managed-runtime-${{ github.run_id }}-${{ github.run_attempt }} + NEMOCLAW_RUN_LIVE_E2E: "1" + OPENSHELL_GATEWAY: nemoclaw + steps: + - name: Validate protected runtime exact-head dispatch + env: + ACTOR: ${{ github.actor }} + BASE_SHA: ${{ inputs.base_sha }} + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + shell: bash + run: | + set -euo pipefail + [[ "$REPOSITORY" == "NVIDIA/NemoClaw" && "$REF" == "refs/heads/main" && "$EVENT_NAME" == "workflow_dispatch" ]] || { + echo "::error::Protected managed-image runtime must run from trusted NVIDIA/NemoClaw main" >&2 + exit 1 + } + [[ "$ACTOR" == "github-actions[bot]" ]] || { + echo "::error::Protected managed-image runtime requires the trusted controller actor" >&2 + exit 1 + } + [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { + echo "::error::Protected managed-image runtime requires exact PR and base SHAs" >&2 + exit 1 + } + [[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA" ]] || { + echo "::error::Protected managed-image runtime requires the exact trusted workflow SHA" >&2 + exit 1 + } + [[ "$RUNNER_ARCH_KIND" == "X64" ]] || { + echo "::error::Protected managed-image runtime requires a native linux/amd64 GPU runner" >&2 + exit 1 + } + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + + - *dockerhub-auth + + - name: Set up protected runtime Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + with: + driver-opts: network=host + buildkitd-config-inline: | + [registry."localhost:5000"] + http = true + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + with: + build-cli: "false" + + - name: Validate protected runtime activation contract + env: + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + shell: bash + run: | + set -euo pipefail + activation="ci/protected-managed-image-runtime-activation-v1.json" + [[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { + echo "::error::Protected managed-image runtime checkout does not match the exact PR SHA" >&2 + exit 1 + } + [[ -f "$activation" && ! -L "$activation" ]] || { + echo "::error::Protected managed-image runtime activation contract is absent" >&2 + exit 1 + } + jq -e ' + (keys | sort) == ["agents", "contractVersion", "jobId", "platform", "providers"] and + .contractVersion == 1 and + .jobId == "managed-image-protected-runtime" and + .agents == ["openclaw", "hermes", "langchain-deepagents-code"] and + .platform == "linux/amd64" and + .providers == ["ollama", "nim", "vllm"] + ' "$activation" >/dev/null || { + echo "::error::Protected managed-image runtime activation contract is invalid" >&2 + exit 1 + } + install -d -m 0700 "$E2E_ARTIFACT_DIR" + + - id: runtime-bases + name: Resolve exact amd64 runtime base images + shell: bash + run: | + set -euo pipefail + work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-runtime-bases.XXXXXX")" + trap 'rm -rf -- "$work_dir"' EXIT + + resolve_base() { + local output_name="$1" + local alias="$2" + local repository="$3" + local alias_raw="$work_dir/${output_name}-alias.raw" + local exact_raw="$work_dir/${output_name}-exact.raw" + docker buildx imagetools inspect "$alias" --raw > "$alias_raw" + local digest + digest="$( + jq -er ' + if ( + .mediaType == "application/vnd.oci.image.index.v1+json" or + .mediaType == "application/vnd.docker.distribution.manifest.list.v2+json" + ) then + [.manifests[] | select(.platform.os == "linux" and .platform.architecture == "amd64")] + | if length == 1 then .[0].digest else error("not one exact amd64 descriptor") end + else + error("base alias is not a platform index") + end + ' "$alias_raw" + )" + [[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]] || { + echo "::error::${output_name} base alias returned an invalid digest" >&2 + exit 1 + } + local reference="${repository}@${digest}" + docker buildx imagetools inspect "$reference" --raw > "$exact_raw" + [[ "sha256:$(sha256sum "$exact_raw" | awk '{print $1}')" == "$digest" ]] || { + echo "::error::${output_name} exact base bytes do not match the selected digest" >&2 + exit 1 + } + printf '%s=%s\n' "$output_name" "$reference" >> "$GITHUB_OUTPUT" + } + + resolve_base openclaw \ + ghcr.io/nvidia/nemoclaw/sandbox-base:latest \ + ghcr.io/nvidia/nemoclaw/sandbox-base + resolve_base hermes \ + ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest \ + ghcr.io/nvidia/nemoclaw/hermes-sandbox-base + resolve_base dcode \ + ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:latest \ + ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base + + - name: Start isolated protected runtime registry + shell: bash + run: | + set -euo pipefail + if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then + echo "::error::Protected runtime registry name already exists" >&2 + exit 1 + fi + if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then + echo "::error::Refusing to reuse an existing localhost:5000 registry" >&2 + exit 1 + fi + docker run --detach \ + --name "$NEMOCLAW_PROTECTED_REGISTRY_NAME" \ + --label "io.nvidia.nemoclaw.e2e-owner=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT}" \ + --label "io.nvidia.nemoclaw.e2e-platform=linux/amd64" \ + --publish 127.0.0.1:5000:5000 \ + docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373 + for _ in $(seq 1 30); do + if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null; then + exit 0 + fi + sleep 1 + done + docker logs "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >&2 + exit 1 + + - name: Build exact all-agent protected runtime images + env: + BASE_DCODE: ${{ steps.runtime-bases.outputs.dcode }} + BASE_HERMES: ${{ steps.runtime-bases.outputs.hermes }} + BASE_OPENCLAW: ${{ steps.runtime-bases.outputs.openclaw }} + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + shell: bash + run: | + set -euo pipefail + scripts/checks/build-protected-managed-images.sh \ + --output "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" \ + --revision "$CHECKOUT_SHA" \ + --cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \ + --platform linux/amd64 \ + --openclaw-base "$BASE_OPENCLAW" \ + --hermes-base "$BASE_HERMES" \ + --dcode-base "$BASE_DCODE" + + - name: Install OpenShell CLI + run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh + + - name: Run all-agent GPU, local inference, rollback, and cleanup qualification + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + shell: bash + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + export OPENSHELL_BIN="$(command -v openshell)" + "$OPENSHELL_BIN" --version + npx tsx tools/e2e/live-vitest-invocation.mts run \ + --test-path test/e2e/live/managed-image-protected-runtime.test.ts + + - name: Remove isolated protected runtime registry + if: always() + shell: bash + run: | + set -euo pipefail + if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then + owner="$( + docker container inspect \ + --format '{{index .Config.Labels "io.nvidia.nemoclaw.e2e-owner"}}' \ + "$NEMOCLAW_PROTECTED_REGISTRY_NAME" + )" + [[ "$owner" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" ]] || { + echo "::error::Refusing to remove a runtime registry not owned by this protected job" >&2 + exit 1 + } + docker rm -f "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null + fi + if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then + echo "::error::Protected managed-image runtime registry remained after cleanup" >&2 + exit 1 + fi + if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then + echo "::error::Protected managed-image runtime registry listener remained after cleanup" >&2 + exit 1 + fi + + - name: Upload protected managed-image runtime artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: e2e-managed-image-protected-runtime + path: e2e-artifacts/live/managed-image-protected-runtime/ + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + agent-turn-latency: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',agent-turn-latency,') || contains(format(',{0},', inputs.targets), ',agent-turn-latency,') }} @@ -6070,6 +6336,7 @@ jobs: cloud-inference, gpu-e2e, managed-image-multiarch-startup, + managed-image-protected-runtime, agent-turn-latency, kimi-inference-compat, hermes-inference-switch, diff --git a/test/e2e/live/managed-image-protected-runtime-helpers.ts b/test/e2e/live/managed-image-protected-runtime-helpers.ts new file mode 100644 index 00000000000..e04ab3fac29 --- /dev/null +++ b/test/e2e/live/managed-image-protected-runtime-helpers.ts @@ -0,0 +1,408 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { + type ManagedImageLocalInferenceKind, + managedImageProtectedSandboxName, + PROTECTED_MANAGED_IMAGE_AGENTS, + type ProtectedManagedImageContract, + parseProtectedManagedImageContracts, +} from "../../../scripts/checks/managed-image-protected-runtime-contract.ts"; +import { + adoptServedModelId, + dockerLoginNgc, + pullNimImage, + startNimContainerByName, + stopNimContainerByName, + waitForNimHealth, +} from "../../../src/lib/inference/nim.ts"; +import { + getOllamaProxyToken, + killStaleProxy, + persistAndProbeOllamaProxy, + startOllamaAuthProxy, +} from "../../../src/lib/inference/ollama/proxy.ts"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import type { E2ETargetFixtures } from "../fixtures/e2e-test.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import { + assertNvidiaAvailable, + cleanupOllama, + ensureOllama, + env as gpuEnv, + REPO_ROOT, +} from "./gpu-e2e-helpers.ts"; + +const OLLAMA_MODEL = "qwen3.5:9b"; +const VLLM_MODEL = "Qwen/Qwen2.5-0.5B-Instruct"; +const VLLM_IMAGE = + "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416"; +const VLLM_CONTAINER = "nemoclaw-managed-image-vllm-e2e"; +const NIM_CATALOG_MODEL = "nvidia/nemotron-3-nano-30b-a3b"; +const NIM_CONTAINER = "nemoclaw-managed-image-nim-e2e"; + +type RuntimeFixtures = Pick; + +function imageContracts(): ProtectedManagedImageContract[] { + const contractPath = process.env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT; + if (!contractPath || !path.isAbsolute(contractPath)) { + throw new Error("NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT must be an absolute path"); + } + return parseProtectedManagedImageContracts( + JSON.parse(fs.readFileSync(contractPath, "utf8")), + "linux/amd64", + ); +} + +function requiredNgcApiKey(value: string): string { + const key = value.trim(); + if (!key || /[\0\r\n]/u.test(key)) { + throw new Error("protected managed-image NIM qualification requires NVIDIA_API_KEY"); + } + return key; +} + +async function runExactImageQualification( + host: HostCliClient, + contract: ProtectedManagedImageContract, + kind: ManagedImageLocalInferenceKind, + model: string, + extraEnv: NodeJS.ProcessEnv, +): Promise { + const sandboxName = managedImageProtectedSandboxName(contract.agent, kind); + const result = await host.command( + "npx", + [ + "--no-install", + "tsx", + "scripts/checks/run-managed-image-openshell-e2e.ts", + "--agent", + contract.agent, + "--image", + contract.reference, + "--sandbox", + sandboxName, + "--gpu", + "--local-provider", + kind, + "--model", + model, + ], + { + artifactName: `managed-image-${contract.agent}-${kind}`, + cwd: REPO_ROOT, + env: { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_NON_INTERACTIVE: "1", + ...extraEnv, + }, + timeoutMs: 20 * 60_000, + }, + ); + expect(result.exitCode, resultText(result)).toBe(0); + expect(result.stdout).toContain(`exact ${contract.agent} PR image ${contract.reference}`); + expect(result.stdout).toContain("real NVIDIA GPU access"); + expect(result.stdout).toContain(`${kind} inference.local completion`); +} + +async function qualifyEveryAgent( + host: HostCliClient, + contracts: readonly ProtectedManagedImageContract[], + kind: ManagedImageLocalInferenceKind, + model: string, + extraEnv: NodeJS.ProcessEnv, +): Promise { + for (const contract of contracts) { + await runExactImageQualification(host, contract, kind, model, extraEnv); + } +} + +async function startProtectedOllama(host: HostCliClient): Promise { + await ensureOllama(host); + await cleanupOllama(host, "pre-cleanup-managed-image-ollama"); + const start = await host.command( + "bash", + [ + "-lc", + `set -euo pipefail +OLLAMA_HOST=127.0.0.1:11434 nohup ollama serve >"${process.env.RUNNER_TEMP ?? "/tmp"}/managed-image-ollama.log" 2>&1 & +for _ in $(seq 1 120); do + curl -fsS --connect-timeout 2 http://127.0.0.1:11434/api/tags >/dev/null 2>&1 && exit 0 + sleep 1 +done +exit 1`, + ], + { + artifactName: "start-managed-image-ollama", + env: gpuEnv(), + timeoutMs: 150_000, + }, + ); + expect(start.exitCode, resultText(start)).toBe(0); + const pull = await host.command("ollama", ["pull", OLLAMA_MODEL], { + artifactName: "pull-managed-image-ollama-model", + env: gpuEnv(), + timeoutMs: 45 * 60_000, + }); + expect(pull.exitCode, resultText(pull)).toBe(0); + expect(startOllamaAuthProxy(), "Ollama auth proxy must start").toBe(true); + const proxyToken = getOllamaProxyToken(); + expect(proxyToken).toMatch(/^[a-f0-9]{48}$/u); + await persistAndProbeOllamaProxy(proxyToken!); + return proxyToken!; +} + +async function proveOllamaGpuPlacement(host: HostCliClient): Promise { + const result = await host.command( + "bash", + [ + "-lc", + `curl -fsS http://127.0.0.1:11434/api/ps | jq -e --arg model "${OLLAMA_MODEL}" ' + [.models[] | select((.name == $model or .model == $model) and ((.size_vram // 0) > 0))] + | length >= 1 + '`, + ], + { + artifactName: "ollama-gpu-placement", + env: gpuEnv(), + timeoutMs: 30_000, + }, + ); + expect(result.exitCode, resultText(result)).toBe(0); +} + +async function startProtectedVllm(host: HostCliClient): Promise { + await host.command("docker", ["rm", "-f", VLLM_CONTAINER], { + artifactName: "pre-cleanup-vllm", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + const start = await host.command( + "docker", + [ + "run", + "--detach", + "--name", + VLLM_CONTAINER, + "--gpus", + "all", + "--publish", + "8000:8000", + VLLM_IMAGE, + "--model", + VLLM_MODEL, + "--served-model-name", + VLLM_MODEL, + "--max-model-len", + "2048", + "--gpu-memory-utilization", + "0.45", + ], + { + artifactName: "start-vllm", + env: buildAvailabilityProbeEnv(), + timeoutMs: 20 * 60_000, + }, + ); + expect(start.exitCode, resultText(start)).toBe(0); + const ready = await host.command( + "bash", + [ + "-lc", + `set -euo pipefail +for _ in $(seq 1 300); do + curl -fsS --connect-timeout 2 http://127.0.0.1:8000/v1/models >/dev/null 2>&1 && exit 0 + docker container inspect "${VLLM_CONTAINER}" --format '{{.State.Running}}' | grep -Fx true >/dev/null + sleep 2 +done +docker logs "${VLLM_CONTAINER}" >&2 +exit 1`, + ], + { + artifactName: "wait-vllm", + env: buildAvailabilityProbeEnv(), + timeoutMs: 11 * 60_000, + }, + ); + expect(ready.exitCode, resultText(ready)).toBe(0); + const cuda = await host.command( + "docker", + [ + "exec", + VLLM_CONTAINER, + "python3", + "-c", + "import torch; assert torch.cuda.is_available(); print(torch.cuda.get_device_name(0))", + ], + { + artifactName: "vllm-cuda-initialization", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + expect(cuda.exitCode, resultText(cuda)).toBe(0); +} + +async function startProtectedNim(host: HostCliClient, apiKey: string): Promise { + stopNimContainerByName(NIM_CONTAINER, { silent: true }); + expect(dockerLoginNgc(apiKey), "NGC login must succeed for protected NIM qualification").toBe( + true, + ); + pullNimImage(NIM_CATALOG_MODEL); + startNimContainerByName(NIM_CONTAINER, NIM_CATALOG_MODEL, 8000, { ngcApiKey: apiKey }); + expect( + waitForNimHealth(8000, 20 * 60, { container: NIM_CONTAINER }), + "NIM must become healthy", + ).toBe(true); + const servedModel = adoptServedModelId(NIM_CATALOG_MODEL, 8000); + expect(servedModel, "NIM must report one safe served model").toBeTruthy(); + const cuda = await host.command("docker", ["exec", NIM_CONTAINER, "nvidia-smi", "-L"], { + artifactName: "nim-cuda-initialization", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + expect(cuda.exitCode, resultText(cuda)).toBe(0); + return servedModel!; +} + +async function qualifyRollback( + host: HostCliClient, + contract: ProtectedManagedImageContract, +): Promise { + const sandboxName = managedImageProtectedSandboxName(contract.agent, "rollback"); + const result = await host.command( + "npx", + [ + "--no-install", + "tsx", + "scripts/checks/run-managed-image-openshell-e2e.ts", + "--agent", + contract.agent, + "--image", + contract.reference, + "--sandbox", + sandboxName, + "--inject-bootstrap-completion-failure", + ], + { + artifactName: `managed-image-${contract.agent}-bootstrap-rollback`, + cwd: REPO_ROOT, + env: { ...buildAvailabilityProbeEnv(), NEMOCLAW_NON_INTERACTIVE: "1" }, + timeoutMs: 20 * 60_000, + }, + ); + expect(result.exitCode, resultText(result)).toBe(0); + expect(result.stdout).toContain( + `removed the failed exact ${contract.agent} sandbox before harness cleanup`, + ); + expect(result.stdout).toContain( + `left no sandbox, container, network, or harness state orphan for ${contract.agent}`, + ); +} + +async function qualifyEveryRollback( + host: HostCliClient, + contracts: readonly ProtectedManagedImageContract[], +): Promise { + for (const contract of contracts) await qualifyRollback(host, contract); +} + +async function proveOwnedRuntimeInventoryClean(host: HostCliClient): Promise { + const result = await host.command( + "bash", + [ + "-lc", + `set -euo pipefail +containers="$(docker ps -a --format '{{.Label "openshell.ai/sandbox-name"}}' --filter label=openshell.ai/managed-by=openshell | grep '^nemoclaw-managed-' || true)" +networks="$(docker network ls --format '{{.Name}}' | grep '^nemoclaw-managed-pr-' || true)" +test -z "$containers" +test -z "$networks"`, + ], + { + artifactName: "final-managed-image-owned-runtime-inventory", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(result.exitCode, resultText(result)).toBe(0); +} + +export async function qualifyProtectedManagedImageRuntime( + fixtures: RuntimeFixtures, + ngcApiKeyInput: string, +): Promise { + const { artifacts, cleanup, host, progress } = fixtures; + const contracts = imageContracts(); + const ngcApiKey = requiredNgcApiKey(ngcApiKeyInput); + + cleanup.trackDisposable("remove protected vLLM container", async () => { + await host.command("docker", ["rm", "-f", VLLM_CONTAINER], { + artifactName: "cleanup-vllm-container", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + }); + cleanup.trackDisposable("remove protected NIM container", () => { + stopNimContainerByName(NIM_CONTAINER, { silent: true }); + }); + cleanup.trackDisposable("stop protected Ollama runtime", async () => { + killStaleProxy(); + await cleanupOllama(host, "cleanup-managed-image-ollama"); + }); + + const docker = await host.command("docker", ["info"], { + artifactName: "docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(docker.exitCode, resultText(docker)).toBe(0); + const nvidia = await host.command("nvidia-smi", [], { + artifactName: "nvidia-smi", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + assertNvidiaAvailable(nvidia, (message) => { + throw new Error(message ?? "protected GPU runner is unavailable"); + }); + + progress.phase("qualify all managed agents with GPU-backed Ollama"); + const proxyToken = await startProtectedOllama(host); + await qualifyEveryAgent(host, contracts, "ollama", OLLAMA_MODEL, { + NEMOCLAW_OLLAMA_PROXY_TOKEN: proxyToken, + }); + await proveOllamaGpuPlacement(host); + killStaleProxy(); + await cleanupOllama(host, "stop-ollama-before-vllm"); + + progress.phase("qualify all managed agents with GPU-backed vLLM"); + await startProtectedVllm(host); + await qualifyEveryAgent(host, contracts, "vllm", VLLM_MODEL, { + NEMOCLAW_VLLM_LOCAL_TOKEN: "protected-local-vllm", + }); + await host.command("docker", ["rm", "-f", VLLM_CONTAINER], { + artifactName: "stop-vllm-before-nim", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + + progress.phase("qualify all managed agents with GPU-backed NVIDIA NIM"); + const nimModel = await startProtectedNim(host, ngcApiKey); + await qualifyEveryAgent(host, contracts, "nim", nimModel, { + NEMOCLAW_VLLM_LOCAL_TOKEN: "protected-local-nim", + }); + stopNimContainerByName(NIM_CONTAINER, { silent: true }); + + progress.phase("prove all-agent managed bootstrap rollback and exact cleanup"); + await qualifyEveryRollback(host, contracts); + await proveOwnedRuntimeInventoryClean(host); + await artifacts.writeJson("managed-image-protected-runtime-summary.json", { + agents: PROTECTED_MANAGED_IMAGE_AGENTS, + providers: ["ollama", "vllm", "nim"], + rollbackAgents: PROTECTED_MANAGED_IMAGE_AGENTS, + }); +} diff --git a/test/e2e/live/managed-image-protected-runtime.test.ts b/test/e2e/live/managed-image-protected-runtime.test.ts new file mode 100644 index 00000000000..581e08ca094 --- /dev/null +++ b/test/e2e/live/managed-image-protected-runtime.test.ts @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { test } from "../fixtures/e2e-test.ts"; +import { qualifyProtectedManagedImageRuntime } from "./managed-image-protected-runtime-helpers.ts"; + +const TIMEOUT_MS = 220 * 60_000; + +test("exact all-agent managed images retain GPU, Ollama, NIM, vLLM, rollback, and cleanup (#7744)", { + timeout: TIMEOUT_MS, + meta: { + e2ePhases: [ + "qualify all managed agents with GPU-backed Ollama", + "qualify all managed agents with GPU-backed vLLM", + "qualify all managed agents with GPU-backed NVIDIA NIM", + "prove all-agent managed bootstrap rollback and exact cleanup", + ], + }, +}, async ({ artifacts, cleanup, host, progress, secrets }) => { + await artifacts.target.declare({ + id: "managed-image-protected-runtime", + boundary: + "exact PR image digests for every managed agent through Docker/OpenShell GPU, host-local Ollama, NVIDIA NIM, vLLM, transactional rollback, and owned cleanup", + agents: ["openclaw", "hermes", "langchain-deepagents-code"], + providers: ["ollama", "nim", "vllm"], + credentialBoundary: + "The NVIDIA key is staged only to the host-side NGC login and NIM container; managed sandboxes receive only generated local route tokens.", + }); + await qualifyProtectedManagedImageRuntime( + { artifacts, cleanup, host, progress }, + secrets.optional("NVIDIA_API_KEY") ?? "", + ); +}); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 9f912d2fc21..ecb7113e029 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -11,6 +11,16 @@ "test/protected-managed-image-contract.test.ts" ] }, + { + "live": "test/e2e/live/managed-image-protected-runtime.test.ts", + "fast": [ + "src/lib/inference/nim.test.ts", + "src/lib/onboard/sandbox-gpu-create-flow.test.ts", + "test/e2e/support/managed-image-protected-runtime-workflow.test.ts", + "test/managed-image-protected-runtime-contract.test.ts", + "test/pr-risk-plan.test.ts" + ] + }, { "live": "test/e2e/live/hermes-gpu-startup.test.ts", "fast": [ diff --git a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts new file mode 100644 index 00000000000..3b460854084 --- /dev/null +++ b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import YAML from "yaml"; + +import { validateManagedImageProtectedRuntimeWorkflow } from "../../../tools/e2e/managed-image-protected-runtime-workflow-boundary.mts"; + +type WorkflowRecord = Record; + +function workflow(): WorkflowRecord { + return YAML.parse( + fs.readFileSync(path.resolve(__dirname, "../../../.github/workflows/e2e.yaml"), "utf8"), + ) as WorkflowRecord; +} + +function runtimeJob(value: WorkflowRecord): Record { + return (value.jobs as Record>)["managed-image-protected-runtime"]; +} + +function namedStep(value: WorkflowRecord, name: string): Record { + return (runtimeJob(value).steps as Array>).find( + (step) => step.name === name, + )!; +} + +describe("protected managed-image runtime workflow boundary", () => { + it("accepts the exact dormant trusted runtime lane", () => { + expect(validateManagedImageProtectedRuntimeWorkflow(workflow())).toEqual([]); + }); + + it("rejects job-scoped NGC credentials", () => { + const value = workflow(); + runtimeJob(value).env = { + ...(runtimeJob(value).env as Record), + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + }; + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toContain( + "managed-image-protected-runtime must not expose NVIDIA_API_KEY at job scope", + ); + }); + + it("rejects removing NIM from the activation contract", () => { + const value = workflow(); + const step = namedStep(value, "Validate protected runtime activation contract"); + step.run = String(step.run).replace( + '.providers == ["ollama", "nim", "vllm"]', + '.providers == ["ollama", "vllm"]', + ); + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toContain( + 'managed-image-protected-runtime step \'Validate protected runtime activation contract\' must include .providers == ["ollama", "nim", "vllm"]', + ); + }); + + it("rejects qualification before exact all-agent image construction", () => { + const value = workflow(); + const job = runtimeJob(value); + const workflowSteps = job.steps as Array>; + const qualification = namedStep( + value, + "Run all-agent GPU, local inference, rollback, and cleanup qualification", + ); + job.steps = [qualification, ...workflowSteps.filter((step) => step !== qualification)]; + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toContain( + "managed-image-protected-runtime protected qualification and cleanup steps drifted", + ); + }); +}); diff --git a/test/managed-image-protected-runtime-contract.test.ts b/test/managed-image-protected-runtime-contract.test.ts new file mode 100644 index 00000000000..6124a2a7b5a --- /dev/null +++ b/test/managed-image-protected-runtime-contract.test.ts @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { managedStartupE2eProfile } from "../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + MANAGED_IMAGE_LOCAL_INFERENCE_KINDS, + managedImageProtectedSandboxName, + resolveManagedImageLocalInferenceRoute, + withManagedImageLocalInferenceProfile, +} from "../scripts/checks/managed-image-protected-runtime-contract.ts"; +import { + managedImageOpenShellBasePolicyPath, + managedImageOpenShellCommittedProbe, + managedImageOpenShellProbe, + parseManagedImageOpenShellE2eInputs, +} from "../scripts/checks/run-managed-image-openshell-e2e.ts"; + +const IMAGE = `localhost:5000/nemoclaw-managed-protected/openclaw@sha256:${"a".repeat(64)}`; + +describe("protected managed-image runtime contract", () => { + it.each([ + ["ollama", "ollama-local", "NEMOCLAW_OLLAMA_PROXY_TOKEN", 11435], + ["nim", "vllm-local", "NEMOCLAW_VLLM_LOCAL_TOKEN", 8000], + ["vllm", "vllm-local", "NEMOCLAW_VLLM_LOCAL_TOKEN", 8000], + ] as const)("maps %s to its exact host-local route", (kind, provider, credential, port) => { + const route = resolveManagedImageLocalInferenceRoute(kind); + + expect(MANAGED_IMAGE_LOCAL_INFERENCE_KINDS).toContain(kind); + expect(route).toMatchObject({ kind, providerName: provider, credentialEnv: credential }); + expect(new URL(route.defaultBaseUrl)).toMatchObject({ + hostname: "host.openshell.internal", + port: String(port), + pathname: "/v1", + protocol: "http:", + }); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("binds %s to an exact GPU/local-inference launch", (agent) => { + const parsed = parseManagedImageOpenShellE2eInputs([ + "--agent", + agent, + "--image", + IMAGE, + "--sandbox", + managedImageProtectedSandboxName(agent, "nim"), + "--gpu", + "--local-provider", + "nim", + "--model", + "nvidia/nemotron-3-nano", + ]); + + expect(parsed).toEqual({ + agent, + gpu: true, + image: IMAGE, + localProvider: "nim", + model: "nvidia/nemotron-3-nano", + sandbox: managedImageProtectedSandboxName(agent, "nim"), + }); + expect(path.isAbsolute(managedImageOpenShellBasePolicyPath(agent))).toBe(true); + expect(managedImageOpenShellProbe(agent)).toContain("managed-startup-complete.json"); + }); + + it("rewrites only the inference route while preserving the managed agent profile", () => { + const profile = managedStartupE2eProfile("hermes", false, true, true); + const route = resolveManagedImageLocalInferenceRoute("nim"); + const rewritten = withManagedImageLocalInferenceProfile( + profile, + route, + "nvidia/nemotron-3-nano", + ); + + expect(rewritten).toMatchObject({ + agent: "hermes", + inference: { + api: "openai-completions", + model: "nvidia/nemotron-3-nano", + routedBaseUrl: "https://inference.local/v1", + routeProvider: "inference", + upstreamEndpointUrl: null, + upstreamProvider: "vllm-local", + }, + }); + expect(rewritten.agentConfig).toEqual(profile.agentConfig); + }); + + it("rejects mutable images and incomplete GPU provider tuples", () => { + expect(() => + parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + "localhost:5000/openclaw:latest", + "--sandbox", + "managed-openclaw", + ]), + ).toThrow(/immutable repository@sha256/u); + expect(() => + parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + "managed-openclaw", + "--gpu", + ]), + ).toThrow(/--gpu requires/u); + }); + + it("keeps rollback cleanup distinct from initial readiness", () => { + expect(managedImageOpenShellCommittedProbe()).toContain( + "managed-startup-shared-state-transaction-v1", + ); + expect( + parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + "managed-openclaw-rollback", + "--inject-bootstrap-completion-failure", + ]), + ).toMatchObject({ failureInjection: "bootstrap-completion" }); + }); +}); diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index edfdb3a340f..df1825b6eca 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -76,7 +76,7 @@ describe("deterministic PR risk plan", () => { const second = plan("src/lib/onboard.ts", "src/lib/state/registry.ts"); expect(first).toEqual(second); - expect(first.version).toBe(13); + expect(first.version).toBe(14); expect(first.headSha).toBe(HEAD_SHA); expect(first.planHash).toMatch(/^[a-f0-9]{64}$/u); expect(first.changedFiles).toEqual(["src/lib/onboard.ts", "src/lib/state/registry.ts"]); @@ -344,6 +344,29 @@ describe("deterministic PR risk plan", () => { ).toBe(false); }); + it("keeps protected GPU and local-inference qualification activation-only until trusted (#7744)", () => { + const activation = "ci/protected-managed-image-runtime-activation-v1.json"; + const result = plan(activation); + const dormantImplementation = plan( + "scripts/checks/run-managed-image-openshell-e2e.ts", + "test/e2e/live/managed-image-protected-runtime.test.ts", + ); + + expect(result.families).toContainEqual( + expect.objectContaining({ + id: "managed-image-protected-runtime", + matchedFiles: [activation], + requiredJobs: ["managed-image-protected-runtime"], + }), + ); + expect(riskPlanRequiredJobIds(result)).toEqual(["managed-image-protected-runtime"]); + expect( + dormantImplementation.families.some( + (family) => family.id === "managed-image-protected-runtime", + ), + ).toBe(false); + }); + it("runs snapshot commands for restored-gateway pairing runtime changes (#7431)", () => { const runtimeFiles = [ "src/lib/actions/sandbox/restore-gateway-pairing.ts", diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 8c615200ab3..e27216fd602 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; -export const RISK_PLAN_VERSION = 13 as const; +export const RISK_PLAN_VERSION = 14 as const; export const PR_E2E_TYPED_TARGET_IDS = [ "ubuntu-repo-cloud-langchain-deepagents-code", @@ -51,6 +51,8 @@ const HERMES_MANAGED_POLICY_FILES = new Set([ ]); const MANAGED_IMAGE_MULTIARCH_ACTIVATION = "ci/protected-managed-image-multiarch-activation-v1.json"; +const MANAGED_IMAGE_PROTECTED_RUNTIME_ACTIVATION = + "ci/protected-managed-image-runtime-activation-v1.json"; const MANAGED_IMAGE_MULTIARCH_INPUTS = new Set([ MANAGED_IMAGE_MULTIARCH_ACTIVATION, ".dockerignore", @@ -90,6 +92,7 @@ export type RiskFamilyId = | "credentials-security" | "e2e-control-plane" | "managed-image-multiarch" + | "managed-image-protected-runtime" | "sandbox-boundary" | "focused-e2e"; @@ -432,6 +435,23 @@ export const RISK_RULES: readonly RiskRule[] = [ MANAGED_IMAGE_MULTIARCH_CHILD_CREDENTIALS.test(file) || MANAGED_IMAGE_MULTIARCH_INPUT_PREFIXES.some((prefix) => file.startsWith(prefix)), }, + { + id: "managed-image-protected-runtime", + summary: + "Protected managed-image runtime qualification must retain real GPU access, host-local Ollama, NVIDIA NIM, vLLM, transactional rollback, and exact cleanup for every shipped agent.", + tier: 3, + requiredJobs: ["managed-image-protected-runtime"], + invariants: [ + "OpenClaw, Hermes, and Deep Agents Code run from exact PR image digests through the production managed-bootstrap path", + "real NVIDIA GPU access and host-local Ollama, NVIDIA NIM, and vLLM inference.local completions are all required", + "bootstrap completion failure removes the exact failed sandbox, container, network, and transaction state for every agent", + "NGC credentials remain host-scoped and never enter a managed sandbox or persisted artifact", + ], + // The trusted workflow and validator land before activation. The follow-on + // activation slice broadens this boundary to runtime inputs after the + // protected job exists on main and can safely qualify candidate code. + matches: (file) => file === MANAGED_IMAGE_PROTECTED_RUNTIME_ACTIVATION, + }, { id: "sandbox-boundary", summary: diff --git a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts new file mode 100644 index 00000000000..607de081537 --- /dev/null +++ b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts @@ -0,0 +1,274 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +type WorkflowRecord = Record; +type WorkflowStep = WorkflowRecord & { + env?: WorkflowRecord; + name?: string; + run?: string; + uses?: string; + with?: WorkflowRecord; +}; + +const JOB_ID = "managed-image-protected-runtime"; +const SELECTOR = + "${{ contains(format(',{0},', inputs.jobs), ',managed-image-protected-runtime,') || contains(format(',{0},', inputs.targets), ',managed-image-protected-runtime,') }}"; +const ACTIVATION_PATH = "ci/protected-managed-image-runtime-activation-v1.json"; +const LIVE_TEST_PATH = "test/e2e/live/managed-image-protected-runtime.test.ts"; +const REGISTRY_IMAGE = + "docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"; + +function record(value: unknown): WorkflowRecord { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as WorkflowRecord) + : {}; +} + +function steps(value: unknown): WorkflowStep[] { + return Array.isArray(value) ? (value as WorkflowStep[]) : []; +} + +function text(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +function requireStep( + errors: string[], + workflowSteps: readonly WorkflowStep[], + name: string, +): WorkflowStep | undefined { + const matches = workflowSteps.filter((step) => step.name === name); + if (matches.length !== 1) errors.push(`${JOB_ID} must define exactly one '${name}' step`); + return matches[0]; +} + +function requireValues( + errors: string[], + subject: string, + actual: WorkflowRecord, + expected: Readonly>, +): void { + for (const [key, value] of Object.entries(expected)) { + if (actual[key] !== value) errors.push(`${subject} must bind ${key} to ${String(value)}`); + } +} + +function requireFragments( + errors: string[], + step: WorkflowStep | undefined, + fragments: readonly string[], +): void { + const run = text(step?.run); + for (const fragment of fragments) { + if (!run.includes(fragment)) { + errors.push(`${JOB_ID} step '${step?.name ?? "missing"}' must include ${fragment}`); + } + } +} + +function requireOrderedSteps( + errors: string[], + workflowSteps: readonly WorkflowStep[], + names: readonly string[], +): void { + const indexes = names.map((name) => workflowSteps.findIndex((step) => step.name === name)); + if (indexes.some((index) => index < 0)) return; + if (indexes.some((index, offset) => offset > 0 && index <= indexes[offset - 1])) { + errors.push(`${JOB_ID} protected qualification and cleanup steps drifted`); + } +} + +export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowRecord): string[] { + const errors: string[] = []; + const job = record(record(workflow.jobs)[JOB_ID]); + if (Object.keys(job).length === 0) return [`workflow missing ${JOB_ID} job`]; + + if (job.needs !== "generate-matrix") errors.push(`${JOB_ID} must depend on generate-matrix`); + if (job.if !== SELECTOR) errors.push(`${JOB_ID} must remain explicit-only and selector-bound`); + if (job["runs-on"] !== "linux-amd64-gpu-rtxpro6000-latest-1") { + errors.push(`${JOB_ID} must run on the protected amd64 GPU runner`); + } + if (job["timeout-minutes"] !== 300) errors.push(`${JOB_ID} must keep the 300 minute timeout`); + if (record(job.permissions).contents !== "read") { + errors.push(`${JOB_ID} permissions must be contents: read`); + } + if (job["continue-on-error"] !== undefined) { + errors.push(`${JOB_ID} must not weaken failures with continue-on-error`); + } + + const jobEnv = record(job.env); + requireValues(errors, `${JOB_ID} env`, jobEnv, { + E2E_ARTIFACT_DIR: "${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime", + E2E_DEFAULT_ENABLED: "0", + E2E_JOB: "1", + E2E_TARGET_ID: JOB_ID, + RELEASE_E2E_ACTIVATION_PATH: ACTIVATION_PATH, + NEMOCLAW_E2E_SHARD: "linux-amd64-gpu", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: "${{ inputs.base_sha }}", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: + "protected-${{ github.run_id }}-${{ github.run_attempt }}", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT: + "${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime/contracts.json", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM: "linux/amd64", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA: "${{ inputs.workflow_sha }}", + NEMOCLAW_PROTECTED_REGISTRY_NAME: + "nemoclaw-managed-runtime-${{ github.run_id }}-${{ github.run_attempt }}", + NEMOCLAW_RUN_LIVE_E2E: "1", + }); + if (jobEnv.NVIDIA_API_KEY !== undefined) { + errors.push(`${JOB_ID} must not expose NVIDIA_API_KEY at job scope`); + } + + const workflowSteps = steps(job.steps); + const guard = requireStep( + errors, + workflowSteps, + "Validate protected runtime exact-head dispatch", + ); + requireValues(errors, `${JOB_ID} exact-head guard env`, record(guard?.env), { + ACTOR: "${{ github.actor }}", + BASE_SHA: "${{ inputs.base_sha }}", + CHECKOUT_SHA: "${{ inputs.checkout_sha }}", + EVENT_NAME: "${{ github.event_name }}", + EXPECTED_WORKFLOW_SHA: "${{ inputs.workflow_sha }}", + REF: "${{ github.ref }}", + REPOSITORY: "${{ github.repository }}", + RUNNER_ARCH_KIND: "${{ runner.arch }}", + WORKFLOW_SHA: "${{ github.workflow_sha }}", + }); + requireFragments(errors, guard, [ + '"NVIDIA/NemoClaw"', + '"refs/heads/main"', + '"workflow_dispatch"', + '"github-actions[bot]"', + '[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]]', + '"$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA"', + '"$RUNNER_ARCH_KIND" == "X64"', + ]); + + const checkouts = workflowSteps.filter((step) => text(step.uses).startsWith("actions/checkout@")); + if (checkouts.length !== 1) errors.push(`${JOB_ID} must define exactly one candidate checkout`); + requireValues(errors, `${JOB_ID} candidate checkout`, record(checkouts[0]?.with), { + repository: "${{ inputs.checkout_repository || github.repository }}", + ref: "${{ inputs.checkout_sha || github.sha }}", + "fetch-depth": 0, + "persist-credentials": false, + }); + + const buildx = requireStep(errors, workflowSteps, "Set up protected runtime Buildx"); + if (buildx?.uses !== "docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c") { + errors.push(`${JOB_ID} must pin the reviewed Buildx setup action`); + } + requireValues(errors, `${JOB_ID} Buildx setup`, record(buildx?.with), { + "driver-opts": "network=host", + "buildkitd-config-inline": '[registry."localhost:5000"]\n http = true\n', + }); + + const activation = requireStep( + errors, + workflowSteps, + "Validate protected runtime activation contract", + ); + requireFragments(errors, activation, [ + `activation="${ACTIVATION_PATH}"`, + '[[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]]', + '[[ -f "$activation" && ! -L "$activation" ]]', + '(keys | sort) == ["agents", "contractVersion", "jobId", "platform", "providers"]', + '.agents == ["openclaw", "hermes", "langchain-deepagents-code"]', + '.platform == "linux/amd64"', + '.providers == ["ollama", "nim", "vllm"]', + ]); + + const bases = requireStep(errors, workflowSteps, "Resolve exact amd64 runtime base images"); + requireFragments(errors, bases, [ + 'docker buildx imagetools inspect "$alias" --raw', + '.platform.os == "linux" and .platform.architecture == "amd64"', + 'reference="${repository}@${digest}"', + '"sha256:$(sha256sum "$exact_raw" | awk \'{print $1}\')" == "$digest"', + "ghcr.io/nvidia/nemoclaw/sandbox-base:latest", + "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest", + "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:latest", + ]); + + const registry = requireStep(errors, workflowSteps, "Start isolated protected runtime registry"); + requireFragments(errors, registry, [ + 'docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME"', + "http://127.0.0.1:5000/v2/", + "io.nvidia.nemoclaw.e2e-owner=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT}", + "--publish 127.0.0.1:5000:5000", + REGISTRY_IMAGE, + ]); + + const build = requireStep( + errors, + workflowSteps, + "Build exact all-agent protected runtime images", + ); + requireFragments(errors, build, [ + "scripts/checks/build-protected-managed-images.sh", + '--revision "$CHECKOUT_SHA"', + '--cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT"', + "--platform linux/amd64", + '--openclaw-base "$BASE_OPENCLAW"', + '--hermes-base "$BASE_HERMES"', + '--dcode-base "$BASE_DCODE"', + ]); + + const install = requireStep(errors, workflowSteps, "Install OpenShell CLI"); + requireFragments(errors, install, [ + "env -u DOCKER_CONFIG", + "-u NVIDIA_API_KEY", + "-u NVIDIA_INFERENCE_API_KEY", + "bash scripts/install-openshell.sh", + ]); + + const qualification = requireStep( + errors, + workflowSteps, + "Run all-agent GPU, local inference, rollback, and cleanup qualification", + ); + requireValues(errors, `${JOB_ID} qualification env`, record(qualification?.env), { + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + }); + requireFragments(errors, qualification, [ + 'export OPENSHELL_BIN="$(command -v openshell)"', + "tools/e2e/live-vitest-invocation.mts run", + `--test-path ${LIVE_TEST_PATH}`, + ]); + + const cleanup = requireStep(errors, workflowSteps, "Remove isolated protected runtime registry"); + if (cleanup?.if !== "always()") errors.push(`${JOB_ID} registry cleanup must always run`); + requireFragments(errors, cleanup, [ + "io.nvidia.nemoclaw.e2e-owner", + '[[ "$owner" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" ]]', + 'docker rm -f "$NEMOCLAW_PROTECTED_REGISTRY_NAME"', + "http://127.0.0.1:5000/v2/", + ]); + + const upload = requireStep( + errors, + workflowSteps, + "Upload protected managed-image runtime artifacts", + ); + if (upload?.if !== "always()") errors.push(`${JOB_ID} artifact upload must always run`); + requireValues(errors, `${JOB_ID} artifact upload`, record(upload?.with), { + name: "e2e-managed-image-protected-runtime", + path: "e2e-artifacts/live/managed-image-protected-runtime/", + }); + requireStep(errors, workflowSteps, "Clean up Docker auth"); + requireOrderedSteps(errors, workflowSteps, [ + "Validate protected runtime exact-head dispatch", + "Validate protected runtime activation contract", + "Resolve exact amd64 runtime base images", + "Start isolated protected runtime registry", + "Build exact all-agent protected runtime images", + "Install OpenShell CLI", + "Run all-agent GPU, local inference, rollback, and cleanup qualification", + "Remove isolated protected runtime registry", + "Upload protected managed-image runtime artifacts", + "Clean up Docker auth", + ]); + + return errors; +} diff --git a/tools/e2e/prepare-e2e-workflow-boundary.mts b/tools/e2e/prepare-e2e-workflow-boundary.mts index d1c6d4e83ff..79a05ad6ee1 100644 --- a/tools/e2e/prepare-e2e-workflow-boundary.mts +++ b/tools/e2e/prepare-e2e-workflow-boundary.mts @@ -28,6 +28,7 @@ const NO_BUILD_JOBS = new Set([ "generate-matrix", "bootstrap-install-smoke", "managed-image-multiarch-startup", + "managed-image-protected-runtime", "ollama-auth-proxy", "security-posture", "shields-config", diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 481196cad25..774a40b4cef 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -150,6 +150,13 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ path: "e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/", }, ], + [ + "managed-image-protected-runtime", + { + name: "e2e-managed-image-protected-runtime", + path: "e2e-artifacts/live/managed-image-protected-runtime/", + }, + ], [ "network-policy", { diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 02aa66417c1..70d96a69d74 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -27,6 +27,7 @@ import { validateInferenceSwitchWorkflow, } from "./inference-switch-workflow-boundary.mts"; import { validateManagedImageMultiarchWorkflow } from "./managed-image-multiarch-workflow-boundary.mts"; +import { validateManagedImageProtectedRuntimeWorkflow } from "./managed-image-protected-runtime-workflow-boundary.mts"; import { type OpenClawPluginRuntimeExdevWorkflow, validateOpenClawPluginRuntimeExdevWorkflow, @@ -4239,6 +4240,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { errors.push(...validateHermesGpuStartupWorkflow(workflow)); errors.push(...validateInferenceSwitchWorkflow(workflow as unknown as InferenceSwitchWorkflow)); errors.push(...validateManagedImageMultiarchWorkflow(workflow)); + errors.push(...validateManagedImageProtectedRuntimeWorkflow(workflow)); errors.push( ...validateOpenClawPluginRuntimeExdevWorkflow( workflow as unknown as OpenClawPluginRuntimeExdevWorkflow, From ba4a0b9e6746496276c1770a9d94cc271c3f4eea Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 09:08:29 -0700 Subject: [PATCH 14/24] fix(images): close protected qualification review gaps Signed-off-by: Aaron Erickson --- .../scripts/release-e2e-evidence.mts | 14 ++-- .github/workflows/base-image.yaml | 15 +++- .github/workflows/e2e.yaml | 1 + .../protected-managed-image-contract.ts | 13 ++-- scripts/checks/validate-managed-base-index.sh | 51 +++++++++++++ test/dcode-base-image-workflow.test.ts | 1 + ...managed-image-multiarch-startup-helpers.ts | 32 +++++---- .../managed-image-multiarch-startup.test.ts | 7 +- test/e2e/mock-parity.json | 2 + ...ed-image-multiarch-startup-helpers.test.ts | 70 ++++++++++++++++++ ...managed-image-publication-workflow.test.ts | 1 + test/pr-risk-plan.test.ts | 13 +++- test/release-e2e-evidence.test.ts | 11 ++- test/validate-managed-base-index.test.ts | 72 +++++++++++++++++++ tools/advisors/risk-plan.mts | 13 ++-- ...aged-image-multiarch-workflow-boundary.mts | 13 ++-- 16 files changed, 286 insertions(+), 43 deletions(-) create mode 100755 scripts/checks/validate-managed-base-index.sh create mode 100644 test/e2e/support/managed-image-multiarch-startup-helpers.test.ts create mode 100644 test/validate-managed-base-index.test.ts diff --git a/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts b/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts index 0612394901d..3f13ba96570 100644 --- a/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts +++ b/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts @@ -292,13 +292,17 @@ function releaseActivationPath(job: JsonRecord, jobId: string): string | undefin function candidatePathExists(candidateSha: string, candidatePath: string): boolean { try { - execFileSync("git", ["cat-file", "-e", `${candidateSha}:${candidatePath}`], { + const output = execFileSync("git", ["ls-tree", "--name-only", candidateSha, "--", candidatePath], { cwd: REPO_ROOT, - stdio: "ignore", + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], }); - return true; - } catch { - return false; + return output.trim() === candidatePath; + } catch (error) { + throw new Error( + `could not inspect release E2E activation path ${candidatePath} at candidate ${candidateSha}`, + { cause: error }, + ); } } diff --git a/.github/workflows/base-image.yaml b/.github/workflows/base-image.yaml index 524f48e5808..b993207cf32 100644 --- a/.github/workflows/base-image.yaml +++ b/.github/workflows/base-image.yaml @@ -557,7 +557,10 @@ jobs: exit 1 fi reference="$IMAGE@$digest" - docker buildx imagetools inspect "$reference" >/dev/null + scripts/checks/validate-managed-base-index.sh \ + "$reference" \ + "${platform_digests[linux/amd64]}" \ + "${platform_digests[linux/arm64]}" scripts/export-managed-base-image-contract.sh \ "$AGENT" \ @@ -701,7 +704,10 @@ jobs: exit 1 fi reference="$IMAGE@$digest" - docker buildx imagetools inspect "$reference" >/dev/null + scripts/checks/validate-managed-base-index.sh \ + "$reference" \ + "${platform_digests[linux/amd64]}" \ + "${platform_digests[linux/arm64]}" scripts/export-managed-base-image-contract.sh \ "$AGENT" \ @@ -847,7 +853,10 @@ jobs: exit 1 fi reference="$IMAGE@$digest" - docker buildx imagetools inspect "$reference" >/dev/null + scripts/checks/validate-managed-base-index.sh \ + "$reference" \ + "${platform_digests[linux/amd64]}" \ + "${platform_digests[linux/arm64]}" scripts/export-managed-base-image-contract.sh \ "$AGENT" \ diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 7c67bd097a9..0ad3a9fffbf 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1730,6 +1730,7 @@ jobs: E2E_JOB: "1" E2E_TARGET_ID: "managed-image-multiarch-startup" RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-multiarch-activation-v1.json + NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} NEMOCLAW_E2E_SHARD: ${{ matrix.shard }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: ${{ inputs.base_sha }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: protected-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/scripts/checks/protected-managed-image-contract.ts b/scripts/checks/protected-managed-image-contract.ts index 9fe25a9228b..8bff94773d6 100644 --- a/scripts/checks/protected-managed-image-contract.ts +++ b/scripts/checks/protected-managed-image-contract.ts @@ -62,8 +62,9 @@ export type ProtectedManagedImageEvidenceIdentity = Pick< }; const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/u; -const SHA_PATTERN = /^[a-f0-9]{40}$/u; -const COHORT_PATTERN = /^protected-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$/u; +export const PROTECTED_MANAGED_IMAGE_SHA_PATTERN = /^[a-f0-9]{40}$/u; +export const PROTECTED_MANAGED_IMAGE_COHORT_PATTERN = + /^protected-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$/u; const BASE_REPOSITORIES: Readonly> = Object.freeze({ openclaw: "ghcr.io/nvidia/nemoclaw/sandbox-base", hermes: "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base", @@ -230,11 +231,11 @@ export function parseProtectedManagedImageEvidence( typeof evidence.headSha !== "string" || typeof evidence.baseSha !== "string" || typeof evidence.workflowSha !== "string" || - !SHA_PATTERN.test(evidence.headSha) || - !SHA_PATTERN.test(evidence.baseSha) || - !SHA_PATTERN.test(evidence.workflowSha) || + !PROTECTED_MANAGED_IMAGE_SHA_PATTERN.test(evidence.headSha) || + !PROTECTED_MANAGED_IMAGE_SHA_PATTERN.test(evidence.baseSha) || + !PROTECTED_MANAGED_IMAGE_SHA_PATTERN.test(evidence.workflowSha) || typeof evidence.cohort !== "string" || - !COHORT_PATTERN.test(evidence.cohort) || + !PROTECTED_MANAGED_IMAGE_COHORT_PATTERN.test(evidence.cohort) || typeof evidence.contractSha256 !== "string" || !DIGEST_PATTERN.test(evidence.contractSha256) ) { diff --git a/scripts/checks/validate-managed-base-index.sh b/scripts/checks/validate-managed-base-index.sh new file mode 100755 index 00000000000..20f6b398260 --- /dev/null +++ b/scripts/checks/validate-managed-base-index.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [ "$#" -ne 3 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +reference="$1" +expected_amd64="$2" +expected_arm64="$3" + +if [[ ! "$reference" =~ @sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: managed base index reference must be immutable." >&2 + exit 1 +fi +for expected in "$expected_amd64" "$expected_arm64"; do + if [[ ! "$expected" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "ERROR: managed base platform digest is invalid." >&2 + exit 1 + fi +done + +index_json="$(docker buildx imagetools inspect "$reference" --raw)" +if ! jq -e '.manifests | type == "array"' <<<"$index_json" >/dev/null; then + echo "ERROR: managed base index does not contain a manifest array." >&2 + exit 1 +fi + +for arch in amd64 arm64; do + mapfile -t actual_digests < <( + jq -r --arg arch "$arch" \ + '.manifests[] | select(.platform.os == "linux" and .platform.architecture == $arch) | .digest' \ + <<<"$index_json" + ) + if [ "${#actual_digests[@]}" -ne 1 ]; then + echo "ERROR: managed base index must contain exactly one linux/$arch descriptor." >&2 + exit 1 + fi + case "$arch" in + amd64) expected_digest="$expected_amd64" ;; + arm64) expected_digest="$expected_arm64" ;; + esac + if [ "${actual_digests[0]}" != "$expected_digest" ]; then + echo "ERROR: managed base index linux/$arch descriptor does not match this run's platform digest." >&2 + exit 1 + fi +done diff --git a/test/dcode-base-image-workflow.test.ts b/test/dcode-base-image-workflow.test.ts index eaa66f3f7b6..503b790ff5c 100644 --- a/test/dcode-base-image-workflow.test.ts +++ b/test/dcode-base-image-workflow.test.ts @@ -590,6 +590,7 @@ describe("base-image publication behavior", () => { 'docker buildx imagetools create "${tag_args[@]}" "${sources[@]}"', ); expect(manifestScript).toContain('"amd64,arm64"'); + expect(manifestScript).toContain("scripts/checks/validate-managed-base-index.sh"); expect(manifestScript).toContain("scripts/export-managed-base-image-contract.sh"); for (const step of (manifestJob?.steps ?? []).filter((step) => step.uses)) { expect(step.uses, step.name).toMatch(FULL_SHA_ACTION); diff --git a/test/e2e/live/managed-image-multiarch-startup-helpers.ts b/test/e2e/live/managed-image-multiarch-startup-helpers.ts index a863c3dfbe3..82b749c2e5f 100644 --- a/test/e2e/live/managed-image-multiarch-startup-helpers.ts +++ b/test/e2e/live/managed-image-multiarch-startup-helpers.ts @@ -5,13 +5,12 @@ import fs from "node:fs"; import path from "node:path"; import { + PROTECTED_MANAGED_IMAGE_COHORT_PATTERN, PROTECTED_MANAGED_IMAGE_PLATFORMS, + PROTECTED_MANAGED_IMAGE_SHA_PATTERN, type ProtectedManagedImagePlatform, } from "../../../scripts/checks/protected-managed-image-contract.ts"; -const SHA_PATTERN = /^[a-f0-9]{40}$/u; -const COHORT_PATTERN = /^protected-[1-9][0-9]{0,19}-[1-9][0-9]{0,9}$/u; - export interface ProtectedManagedImageDispatchEnvironment { artifactDirectory: string; baseSha: string; @@ -49,10 +48,10 @@ export function protectedManagedImageDispatchEnvironment(): ProtectedManagedImag if ( !(PROTECTED_MANAGED_IMAGE_PLATFORMS as readonly string[]).includes(platform) || - !COHORT_PATTERN.test(cohort) || - !SHA_PATTERN.test(headSha) || - !SHA_PATTERN.test(baseSha) || - !SHA_PATTERN.test(workflowSha) + !PROTECTED_MANAGED_IMAGE_COHORT_PATTERN.test(cohort) || + !PROTECTED_MANAGED_IMAGE_SHA_PATTERN.test(headSha) || + !PROTECTED_MANAGED_IMAGE_SHA_PATTERN.test(baseSha) || + !PROTECTED_MANAGED_IMAGE_SHA_PATTERN.test(workflowSha) ) { throw new Error("protected managed-image dispatch identity is invalid"); } @@ -73,13 +72,22 @@ export function protectedManagedImageDispatchEnvironment(): ProtectedManagedImag } export function readRegularArtifact(file: string, artifactDirectory: string): Buffer { - const relative = path.relative(fs.realpathSync(artifactDirectory), fs.realpathSync(file)); + const root = fs.realpathSync(artifactDirectory); + const parent = fs.realpathSync(path.dirname(file)); + const candidate = path.join(parent, path.basename(file)); + const relative = path.relative(root, candidate); if (!relative || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { throw new Error(`${file} must be a child of the protected artifact directory`); } - const status = fs.lstatSync(file); - if (!status.isFile() || status.isSymbolicLink() || status.size > 1024 * 1024) { - throw new Error(`${file} must be a bounded regular file`); + + const descriptor = fs.openSync(candidate, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + const status = fs.fstatSync(descriptor); + if (!status.isFile() || status.size > 1024 * 1024) { + throw new Error(`${file} must be a bounded regular file`); + } + return fs.readFileSync(descriptor); + } finally { + fs.closeSync(descriptor); } - return fs.readFileSync(file); } diff --git a/test/e2e/live/managed-image-multiarch-startup.test.ts b/test/e2e/live/managed-image-multiarch-startup.test.ts index af4e7e33425..b4629a32150 100644 --- a/test/e2e/live/managed-image-multiarch-startup.test.ts +++ b/test/e2e/live/managed-image-multiarch-startup.test.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { createHash } from "node:crypto"; -import fs from "node:fs"; import path from "node:path"; import { @@ -30,10 +29,8 @@ test("binds protected all-agent direct startup to the exact multiarch dispatch ( const dispatch = protectedManagedImageDispatchEnvironment(); const activationPath = path.join(dispatch.workspace, PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH); - const activationStatus = fs.lstatSync(activationPath); - expect(activationStatus.isFile()).toBe(true); - expect(activationStatus.isSymbolicLink()).toBe(false); - parseProtectedManagedImageActivation(JSON.parse(fs.readFileSync(activationPath, "utf8"))); + const activationBytes = readRegularArtifact(activationPath, dispatch.workspace); + parseProtectedManagedImageActivation(JSON.parse(activationBytes.toString("utf8"))); progress.phase("validate exact all-agent managed-image contracts"); const contractBytes = readRegularArtifact(dispatch.contractFile, dispatch.artifactDirectory); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 9f912d2fc21..93d290e6363 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -6,6 +6,8 @@ "live": "test/e2e/live/managed-image-multiarch-startup.test.ts", "fast": [ "test/e2e/support/base-image-publication.test.ts", + "test/e2e/support/e2e-workflow.test.ts", + "test/e2e/support/managed-image-multiarch-startup-helpers.test.ts", "test/managed-image-publication-workflow.test.ts", "test/pr-risk-plan.test.ts", "test/protected-managed-image-contract.test.ts" diff --git a/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts new file mode 100644 index 00000000000..ed5de9a4929 --- /dev/null +++ b/test/e2e/support/managed-image-multiarch-startup-helpers.test.ts @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + protectedManagedImageDispatchEnvironment, + readRegularArtifact, +} from "../live/managed-image-multiarch-startup-helpers.ts"; + +const sha = "a".repeat(40); +let temporaryRoot = ""; + +beforeEach(() => { + temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-multiarch-helper-")); + const artifacts = path.join(temporaryRoot, "artifacts"); + fs.mkdirSync(artifacts); + vi.stubEnv("E2E_ARTIFACT_DIR", artifacts); + vi.stubEnv("GITHUB_RUN_ATTEMPT", "1"); + vi.stubEnv("GITHUB_RUN_ID", "123"); + vi.stubEnv("GITHUB_WORKSPACE", temporaryRoot); + vi.stubEnv("NEMOCLAW_E2E_EXPECTED_SHA", sha); + vi.stubEnv("NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA", sha); + vi.stubEnv("NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT", "protected-123-1"); + vi.stubEnv("NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT", path.join(artifacts, "contract.json")); + vi.stubEnv("NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE", path.join(artifacts, "evidence.json")); + vi.stubEnv("NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM", "linux/amd64"); + vi.stubEnv("NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA", sha); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + fs.rmSync(temporaryRoot, { force: true, recursive: true }); +}); + +describe("protected managed-image startup helpers", () => { + it("parses exact protected dispatch identity", () => { + expect(protectedManagedImageDispatchEnvironment()).toMatchObject({ + baseSha: sha, + cohort: "protected-123-1", + headSha: sha, + platform: "linux/amd64", + runAttempt: 1, + runId: 123, + workflowSha: sha, + }); + }); + + it("rejects identity values outside the canonical contract", () => { + vi.stubEnv("NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT", "other-123-1"); + expect(() => protectedManagedImageDispatchEnvironment()).toThrow( + "protected managed-image dispatch identity is invalid", + ); + }); + + it("reads a bounded file through its opened descriptor and rejects a symlink", () => { + const artifacts = path.join(temporaryRoot, "artifacts"); + const artifact = path.join(artifacts, "contract.json"); + const symlink = path.join(artifacts, "contract-link.json"); + fs.writeFileSync(artifact, '{"contractVersion":1}'); + fs.symlinkSync(artifact, symlink); + + expect(readRegularArtifact(artifact, artifacts).toString("utf8")).toBe('{"contractVersion":1}'); + expect(() => readRegularArtifact(symlink, artifacts)).toThrow(); + }); +}); diff --git a/test/managed-image-publication-workflow.test.ts b/test/managed-image-publication-workflow.test.ts index 95e7dd6df33..86d532add15 100644 --- a/test/managed-image-publication-workflow.test.ts +++ b/test/managed-image-publication-workflow.test.ts @@ -266,6 +266,7 @@ describe("complete managed-image publication workflow", () => { expect(manifest.env?.AGENT).toBe(expectedPublisher.agent); expect(manifest.run).toContain('reference="$IMAGE@$digest"'); expect(manifest.run).toContain("--format '{{.Manifest.Digest}}'"); + expect(manifest.run).toContain("scripts/checks/validate-managed-base-index.sh"); expect(manifest.run).toContain("scripts/export-managed-base-image-contract.sh"); expect(manifest.run).toContain('"${platform_digests[linux/amd64]}"'); expect(manifest.run).toContain('"${platform_digests[linux/arm64]}"'); diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index 07fad134f43..b33391ca980 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -316,7 +316,12 @@ describe("deterministic PR risk plan", () => { it("keeps the protected managed-image lane dormant until its trusted activation marker (#7744)", () => { const activation = "ci/protected-managed-image-multiarch-activation-v1.json"; const result = plan(activation); - const preActivationRuntime = plan("scripts/checks/run-managed-image-direct-e2e.ts"); + const preActivationPaths = [ + "scripts/checks/run-managed-image-direct-e2e.ts", + "scripts/checks/build-protected-managed-images.sh", + "scripts/checks/protected-managed-image-contract.ts", + "test/e2e/live/managed-image-multiarch-startup.test.ts", + ]; expect(result.families).toContainEqual( expect.objectContaining({ @@ -326,7 +331,11 @@ describe("deterministic PR risk plan", () => { }), ); expect(riskPlanRequiredJobIds(result)).toEqual(["managed-image-multiarch-startup"]); - expect(preActivationRuntime.families).toEqual([]); + for (const file of preActivationPaths) { + expect(plan(file).families.map((family) => family.id)).not.toContain( + "managed-image-multiarch", + ); + } }); it("runs snapshot commands for restored-gateway pairing runtime changes (#7431)", () => { diff --git a/test/release-e2e-evidence.test.ts b/test/release-e2e-evidence.test.ts index 1431a359414..8232806ed3d 100644 --- a/test/release-e2e-evidence.test.ts +++ b/test/release-e2e-evidence.test.ts @@ -21,7 +21,7 @@ function preflight( ) { return buildReleaseE2ePreflight({ candidateSha, - candidatePathExists: input.candidatePathExists, + candidatePathExists: input.candidatePathExists ?? (() => false), jetsonRunnerOnline: input.jetsonRunnerOnline ?? "true", }); } @@ -125,6 +125,15 @@ describe("release E2E evidence", () => { ).toHaveLength(2); }); + it("fails when the candidate commit cannot be inspected for activation paths", () => { + expect(() => + buildReleaseE2ePreflight({ + candidateSha: "0".repeat(40), + jetsonRunnerOnline: "true", + }), + ).toThrow("could not inspect release E2E activation path"); + }); + it("keeps every static and dynamic matrix row as a distinct execution", () => { const plan = preflight(); const ids = plan.executions.map((execution) => execution.id); diff --git a/test/validate-managed-base-index.test.ts b/test/validate-managed-base-index.test.ts new file mode 100644 index 00000000000..cea24bda674 --- /dev/null +++ b/test/validate-managed-base-index.test.ts @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const repoRoot = path.resolve(import.meta.dirname, ".."); +const validator = path.join(repoRoot, "scripts/checks/validate-managed-base-index.sh"); +const amd64Digest = `sha256:${"a".repeat(64)}`; +const arm64Digest = `sha256:${"b".repeat(64)}`; +const indexDigest = `sha256:${"c".repeat(64)}`; + +function index(amd64: string, arm64: string): string { + return JSON.stringify({ + schemaVersion: 2, + manifests: [ + { digest: amd64, platform: { architecture: "amd64", os: "linux" } }, + { digest: arm64, platform: { architecture: "arm64", os: "linux" } }, + ], + }); +} + +describe("managed base index validation", () => { + it("rejects a retagged index whose platform descriptors came from another run", () => { + const temporaryRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-base-index-")); + const fakeBin = path.join(temporaryRoot, "bin"); + const rawIndex = path.join(temporaryRoot, "index.json"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "docker"), + `#!/usr/bin/env bash +set -euo pipefail +test "\${1:-} \${2:-} \${3:-} \${5:-}" = "buildx imagetools inspect --raw" +cat "$RAW_INDEX" +`, + { mode: 0o755 }, + ); + + const run = () => + spawnSync( + validator, + [`ghcr.io/nvidia/nemoclaw/base@${indexDigest}`, amd64Digest, arm64Digest], + { + encoding: "utf8", + env: { + ...process.env, + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + RAW_INDEX: rawIndex, + }, + }, + ); + + try { + fs.writeFileSync(rawIndex, index(amd64Digest, arm64Digest)); + const accepted = run(); + expect(accepted.status, accepted.stderr).toBe(0); + + fs.writeFileSync(rawIndex, index(`sha256:${"d".repeat(64)}`, arm64Digest)); + const retagged = run(); + expect(retagged.status).not.toBe(0); + expect(retagged.stderr).toContain( + "linux/amd64 descriptor does not match this run's platform digest", + ); + } finally { + fs.rmSync(temporaryRoot, { force: true, recursive: true }); + } + }); +}); diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 8655c85193f..26576243083 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -3,6 +3,11 @@ import { createHash } from "node:crypto"; +import { + PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, + PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID, +} from "../../scripts/checks/protected-managed-image-contract.ts"; + export const RISK_PLAN_VERSION = 13 as const; export const PR_E2E_TYPED_TARGET_IDS = [ @@ -49,8 +54,6 @@ const HERMES_MANAGED_POLICY_FILES = new Set([ "agents/hermes/start.sh", "src/lib/hermes-managed-route.ts", ]); -const MANAGED_IMAGE_MULTIARCH_ACTIVATION = - "ci/protected-managed-image-multiarch-activation-v1.json"; export type RiskTier = 0 | 1 | 2 | 3; export type RiskFamilyId = @@ -390,11 +393,11 @@ export const RISK_RULES: readonly RiskRule[] = [ summary: "Protected managed-image qualification must build and directly start every shipped agent on each supported architecture from exact base and candidate digests.", tier: 3, - requiredJobs: ["managed-image-multiarch-startup"], + requiredJobs: [PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID], invariants: [ "OpenClaw, Hermes, and Deep Agents Code use platform-specific digest-pinned bases from one exact PR head and cohort", "each built image is addressed by its isolated-registry digest and exercises the managed root-stdin and sandbox-hold startup boundary", - "amd64 and arm64 shards emit exact head, base, platform, cohort, base, image, and direct-start evidence before cleanup", + "amd64 and arm64 shards emit exact head, base, platform, cohort, image, and direct-start evidence before cleanup", "the isolated registry is removed before a shard can publish passing risk evidence", ], // Bootstrap contract: this first trusted-controller slice recognizes only @@ -402,7 +405,7 @@ export const RISK_RULES: readonly RiskRule[] = [ // broadens the runtime paths after this job exists on trusted main, which // lets the follow-on prove its own exact head without loading PR-authored // workflow structure into the controller. - matches: (file) => file === MANAGED_IMAGE_MULTIARCH_ACTIVATION, + matches: (file) => file === PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, }, { id: "sandbox-boundary", diff --git a/tools/e2e/managed-image-multiarch-workflow-boundary.mts b/tools/e2e/managed-image-multiarch-workflow-boundary.mts index a828feb39a8..e76b4ab6f1a 100644 --- a/tools/e2e/managed-image-multiarch-workflow-boundary.mts +++ b/tools/e2e/managed-image-multiarch-workflow-boundary.mts @@ -3,6 +3,11 @@ import { isDeepStrictEqual } from "node:util"; +import { + PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, + PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID, +} from "../../scripts/checks/protected-managed-image-contract.ts"; + type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { env?: WorkflowRecord; @@ -12,10 +17,9 @@ type WorkflowStep = WorkflowRecord & { with?: WorkflowRecord; }; -const JOB_ID = "managed-image-multiarch-startup"; -const SELECTOR = - "${{ contains(format(',{0},', inputs.jobs), ',managed-image-multiarch-startup,') || contains(format(',{0},', inputs.targets), ',managed-image-multiarch-startup,') }}"; -const ACTIVATION_PATH = "ci/protected-managed-image-multiarch-activation-v1.json"; +const JOB_ID = PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID; +const SELECTOR = `\${{ contains(format(',{0},', inputs.jobs), ',${JOB_ID},') || contains(format(',{0},', inputs.targets), ',${JOB_ID},') }}`; +const ACTIVATION_PATH = PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH; const DIRECT_TEST_PATH = "test/e2e/live/managed-image-multiarch-startup.test.ts"; const REGISTRY_IMAGE = "docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"; @@ -121,6 +125,7 @@ export function validateManagedImageMultiarchWorkflow(workflow: WorkflowRecord): E2E_DEFAULT_ENABLED: "0", E2E_JOB: "1", E2E_TARGET_ID: JOB_ID, + NEMOCLAW_E2E_EXPECTED_SHA: "${{ inputs.checkout_sha }}", NEMOCLAW_E2E_SHARD: "${{ matrix.shard }}", NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: "${{ inputs.base_sha }}", NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: From 52954545ee9fc8e969d7c099edccad03376c5160 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 09:28:35 -0700 Subject: [PATCH 15/24] fix(ci): normalize protected contract loading Signed-off-by: Aaron Erickson --- tools/advisors/risk-plan.mts | 18 ++++++++++++++---- ...naged-image-multiarch-workflow-boundary.mts | 18 ++++++++++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 26576243083..1bfc6a03c66 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -3,10 +3,20 @@ import { createHash } from "node:crypto"; -import { - PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, - PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID, -} from "../../scripts/checks/protected-managed-image-contract.ts"; +import * as importedProtectedManagedImageContract from "../../scripts/checks/protected-managed-image-contract.ts"; + +// The root TypeScript package is exposed as CJS under the exact +// `node --import tsx` workflow execution mode, but as an ESM namespace under +// Vitest. Normalize both representations before reading shared identifiers. +const protectedManagedImageContract = ( + "default" in importedProtectedManagedImageContract && + importedProtectedManagedImageContract.default + ? importedProtectedManagedImageContract.default + : importedProtectedManagedImageContract +) as typeof import("../../scripts/checks/protected-managed-image-contract.ts"); + +const { PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID } = + protectedManagedImageContract; export const RISK_PLAN_VERSION = 13 as const; diff --git a/tools/e2e/managed-image-multiarch-workflow-boundary.mts b/tools/e2e/managed-image-multiarch-workflow-boundary.mts index e76b4ab6f1a..151506d6f7a 100644 --- a/tools/e2e/managed-image-multiarch-workflow-boundary.mts +++ b/tools/e2e/managed-image-multiarch-workflow-boundary.mts @@ -3,10 +3,20 @@ import { isDeepStrictEqual } from "node:util"; -import { - PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, - PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID, -} from "../../scripts/checks/protected-managed-image-contract.ts"; +import * as importedProtectedManagedImageContract from "../../scripts/checks/protected-managed-image-contract.ts"; + +// The root TypeScript package is exposed as CJS under the exact +// `node --import tsx` workflow execution mode, but as an ESM namespace under +// Vitest. Normalize both representations before reading shared identifiers. +const protectedManagedImageContract = ( + "default" in importedProtectedManagedImageContract && + importedProtectedManagedImageContract.default + ? importedProtectedManagedImageContract.default + : importedProtectedManagedImageContract +) as typeof import("../../scripts/checks/protected-managed-image-contract.ts"); + +const { PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID } = + protectedManagedImageContract; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { From 3ad3fffac5948dc258d36fbb691363431a14efd0 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 09:29:48 -0700 Subject: [PATCH 16/24] fix(ci): remove unused runtime contract import Signed-off-by: Aaron Erickson --- scripts/checks/managed-image-protected-runtime-contract.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/checks/managed-image-protected-runtime-contract.ts b/scripts/checks/managed-image-protected-runtime-contract.ts index 79d9aa05c84..9471255277c 100644 --- a/scripts/checks/managed-image-protected-runtime-contract.ts +++ b/scripts/checks/managed-image-protected-runtime-contract.ts @@ -5,7 +5,6 @@ import type { ManagedStartupAgent, ManagedStartupProfile, } from "../../src/lib/onboard/managed-startup/profile.ts"; -import { PROTECTED_MANAGED_IMAGE_AGENTS } from "./protected-managed-image-contract.ts"; export { PROTECTED_MANAGED_IMAGE_AGENTS, From 18ec2e2940cf502d608073b4ae16354d7ef6938c Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 09:56:15 -0700 Subject: [PATCH 17/24] fix(ci): isolate protected NIM qualification Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 25 +++- .../checks/build-protected-managed-images.sh | 17 ++- .../checks/run-managed-image-openshell-e2e.ts | 25 +++- ...managed-image-protected-runtime-helpers.ts | 113 ++++++++++-------- .../managed-image-protected-runtime.test.ts | 3 +- ...d-image-protected-runtime-workflow.test.ts | 45 ++++++- test/pr-risk-plan.test.ts | 35 ++++++ tools/advisors/risk-plan.mts | 7 +- ...ge-protected-runtime-workflow-boundary.mts | 57 ++++++++- tools/e2e/operations-workflow-boundary.mts | 8 +- tools/e2e/prepare-e2e-workflow-boundary.mts | 1 - tools/e2e/workflow-boundary.mts | 4 +- 12 files changed, 262 insertions(+), 78 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d4a3870a984..5b689051526 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2099,10 +2099,20 @@ jobs: exit 1 } - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Checkout trusted protected runtime qualification + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ github.repository }} + ref: ${{ inputs.workflow_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Checkout exact protected runtime candidate source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} + path: .candidate-runtime fetch-depth: 0 persist-credentials: false @@ -2118,8 +2128,6 @@ jobs: - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 - with: - build-cli: "false" - name: Validate protected runtime activation contract env: @@ -2127,8 +2135,9 @@ jobs: shell: bash run: | set -euo pipefail - activation="ci/protected-managed-image-runtime-activation-v1.json" - [[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { + candidate_root=".candidate-runtime" + activation="$candidate_root/ci/protected-managed-image-runtime-activation-v1.json" + [[ "$(git -C "$candidate_root" rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::Protected managed-image runtime checkout does not match the exact PR SHA" >&2 exit 1 } @@ -2242,11 +2251,13 @@ jobs: --revision "$CHECKOUT_SHA" \ --cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \ --platform linux/amd64 \ + --source-root "$GITHUB_WORKSPACE/.candidate-runtime" \ --openclaw-base "$BASE_OPENCLAW" \ --hermes-base "$BASE_HERMES" \ --dcode-base "$BASE_DCODE" - name: Install OpenShell CLI + shell: bash run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh - name: Run all-agent GPU, local inference, rollback, and cleanup qualification @@ -2255,6 +2266,10 @@ jobs: shell: bash run: | set -euo pipefail + [[ "$(git rev-parse --verify HEAD)" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA" ]] || { + echo "::error::Protected NIM qualification must execute trusted workflow code" >&2 + exit 1 + } export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" export OPENSHELL_BIN="$(command -v openshell)" "$OPENSHELL_BIN" --version diff --git a/scripts/checks/build-protected-managed-images.sh b/scripts/checks/build-protected-managed-images.sh index 127f2088d19..a7049cc785f 100755 --- a/scripts/checks/build-protected-managed-images.sh +++ b/scripts/checks/build-protected-managed-images.sh @@ -5,7 +5,7 @@ set -euo pipefail usage() { - echo "usage: $0 --output --revision --cohort --platform --openclaw-base --hermes-base --dcode-base " >&2 + echo "usage: $0 --output --revision --cohort --platform --openclaw-base --hermes-base --dcode-base [--source-root ]" >&2 exit 2 } @@ -16,6 +16,7 @@ platform="" openclaw_base="" hermes_base="" dcode_base="" +source_root="$PWD" while (($# > 0)); do case "$1" in --output) @@ -53,6 +54,11 @@ while (($# > 0)); do dcode_base="$2" shift 2 ;; + --source-root) + (($# >= 2)) || usage + source_root="$2" + shift 2 + ;; *) usage ;; @@ -66,6 +72,8 @@ done [[ "$openclaw_base" =~ ^ghcr[.]io/nvidia/nemoclaw/sandbox-base@sha256:[a-f0-9]{64}$ ]] || usage [[ "$hermes_base" =~ ^ghcr[.]io/nvidia/nemoclaw/hermes-sandbox-base@sha256:[a-f0-9]{64}$ ]] || usage [[ "$dcode_base" =~ ^ghcr[.]io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base@sha256:[a-f0-9]{64}$ ]] || usage +[[ "$source_root" == /* && "$source_root" != *$'\n'* && -d "$source_root" && ! -L "$source_root" ]] || usage +source_root="$(cd -- "$source_root" && pwd -P)" for command in docker jq sha256sum; do command -v "$command" >/dev/null 2>&1 || { @@ -83,6 +91,7 @@ build_agent() { local agent="$1" local dockerfile="$2" local base_reference="$3" + local dockerfile_path="$source_root/$dockerfile" local image_repository="localhost:5000/nemoclaw-managed-protected/${agent}" local exact_base_raw="$work_dir/${agent}-base-exact.raw" local metadata="$work_dir/${agent}-build-metadata.json" @@ -98,13 +107,13 @@ build_agent() { } scripts/check-production-build-args.sh \ - -f "$dockerfile" \ + -f "$dockerfile_path" \ --build-arg "BASE_IMAGE=${base_reference}" \ --build-arg "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1" \ --build-arg "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root" docker buildx build \ - --file "$dockerfile" \ + --file "$dockerfile_path" \ --platform "$platform" \ --push \ --provenance=false \ @@ -122,7 +131,7 @@ build_agent() { --build-arg "BASE_IMAGE=${base_reference}" \ --build-arg "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1" \ --build-arg "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root" \ - . + "$source_root" local digest digest="$(jq -er '."containerimage.digest"' "$metadata")" diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 2c81c9cc53b..9a7e86b44c3 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -6,6 +6,7 @@ import fs from "node:fs"; import net from "node:net"; import os from "node:os"; import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { resolveAgent } from "../../src/lib/agent/onboard.ts"; import { type InitialSandboxPolicy, @@ -152,7 +153,12 @@ export function parseManagedImageOpenShellE2eInputs(argv: readonly string[]): In } export function managedImageOpenShellBasePolicyPath(agent: ManagedStartupAgent): string { - return path.resolve(__dirname, "..", "..", ...MANAGED_AGENT_BASE_POLICIES[agent]); + return path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "..", + "..", + ...MANAGED_AGENT_BASE_POLICIES[agent], + ); } function commandResult(argv: readonly string[], env: NodeJS.ProcessEnv, timeout = 20_000) { @@ -647,6 +653,7 @@ async function run(input: Inputs): Promise { let ownedContainerId: string | null = null; let initialSandboxPolicy: InitialSandboxPolicy | null = null; let failureInjectionQualified = false; + let primaryError: unknown = null; try { await assertGatewayPortAvailable(); const image = parseImmutableManifestReference(input.image); @@ -712,9 +719,6 @@ async function run(input: Inputs): Promise { }); const prebuild = { createArgs: [...createArgs], imageRef: null, imageId: null }; if ( - prebuild.imageId !== null || - prebuild.imageRef !== null || - prebuild.createArgs.join("\0") !== createArgs.join("\0") || launch.createArgv.filter((value) => value === "--from").length !== 1 || launch.createArgv[launch.createArgv.indexOf("--from") + 1] !== input.image || launch.createArgv.filter((value) => value === "--policy").length !== 1 || @@ -845,6 +849,9 @@ async function run(input: Inputs): Promise { process.stdout.write( `OpenShell launched exact ${input.agent} PR image ${input.image} through the production managed-bootstrap sequence${gpuEnabled ? ` with real NVIDIA GPU access and ${input.localProvider} inference.local completion` : ""}.\n`, ); + } catch (error) { + primaryError = error; + throw error; } finally { const cleanupErrors: string[] = []; if (onboard) { @@ -940,7 +947,13 @@ async function run(input: Inputs): Promise { } fs.rmSync(stateDir, { recursive: true, force: true }); if (cleanupErrors.length > 0) { - throw new Error(`managed-image OpenShell cleanup failed: ${cleanupErrors.join("; ")}`); + const cleanupDetail = `managed-image OpenShell cleanup failed: ${cleanupErrors.join("; ")}`; + if (primaryError) { + const primaryDetail = + primaryError instanceof Error ? primaryError.message : String(primaryError); + throw new Error(`${primaryDetail}; ${cleanupDetail}`, { cause: primaryError }); + } + throw new Error(cleanupDetail); } if (failureInjectionQualified) { process.stdout.write( @@ -950,7 +963,7 @@ async function run(input: Inputs): Promise { } } -if (require.main === module) { +if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) { run(parseManagedImageOpenShellE2eInputs(process.argv.slice(2))).catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; diff --git a/test/e2e/live/managed-image-protected-runtime-helpers.ts b/test/e2e/live/managed-image-protected-runtime-helpers.ts index e04ab3fac29..4ef2bd7d13b 100644 --- a/test/e2e/live/managed-image-protected-runtime-helpers.ts +++ b/test/e2e/live/managed-image-protected-runtime-helpers.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { randomBytes } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; @@ -45,6 +46,8 @@ const VLLM_IMAGE = const VLLM_CONTAINER = "nemoclaw-managed-image-vllm-e2e"; const NIM_CATALOG_MODEL = "nvidia/nemotron-3-nano-30b-a3b"; const NIM_CONTAINER = "nemoclaw-managed-image-nim-e2e"; +const AGENT_QUALIFICATION_TIMEOUT_MS = 10 * 60_000; +const ROLLBACK_QUALIFICATION_TIMEOUT_MS = 10 * 60_000; type RuntimeFixtures = Pick; @@ -101,7 +104,7 @@ async function runExactImageQualification( NEMOCLAW_NON_INTERACTIVE: "1", ...extraEnv, }, - timeoutMs: 20 * 60_000, + timeoutMs: AGENT_QUALIFICATION_TIMEOUT_MS, }, ); expect(result.exitCode, resultText(result)).toBe(0); @@ -220,7 +223,7 @@ for _ in $(seq 1 300); do docker container inspect "${VLLM_CONTAINER}" --format '{{.State.Running}}' | grep -Fx true >/dev/null sleep 2 done -docker logs "${VLLM_CONTAINER}" >&2 +docker logs --tail 200 "${VLLM_CONTAINER}" >&2 exit 1`, ], { @@ -293,7 +296,7 @@ async function qualifyRollback( artifactName: `managed-image-${contract.agent}-bootstrap-rollback`, cwd: REPO_ROOT, env: { ...buildAvailabilityProbeEnv(), NEMOCLAW_NON_INTERACTIVE: "1" }, - timeoutMs: 20 * 60_000, + timeoutMs: ROLLBACK_QUALIFICATION_TIMEOUT_MS, }, ); expect(result.exitCode, resultText(result)).toBe(0); @@ -355,54 +358,66 @@ export async function qualifyProtectedManagedImageRuntime( await cleanupOllama(host, "cleanup-managed-image-ollama"); }); - const docker = await host.command("docker", ["info"], { - artifactName: "docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - expect(docker.exitCode, resultText(docker)).toBe(0); - const nvidia = await host.command("nvidia-smi", [], { - artifactName: "nvidia-smi", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - assertNvidiaAvailable(nvidia, (message) => { - throw new Error(message ?? "protected GPU runner is unavailable"); - }); + let activePhase = "validate protected host runtime"; + try { + const docker = await host.command("docker", ["info"], { + artifactName: "docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(docker.exitCode, resultText(docker)).toBe(0); + const nvidia = await host.command("nvidia-smi", [], { + artifactName: "nvidia-smi", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + assertNvidiaAvailable(nvidia, (message) => { + throw new Error(message ?? "protected GPU runner is unavailable"); + }); - progress.phase("qualify all managed agents with GPU-backed Ollama"); - const proxyToken = await startProtectedOllama(host); - await qualifyEveryAgent(host, contracts, "ollama", OLLAMA_MODEL, { - NEMOCLAW_OLLAMA_PROXY_TOKEN: proxyToken, - }); - await proveOllamaGpuPlacement(host); - killStaleProxy(); - await cleanupOllama(host, "stop-ollama-before-vllm"); + activePhase = "qualify all managed agents with GPU-backed Ollama"; + progress.phase("qualify all managed agents with GPU-backed Ollama"); + const proxyToken = await startProtectedOllama(host); + await qualifyEveryAgent(host, contracts, "ollama", OLLAMA_MODEL, { + NEMOCLAW_OLLAMA_PROXY_TOKEN: proxyToken, + }); + await proveOllamaGpuPlacement(host); + killStaleProxy(); + await cleanupOllama(host, "stop-ollama-before-vllm"); - progress.phase("qualify all managed agents with GPU-backed vLLM"); - await startProtectedVllm(host); - await qualifyEveryAgent(host, contracts, "vllm", VLLM_MODEL, { - NEMOCLAW_VLLM_LOCAL_TOKEN: "protected-local-vllm", - }); - await host.command("docker", ["rm", "-f", VLLM_CONTAINER], { - artifactName: "stop-vllm-before-nim", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }); + activePhase = "qualify all managed agents with GPU-backed vLLM"; + progress.phase("qualify all managed agents with GPU-backed vLLM"); + await startProtectedVllm(host); + await qualifyEveryAgent(host, contracts, "vllm", VLLM_MODEL, { + NEMOCLAW_VLLM_LOCAL_TOKEN: randomBytes(24).toString("hex"), + }); + await host.command("docker", ["rm", "-f", VLLM_CONTAINER], { + artifactName: "stop-vllm-before-nim", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); - progress.phase("qualify all managed agents with GPU-backed NVIDIA NIM"); - const nimModel = await startProtectedNim(host, ngcApiKey); - await qualifyEveryAgent(host, contracts, "nim", nimModel, { - NEMOCLAW_VLLM_LOCAL_TOKEN: "protected-local-nim", - }); - stopNimContainerByName(NIM_CONTAINER, { silent: true }); + activePhase = "qualify all managed agents with GPU-backed NVIDIA NIM"; + progress.phase("qualify all managed agents with GPU-backed NVIDIA NIM"); + const nimModel = await startProtectedNim(host, ngcApiKey); + await qualifyEveryAgent(host, contracts, "nim", nimModel, { + NEMOCLAW_VLLM_LOCAL_TOKEN: randomBytes(24).toString("hex"), + }); + stopNimContainerByName(NIM_CONTAINER, { silent: true }); - progress.phase("prove all-agent managed bootstrap rollback and exact cleanup"); - await qualifyEveryRollback(host, contracts); - await proveOwnedRuntimeInventoryClean(host); - await artifacts.writeJson("managed-image-protected-runtime-summary.json", { - agents: PROTECTED_MANAGED_IMAGE_AGENTS, - providers: ["ollama", "vllm", "nim"], - rollbackAgents: PROTECTED_MANAGED_IMAGE_AGENTS, - }); + activePhase = "prove all-agent managed bootstrap rollback and exact cleanup"; + progress.phase("prove all-agent managed bootstrap rollback and exact cleanup"); + await qualifyEveryRollback(host, contracts); + await proveOwnedRuntimeInventoryClean(host); + await artifacts.writeJson("managed-image-protected-runtime-summary.json", { + agents: PROTECTED_MANAGED_IMAGE_AGENTS, + providers: ["ollama", "vllm", "nim"], + rollbackAgents: PROTECTED_MANAGED_IMAGE_AGENTS, + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`protected managed-image runtime phase '${activePhase}' failed: ${detail}`, { + cause: error, + }); + } } diff --git a/test/e2e/live/managed-image-protected-runtime.test.ts b/test/e2e/live/managed-image-protected-runtime.test.ts index 581e08ca094..c42b6d0325c 100644 --- a/test/e2e/live/managed-image-protected-runtime.test.ts +++ b/test/e2e/live/managed-image-protected-runtime.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { PROTECTED_MANAGED_IMAGE_AGENTS } from "../../../scripts/checks/managed-image-protected-runtime-contract.ts"; import { test } from "../fixtures/e2e-test.ts"; import { qualifyProtectedManagedImageRuntime } from "./managed-image-protected-runtime-helpers.ts"; @@ -21,7 +22,7 @@ test("exact all-agent managed images retain GPU, Ollama, NIM, vLLM, rollback, an id: "managed-image-protected-runtime", boundary: "exact PR image digests for every managed agent through Docker/OpenShell GPU, host-local Ollama, NVIDIA NIM, vLLM, transactional rollback, and owned cleanup", - agents: ["openclaw", "hermes", "langchain-deepagents-code"], + agents: [...PROTECTED_MANAGED_IMAGE_AGENTS], providers: ["ollama", "nim", "vllm"], credentialBoundary: "The NVIDIA key is staged only to the host-side NGC login and NIM container; managed sandboxes receive only generated local route tokens.", diff --git a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts index 3b460854084..767295db634 100644 --- a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts +++ b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts @@ -13,7 +13,10 @@ type WorkflowRecord = Record; function workflow(): WorkflowRecord { return YAML.parse( - fs.readFileSync(path.resolve(__dirname, "../../../.github/workflows/e2e.yaml"), "utf8"), + fs.readFileSync( + path.resolve(import.meta.dirname, "../../../.github/workflows/e2e.yaml"), + "utf8", + ), ) as WorkflowRecord; } @@ -22,9 +25,11 @@ function runtimeJob(value: WorkflowRecord): Record { } function namedStep(value: WorkflowRecord, name: string): Record { - return (runtimeJob(value).steps as Array>).find( + const step = (runtimeJob(value).steps as Array>).find( (step) => step.name === name, - )!; + ); + expect(step, `workflow step '${name}' is missing`).toBeDefined(); + return step as Record; } describe("protected managed-image runtime workflow boundary", () => { @@ -44,6 +49,40 @@ describe("protected managed-image runtime workflow boundary", () => { ); }); + it("rejects checking candidate source out over trusted qualification code", () => { + const value = workflow(); + const candidateCheckout = namedStep(value, "Checkout exact protected runtime candidate source"); + (candidateCheckout.with as Record).path = "."; + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toContain( + "managed-image-protected-runtime candidate checkout must bind path to .candidate-runtime", + ); + }); + + it("rejects exposing the NGC credential to candidate-controlled steps", () => { + const value = workflow(); + namedStep(value, "Validate protected runtime activation contract").env = { + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + }; + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toContain( + "managed-image-protected-runtime must expose NVIDIA_API_KEY only to trusted qualification code", + ); + }); + + it("rejects executing candidate checkout paths in the secret-bearing qualification step", () => { + const value = workflow(); + const qualification = namedStep( + value, + "Run all-agent GPU, local inference, rollback, and cleanup qualification", + ); + qualification.run = `${String(qualification.run)}\nnpx tsx .candidate-runtime/leak.ts`; + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toContain( + "managed-image-protected-runtime trusted qualification must not execute candidate checkout paths", + ); + }); + it("rejects removing NIM from the activation contract", () => { const value = workflow(); const step = namedStep(value, "Validate protected runtime activation contract"); diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index df1825b6eca..41c8a4adf8e 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -344,6 +344,41 @@ describe("deterministic PR risk plan", () => { ).toBe(false); }); + it.each([ + ".github/workflows/managed-images.yaml", + ".dockerignore", + "Dockerfile", + "agents/hermes/Dockerfile", + "ci/npm-audit-exceptions.json", + "nemoclaw/src/index.ts", + "nemoclaw-blueprint/blueprint.yaml", + "scripts/checks/build-protected-managed-images.sh", + "src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.99.json", + "src/lib/core/json-types.ts", + "src/lib/core/ports.ts", + "src/lib/messaging/runtime.ts", + "src/lib/onboard/managed-bootstrap/envelope.ts", + "src/lib/onboard/managed-startup/image-runtime.ts", + "src/lib/security/credential-hash.ts", + "src/lib/state/paths.ts", + "src/lib/state/state-root.ts", + "src/lib/tool-disclosure.ts", + "tools/mcp-tool-discovery-runtime/index.ts", + "tsconfig.runtime-preloads.json", + ])("selects protected multiarch qualification for managed-image input %s (#7744)", (file) => { + expect(riskPlanRequiredJobIds(plan(file))).toContain("managed-image-multiarch-startup"); + }); + + it("does not select protected multiarch qualification for adjacent changes (#7744)", () => { + expect( + plan( + ".github/workflows/e2e.yaml", + "docs/get-started/quickstart.mdx", + "src/lib/onboard/provider-selection.ts", + ).families.some((family) => family.id === "managed-image-multiarch"), + ).toBe(false); + }); + it("keeps protected GPU and local-inference qualification activation-only until trusted (#7744)", () => { const activation = "ci/protected-managed-image-runtime-activation-v1.json"; const result = plan(activation); diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index c7ad3b19e40..3f29a23e8c1 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -66,6 +66,7 @@ const HERMES_MANAGED_POLICY_FILES = new Set([ ]); const MANAGED_IMAGE_PROTECTED_RUNTIME_ACTIVATION = "ci/protected-managed-image-runtime-activation-v1.json"; +const MANAGED_IMAGE_PROTECTED_RUNTIME_JOB_ID = "managed-image-protected-runtime" as const; const MANAGED_IMAGE_MULTIARCH_INPUTS = new Set([ PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, ".dockerignore", @@ -105,7 +106,7 @@ export type RiskFamilyId = | "credentials-security" | "e2e-control-plane" | "managed-image-multiarch" - | "managed-image-protected-runtime" + | typeof MANAGED_IMAGE_PROTECTED_RUNTIME_JOB_ID | "sandbox-boundary" | "focused-e2e"; @@ -449,11 +450,11 @@ export const RISK_RULES: readonly RiskRule[] = [ MANAGED_IMAGE_MULTIARCH_INPUT_PREFIXES.some((prefix) => file.startsWith(prefix)), }, { - id: "managed-image-protected-runtime", + id: MANAGED_IMAGE_PROTECTED_RUNTIME_JOB_ID, summary: "Protected managed-image runtime qualification must retain real GPU access, host-local Ollama, NVIDIA NIM, vLLM, transactional rollback, and exact cleanup for every shipped agent.", tier: 3, - requiredJobs: ["managed-image-protected-runtime"], + requiredJobs: [MANAGED_IMAGE_PROTECTED_RUNTIME_JOB_ID], invariants: [ "OpenClaw, Hermes, and Deep Agents Code run from exact PR image digests through the production managed-bootstrap path", "real NVIDIA GPU access and host-local Ollama, NVIDIA NIM, and vLLM inference.local completions are all required", diff --git a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts index 607de081537..177ee360459 100644 --- a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts +++ b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts @@ -148,10 +148,33 @@ export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowR ]); const checkouts = workflowSteps.filter((step) => text(step.uses).startsWith("actions/checkout@")); - if (checkouts.length !== 1) errors.push(`${JOB_ID} must define exactly one candidate checkout`); - requireValues(errors, `${JOB_ID} candidate checkout`, record(checkouts[0]?.with), { + if (checkouts.length !== 2) { + errors.push(`${JOB_ID} must define one trusted checkout and one isolated candidate checkout`); + } + const trustedCheckout = requireStep( + errors, + workflowSteps, + "Checkout trusted protected runtime qualification", + ); + const candidateCheckout = requireStep( + errors, + workflowSteps, + "Checkout exact protected runtime candidate source", + ); + const checkoutAction = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; + if (trustedCheckout?.uses !== checkoutAction || candidateCheckout?.uses !== checkoutAction) { + errors.push(`${JOB_ID} must pin both trusted and candidate checkouts`); + } + requireValues(errors, `${JOB_ID} trusted checkout`, record(trustedCheckout?.with), { + repository: "${{ github.repository }}", + ref: "${{ inputs.workflow_sha }}", + "fetch-depth": 0, + "persist-credentials": false, + }); + requireValues(errors, `${JOB_ID} candidate checkout`, record(candidateCheckout?.with), { repository: "${{ inputs.checkout_repository || github.repository }}", ref: "${{ inputs.checkout_sha || github.sha }}", + path: ".candidate-runtime", "fetch-depth": 0, "persist-credentials": false, }); @@ -165,14 +188,26 @@ export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowR "buildkitd-config-inline": '[registry."localhost:5000"]\n http = true\n', }); + const prepare = requireStep(errors, workflowSteps, "Prepare E2E workspace"); + if ( + prepare?.uses !== + "NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75" + ) { + errors.push(`${JOB_ID} must pin the trusted E2E preparation action`); + } + if (prepare?.with !== undefined) { + errors.push(`${JOB_ID} must use the default CLI build`); + } + const activation = requireStep( errors, workflowSteps, "Validate protected runtime activation contract", ); requireFragments(errors, activation, [ - `activation="${ACTIVATION_PATH}"`, - '[[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]]', + 'candidate_root=".candidate-runtime"', + `activation="$candidate_root/${ACTIVATION_PATH}"`, + '[[ "$(git -C "$candidate_root" rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]]', '[[ -f "$activation" && ! -L "$activation" ]]', '(keys | sort) == ["agents", "contractVersion", "jobId", "platform", "providers"]', '.agents == ["openclaw", "hermes", "langchain-deepagents-code"]', @@ -210,6 +245,7 @@ export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowR '--revision "$CHECKOUT_SHA"', '--cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT"', "--platform linux/amd64", + '--source-root "$GITHUB_WORKSPACE/.candidate-runtime"', '--openclaw-base "$BASE_OPENCLAW"', '--hermes-base "$BASE_HERMES"', '--dcode-base "$BASE_DCODE"', @@ -231,11 +267,21 @@ export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowR requireValues(errors, `${JOB_ID} qualification env`, record(qualification?.env), { NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", }); + const secretBearingSteps = workflowSteps.filter( + (step) => record(step.env).NVIDIA_API_KEY !== undefined, + ); + if (secretBearingSteps.length !== 1 || secretBearingSteps[0] !== qualification) { + errors.push(`${JOB_ID} must expose NVIDIA_API_KEY only to trusted qualification code`); + } requireFragments(errors, qualification, [ + '[[ "$(git rev-parse --verify HEAD)" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA" ]]', 'export OPENSHELL_BIN="$(command -v openshell)"', "tools/e2e/live-vitest-invocation.mts run", `--test-path ${LIVE_TEST_PATH}`, ]); + if (text(qualification?.run).includes(".candidate-runtime")) { + errors.push(`${JOB_ID} trusted qualification must not execute candidate checkout paths`); + } const cleanup = requireStep(errors, workflowSteps, "Remove isolated protected runtime registry"); if (cleanup?.if !== "always()") errors.push(`${JOB_ID} registry cleanup must always run`); @@ -259,6 +305,9 @@ export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowR requireStep(errors, workflowSteps, "Clean up Docker auth"); requireOrderedSteps(errors, workflowSteps, [ "Validate protected runtime exact-head dispatch", + "Checkout trusted protected runtime qualification", + "Checkout exact protected runtime candidate source", + "Prepare E2E workspace", "Validate protected runtime activation contract", "Resolve exact amd64 runtime base images", "Start isolated protected runtime registry", diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 33615918ce6..6421505dea2 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -419,11 +419,17 @@ function validatePrGateDispatch(errors: string[], workflow: OperationsWorkflow): step.name === "Check out trusted E2E workflow" && step.if === PUBLICATION_REQUIRED_CONDITION && step.with?.ref === "${{ github.sha }}"; + const trustedManagedImageRuntimeCheckout = + jobName === "managed-image-protected-runtime" && + step.name === "Checkout trusted protected runtime qualification" && + step.with?.repository === "${{ github.repository }}" && + step.with?.ref === "${{ inputs.workflow_sha }}"; const trustedCheckout = trustedHermesFixtureCheckout || trustedReportHelperCheckout || trustedLaunchableLaneCheckout || - trustedPublicationCheckout; + trustedPublicationCheckout || + trustedManagedImageRuntimeCheckout; if ( step.uses?.startsWith("actions/checkout@") && step.with?.ref !== "${{ inputs.checkout_sha || github.sha }}" && diff --git a/tools/e2e/prepare-e2e-workflow-boundary.mts b/tools/e2e/prepare-e2e-workflow-boundary.mts index 79a05ad6ee1..d1c6d4e83ff 100644 --- a/tools/e2e/prepare-e2e-workflow-boundary.mts +++ b/tools/e2e/prepare-e2e-workflow-boundary.mts @@ -28,7 +28,6 @@ const NO_BUILD_JOBS = new Set([ "generate-matrix", "bootstrap-install-smoke", "managed-image-multiarch-startup", - "managed-image-protected-runtime", "ollama-auth-proxy", "security-posture", "shields-config", diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 70d96a69d74..aff98099001 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -2494,7 +2494,9 @@ function validateDockerHubAuthBoundary(errors: string[], jobs: WorkflowRecord): requireCanonicalDockerHubCleanupRun(errors, jobName, cleanup); const checkoutIndex = steps.findIndex((step) => - stringValue(step.uses).startsWith("actions/checkout@"), + jobName === "managed-image-protected-runtime" + ? step.name === "Checkout exact protected runtime candidate source" + : stringValue(step.uses).startsWith("actions/checkout@"), ); const authIndex = steps.indexOf(auth); const cleanupIndex = steps.indexOf(cleanup); From 99b09aef4944218d292f0845bbfcb6a692a9a053 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 10:04:06 -0700 Subject: [PATCH 18/24] test(ci): prove workflow loader compatibility Signed-off-by: Aaron Erickson --- test/pr-risk-plan.test.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index b33391ca980..2c1eaf1ae60 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -1,6 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; +import path from "node:path"; + import { describe, expect, it } from "vitest"; import { buildRiskPlan, @@ -16,6 +19,7 @@ import { import { classifyTestDepth } from "../tools/pr-review-advisor/analyze.mts"; const HEAD_SHA = "a".repeat(40); +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); const HERMES_SANDBOX_BOUNDARY_JOBS = [ "full-e2e", "hermes-e2e", @@ -338,6 +342,31 @@ describe("deterministic PR risk plan", () => { } }); + it("loads protected multiarch identifiers through the workflow node loader (#7744)", () => { + const source = [ + 'const risk = await import("./tools/advisors/risk-plan.mts");', + 'const boundary = await import("./tools/e2e/managed-image-multiarch-workflow-boundary.mts");', + 'const activation = "ci/protected-managed-image-multiarch-activation-v1.json";', + 'const job = "managed-image-multiarch-startup";', + 'const plan = risk.buildRiskPlan({ headSha: "a".repeat(40), changedFiles: [activation] });', + 'if (!plan.requiredJobs.some((value) => value.id === job)) throw new Error("risk plan loader contract failed");', + 'const errors = boundary.validateManagedImageMultiarchWorkflow({ jobs: { [job]: { steps: [{ name: "Validate candidate activation contract", run: "" }] } } });', + 'if (!errors.some((value) => value.includes(activation))) throw new Error("workflow boundary loader contract failed");', + "console.log(JSON.stringify({ activation, job }));", + ].join("\n"); + const result = spawnSync( + process.execPath, + ["--import", "tsx", "--input-type=module", "-e", source], + { cwd: REPO_ROOT, encoding: "utf8" }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + activation: "ci/protected-managed-image-multiarch-activation-v1.json", + job: "managed-image-multiarch-startup", + }); + }); + it("runs snapshot commands for restored-gateway pairing runtime changes (#7431)", () => { const runtimeFiles = [ "src/lib/actions/sandbox/restore-gateway-pairing.ts", From 5e70b7bf76a2933d74e153259beb99970f8e8a9d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 10:16:34 -0700 Subject: [PATCH 19/24] fix(e2e): preserve cleanup failure selection Signed-off-by: Aaron Erickson --- .../checks/run-managed-image-openshell-e2e.ts | 86 +++++++++++-------- 1 file changed, 52 insertions(+), 34 deletions(-) diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 9a7e86b44c3..1c33603e507 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -653,7 +653,9 @@ async function run(input: Inputs): Promise { let ownedContainerId: string | null = null; let initialSandboxPolicy: InitialSandboxPolicy | null = null; let failureInjectionQualified = false; - let primaryError: unknown = null; + let primaryError: unknown; + let hasPrimaryError = false; + const cleanupErrors: string[] = []; try { await assertGatewayPortAvailable(); const image = parseImmutableManifestReference(input.image); @@ -767,7 +769,7 @@ async function run(input: Inputs): Promise { } as RuntimeProviderBundle & { readonly bootstrap: Extract; }; - let flow: Awaited>; + let flow: Awaited> | null = null; try { flow = await runSandboxGpuCreateFlow( { @@ -828,32 +830,37 @@ async function run(input: Inputs): Promise { process.stdout.write( `Injected managed-bootstrap completion failure removed the failed exact ${input.agent} sandbox before harness cleanup.\n`, ); - return; + } else { + throw error; } - throw error; - } - const expectedRoute = gpuEnabled ? "native" : "none"; - if (flow.route !== expectedRoute || flow.createResult.status !== 0) { - throw new Error( - `production managed-bootstrap flow did not complete the exact PR image create: route=${flow.route} status=${flow.createResult.status}`, - ); } - await waitForCommittedSandboxProbe(onboard, input, launch.sandboxEnv, !gpuEnabled); - ownedContainerId = assertExactSandboxImage(input, networkName, launch.sandboxEnv); - if (gpuEnabled) { - assertProtectedLocalInference(onboard, input, launch.sandboxEnv); - await flow.runtimePatch.commitAfterReady(); - await waitForCommittedSandboxProbe(onboard, input, launch.sandboxEnv); + if (!failureInjectionQualified) { + if (!flow) { + throw new Error("production managed-bootstrap flow returned no result"); + } + const expectedRoute = gpuEnabled ? "native" : "none"; + if (flow.route !== expectedRoute || flow.createResult.status !== 0) { + throw new Error( + `production managed-bootstrap flow did not complete the exact PR image create: route=${flow.route} status=${flow.createResult.status}`, + ); + } + + await waitForCommittedSandboxProbe(onboard, input, launch.sandboxEnv, !gpuEnabled); + ownedContainerId = assertExactSandboxImage(input, networkName, launch.sandboxEnv); + if (gpuEnabled) { + assertProtectedLocalInference(onboard, input, launch.sandboxEnv); + await flow.runtimePatch.commitAfterReady(); + await waitForCommittedSandboxProbe(onboard, input, launch.sandboxEnv); + } + process.stdout.write( + `OpenShell launched exact ${input.agent} PR image ${input.image} through the production managed-bootstrap sequence${gpuEnabled ? ` with real NVIDIA GPU access and ${input.localProvider} inference.local completion` : ""}.\n`, + ); } - process.stdout.write( - `OpenShell launched exact ${input.agent} PR image ${input.image} through the production managed-bootstrap sequence${gpuEnabled ? ` with real NVIDIA GPU access and ${input.localProvider} inference.local completion` : ""}.\n`, - ); } catch (error) { primaryError = error; - throw error; + hasPrimaryError = true; } finally { - const cleanupErrors: string[] = []; if (onboard) { commandResult( onboard.openshellArgv(["sandbox", "delete", input.sandbox]), @@ -945,21 +952,32 @@ async function run(input: Inputs): Promise { } catch (error) { cleanupErrors.push(error instanceof Error ? error.message : String(error)); } - fs.rmSync(stateDir, { recursive: true, force: true }); - if (cleanupErrors.length > 0) { - const cleanupDetail = `managed-image OpenShell cleanup failed: ${cleanupErrors.join("; ")}`; - if (primaryError) { - const primaryDetail = - primaryError instanceof Error ? primaryError.message : String(primaryError); - throw new Error(`${primaryDetail}; ${cleanupDetail}`, { cause: primaryError }); - } - throw new Error(cleanupDetail); + try { + fs.rmSync(stateDir, { recursive: true, force: true }); + } catch (error) { + cleanupErrors.push(error instanceof Error ? error.message : String(error)); } - if (failureInjectionQualified) { - process.stdout.write( - `Managed-bootstrap failure injection left no sandbox, container, network, or harness state orphan for ${input.agent}.\n`, - ); + } + + const cleanupDetail = + cleanupErrors.length > 0 + ? `managed-image OpenShell cleanup failed: ${cleanupErrors.join("; ")}` + : null; + if (hasPrimaryError) { + if (cleanupDetail) { + const primaryDetail = + primaryError instanceof Error ? primaryError.message : String(primaryError); + throw new Error(`${primaryDetail}; ${cleanupDetail}`, { cause: primaryError }); } + throw primaryError; + } + if (cleanupDetail) { + throw new Error(cleanupDetail); + } + if (failureInjectionQualified) { + process.stdout.write( + `Managed-bootstrap failure injection left no sandbox, container, network, or harness state orphan for ${input.agent}.\n`, + ); } } From 6a18088434756c13763f30b9d05812741884f9d8 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 10:27:00 -0700 Subject: [PATCH 20/24] test(ci): copy managed-image contract fixture Signed-off-by: Aaron Erickson --- test/e2e-recommendations.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e-recommendations.test.ts b/test/e2e-recommendations.test.ts index 142059c4390..80fbf4a57c0 100644 --- a/test/e2e-recommendations.test.ts +++ b/test/e2e-recommendations.test.ts @@ -75,6 +75,7 @@ describe("E2E recommendation normalizer", () => { "tools/advisors/e2e-text.mts", "tools/advisors/json.mts", "tools/advisors/risk-plan.mts", + "scripts/checks/protected-managed-image-contract.ts", "tools/e2e/module-tags.mts", ".github/workflows/e2e.yaml", "test/vllm-docker-storage.test.ts", From 59fbc1af816b759c102f391ec289cc0f2023a231 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 11:31:29 -0700 Subject: [PATCH 21/24] test(e2e): refresh runtime compatibility hashes Signed-off-by: Aaron Erickson --- test/e2e/support/e2e-cross-runtime-compatibility.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/support/e2e-cross-runtime-compatibility.test.ts b/test/e2e/support/e2e-cross-runtime-compatibility.test.ts index 787a63f5024..d45722b8ba2 100644 --- a/test/e2e/support/e2e-cross-runtime-compatibility.test.ts +++ b/test/e2e/support/e2e-cross-runtime-compatibility.test.ts @@ -29,7 +29,7 @@ describe("cross-runtime foundation compatibility", () => { ), ).toBe("6272aab16cf4b9555bdc4b3f4c0cdd24b5faa55118cbd61cbb4b30a3d418a63a"); expect(digestOutput(buildE2eWorkflowPlan())).toBe( - "36795de73b09280ad17f7a6296d5690572e23dabe40b374e754836066589d145", + "00dddd726f979dfddceef7b61b7e4937d48394af3e7224b22bb4014d5b362106", ); }); @@ -45,7 +45,7 @@ describe("cross-runtime foundation compatibility", () => { ]; expect(digestOutput(cases.map(buildRiskPlan))).toBe( - "311bd367e8d6ee469a9ec99aba13ab9b806ac679f7e71381c09d4fc4beafd4a2", + "7f55218cbfc184b0c2478ae435075c0fc2b9b053b01dda3fdb15c4697bf427e0", ); }); }); From 1dcfbfd8f844ea27ed0cb2b2c13a75aac487b131 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 11:46:50 -0700 Subject: [PATCH 22/24] test(e2e): activate protected managed runtime Signed-off-by: Aaron Erickson --- .github/workflows/e2e.yaml | 7 +++--- ...d-managed-image-runtime-activation-v1.json | 7 ++++++ ...d-image-protected-runtime-workflow.test.ts | 22 ++++++++++++++++++- 3 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 ci/protected-managed-image-runtime-activation-v1.json diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 5b689051526..1870a43e337 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2033,10 +2033,9 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - # This explicit-only lane remains dormant until its trusted workflow and - # validation boundary land on main. A follow-on candidate activates it with - # ci/protected-managed-image-runtime-activation-v1.json so the trusted - # controller can qualify that candidate's exact head without executing + # This explicit-only lane runs only when the exact candidate includes + # ci/protected-managed-image-runtime-activation-v1.json. The trusted + # controller can then qualify that candidate's exact head without executing # PR-controlled workflow code. managed-image-protected-runtime: name: Protected managed-image GPU and local inference diff --git a/ci/protected-managed-image-runtime-activation-v1.json b/ci/protected-managed-image-runtime-activation-v1.json new file mode 100644 index 00000000000..97260b05fe6 --- /dev/null +++ b/ci/protected-managed-image-runtime-activation-v1.json @@ -0,0 +1,7 @@ +{ + "agents": ["openclaw", "hermes", "langchain-deepagents-code"], + "contractVersion": 1, + "jobId": "managed-image-protected-runtime", + "platform": "linux/amd64", + "providers": ["ollama", "nim", "vllm"] +} diff --git a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts index 767295db634..168666adf56 100644 --- a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts +++ b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts @@ -33,10 +33,30 @@ function namedStep(value: WorkflowRecord, name: string): Record } describe("protected managed-image runtime workflow boundary", () => { - it("accepts the exact dormant trusted runtime lane", () => { + it("accepts the exact activated trusted runtime lane", () => { expect(validateManagedImageProtectedRuntimeWorkflow(workflow())).toEqual([]); }); + it("ships the exact activation contract consumed by the trusted lane (#7744)", () => { + const activation = JSON.parse( + fs.readFileSync( + path.resolve( + import.meta.dirname, + "../../../ci/protected-managed-image-runtime-activation-v1.json", + ), + "utf8", + ), + ) as unknown; + + expect(activation).toEqual({ + agents: ["openclaw", "hermes", "langchain-deepagents-code"], + contractVersion: 1, + jobId: "managed-image-protected-runtime", + platform: "linux/amd64", + providers: ["ollama", "nim", "vllm"], + }); + }); + it("rejects job-scoped NGC credentials", () => { const value = workflow(); runtimeJob(value).env = { From 8820c0bca2e78d63442d42ebb9b2d8b944592ec3 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 12:05:34 -0700 Subject: [PATCH 23/24] fix(ci): keep protected qualification fail closed Signed-off-by: Aaron Erickson --- scripts/checks/run-managed-image-openshell-e2e.ts | 5 +++++ tools/advisors/risk-plan.mts | 4 ++++ tools/e2e/managed-image-multiarch-workflow-boundary.mts | 3 +++ .../managed-image-protected-runtime-workflow-boundary.mts | 6 ++++++ 4 files changed, 18 insertions(+) diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 1c33603e507..85525a91c7e 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -46,6 +46,11 @@ import { withManagedImageLocalInferenceProfile, } from "./managed-image-protected-runtime-contract.ts"; +// This executable owns one protected qualification transaction from sandbox +// creation through exact cleanup. Keep its stateful orchestration and cleanup +// together so no cross-module return path can bypass rollback; stateless route +// and profile policy remains in managed-image-protected-runtime-contract.ts. + const MANAGED_AGENTS = new Set([ "openclaw", "hermes", diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index ada59bb6c41..31ebbbbb576 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -68,6 +68,10 @@ const HERMES_MANAGED_POLICY_FILES = new Set([ const MANAGED_IMAGE_PROTECTED_RUNTIME_ACTIVATION = "ci/protected-managed-image-runtime-activation-v1.json"; const MANAGED_IMAGE_PROTECTED_RUNTIME_JOB_ID = "managed-image-protected-runtime" as const; +// The activation-only phase is complete. Any input that can change bytes or +// startup policy in a shipped managed image must requalify the exact all-agent +// amd64/arm64 cohort; the positive and adjacent-path cases in +// test/pr-risk-plan.test.ts keep this inventory intentional and bounded. const MANAGED_IMAGE_MULTIARCH_INPUTS = new Set([ PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, ".dockerignore", diff --git a/tools/e2e/managed-image-multiarch-workflow-boundary.mts b/tools/e2e/managed-image-multiarch-workflow-boundary.mts index 151506d6f7a..4a17e307308 100644 --- a/tools/e2e/managed-image-multiarch-workflow-boundary.mts +++ b/tools/e2e/managed-image-multiarch-workflow-boundary.mts @@ -115,6 +115,9 @@ export function validateManagedImageMultiarchWorkflow(workflow: WorkflowRecord): if (record(job.permissions).contents !== "read") { errors.push(`${JOB_ID} permissions must be contents: read`); } + if (job["continue-on-error"] !== undefined) { + errors.push(`${JOB_ID} must not weaken failures with continue-on-error`); + } const expectedStrategy = { "fail-fast": false, diff --git a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts index 177ee360459..7521d09c0d1 100644 --- a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts +++ b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts @@ -18,6 +18,12 @@ const LIVE_TEST_PATH = "test/e2e/live/managed-image-protected-runtime.test.ts"; const REGISTRY_IMAGE = "docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"; +// Keep lane-specific trust assertions explicit: the multiarch lane executes +// candidate code directly, while this GPU lane keeps secrets in trusted code +// and isolates candidate source. The workflow-boundary aggregate runs both +// validators, which fail closed on the common job invariants without weakening +// either boundary into a generic lowest-common-denominator validator. + function record(value: unknown): WorkflowRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as WorkflowRecord) From ff85f4851494782b66f955fe8cb1831feeaa9947 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 4 Aug 2026 12:18:30 -0700 Subject: [PATCH 24/24] test(e2e): cover protected runtime input boundaries Signed-off-by: Aaron Erickson --- .../checks/run-managed-image-openshell-e2e.ts | 15 ++- ...d-image-protected-runtime-contract.test.ts | 23 ++++ ...otected-managed-image-build-script.test.ts | 120 ++++++++++++++++++ 3 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 test/protected-managed-image-build-script.test.ts diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 85525a91c7e..e5f2734a1f4 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -383,10 +383,12 @@ async function waitForCommittedSandboxProbe( ); } -function localInferenceBaseUrl(input: Inputs): string { - if (!input.localProvider) throw new Error("local provider is required"); - const route = resolveManagedImageLocalInferenceRoute(input.localProvider); - const configured = String(process.env.NEMOCLAW_E2E_LOCAL_INFERENCE_BASE_URL ?? "").trim(); +export function managedImageLocalInferenceBaseUrl( + localProvider: ManagedImageLocalInferenceKind, + configuredValue = process.env.NEMOCLAW_E2E_LOCAL_INFERENCE_BASE_URL, +): string { + const route = resolveManagedImageLocalInferenceRoute(localProvider); + const configured = String(configuredValue ?? "").trim(); const value = configured || route.defaultBaseUrl; let parsed: URL; try { @@ -409,6 +411,11 @@ function localInferenceBaseUrl(input: Inputs): string { return value.replace(/\/+$/u, ""); } +function localInferenceBaseUrl(input: Inputs): string { + if (!input.localProvider) throw new Error("local provider is required"); + return managedImageLocalInferenceBaseUrl(input.localProvider); +} + function configureLocalInferenceRoute( onboard: OnboardModule, input: Inputs, diff --git a/test/managed-image-protected-runtime-contract.test.ts b/test/managed-image-protected-runtime-contract.test.ts index 6124a2a7b5a..4c81e93d9b1 100644 --- a/test/managed-image-protected-runtime-contract.test.ts +++ b/test/managed-image-protected-runtime-contract.test.ts @@ -13,6 +13,7 @@ import { withManagedImageLocalInferenceProfile, } from "../scripts/checks/managed-image-protected-runtime-contract.ts"; import { + managedImageLocalInferenceBaseUrl, managedImageOpenShellBasePolicyPath, managedImageOpenShellCommittedProbe, managedImageOpenShellProbe, @@ -39,6 +40,28 @@ describe("protected managed-image runtime contract", () => { }); }); + it("accepts an exact protected local-inference URL override", () => { + expect( + managedImageLocalInferenceBaseUrl("ollama", "http://host.openshell.internal:11435/v1/"), + ).toBe("http://host.openshell.internal:11435/v1"); + }); + + it.each([ + ["HTTPS", "https://host.openshell.internal:11435/v1"], + ["another host", "http://example.invalid:11435/v1"], + ["a missing port", "http://host.openshell.internal/v1"], + ["port zero", "http://host.openshell.internal:0/v1"], + ["an out-of-range port", "http://host.openshell.internal:65536/v1"], + ["another path", "http://host.openshell.internal:11435/v2"], + ["credentials", "http://user:secret@host.openshell.internal:11435/v1"], + ["a query", "http://host.openshell.internal:11435/v1?model=other"], + ["a fragment", "http://host.openshell.internal:11435/v1#other"], + ])("rejects a protected local-inference override with %s", (_case, value) => { + expect(() => managedImageLocalInferenceBaseUrl("ollama", value)).toThrow( + /protected local inference/u, + ); + }); + it.each([ "openclaw", "hermes", diff --git a/test/protected-managed-image-build-script.test.ts b/test/protected-managed-image-build-script.test.ts new file mode 100644 index 00000000000..5dafc61e0fe --- /dev/null +++ b/test/protected-managed-image-build-script.test.ts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const REPO_ROOT = fileURLToPath(new URL("..", import.meta.url)); +const SCRIPT = path.join(REPO_ROOT, "scripts/checks/build-protected-managed-images.sh"); +const REVISION = "a".repeat(40); +const DIGEST = "b".repeat(64); + +let testRoot = ""; +let stubBin = ""; +let dockerLog = ""; + +function writeExecutable(name: string, source: string): void { + const target = path.join(stubBin, name); + writeFileSync(target, source, "utf8"); + chmodSync(target, 0o755); +} + +function runBuild(sourceRoot: string) { + const output = path.join(testRoot, "contracts.json"); + return spawnSync( + "bash", + [ + SCRIPT, + "--output", + output, + "--revision", + REVISION, + "--cohort", + "protected-1-1", + "--platform", + "linux/amd64", + "--openclaw-base", + `ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:${DIGEST}`, + "--hermes-base", + `ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@sha256:${DIGEST}`, + "--dcode-base", + `ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base@sha256:${DIGEST}`, + "--source-root", + sourceRoot, + ], + { + cwd: REPO_ROOT, + encoding: "utf8", + env: { + ...process.env, + NEMOCLAW_TEST_DOCKER_LOG: dockerLog, + PATH: `${stubBin}:${process.env.PATH ?? ""}`, + RUNNER_TEMP: testRoot, + }, + }, + ); +} + +beforeEach(() => { + testRoot = mkdtempSync(path.join(os.tmpdir(), "nemoclaw-protected-build-")); + stubBin = path.join(testRoot, "bin"); + dockerLog = path.join(testRoot, "docker.log"); + mkdirSync(stubBin); + writeExecutable( + "docker", + '#!/usr/bin/env bash\nprintf "%s\\n" "$*" >> "$NEMOCLAW_TEST_DOCKER_LOG"\nexit 88\n', + ); + writeExecutable("jq", "#!/usr/bin/env bash\nexit 89\n"); + writeExecutable("sha256sum", "#!/usr/bin/env bash\nexit 90\n"); +}); + +afterEach(() => { + rmSync(testRoot, { force: true, recursive: true }); +}); + +describe("protected managed-image source-root boundary", () => { + it("accepts one absolute non-symlink source root before invoking Docker", () => { + const sourceRoot = path.join(testRoot, "candidate"); + mkdirSync(sourceRoot); + + const result = runBuild(sourceRoot); + + expect(result.status, result.stderr).toBe(88); + expect(readFileSync(dockerLog, "utf8")).toContain("buildx imagetools inspect"); + }); + + it.each([ + ["relative", () => "."], + ["newline-bearing", () => `${testRoot}/candidate\n`], + ["missing", () => path.join(testRoot, "missing")], + [ + "symlink", + () => { + const target = path.join(testRoot, "candidate"); + const link = path.join(testRoot, "candidate-link"); + mkdirSync(target); + symlinkSync(target, link, "dir"); + return link; + }, + ], + ])("rejects a %s source root before invoking Docker", (_case, sourceRoot) => { + const result = runBuild(sourceRoot()); + + expect(result.status, result.stderr).toBe(2); + expect(existsSync(dockerLog)).toBe(false); + }); +});