diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 0ad3a9fffbf..1870a43e337 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2033,6 +2033,285 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + # This explicit-only lane runs only when the exact candidate includes + # ci/protected-managed-image-runtime-activation-v1.json. The trusted + # controller can then qualify that candidate's exact head without executing + # PR-controlled workflow code. + managed-image-protected-runtime: + name: Protected managed-image GPU and local inference + needs: generate-matrix + if: ${{ contains(format(',{0},', inputs.jobs), ',managed-image-protected-runtime,') || contains(format(',{0},', inputs.targets), ',managed-image-protected-runtime,') }} + runs-on: linux-amd64-gpu-rtxpro6000-latest-1 + timeout-minutes: 300 + permissions: + contents: read + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime + E2E_DEFAULT_ENABLED: "0" + E2E_JOB: "1" + E2E_TARGET_ID: "managed-image-protected-runtime" + RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-runtime-activation-v1.json + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_E2E_SHARD: linux-amd64-gpu + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: ${{ inputs.base_sha }} + NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: protected-${{ github.run_id }}-${{ github.run_attempt }} + NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime/contracts.json + NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM: linux/amd64 + NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + NEMOCLAW_PROTECTED_REGISTRY_NAME: nemoclaw-managed-runtime-${{ github.run_id }}-${{ github.run_attempt }} + NEMOCLAW_RUN_LIVE_E2E: "1" + OPENSHELL_GATEWAY: nemoclaw + steps: + - name: Validate protected runtime exact-head dispatch + env: + ACTOR: ${{ github.actor }} + BASE_SHA: ${{ inputs.base_sha }} + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + shell: bash + run: | + set -euo pipefail + [[ "$REPOSITORY" == "NVIDIA/NemoClaw" && "$REF" == "refs/heads/main" && "$EVENT_NAME" == "workflow_dispatch" ]] || { + echo "::error::Protected managed-image runtime must run from trusted NVIDIA/NemoClaw main" >&2 + exit 1 + } + [[ "$ACTOR" == "github-actions[bot]" ]] || { + echo "::error::Protected managed-image runtime requires the trusted controller actor" >&2 + exit 1 + } + [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { + echo "::error::Protected managed-image runtime requires exact PR and base SHAs" >&2 + exit 1 + } + [[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA" ]] || { + echo "::error::Protected managed-image runtime requires the exact trusted workflow SHA" >&2 + exit 1 + } + [[ "$RUNNER_ARCH_KIND" == "X64" ]] || { + echo "::error::Protected managed-image runtime requires a native linux/amd64 GPU runner" >&2 + exit 1 + } + + - name: Checkout trusted protected runtime qualification + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ github.repository }} + ref: ${{ inputs.workflow_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Checkout exact protected runtime candidate source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_sha || github.sha }} + path: .candidate-runtime + fetch-depth: 0 + persist-credentials: false + + - *dockerhub-auth + + - name: Set up protected runtime Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + with: + driver-opts: network=host + buildkitd-config-inline: | + [registry."localhost:5000"] + http = true + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + + - name: Validate protected runtime activation contract + env: + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + shell: bash + run: | + set -euo pipefail + candidate_root=".candidate-runtime" + activation="$candidate_root/ci/protected-managed-image-runtime-activation-v1.json" + [[ "$(git -C "$candidate_root" rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { + echo "::error::Protected managed-image runtime checkout does not match the exact PR SHA" >&2 + exit 1 + } + [[ -f "$activation" && ! -L "$activation" ]] || { + echo "::error::Protected managed-image runtime activation contract is absent" >&2 + exit 1 + } + jq -e ' + (keys | sort) == ["agents", "contractVersion", "jobId", "platform", "providers"] and + .contractVersion == 1 and + .jobId == "managed-image-protected-runtime" and + .agents == ["openclaw", "hermes", "langchain-deepagents-code"] and + .platform == "linux/amd64" and + .providers == ["ollama", "nim", "vllm"] + ' "$activation" >/dev/null || { + echo "::error::Protected managed-image runtime activation contract is invalid" >&2 + exit 1 + } + install -d -m 0700 "$E2E_ARTIFACT_DIR" + + - id: runtime-bases + name: Resolve exact amd64 runtime base images + shell: bash + run: | + set -euo pipefail + work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-runtime-bases.XXXXXX")" + trap 'rm -rf -- "$work_dir"' EXIT + + resolve_base() { + local output_name="$1" + local alias="$2" + local repository="$3" + local alias_raw="$work_dir/${output_name}-alias.raw" + local exact_raw="$work_dir/${output_name}-exact.raw" + docker buildx imagetools inspect "$alias" --raw > "$alias_raw" + local digest + digest="$( + jq -er ' + if ( + .mediaType == "application/vnd.oci.image.index.v1+json" or + .mediaType == "application/vnd.docker.distribution.manifest.list.v2+json" + ) then + [.manifests[] | select(.platform.os == "linux" and .platform.architecture == "amd64")] + | if length == 1 then .[0].digest else error("not one exact amd64 descriptor") end + else + error("base alias is not a platform index") + end + ' "$alias_raw" + )" + [[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]] || { + echo "::error::${output_name} base alias returned an invalid digest" >&2 + exit 1 + } + local reference="${repository}@${digest}" + docker buildx imagetools inspect "$reference" --raw > "$exact_raw" + [[ "sha256:$(sha256sum "$exact_raw" | awk '{print $1}')" == "$digest" ]] || { + echo "::error::${output_name} exact base bytes do not match the selected digest" >&2 + exit 1 + } + printf '%s=%s\n' "$output_name" "$reference" >> "$GITHUB_OUTPUT" + } + + resolve_base openclaw \ + ghcr.io/nvidia/nemoclaw/sandbox-base:latest \ + ghcr.io/nvidia/nemoclaw/sandbox-base + resolve_base hermes \ + ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest \ + ghcr.io/nvidia/nemoclaw/hermes-sandbox-base + resolve_base dcode \ + ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:latest \ + ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base + + - name: Start isolated protected runtime registry + shell: bash + run: | + set -euo pipefail + if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then + echo "::error::Protected runtime registry name already exists" >&2 + exit 1 + fi + if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then + echo "::error::Refusing to reuse an existing localhost:5000 registry" >&2 + exit 1 + fi + docker run --detach \ + --name "$NEMOCLAW_PROTECTED_REGISTRY_NAME" \ + --label "io.nvidia.nemoclaw.e2e-owner=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT}" \ + --label "io.nvidia.nemoclaw.e2e-platform=linux/amd64" \ + --publish 127.0.0.1:5000:5000 \ + docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373 + for _ in $(seq 1 30); do + if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null; then + exit 0 + fi + sleep 1 + done + docker logs "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >&2 + exit 1 + + - name: Build exact all-agent protected runtime images + env: + BASE_DCODE: ${{ steps.runtime-bases.outputs.dcode }} + BASE_HERMES: ${{ steps.runtime-bases.outputs.hermes }} + BASE_OPENCLAW: ${{ steps.runtime-bases.outputs.openclaw }} + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + shell: bash + run: | + set -euo pipefail + scripts/checks/build-protected-managed-images.sh \ + --output "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" \ + --revision "$CHECKOUT_SHA" \ + --cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \ + --platform linux/amd64 \ + --source-root "$GITHUB_WORKSPACE/.candidate-runtime" \ + --openclaw-base "$BASE_OPENCLAW" \ + --hermes-base "$BASE_HERMES" \ + --dcode-base "$BASE_DCODE" + + - name: Install OpenShell CLI + shell: bash + run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh + + - name: Run all-agent GPU, local inference, rollback, and cleanup qualification + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + shell: bash + run: | + set -euo pipefail + [[ "$(git rev-parse --verify HEAD)" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA" ]] || { + echo "::error::Protected NIM qualification must execute trusted workflow code" >&2 + exit 1 + } + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + export OPENSHELL_BIN="$(command -v openshell)" + "$OPENSHELL_BIN" --version + npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/managed-image-protected-runtime.test.ts + + - name: Remove isolated protected runtime registry + if: always() + shell: bash + run: | + set -euo pipefail + if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then + owner="$( + docker container inspect \ + --format '{{index .Config.Labels "io.nvidia.nemoclaw.e2e-owner"}}' \ + "$NEMOCLAW_PROTECTED_REGISTRY_NAME" + )" + [[ "$owner" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" ]] || { + echo "::error::Refusing to remove a runtime registry not owned by this protected job" >&2 + exit 1 + } + docker rm -f "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null + fi + if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then + echo "::error::Protected managed-image runtime registry remained after cleanup" >&2 + exit 1 + fi + if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then + echo "::error::Protected managed-image runtime registry listener remained after cleanup" >&2 + exit 1 + fi + + - name: Upload protected managed-image runtime artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: e2e-managed-image-protected-runtime + path: e2e-artifacts/live/managed-image-protected-runtime/ + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + agent-turn-latency: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',agent-turn-latency,') || contains(format(',{0},', inputs.targets), ',agent-turn-latency,') }} @@ -6071,6 +6350,7 @@ jobs: cloud-inference, gpu-e2e, managed-image-multiarch-startup, + managed-image-protected-runtime, agent-turn-latency, kimi-inference-compat, hermes-inference-switch, diff --git a/ci/protected-managed-image-multiarch-activation-v1.json b/ci/protected-managed-image-multiarch-activation-v1.json new file mode 100644 index 00000000000..5960e13e340 --- /dev/null +++ b/ci/protected-managed-image-multiarch-activation-v1.json @@ -0,0 +1,6 @@ +{ + "agents": ["openclaw", "hermes", "langchain-deepagents-code"], + "contractVersion": 1, + "jobId": "managed-image-multiarch-startup", + "platforms": ["linux/amd64", "linux/arm64"] +} diff --git a/ci/protected-managed-image-runtime-activation-v1.json b/ci/protected-managed-image-runtime-activation-v1.json new file mode 100644 index 00000000000..97260b05fe6 --- /dev/null +++ b/ci/protected-managed-image-runtime-activation-v1.json @@ -0,0 +1,7 @@ +{ + "agents": ["openclaw", "hermes", "langchain-deepagents-code"], + "contractVersion": 1, + "jobId": "managed-image-protected-runtime", + "platform": "linux/amd64", + "providers": ["ollama", "nim", "vllm"] +} diff --git a/scripts/checks/build-protected-managed-images.sh b/scripts/checks/build-protected-managed-images.sh index 127f2088d19..a7049cc785f 100755 --- a/scripts/checks/build-protected-managed-images.sh +++ b/scripts/checks/build-protected-managed-images.sh @@ -5,7 +5,7 @@ set -euo pipefail usage() { - echo "usage: $0 --output --revision --cohort --platform --openclaw-base --hermes-base --dcode-base " >&2 + echo "usage: $0 --output --revision --cohort --platform --openclaw-base --hermes-base --dcode-base [--source-root ]" >&2 exit 2 } @@ -16,6 +16,7 @@ platform="" openclaw_base="" hermes_base="" dcode_base="" +source_root="$PWD" while (($# > 0)); do case "$1" in --output) @@ -53,6 +54,11 @@ while (($# > 0)); do dcode_base="$2" shift 2 ;; + --source-root) + (($# >= 2)) || usage + source_root="$2" + shift 2 + ;; *) usage ;; @@ -66,6 +72,8 @@ done [[ "$openclaw_base" =~ ^ghcr[.]io/nvidia/nemoclaw/sandbox-base@sha256:[a-f0-9]{64}$ ]] || usage [[ "$hermes_base" =~ ^ghcr[.]io/nvidia/nemoclaw/hermes-sandbox-base@sha256:[a-f0-9]{64}$ ]] || usage [[ "$dcode_base" =~ ^ghcr[.]io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base@sha256:[a-f0-9]{64}$ ]] || usage +[[ "$source_root" == /* && "$source_root" != *$'\n'* && -d "$source_root" && ! -L "$source_root" ]] || usage +source_root="$(cd -- "$source_root" && pwd -P)" for command in docker jq sha256sum; do command -v "$command" >/dev/null 2>&1 || { @@ -83,6 +91,7 @@ build_agent() { local agent="$1" local dockerfile="$2" local base_reference="$3" + local dockerfile_path="$source_root/$dockerfile" local image_repository="localhost:5000/nemoclaw-managed-protected/${agent}" local exact_base_raw="$work_dir/${agent}-base-exact.raw" local metadata="$work_dir/${agent}-build-metadata.json" @@ -98,13 +107,13 @@ build_agent() { } scripts/check-production-build-args.sh \ - -f "$dockerfile" \ + -f "$dockerfile_path" \ --build-arg "BASE_IMAGE=${base_reference}" \ --build-arg "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1" \ --build-arg "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root" docker buildx build \ - --file "$dockerfile" \ + --file "$dockerfile_path" \ --platform "$platform" \ --push \ --provenance=false \ @@ -122,7 +131,7 @@ build_agent() { --build-arg "BASE_IMAGE=${base_reference}" \ --build-arg "NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION=1" \ --build-arg "NEMOCLAW_MANAGED_IMAGE_RUNTIME_USER=root" \ - . + "$source_root" local digest digest="$(jq -er '."containerimage.digest"' "$metadata")" diff --git a/scripts/checks/managed-image-protected-runtime-contract.ts b/scripts/checks/managed-image-protected-runtime-contract.ts new file mode 100644 index 00000000000..9471255277c --- /dev/null +++ b/scripts/checks/managed-image-protected-runtime-contract.ts @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + ManagedStartupAgent, + ManagedStartupProfile, +} from "../../src/lib/onboard/managed-startup/profile.ts"; + +export { + PROTECTED_MANAGED_IMAGE_AGENTS, + type ProtectedManagedImageContract, + parseProtectedManagedImageContracts, +} from "./protected-managed-image-contract.ts"; + +export const MANAGED_IMAGE_LOCAL_INFERENCE_KINDS = ["ollama", "nim", "vllm"] as const; + +export type ManagedImageLocalInferenceKind = (typeof MANAGED_IMAGE_LOCAL_INFERENCE_KINDS)[number]; + +export type ManagedImageLocalInferenceRoute = { + readonly kind: ManagedImageLocalInferenceKind; + readonly providerName: "ollama-local" | "vllm-local"; + readonly credentialEnv: "NEMOCLAW_OLLAMA_PROXY_TOKEN" | "NEMOCLAW_VLLM_LOCAL_TOKEN"; + readonly defaultBaseUrl: string; +}; + +const LOCAL_INFERENCE_ROUTES: Readonly< + Record +> = Object.freeze({ + ollama: Object.freeze({ + kind: "ollama", + providerName: "ollama-local", + credentialEnv: "NEMOCLAW_OLLAMA_PROXY_TOKEN", + defaultBaseUrl: "http://host.openshell.internal:11435/v1", + }), + // Local NIM exposes the same OpenAI-compatible host route as local vLLM. + // Keep the source kinds distinct even though OpenShell intentionally binds + // both to vllm-local; this prevents a future engine-specific route change + // from being silently treated as equivalent. + nim: Object.freeze({ + kind: "nim", + providerName: "vllm-local", + credentialEnv: "NEMOCLAW_VLLM_LOCAL_TOKEN", + defaultBaseUrl: "http://host.openshell.internal:8000/v1", + }), + vllm: Object.freeze({ + kind: "vllm", + providerName: "vllm-local", + credentialEnv: "NEMOCLAW_VLLM_LOCAL_TOKEN", + defaultBaseUrl: "http://host.openshell.internal:8000/v1", + }), +}); + +export function isManagedImageLocalInferenceKind( + value: string, +): value is ManagedImageLocalInferenceKind { + return (MANAGED_IMAGE_LOCAL_INFERENCE_KINDS as readonly string[]).includes(value); +} + +export function resolveManagedImageLocalInferenceRoute( + kind: ManagedImageLocalInferenceKind, +): ManagedImageLocalInferenceRoute { + return LOCAL_INFERENCE_ROUTES[kind]; +} + +export function withManagedImageLocalInferenceProfile( + profile: ManagedStartupProfile, + route: ManagedImageLocalInferenceRoute, + model: string, +): ManagedStartupProfile { + const primaryModelRef = + profile.agent === "openclaw" ? `inference/${model}` : profile.inference.primaryModelRef; + return { + ...profile, + inference: { + ...profile.inference, + routeProvider: "inference", + upstreamProvider: route.providerName, + model, + primaryModelRef, + routedBaseUrl: "https://inference.local/v1", + upstreamEndpointUrl: null, + api: "openai-completions", + }, + } as ManagedStartupProfile; +} + +export function managedImageProtectedSandboxName( + agent: ManagedStartupAgent, + routeKind: ManagedImageLocalInferenceKind | "rollback", +): string { + const agentToken = + agent === "langchain-deepagents-code" ? "dcode" : agent.replace(/[^a-z0-9-]+/gu, "-"); + return `nemoclaw-managed-${agentToken}-${routeKind}`; +} diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts new file mode 100644 index 00000000000..e5f2734a1f4 --- /dev/null +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -0,0 +1,1003 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { resolveAgent } from "../../src/lib/agent/onboard.ts"; +import { + type InitialSandboxPolicy, + prepareInitialSandboxCreatePolicy, +} from "../../src/lib/onboard/initial-policy.ts"; +import { + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + type ManagedBootstrapAdapter, + type ManagedBootstrapAuthorityStore, +} from "../../src/lib/onboard/managed-bootstrap/adapter.ts"; +import { createDockerManagedBootstrapAdapter } from "../../src/lib/onboard/managed-bootstrap/docker.ts"; +import { createDockerManagedBootstrapSurface } from "../../src/lib/onboard/managed-bootstrap/docker-runtime.ts"; +import { + encodeManagedStartupProfile, + type ManagedStartupAgent, +} from "../../src/lib/onboard/managed-startup/profile.ts"; +import { createManagedStartupRootApplyRequest } from "../../src/lib/onboard/managed-startup/root-apply.ts"; +import type { + RuntimeProviderBootstrapSurface, + RuntimeProviderBundle, +} from "../../src/lib/onboard/runtime-provider/contract.ts"; +import { createDockerRuntimeProviderBundle } from "../../src/lib/onboard/runtime-provider/docker.ts"; +import { prepareSandboxCreateLaunch } from "../../src/lib/onboard/sandbox-create-launch.ts"; +import { + resolveDockerStartupCommandPatch, + runSandboxGpuCreateFlow, +} from "../../src/lib/onboard/sandbox-gpu-create-flow.ts"; +import { createDirectSandboxGpuVerifier } from "../../src/lib/onboard/sandbox-gpu-preflight.ts"; +import { + MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, + managedStartupE2eProfile, +} from "./generate-managed-startup-profile-fixture.mts"; +import { + isManagedImageLocalInferenceKind, + type ManagedImageLocalInferenceKind, + resolveManagedImageLocalInferenceRoute, + withManagedImageLocalInferenceProfile, +} from "./managed-image-protected-runtime-contract.ts"; + +// This executable owns one protected qualification transaction from sandbox +// creation through exact cleanup. Keep its stateful orchestration and cleanup +// together so no cross-module return path can bypass rollback; stateless route +// and profile policy remains in managed-image-protected-runtime-contract.ts. + +const MANAGED_AGENTS = new Set([ + "openclaw", + "hermes", + "langchain-deepagents-code", +]); +const MODEL = "nvidia/nemotron-3-ultra-550b-a55b"; +const GATEWAY_PORT = 8080; +const IMMUTABLE_MANIFEST_REFERENCE_RE = /^([^\s@]+)@(sha256:[a-f0-9]{64})$/u; +const MANAGED_AGENT_BASE_POLICIES: Record = { + openclaw: ["nemoclaw-blueprint", "policies", "openclaw-sandbox.yaml"], + hermes: ["agents", "hermes", "policy-additions.yaml"], + "langchain-deepagents-code": ["agents", "langchain-deepagents-code", "policy-additions.yaml"], +}; + +function compactText(value = ""): string { + return String(value).replace(/\s+/gu, " ").trim(); +} + +function redactProtectedGpuProof(value: string): string { + return String(value) + .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/giu, "Bearer ") + .replace(/\b([A-Z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD))=([^\s]*)/giu, "$1="); +} + +type Inputs = { + agent: ManagedStartupAgent; + image: string; + sandbox: string; + gpu?: true; + localProvider?: ManagedImageLocalInferenceKind; + model?: string; + failureInjection?: "bootstrap-completion"; +}; + +type OnboardModule = { + openshellArgv(args: string[]): string[]; + runOpenshell(args: string[], opts?: Record): ReturnType; + runCaptureOpenshell(args: string[], opts?: Record): string; + sleepSeconds(seconds: number): void; + startGatewayForRecovery(options: { gatewayName: string; gatewayPort: number }): Promise; +}; + +function requiredValue(argv: readonly string[], flag: string): string { + const index = argv.indexOf(flag); + const value = index >= 0 ? argv[index + 1] : undefined; + if (!value || value.startsWith("--")) throw new Error(`${flag} is required`); + return value; +} + +export function parseManagedImageOpenShellE2eInputs(argv: readonly string[]): Inputs { + const valueFlags = new Set(["--agent", "--image", "--sandbox", "--local-provider", "--model"]); + const booleanFlags = new Set(["--gpu", "--inject-bootstrap-completion-failure"]); + for (let index = 0; index < argv.length; index += 1) { + const value = argv[index] ?? ""; + if (booleanFlags.has(value)) continue; + if (!valueFlags.has(value)) throw new Error(`unsupported arguments: ${value}`); + const next = argv[index + 1]; + if (!next || next.startsWith("--")) throw new Error(`${value} is required`); + index += 1; + } + const agentValue = requiredValue(argv, "--agent"); + if (!MANAGED_AGENTS.has(agentValue as ManagedStartupAgent)) { + throw new Error("--agent must identify a shipped managed-image agent"); + } + const image = requiredValue(argv, "--image"); + if (!IMMUTABLE_MANIFEST_REFERENCE_RE.test(image)) { + throw new Error("--image must be an immutable repository@sha256 manifest reference"); + } + const sandbox = requiredValue(argv, "--sandbox"); + if (!/^[a-z0-9](?:[a-z0-9.-]{0,61}[a-z0-9])?$/u.test(sandbox)) { + throw new Error("--sandbox must be a valid RFC 1123 label"); + } + const gpu = argv.includes("--gpu"); + const localProviderValue = argv.includes("--local-provider") + ? requiredValue(argv, "--local-provider") + : null; + if (localProviderValue && !isManagedImageLocalInferenceKind(localProviderValue)) { + throw new Error("--local-provider must be one of: ollama, nim, vllm"); + } + const model = argv.includes("--model") ? requiredValue(argv, "--model") : null; + if (model && !/^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,255}$/u.test(model)) { + throw new Error("--model must be one bounded model identifier"); + } + const failureInjection = argv.includes("--inject-bootstrap-completion-failure"); + if (gpu && (!localProviderValue || !model)) { + throw new Error("--gpu requires --local-provider and --model"); + } + if (!gpu && (localProviderValue || model)) { + throw new Error("--local-provider and --model require --gpu"); + } + if (failureInjection && gpu) { + throw new Error("bootstrap failure injection cannot be combined with the GPU qualification"); + } + return { + agent: agentValue as ManagedStartupAgent, + image, + sandbox, + ...(gpu ? { gpu: true as const } : {}), + ...(localProviderValue + ? { localProvider: localProviderValue as ManagedImageLocalInferenceKind } + : {}), + ...(model ? { model } : {}), + ...(failureInjection ? { failureInjection: "bootstrap-completion" as const } : {}), + }; +} + +export function managedImageOpenShellBasePolicyPath(agent: ManagedStartupAgent): string { + return path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "..", + "..", + ...MANAGED_AGENT_BASE_POLICIES[agent], + ); +} + +function commandResult(argv: readonly string[], env: NodeJS.ProcessEnv, timeout = 20_000) { + const [command, ...args] = argv; + if (!command) throw new Error("command argv must not be empty"); + return spawnSync(command, args, { + encoding: "utf8", + env, + killSignal: "SIGKILL", + stdio: ["ignore", "pipe", "pipe"], + timeout, + }); +} + +function commandDetail(result: ReturnType): string { + return `${result.error?.message ?? ""}\n${result.stdout ?? ""}\n${result.stderr ?? ""}` + .trim() + .slice(-8_000); +} + +function isDockerNotFound(result: ReturnType): boolean { + return ( + result.status !== 0 && + /(?:no such (?:container|network|object)|not found)/iu.test(commandDetail(result)) + ); +} + +function readGatewayPid(stateDir: string): number | null { + try { + const value = Number.parseInt( + fs.readFileSync(path.join(stateDir, "openshell-gateway.pid"), "utf8").trim(), + 10, + ); + return Number.isSafeInteger(value) && value > 1 ? value : null; + } catch { + return null; + } +} + +function stopProcess(pid: number | null): void { + if (!pid) return; + try { + process.kill(pid, "SIGTERM"); + } catch { + return; + } + for (let attempt = 0; attempt < 50; attempt += 1) { + try { + process.kill(pid, 0); + } catch { + return; + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100); + } + try { + process.kill(pid, "SIGKILL"); + } catch { + // The process exited between the liveness probe and the final signal. + } +} + +function createProtectedAuthorityStore(stateDir: string): ManagedBootstrapAuthorityStore { + const authorityDir = path.join(stateDir, "managed-bootstrap-authority"); + fs.mkdirSync(authorityDir, { mode: 0o700, recursive: true }); + return { + async recordPreparedAuthority(authority) { + const finalPath = path.join(authorityDir, `${authority.bootstrapIdentity}.json`); + const temporaryPath = `${finalPath}.tmp-${process.pid}`; + const serialized = `${JSON.stringify(authority)}\n`; + const file = fs.openSync(temporaryPath, "wx", 0o600); + try { + fs.writeFileSync(file, serialized, "utf8"); + fs.fsyncSync(file); + } finally { + fs.closeSync(file); + } + fs.renameSync(temporaryPath, finalPath); + const directory = fs.openSync(authorityDir, "r"); + try { + fs.fsyncSync(directory); + } finally { + fs.closeSync(directory); + } + if (fs.readFileSync(finalPath, "utf8") !== serialized) { + throw new Error("protected managed-bootstrap authority was not durably re-readable"); + } + return { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: authority.sandbox, + bootstrapIdentity: authority.bootstrapIdentity, + authorityFingerprint: authority.authorityFingerprint, + recordId: `protected-${authority.bootstrapIdentity}`, + recordedAt: new Date().toISOString(), + }; + }, + }; +} + +async function assertGatewayPortAvailable(): Promise { + await new Promise((resolve, reject) => { + const server = net.createServer(); + server.unref(); + server.once("error", () => { + reject( + new Error( + `refusing to disturb an existing listener on the managed-image E2E gateway port ${GATEWAY_PORT}`, + ), + ); + }); + server.listen(GATEWAY_PORT, "127.0.0.1", () => { + server.close((error) => { + if (error) reject(error); + else resolve(); + }); + }); + }); +} + +function managedConfigPath(agent: ManagedStartupAgent): string { + switch (agent) { + case "openclaw": + return "/sandbox/.openclaw/openclaw.json"; + case "hermes": + return "/sandbox/.hermes/config.yaml"; + case "langchain-deepagents-code": + return "/sandbox/.deepagents/config.toml"; + } +} + +export function managedImageOpenShellProbe( + agent: ManagedStartupAgent, + model: string = MODEL, +): string { + const healthProbe = + agent === "openclaw" + ? "/usr/bin/curl -fsS --max-time 5 http://127.0.0.1:18789/health >/dev/null" + : agent === "hermes" + ? "/usr/bin/curl -fsS --max-time 5 http://127.0.0.1:8642/health >/dev/null" + : "/usr/local/bin/dcode --version >/dev/null"; + return [ + "set -eu", + `test -x ${ + agent === "openclaw" + ? "/usr/local/bin/openclaw" + : agent === "hermes" + ? "/usr/local/bin/hermes" + : "/usr/local/bin/dcode" + }`, + `grep -F ${JSON.stringify(model)} ${JSON.stringify(managedConfigPath(agent))} >/dev/null`, + "test ! -L /run/nemoclaw/managed-startup-runtime.env", + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-runtime.env)" = "0:0:444"', + "test ! -L /run/nemoclaw/managed-startup-complete.json", + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-complete.json)" = "0:0:444"', + "test -s /usr/local/share/nemoclaw/corporate-ca.pem", + 'test "$(stat -c "%u:%g:%a" /usr/local/share/nemoclaw/corporate-ca.pem)" = "0:0:444"', + "test -s /run/nemoclaw/managed-startup-ca-bundle.pem", + 'test "$(stat -c "%u:%g:%a" /run/nemoclaw/managed-startup-ca-bundle.pem)" = "0:0:444"', + healthProbe, + ].join("\n"); +} + +export function managedImageOpenShellCommittedProbe(): string { + return [ + "set -eu", + "test ! -e /var/lib/nemoclaw/managed-startup-shared-state-transaction-v1", + ].join("\n"); +} + +async function waitForCommittedSandboxProbe( + onboard: OnboardModule, + input: Inputs, + env: NodeJS.ProcessEnv, + requireCommitted = true, +): Promise { + const healthProbe = managedImageOpenShellProbe(input.agent, input.model ?? MODEL); + const committedProbe = managedImageOpenShellCommittedProbe(); + const deadline = Date.now() + 240_000; + const runProbe = (probe: string, timeoutMs: number) => + commandResult( + onboard.openshellArgv([ + "sandbox", + "exec", + "--name", + input.sandbox, + "--", + "/bin/sh", + "-eu", + "-c", + probe, + ]), + env, + timeoutMs, + ); + let lastHealthDetail = ""; + while (Date.now() < deadline) { + const remainingMs = deadline - Date.now(); + const health = runProbe(healthProbe, Math.max(1, Math.min(15_000, remainingMs))); + if (health.status === 0) { + if (!requireCommitted) return; + const committed = runProbe( + committedProbe, + Math.max(1, Math.min(15_000, deadline - Date.now())), + ); + if (committed.status !== 0) { + throw new Error( + `managed bootstrap committed, but transaction cleanup was not observable through the exact sandbox: ${commandDetail(committed)}`, + ); + } + return; + } + lastHealthDetail = commandDetail(health); + const sleepMs = Math.min(2_000, Math.max(0, deadline - Date.now())); + if (sleepMs > 0) await new Promise((resolve) => setTimeout(resolve, sleepMs)); + } + throw new Error( + `OpenShell sandbox did not pass the exact-image managed-bootstrap probe within 240s: ${lastHealthDetail}`, + ); +} + +export function managedImageLocalInferenceBaseUrl( + localProvider: ManagedImageLocalInferenceKind, + configuredValue = process.env.NEMOCLAW_E2E_LOCAL_INFERENCE_BASE_URL, +): string { + const route = resolveManagedImageLocalInferenceRoute(localProvider); + const configured = String(configuredValue ?? "").trim(); + const value = configured || route.defaultBaseUrl; + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw new Error("protected local inference base URL is invalid"); + } + if ( + parsed.protocol !== "http:" || + parsed.hostname !== "host.openshell.internal" || + !/^[1-9][0-9]{0,4}$/u.test(parsed.port) || + parsed.pathname.replace(/\/+$/u, "") !== "/v1" || + parsed.username || + parsed.password || + parsed.search || + parsed.hash + ) { + throw new Error("protected local inference must use http://host.openshell.internal:/v1"); + } + return value.replace(/\/+$/u, ""); +} + +function localInferenceBaseUrl(input: Inputs): string { + if (!input.localProvider) throw new Error("local provider is required"); + return managedImageLocalInferenceBaseUrl(input.localProvider); +} + +function configureLocalInferenceRoute( + onboard: OnboardModule, + input: Inputs, + env: NodeJS.ProcessEnv, +): void { + if (!input.localProvider || !input.model) return; + const route = resolveManagedImageLocalInferenceRoute(input.localProvider); + const credential = String(env[route.credentialEnv] ?? "").trim(); + if (!credential || /[\0\r\n]/u.test(credential)) { + throw new Error(`${route.credentialEnv} is required for protected local inference`); + } + const commandEnv = { ...env, [route.credentialEnv]: credential }; + const create = onboard.runOpenshell( + [ + "provider", + "create", + "--name", + route.providerName, + "--type", + "openai", + "--credential", + route.credentialEnv, + "--config", + `OPENAI_BASE_URL=${localInferenceBaseUrl(input)}`, + ], + { ignoreError: true, env: commandEnv, stdio: ["ignore", "pipe", "pipe"] }, + ); + if (create.status !== 0) { + throw new Error(`protected local inference provider creation failed: ${commandDetail(create)}`); + } + const setRoute = onboard.runOpenshell( + [ + "inference", + "set", + "--no-verify", + "--provider", + route.providerName, + "--model", + input.model, + "--timeout", + "120", + ], + { ignoreError: true, env: commandEnv, stdio: ["ignore", "pipe", "pipe"] }, + ); + if (setRoute.status !== 0) { + throw new Error(`protected local inference route failed: ${commandDetail(setRoute)}`); + } +} + +function localInferenceProbe(input: Inputs): string { + if (!input.model) throw new Error("local inference model is required"); + const payload = JSON.stringify({ + model: input.model, + messages: [{ role: "user", content: "Reply with exactly one word: PONG" }], + reasoning_effort: "none", + max_tokens: 32, + }); + return [ + "set -eu", + "response=/tmp/nemoclaw-managed-image-inference.json", + `curl -fsS --max-time 180 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' --data ${JSON.stringify(payload)} > "$response"`, + "node - \"$response\" <<'NODE'", + 'const fs = require("node:fs");', + 'const body = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));', + "const choice = Array.isArray(body.choices) ? body.choices[0] : null;", + 'const text = choice && choice.message && typeof choice.message.content === "string"', + " ? choice.message.content", + ' : choice && typeof choice.text === "string" ? choice.text : "";', + 'if (!/pong/i.test(text)) throw new Error("local inference did not return PONG");', + "NODE", + 'rm -f "$response"', + ].join("\n"); +} + +function assertProtectedLocalInference( + onboard: OnboardModule, + input: Inputs, + env: NodeJS.ProcessEnv, +): void { + const result = commandResult( + onboard.openshellArgv([ + "sandbox", + "exec", + "--name", + input.sandbox, + "--", + "/bin/sh", + "-eu", + "-c", + localInferenceProbe(input), + ]), + env, + 210_000, + ); + if (result.status !== 0) { + throw new Error(`sandbox inference.local completion failed: ${commandDetail(result)}`); + } +} + +function failureInjectingAdapter(onboard: OnboardModule): ManagedBootstrapAdapter { + const adapter = createDockerManagedBootstrapAdapter({ + runCaptureOpenshell: onboard.runCaptureOpenshell, + runOpenshell: onboard.runOpenshell, + sleep: onboard.sleepSeconds, + }); + return { + ...adapter, + async awaitBootstrap(input) { + await adapter.awaitBootstrap(input); + throw new Error("protected-e2e-injected-bootstrap-completion-failure"); + }, + }; +} + +function parseImmutableManifestReference(image: string): { + repository: string; + manifestDigest: `sha256:${string}`; +} { + const match = IMMUTABLE_MANIFEST_REFERENCE_RE.exec(image); + if (!match?.[1] || !match[2]) { + throw new Error("--image must be an immutable repository@sha256 manifest reference"); + } + return { + repository: match[1], + manifestDigest: match[2] as `sha256:${string}`, + }; +} + +function resolveLocalImageContentId(image: string, env: NodeJS.ProcessEnv): string { + const inspect = commandResult(["docker", "image", "inspect", "--format", "{{.Id}}", image], env); + const contentId = String(inspect.stdout ?? "").trim(); + if (inspect.status !== 0 || !/^sha256:[a-f0-9]{64}$/u.test(contentId)) { + throw new Error( + `--image does not resolve to one immutable local image content ID: ${commandDetail(inspect)}`, + ); + } + return contentId; +} + +function exactHarnessContainerIds( + input: Inputs, + networkName: string, + env: NodeJS.ProcessEnv, +): { candidateCount: number; exactIds: string[] } { + const expectedContentId = resolveLocalImageContentId(input.image, env); + const list = commandResult( + [ + "docker", + "ps", + "-aq", + "--no-trunc", + "--filter", + "label=openshell.ai/managed-by=openshell", + "--filter", + `label=openshell.ai/sandbox-name=${input.sandbox}`, + ], + env, + ); + if (list.status !== 0) { + throw new Error(`could not resolve the OpenShell sandbox container: ${commandDetail(list)}`); + } + const candidates = String(list.stdout ?? "") + .trim() + .split(/\s+/u) + .filter(Boolean); + const exactIds: string[] = []; + for (const candidate of candidates) { + const inspect = commandResult(["docker", "inspect", candidate], env); + if (inspect.status !== 0) continue; + try { + const records = JSON.parse(String(inspect.stdout ?? "")) as Array<{ + Config?: { Labels?: Record }; + Image?: string; + NetworkSettings?: { Networks?: Record }; + }>; + const record = records.length === 1 ? records[0] : undefined; + if ( + record?.Config?.Labels?.["openshell.ai/managed-by"] === "openshell" && + record.Config.Labels["openshell.ai/sandbox-name"] === input.sandbox && + record.Image === expectedContentId && + Object.hasOwn(record.NetworkSettings?.Networks ?? {}, networkName) + ) { + exactIds.push(candidate); + } + } catch { + // An unparseable inspection result cannot establish cleanup ownership. + } + } + return { candidateCount: candidates.length, exactIds }; +} + +function assertExactSandboxImage( + input: Inputs, + networkName: string, + env: NodeJS.ProcessEnv, +): string { + const resolved = exactHarnessContainerIds(input, networkName, env); + if (resolved.candidateCount !== 1 || resolved.exactIds.length !== 1) { + throw new Error( + `OpenShell did not launch exactly one harness-owned PR image container: found ${resolved.candidateCount} labeled and ${resolved.exactIds.length} exact`, + ); + } + return resolved.exactIds[0] ?? ""; +} + +function assertFailedSandboxAbsent( + onboard: OnboardModule, + input: Inputs, + env: NodeJS.ProcessEnv, +): void { + const get = onboard.runOpenshell(["sandbox", "get", input.sandbox], { + ignoreError: true, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + const list = onboard.runOpenshell(["sandbox", "list"], { + ignoreError: true, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + if ( + get.status === 0 || + list.status !== 0 || + `${list.stdout ?? ""}\n${list.stderr ?? ""}`.includes(input.sandbox) + ) { + throw new Error( + `managed-bootstrap rollback retained failed OpenShell sandbox state: get=${commandDetail(get)} list=${commandDetail(list)}`, + ); + } +} + +async function run(input: Inputs): Promise { + const stateParent = process.env.RUNNER_TEMP || os.tmpdir(); + const stateDir = fs.mkdtempSync(path.join(stateParent, "nemoclaw-managed-openshell-")); + const networkName = `nemoclaw-managed-pr-${process.pid}-${Date.now().toString(36)}`; + process.env.NEMOCLAW_NON_INTERACTIVE = "1"; + process.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR = stateDir; + process.env.NEMOCLAW_GATEWAY_PORT = String(GATEWAY_PORT); + process.env.NEMOCLAW_DOCKER_GPU_SUPERVISOR_RECONNECT_TIMEOUT = "240"; + process.env.OPENSHELL_DOCKER_NETWORK_NAME = networkName; + process.env.XDG_CONFIG_HOME = path.join(stateDir, "xdg-config"); + process.env.XDG_DATA_HOME = path.join(stateDir, "xdg-data"); + process.env.XDG_STATE_HOME = path.join(stateDir, "xdg-state"); + process.env.PATH = `${path.join(os.homedir(), ".local", "bin")}:${process.env.PATH ?? ""}`; + + let onboard: OnboardModule | null = null; + let ownedContainerId: string | null = null; + let initialSandboxPolicy: InitialSandboxPolicy | null = null; + let failureInjectionQualified = false; + let primaryError: unknown; + let hasPrimaryError = false; + const cleanupErrors: string[] = []; + try { + await assertGatewayPortAvailable(); + const image = parseImmutableManifestReference(input.image); + resolveLocalImageContentId(input.image, process.env); + + const onboardImport = (await import("../../src/lib/onboard.ts")) as unknown as + | OnboardModule + | { default: OnboardModule }; + onboard = "default" in onboardImport ? onboardImport.default : onboardImport; + await onboard.startGatewayForRecovery({ + gatewayName: "nemoclaw", + gatewayPort: GATEWAY_PORT, + }); + configureLocalInferenceRoute(onboard, input, process.env); + + const baseProfile = managedStartupE2eProfile(input.agent, false, true, true); + const protectedProfile = + input.localProvider && input.model + ? withManagedImageLocalInferenceProfile( + baseProfile, + resolveManagedImageLocalInferenceRoute(input.localProvider), + input.model, + ) + : baseProfile; + const profile = encodeManagedStartupProfile(protectedProfile); + const rootApplyRequest = createManagedStartupRootApplyRequest({ + agent: input.agent, + encodedProfile: profile, + corporateCaB64: Buffer.from(MANAGED_STARTUP_E2E_CORPORATE_CA_PEM, "utf8").toString("base64"), + }); + initialSandboxPolicy = prepareInitialSandboxCreatePolicy( + managedImageOpenShellBasePolicyPath(input.agent), + [], + { + agentName: input.agent, + directGpu: input.gpu === true, + hostGpuAvailable: input.gpu === true, + additionalPresets: input.localProvider ? ["local-inference"] : [], + }, + ); + const createArgs = [ + "--from", + input.image, + "--name", + input.sandbox, + "--policy", + initialSandboxPolicy.policyPath, + ...(input.gpu ? ["--gpu"] : []), + ]; + const launch = prepareSandboxCreateLaunch({ + agent: resolveAgent({ agentFlag: input.agent }), + sandboxName: input.sandbox, + chatUiUrl: "", + createArgs, + env: {}, + extraPlaceholderKeys: [], + getDashboardForwardPort: () => "0", + hermesDashboardState: { config: null, enabled: false }, + manageDashboard: false, + openshellShellCommand: (args: string[]) => args.map((arg) => JSON.stringify(arg)).join(" "), + openshellArgv: onboard.openshellArgv, + managedStartupRootApplyRequest: rootApplyRequest, + }); + const prebuild = { createArgs: [...createArgs], imageRef: null, imageId: null }; + if ( + launch.createArgv.filter((value) => value === "--from").length !== 1 || + launch.createArgv[launch.createArgv.indexOf("--from") + 1] !== input.image || + launch.createArgv.filter((value) => value === "--policy").length !== 1 || + launch.createArgv[launch.createArgv.indexOf("--policy") + 1] !== + initialSandboxPolicy.policyPath + ) { + throw new Error("managed-image launch renderer altered the exact PR image identity"); + } + const startupPlan = resolveDockerStartupCommandPatch( + { name: input.agent } as Parameters[0], + true, + ); + if ( + !launch.managedStartupRootApplyRequest || + !launch.managedBootstrapIdentity || + !launch.intendedSandboxStartupCommand + ) { + throw new Error("managed-image launch did not retain its identity-bound bootstrap contract"); + } + + const gpuEnabled = input.gpu === true; + const gpuConfig = { + mode: gpuEnabled ? ("1" as const) : ("0" as const), + hostGpuDetected: gpuEnabled, + hostGpuPlatform: gpuEnabled ? ("linux" as const) : null, + sandboxGpuEnabled: gpuEnabled, + sandboxGpuDevice: null, + errors: [], + }; + const verifyDirectSandboxGpu = gpuEnabled + ? createDirectSandboxGpuVerifier({ + runOpenshell: onboard.runOpenshell, + compactText, + redact: redactProtectedGpuProof, + }) + : () => ({ + status: "unverified" as const, + cudaVerified: false, + label: "disabled", + detail: null, + at: new Date().toISOString(), + }); + const runtimeProvider = { + ...createDockerRuntimeProviderBundle(), + bootstrap: createDockerManagedBootstrapSurface("docker"), + } as RuntimeProviderBundle & { + readonly bootstrap: Extract; + }; + let flow: Awaited> | null = null; + try { + flow = await runSandboxGpuCreateFlow( + { + sandboxName: input.sandbox, + provider: input.localProvider + ? resolveManagedImageLocalInferenceRoute(input.localProvider).providerName + : "nvidia", + sandboxGpuConfig: gpuConfig, + gpuRoutePlan: gpuEnabled ? "native-only" : "none", + initialGpuRoute: gpuEnabled ? "native" : "none", + compatibilityPolicyPath: null, + dockerDriverGateway: true, + gatewayPort: GATEWAY_PORT, + sandboxReadyTimeoutSecs: 240, + createArgv: launch.createArgv, + sandboxEnv: launch.sandboxEnv, + sandboxStartupCommand: launch.sandboxStartupCommand, + prebuild, + restoreBackupPath: null, + terminalAgent: input.agent === "langchain-deepagents-code", + managedBootstrap: { + bootstrapIdentity: launch.managedBootstrapIdentity, + runtimeProvider, + authorityStore: createProtectedAuthorityStore(stateDir), + request: launch.managedStartupRootApplyRequest, + image, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + intendedWorkloadArgv: launch.intendedSandboxStartupCommand, + expectedSupervisorArgv: ["/opt/openshell/bin/openshell-sandbox"], + }, + ...startupPlan, + }, + { + runOpenshell: onboard.runOpenshell, + runCaptureOpenshell: onboard.runCaptureOpenshell, + sleep: onboard.sleepSeconds, + openshellArgv: onboard.openshellArgv, + verifyDirectSandboxGpu, + ...(input.failureInjection + ? { createManagedBootstrapAdapter: () => failureInjectingAdapter(onboard!) } + : {}), + }, + ); + } catch (error) { + if ( + input.failureInjection === "bootstrap-completion" && + error instanceof Error && + error.message.includes("protected-e2e-injected-bootstrap-completion-failure") + ) { + const resolved = exactHarnessContainerIds(input, networkName, launch.sandboxEnv); + if (resolved.candidateCount !== 0 || resolved.exactIds.length !== 0) { + throw new Error( + `managed-bootstrap rollback retained a failed held sandbox: found ${resolved.candidateCount} labeled and ${resolved.exactIds.length} exact containers`, + ); + } + assertFailedSandboxAbsent(onboard, input, launch.sandboxEnv); + failureInjectionQualified = true; + process.stdout.write( + `Injected managed-bootstrap completion failure removed the failed exact ${input.agent} sandbox before harness cleanup.\n`, + ); + } else { + throw error; + } + } + + if (!failureInjectionQualified) { + if (!flow) { + throw new Error("production managed-bootstrap flow returned no result"); + } + const expectedRoute = gpuEnabled ? "native" : "none"; + if (flow.route !== expectedRoute || flow.createResult.status !== 0) { + throw new Error( + `production managed-bootstrap flow did not complete the exact PR image create: route=${flow.route} status=${flow.createResult.status}`, + ); + } + + await waitForCommittedSandboxProbe(onboard, input, launch.sandboxEnv, !gpuEnabled); + ownedContainerId = assertExactSandboxImage(input, networkName, launch.sandboxEnv); + if (gpuEnabled) { + assertProtectedLocalInference(onboard, input, launch.sandboxEnv); + await flow.runtimePatch.commitAfterReady(); + await waitForCommittedSandboxProbe(onboard, input, launch.sandboxEnv); + } + process.stdout.write( + `OpenShell launched exact ${input.agent} PR image ${input.image} through the production managed-bootstrap sequence${gpuEnabled ? ` with real NVIDIA GPU access and ${input.localProvider} inference.local completion` : ""}.\n`, + ); + } + } catch (error) { + primaryError = error; + hasPrimaryError = true; + } finally { + if (onboard) { + commandResult( + onboard.openshellArgv(["sandbox", "delete", input.sandbox]), + process.env, + 15_000, + ); + } + stopProcess(readGatewayPid(stateDir)); + if (onboard) { + commandResult(onboard.openshellArgv(["gateway", "remove", "nemoclaw"]), process.env, 15_000); + } + try { + const resolved = exactHarnessContainerIds(input, networkName, process.env); + const cleanupContainerId = + resolved.exactIds.length === 1 ? (resolved.exactIds[0] ?? null) : null; + if (cleanupContainerId) { + const remove = commandResult( + ["docker", "rm", "-f", cleanupContainerId], + process.env, + 15_000, + ); + const verify = commandResult( + ["docker", "container", "inspect", cleanupContainerId], + process.env, + 15_000, + ); + if (verify.status === 0 || !isDockerNotFound(verify)) { + cleanupErrors.push( + `exact harness container ${cleanupContainerId} was not removed: ${commandDetail(remove)} ${commandDetail(verify)}`.trim(), + ); + } + } else if (resolved.exactIds.length > 1) { + cleanupErrors.push( + `refusing ambiguous exact harness container cleanup: ${resolved.exactIds.length} matches`, + ); + } else if (ownedContainerId) { + const verify = commandResult( + ["docker", "container", "inspect", ownedContainerId], + process.env, + 15_000, + ); + if (verify.status === 0 || !isDockerNotFound(verify)) { + cleanupErrors.push( + `could not prove exact harness container ${ownedContainerId} was removed: ${commandDetail(verify)}`, + ); + } + } + } catch (error) { + cleanupErrors.push(error instanceof Error ? error.message : String(error)); + } + const removeNetwork = commandResult( + ["docker", "network", "rm", networkName], + process.env, + 15_000, + ); + const verifyNetwork = commandResult( + ["docker", "network", "inspect", networkName], + process.env, + 15_000, + ); + if (verifyNetwork.status === 0 || !isDockerNotFound(verifyNetwork)) { + cleanupErrors.push( + `harness network ${networkName} was not removed: ${commandDetail(removeNetwork)} ${commandDetail(verifyNetwork)}`.trim(), + ); + } + const remainingSandboxContainers = commandResult( + [ + "docker", + "ps", + "-aq", + "--filter", + "label=openshell.ai/managed-by=openshell", + "--filter", + `label=openshell.ai/sandbox-name=${input.sandbox}`, + ], + process.env, + 15_000, + ); + if ( + remainingSandboxContainers.status !== 0 || + String(remainingSandboxContainers.stdout ?? "").trim() !== "" + ) { + cleanupErrors.push( + `managed-image sandbox/container orphan remained after cleanup: ${commandDetail(remainingSandboxContainers)}`, + ); + } + try { + initialSandboxPolicy?.cleanup?.(); + } catch (error) { + cleanupErrors.push(error instanceof Error ? error.message : String(error)); + } + try { + fs.rmSync(stateDir, { recursive: true, force: true }); + } catch (error) { + cleanupErrors.push(error instanceof Error ? error.message : String(error)); + } + } + + const cleanupDetail = + cleanupErrors.length > 0 + ? `managed-image OpenShell cleanup failed: ${cleanupErrors.join("; ")}` + : null; + if (hasPrimaryError) { + if (cleanupDetail) { + const primaryDetail = + primaryError instanceof Error ? primaryError.message : String(primaryError); + throw new Error(`${primaryDetail}; ${cleanupDetail}`, { cause: primaryError }); + } + throw primaryError; + } + if (cleanupDetail) { + throw new Error(cleanupDetail); + } + if (failureInjectionQualified) { + process.stdout.write( + `Managed-bootstrap failure injection left no sandbox, container, network, or harness state orphan for ${input.agent}.\n`, + ); + } +} + +if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) { + run(parseManagedImageOpenShellE2eInputs(process.argv.slice(2))).catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} + +export { run as runManagedImageOpenShellE2e }; diff --git a/test/e2e/live/managed-image-protected-runtime-helpers.ts b/test/e2e/live/managed-image-protected-runtime-helpers.ts new file mode 100644 index 00000000000..4ef2bd7d13b --- /dev/null +++ b/test/e2e/live/managed-image-protected-runtime-helpers.ts @@ -0,0 +1,423 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { randomBytes } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +import { + type ManagedImageLocalInferenceKind, + managedImageProtectedSandboxName, + PROTECTED_MANAGED_IMAGE_AGENTS, + type ProtectedManagedImageContract, + parseProtectedManagedImageContracts, +} from "../../../scripts/checks/managed-image-protected-runtime-contract.ts"; +import { + adoptServedModelId, + dockerLoginNgc, + pullNimImage, + startNimContainerByName, + stopNimContainerByName, + waitForNimHealth, +} from "../../../src/lib/inference/nim.ts"; +import { + getOllamaProxyToken, + killStaleProxy, + persistAndProbeOllamaProxy, + startOllamaAuthProxy, +} from "../../../src/lib/inference/ollama/proxy.ts"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import type { E2ETargetFixtures } from "../fixtures/e2e-test.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import { + assertNvidiaAvailable, + cleanupOllama, + ensureOllama, + env as gpuEnv, + REPO_ROOT, +} from "./gpu-e2e-helpers.ts"; + +const OLLAMA_MODEL = "qwen3.5:9b"; +const VLLM_MODEL = "Qwen/Qwen2.5-0.5B-Instruct"; +const VLLM_IMAGE = + "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416"; +const VLLM_CONTAINER = "nemoclaw-managed-image-vllm-e2e"; +const NIM_CATALOG_MODEL = "nvidia/nemotron-3-nano-30b-a3b"; +const NIM_CONTAINER = "nemoclaw-managed-image-nim-e2e"; +const AGENT_QUALIFICATION_TIMEOUT_MS = 10 * 60_000; +const ROLLBACK_QUALIFICATION_TIMEOUT_MS = 10 * 60_000; + +type RuntimeFixtures = Pick; + +function imageContracts(): ProtectedManagedImageContract[] { + const contractPath = process.env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT; + if (!contractPath || !path.isAbsolute(contractPath)) { + throw new Error("NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT must be an absolute path"); + } + return parseProtectedManagedImageContracts( + JSON.parse(fs.readFileSync(contractPath, "utf8")), + "linux/amd64", + ); +} + +function requiredNgcApiKey(value: string): string { + const key = value.trim(); + if (!key || /[\0\r\n]/u.test(key)) { + throw new Error("protected managed-image NIM qualification requires NVIDIA_API_KEY"); + } + return key; +} + +async function runExactImageQualification( + host: HostCliClient, + contract: ProtectedManagedImageContract, + kind: ManagedImageLocalInferenceKind, + model: string, + extraEnv: NodeJS.ProcessEnv, +): Promise { + const sandboxName = managedImageProtectedSandboxName(contract.agent, kind); + const result = await host.command( + "npx", + [ + "--no-install", + "tsx", + "scripts/checks/run-managed-image-openshell-e2e.ts", + "--agent", + contract.agent, + "--image", + contract.reference, + "--sandbox", + sandboxName, + "--gpu", + "--local-provider", + kind, + "--model", + model, + ], + { + artifactName: `managed-image-${contract.agent}-${kind}`, + cwd: REPO_ROOT, + env: { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_NON_INTERACTIVE: "1", + ...extraEnv, + }, + timeoutMs: AGENT_QUALIFICATION_TIMEOUT_MS, + }, + ); + expect(result.exitCode, resultText(result)).toBe(0); + expect(result.stdout).toContain(`exact ${contract.agent} PR image ${contract.reference}`); + expect(result.stdout).toContain("real NVIDIA GPU access"); + expect(result.stdout).toContain(`${kind} inference.local completion`); +} + +async function qualifyEveryAgent( + host: HostCliClient, + contracts: readonly ProtectedManagedImageContract[], + kind: ManagedImageLocalInferenceKind, + model: string, + extraEnv: NodeJS.ProcessEnv, +): Promise { + for (const contract of contracts) { + await runExactImageQualification(host, contract, kind, model, extraEnv); + } +} + +async function startProtectedOllama(host: HostCliClient): Promise { + await ensureOllama(host); + await cleanupOllama(host, "pre-cleanup-managed-image-ollama"); + const start = await host.command( + "bash", + [ + "-lc", + `set -euo pipefail +OLLAMA_HOST=127.0.0.1:11434 nohup ollama serve >"${process.env.RUNNER_TEMP ?? "/tmp"}/managed-image-ollama.log" 2>&1 & +for _ in $(seq 1 120); do + curl -fsS --connect-timeout 2 http://127.0.0.1:11434/api/tags >/dev/null 2>&1 && exit 0 + sleep 1 +done +exit 1`, + ], + { + artifactName: "start-managed-image-ollama", + env: gpuEnv(), + timeoutMs: 150_000, + }, + ); + expect(start.exitCode, resultText(start)).toBe(0); + const pull = await host.command("ollama", ["pull", OLLAMA_MODEL], { + artifactName: "pull-managed-image-ollama-model", + env: gpuEnv(), + timeoutMs: 45 * 60_000, + }); + expect(pull.exitCode, resultText(pull)).toBe(0); + expect(startOllamaAuthProxy(), "Ollama auth proxy must start").toBe(true); + const proxyToken = getOllamaProxyToken(); + expect(proxyToken).toMatch(/^[a-f0-9]{48}$/u); + await persistAndProbeOllamaProxy(proxyToken!); + return proxyToken!; +} + +async function proveOllamaGpuPlacement(host: HostCliClient): Promise { + const result = await host.command( + "bash", + [ + "-lc", + `curl -fsS http://127.0.0.1:11434/api/ps | jq -e --arg model "${OLLAMA_MODEL}" ' + [.models[] | select((.name == $model or .model == $model) and ((.size_vram // 0) > 0))] + | length >= 1 + '`, + ], + { + artifactName: "ollama-gpu-placement", + env: gpuEnv(), + timeoutMs: 30_000, + }, + ); + expect(result.exitCode, resultText(result)).toBe(0); +} + +async function startProtectedVllm(host: HostCliClient): Promise { + await host.command("docker", ["rm", "-f", VLLM_CONTAINER], { + artifactName: "pre-cleanup-vllm", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + const start = await host.command( + "docker", + [ + "run", + "--detach", + "--name", + VLLM_CONTAINER, + "--gpus", + "all", + "--publish", + "8000:8000", + VLLM_IMAGE, + "--model", + VLLM_MODEL, + "--served-model-name", + VLLM_MODEL, + "--max-model-len", + "2048", + "--gpu-memory-utilization", + "0.45", + ], + { + artifactName: "start-vllm", + env: buildAvailabilityProbeEnv(), + timeoutMs: 20 * 60_000, + }, + ); + expect(start.exitCode, resultText(start)).toBe(0); + const ready = await host.command( + "bash", + [ + "-lc", + `set -euo pipefail +for _ in $(seq 1 300); do + curl -fsS --connect-timeout 2 http://127.0.0.1:8000/v1/models >/dev/null 2>&1 && exit 0 + docker container inspect "${VLLM_CONTAINER}" --format '{{.State.Running}}' | grep -Fx true >/dev/null + sleep 2 +done +docker logs --tail 200 "${VLLM_CONTAINER}" >&2 +exit 1`, + ], + { + artifactName: "wait-vllm", + env: buildAvailabilityProbeEnv(), + timeoutMs: 11 * 60_000, + }, + ); + expect(ready.exitCode, resultText(ready)).toBe(0); + const cuda = await host.command( + "docker", + [ + "exec", + VLLM_CONTAINER, + "python3", + "-c", + "import torch; assert torch.cuda.is_available(); print(torch.cuda.get_device_name(0))", + ], + { + artifactName: "vllm-cuda-initialization", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + expect(cuda.exitCode, resultText(cuda)).toBe(0); +} + +async function startProtectedNim(host: HostCliClient, apiKey: string): Promise { + stopNimContainerByName(NIM_CONTAINER, { silent: true }); + expect(dockerLoginNgc(apiKey), "NGC login must succeed for protected NIM qualification").toBe( + true, + ); + pullNimImage(NIM_CATALOG_MODEL); + startNimContainerByName(NIM_CONTAINER, NIM_CATALOG_MODEL, 8000, { ngcApiKey: apiKey }); + expect( + waitForNimHealth(8000, 20 * 60, { container: NIM_CONTAINER }), + "NIM must become healthy", + ).toBe(true); + const servedModel = adoptServedModelId(NIM_CATALOG_MODEL, 8000); + expect(servedModel, "NIM must report one safe served model").toBeTruthy(); + const cuda = await host.command("docker", ["exec", NIM_CONTAINER, "nvidia-smi", "-L"], { + artifactName: "nim-cuda-initialization", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + expect(cuda.exitCode, resultText(cuda)).toBe(0); + return servedModel!; +} + +async function qualifyRollback( + host: HostCliClient, + contract: ProtectedManagedImageContract, +): Promise { + const sandboxName = managedImageProtectedSandboxName(contract.agent, "rollback"); + const result = await host.command( + "npx", + [ + "--no-install", + "tsx", + "scripts/checks/run-managed-image-openshell-e2e.ts", + "--agent", + contract.agent, + "--image", + contract.reference, + "--sandbox", + sandboxName, + "--inject-bootstrap-completion-failure", + ], + { + artifactName: `managed-image-${contract.agent}-bootstrap-rollback`, + cwd: REPO_ROOT, + env: { ...buildAvailabilityProbeEnv(), NEMOCLAW_NON_INTERACTIVE: "1" }, + timeoutMs: ROLLBACK_QUALIFICATION_TIMEOUT_MS, + }, + ); + expect(result.exitCode, resultText(result)).toBe(0); + expect(result.stdout).toContain( + `removed the failed exact ${contract.agent} sandbox before harness cleanup`, + ); + expect(result.stdout).toContain( + `left no sandbox, container, network, or harness state orphan for ${contract.agent}`, + ); +} + +async function qualifyEveryRollback( + host: HostCliClient, + contracts: readonly ProtectedManagedImageContract[], +): Promise { + for (const contract of contracts) await qualifyRollback(host, contract); +} + +async function proveOwnedRuntimeInventoryClean(host: HostCliClient): Promise { + const result = await host.command( + "bash", + [ + "-lc", + `set -euo pipefail +containers="$(docker ps -a --format '{{.Label "openshell.ai/sandbox-name"}}' --filter label=openshell.ai/managed-by=openshell | grep '^nemoclaw-managed-' || true)" +networks="$(docker network ls --format '{{.Name}}' | grep '^nemoclaw-managed-pr-' || true)" +test -z "$containers" +test -z "$networks"`, + ], + { + artifactName: "final-managed-image-owned-runtime-inventory", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(result.exitCode, resultText(result)).toBe(0); +} + +export async function qualifyProtectedManagedImageRuntime( + fixtures: RuntimeFixtures, + ngcApiKeyInput: string, +): Promise { + const { artifacts, cleanup, host, progress } = fixtures; + const contracts = imageContracts(); + const ngcApiKey = requiredNgcApiKey(ngcApiKeyInput); + + cleanup.trackDisposable("remove protected vLLM container", async () => { + await host.command("docker", ["rm", "-f", VLLM_CONTAINER], { + artifactName: "cleanup-vllm-container", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + }); + cleanup.trackDisposable("remove protected NIM container", () => { + stopNimContainerByName(NIM_CONTAINER, { silent: true }); + }); + cleanup.trackDisposable("stop protected Ollama runtime", async () => { + killStaleProxy(); + await cleanupOllama(host, "cleanup-managed-image-ollama"); + }); + + let activePhase = "validate protected host runtime"; + try { + const docker = await host.command("docker", ["info"], { + artifactName: "docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(docker.exitCode, resultText(docker)).toBe(0); + const nvidia = await host.command("nvidia-smi", [], { + artifactName: "nvidia-smi", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + assertNvidiaAvailable(nvidia, (message) => { + throw new Error(message ?? "protected GPU runner is unavailable"); + }); + + activePhase = "qualify all managed agents with GPU-backed Ollama"; + progress.phase("qualify all managed agents with GPU-backed Ollama"); + const proxyToken = await startProtectedOllama(host); + await qualifyEveryAgent(host, contracts, "ollama", OLLAMA_MODEL, { + NEMOCLAW_OLLAMA_PROXY_TOKEN: proxyToken, + }); + await proveOllamaGpuPlacement(host); + killStaleProxy(); + await cleanupOllama(host, "stop-ollama-before-vllm"); + + activePhase = "qualify all managed agents with GPU-backed vLLM"; + progress.phase("qualify all managed agents with GPU-backed vLLM"); + await startProtectedVllm(host); + await qualifyEveryAgent(host, contracts, "vllm", VLLM_MODEL, { + NEMOCLAW_VLLM_LOCAL_TOKEN: randomBytes(24).toString("hex"), + }); + await host.command("docker", ["rm", "-f", VLLM_CONTAINER], { + artifactName: "stop-vllm-before-nim", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + + activePhase = "qualify all managed agents with GPU-backed NVIDIA NIM"; + progress.phase("qualify all managed agents with GPU-backed NVIDIA NIM"); + const nimModel = await startProtectedNim(host, ngcApiKey); + await qualifyEveryAgent(host, contracts, "nim", nimModel, { + NEMOCLAW_VLLM_LOCAL_TOKEN: randomBytes(24).toString("hex"), + }); + stopNimContainerByName(NIM_CONTAINER, { silent: true }); + + activePhase = "prove all-agent managed bootstrap rollback and exact cleanup"; + progress.phase("prove all-agent managed bootstrap rollback and exact cleanup"); + await qualifyEveryRollback(host, contracts); + await proveOwnedRuntimeInventoryClean(host); + await artifacts.writeJson("managed-image-protected-runtime-summary.json", { + agents: PROTECTED_MANAGED_IMAGE_AGENTS, + providers: ["ollama", "vllm", "nim"], + rollbackAgents: PROTECTED_MANAGED_IMAGE_AGENTS, + }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`protected managed-image runtime phase '${activePhase}' failed: ${detail}`, { + cause: error, + }); + } +} diff --git a/test/e2e/live/managed-image-protected-runtime.test.ts b/test/e2e/live/managed-image-protected-runtime.test.ts new file mode 100644 index 00000000000..c42b6d0325c --- /dev/null +++ b/test/e2e/live/managed-image-protected-runtime.test.ts @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { PROTECTED_MANAGED_IMAGE_AGENTS } from "../../../scripts/checks/managed-image-protected-runtime-contract.ts"; +import { test } from "../fixtures/e2e-test.ts"; +import { qualifyProtectedManagedImageRuntime } from "./managed-image-protected-runtime-helpers.ts"; + +const TIMEOUT_MS = 220 * 60_000; + +test("exact all-agent managed images retain GPU, Ollama, NIM, vLLM, rollback, and cleanup (#7744)", { + timeout: TIMEOUT_MS, + meta: { + e2ePhases: [ + "qualify all managed agents with GPU-backed Ollama", + "qualify all managed agents with GPU-backed vLLM", + "qualify all managed agents with GPU-backed NVIDIA NIM", + "prove all-agent managed bootstrap rollback and exact cleanup", + ], + }, +}, async ({ artifacts, cleanup, host, progress, secrets }) => { + await artifacts.target.declare({ + id: "managed-image-protected-runtime", + boundary: + "exact PR image digests for every managed agent through Docker/OpenShell GPU, host-local Ollama, NVIDIA NIM, vLLM, transactional rollback, and owned cleanup", + agents: [...PROTECTED_MANAGED_IMAGE_AGENTS], + providers: ["ollama", "nim", "vllm"], + credentialBoundary: + "The NVIDIA key is staged only to the host-side NGC login and NIM container; managed sandboxes receive only generated local route tokens.", + }); + await qualifyProtectedManagedImageRuntime( + { artifacts, cleanup, host, progress }, + secrets.optional("NVIDIA_API_KEY") ?? "", + ); +}); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 93d290e6363..aab551a2c6c 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -13,6 +13,16 @@ "test/protected-managed-image-contract.test.ts" ] }, + { + "live": "test/e2e/live/managed-image-protected-runtime.test.ts", + "fast": [ + "src/lib/inference/nim.test.ts", + "src/lib/onboard/sandbox-gpu-create-flow.test.ts", + "test/e2e/support/managed-image-protected-runtime-workflow.test.ts", + "test/managed-image-protected-runtime-contract.test.ts", + "test/pr-risk-plan.test.ts" + ] + }, { "live": "test/e2e/live/hermes-gpu-startup.test.ts", "fast": [ diff --git a/test/e2e/support/e2e-cross-runtime-compatibility.test.ts b/test/e2e/support/e2e-cross-runtime-compatibility.test.ts index 787a63f5024..d45722b8ba2 100644 --- a/test/e2e/support/e2e-cross-runtime-compatibility.test.ts +++ b/test/e2e/support/e2e-cross-runtime-compatibility.test.ts @@ -29,7 +29,7 @@ describe("cross-runtime foundation compatibility", () => { ), ).toBe("6272aab16cf4b9555bdc4b3f4c0cdd24b5faa55118cbd61cbb4b30a3d418a63a"); expect(digestOutput(buildE2eWorkflowPlan())).toBe( - "36795de73b09280ad17f7a6296d5690572e23dabe40b374e754836066589d145", + "00dddd726f979dfddceef7b61b7e4937d48394af3e7224b22bb4014d5b362106", ); }); @@ -45,7 +45,7 @@ describe("cross-runtime foundation compatibility", () => { ]; expect(digestOutput(cases.map(buildRiskPlan))).toBe( - "311bd367e8d6ee469a9ec99aba13ab9b806ac679f7e71381c09d4fc4beafd4a2", + "7f55218cbfc184b0c2478ae435075c0fc2b9b053b01dda3fdb15c4697bf427e0", ); }); }); diff --git a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts new file mode 100644 index 00000000000..168666adf56 --- /dev/null +++ b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import YAML from "yaml"; + +import { validateManagedImageProtectedRuntimeWorkflow } from "../../../tools/e2e/managed-image-protected-runtime-workflow-boundary.mts"; + +type WorkflowRecord = Record; + +function workflow(): WorkflowRecord { + return YAML.parse( + fs.readFileSync( + path.resolve(import.meta.dirname, "../../../.github/workflows/e2e.yaml"), + "utf8", + ), + ) as WorkflowRecord; +} + +function runtimeJob(value: WorkflowRecord): Record { + return (value.jobs as Record>)["managed-image-protected-runtime"]; +} + +function namedStep(value: WorkflowRecord, name: string): Record { + const step = (runtimeJob(value).steps as Array>).find( + (step) => step.name === name, + ); + expect(step, `workflow step '${name}' is missing`).toBeDefined(); + return step as Record; +} + +describe("protected managed-image runtime workflow boundary", () => { + it("accepts the exact activated trusted runtime lane", () => { + expect(validateManagedImageProtectedRuntimeWorkflow(workflow())).toEqual([]); + }); + + it("ships the exact activation contract consumed by the trusted lane (#7744)", () => { + const activation = JSON.parse( + fs.readFileSync( + path.resolve( + import.meta.dirname, + "../../../ci/protected-managed-image-runtime-activation-v1.json", + ), + "utf8", + ), + ) as unknown; + + expect(activation).toEqual({ + agents: ["openclaw", "hermes", "langchain-deepagents-code"], + contractVersion: 1, + jobId: "managed-image-protected-runtime", + platform: "linux/amd64", + providers: ["ollama", "nim", "vllm"], + }); + }); + + it("rejects job-scoped NGC credentials", () => { + const value = workflow(); + runtimeJob(value).env = { + ...(runtimeJob(value).env as Record), + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + }; + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toContain( + "managed-image-protected-runtime must not expose NVIDIA_API_KEY at job scope", + ); + }); + + it("rejects checking candidate source out over trusted qualification code", () => { + const value = workflow(); + const candidateCheckout = namedStep(value, "Checkout exact protected runtime candidate source"); + (candidateCheckout.with as Record).path = "."; + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toContain( + "managed-image-protected-runtime candidate checkout must bind path to .candidate-runtime", + ); + }); + + it("rejects exposing the NGC credential to candidate-controlled steps", () => { + const value = workflow(); + namedStep(value, "Validate protected runtime activation contract").env = { + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + }; + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toContain( + "managed-image-protected-runtime must expose NVIDIA_API_KEY only to trusted qualification code", + ); + }); + + it("rejects executing candidate checkout paths in the secret-bearing qualification step", () => { + const value = workflow(); + const qualification = namedStep( + value, + "Run all-agent GPU, local inference, rollback, and cleanup qualification", + ); + qualification.run = `${String(qualification.run)}\nnpx tsx .candidate-runtime/leak.ts`; + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toContain( + "managed-image-protected-runtime trusted qualification must not execute candidate checkout paths", + ); + }); + + it("rejects removing NIM from the activation contract", () => { + const value = workflow(); + const step = namedStep(value, "Validate protected runtime activation contract"); + step.run = String(step.run).replace( + '.providers == ["ollama", "nim", "vllm"]', + '.providers == ["ollama", "vllm"]', + ); + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toContain( + 'managed-image-protected-runtime step \'Validate protected runtime activation contract\' must include .providers == ["ollama", "nim", "vllm"]', + ); + }); + + it("rejects qualification before exact all-agent image construction", () => { + const value = workflow(); + const job = runtimeJob(value); + const workflowSteps = job.steps as Array>; + const qualification = namedStep( + value, + "Run all-agent GPU, local inference, rollback, and cleanup qualification", + ); + job.steps = [qualification, ...workflowSteps.filter((step) => step !== qualification)]; + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toContain( + "managed-image-protected-runtime protected qualification and cleanup steps drifted", + ); + }); +}); diff --git a/test/managed-image-protected-runtime-contract.test.ts b/test/managed-image-protected-runtime-contract.test.ts new file mode 100644 index 00000000000..4c81e93d9b1 --- /dev/null +++ b/test/managed-image-protected-runtime-contract.test.ts @@ -0,0 +1,159 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { managedStartupE2eProfile } from "../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { + MANAGED_IMAGE_LOCAL_INFERENCE_KINDS, + managedImageProtectedSandboxName, + resolveManagedImageLocalInferenceRoute, + withManagedImageLocalInferenceProfile, +} from "../scripts/checks/managed-image-protected-runtime-contract.ts"; +import { + managedImageLocalInferenceBaseUrl, + managedImageOpenShellBasePolicyPath, + managedImageOpenShellCommittedProbe, + managedImageOpenShellProbe, + parseManagedImageOpenShellE2eInputs, +} from "../scripts/checks/run-managed-image-openshell-e2e.ts"; + +const IMAGE = `localhost:5000/nemoclaw-managed-protected/openclaw@sha256:${"a".repeat(64)}`; + +describe("protected managed-image runtime contract", () => { + it.each([ + ["ollama", "ollama-local", "NEMOCLAW_OLLAMA_PROXY_TOKEN", 11435], + ["nim", "vllm-local", "NEMOCLAW_VLLM_LOCAL_TOKEN", 8000], + ["vllm", "vllm-local", "NEMOCLAW_VLLM_LOCAL_TOKEN", 8000], + ] as const)("maps %s to its exact host-local route", (kind, provider, credential, port) => { + const route = resolveManagedImageLocalInferenceRoute(kind); + + expect(MANAGED_IMAGE_LOCAL_INFERENCE_KINDS).toContain(kind); + expect(route).toMatchObject({ kind, providerName: provider, credentialEnv: credential }); + expect(new URL(route.defaultBaseUrl)).toMatchObject({ + hostname: "host.openshell.internal", + port: String(port), + pathname: "/v1", + protocol: "http:", + }); + }); + + it("accepts an exact protected local-inference URL override", () => { + expect( + managedImageLocalInferenceBaseUrl("ollama", "http://host.openshell.internal:11435/v1/"), + ).toBe("http://host.openshell.internal:11435/v1"); + }); + + it.each([ + ["HTTPS", "https://host.openshell.internal:11435/v1"], + ["another host", "http://example.invalid:11435/v1"], + ["a missing port", "http://host.openshell.internal/v1"], + ["port zero", "http://host.openshell.internal:0/v1"], + ["an out-of-range port", "http://host.openshell.internal:65536/v1"], + ["another path", "http://host.openshell.internal:11435/v2"], + ["credentials", "http://user:secret@host.openshell.internal:11435/v1"], + ["a query", "http://host.openshell.internal:11435/v1?model=other"], + ["a fragment", "http://host.openshell.internal:11435/v1#other"], + ])("rejects a protected local-inference override with %s", (_case, value) => { + expect(() => managedImageLocalInferenceBaseUrl("ollama", value)).toThrow( + /protected local inference/u, + ); + }); + + it.each([ + "openclaw", + "hermes", + "langchain-deepagents-code", + ] as const)("binds %s to an exact GPU/local-inference launch", (agent) => { + const parsed = parseManagedImageOpenShellE2eInputs([ + "--agent", + agent, + "--image", + IMAGE, + "--sandbox", + managedImageProtectedSandboxName(agent, "nim"), + "--gpu", + "--local-provider", + "nim", + "--model", + "nvidia/nemotron-3-nano", + ]); + + expect(parsed).toEqual({ + agent, + gpu: true, + image: IMAGE, + localProvider: "nim", + model: "nvidia/nemotron-3-nano", + sandbox: managedImageProtectedSandboxName(agent, "nim"), + }); + expect(path.isAbsolute(managedImageOpenShellBasePolicyPath(agent))).toBe(true); + expect(managedImageOpenShellProbe(agent)).toContain("managed-startup-complete.json"); + }); + + it("rewrites only the inference route while preserving the managed agent profile", () => { + const profile = managedStartupE2eProfile("hermes", false, true, true); + const route = resolveManagedImageLocalInferenceRoute("nim"); + const rewritten = withManagedImageLocalInferenceProfile( + profile, + route, + "nvidia/nemotron-3-nano", + ); + + expect(rewritten).toMatchObject({ + agent: "hermes", + inference: { + api: "openai-completions", + model: "nvidia/nemotron-3-nano", + routedBaseUrl: "https://inference.local/v1", + routeProvider: "inference", + upstreamEndpointUrl: null, + upstreamProvider: "vllm-local", + }, + }); + expect(rewritten.agentConfig).toEqual(profile.agentConfig); + }); + + it("rejects mutable images and incomplete GPU provider tuples", () => { + expect(() => + parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + "localhost:5000/openclaw:latest", + "--sandbox", + "managed-openclaw", + ]), + ).toThrow(/immutable repository@sha256/u); + expect(() => + parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + "managed-openclaw", + "--gpu", + ]), + ).toThrow(/--gpu requires/u); + }); + + it("keeps rollback cleanup distinct from initial readiness", () => { + expect(managedImageOpenShellCommittedProbe()).toContain( + "managed-startup-shared-state-transaction-v1", + ); + expect( + parseManagedImageOpenShellE2eInputs([ + "--agent", + "openclaw", + "--image", + IMAGE, + "--sandbox", + "managed-openclaw-rollback", + "--inject-bootstrap-completion-failure", + ]), + ).toMatchObject({ failureInjection: "bootstrap-completion" }); + }); +}); diff --git a/test/pr-e2e-gate-signal-shards.test.ts b/test/pr-e2e-gate-signal-shards.test.ts index c6b0666d0c3..edd01697f6a 100644 --- a/test/pr-e2e-gate-signal-shards.test.ts +++ b/test/pr-e2e-gate-signal-shards.test.ts @@ -47,8 +47,8 @@ describe("PR E2E signal shard policy", () => { }); const broadPlan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: BROAD_FILES }); const broadShards = expectedSignalShards(riskPlanRequiredJobIds(broadPlan)); - expect(Object.keys(broadShards)).toHaveLength(13); - expect(Object.values(broadShards).flat()).toHaveLength(15); + expect(Object.keys(broadShards)).toHaveLength(14); + expect(Object.values(broadShards).flat()).toHaveLength(17); expect(() => expectedSignalShards(["not-a-workflow-job"])).toThrow(/does not define/u); }); diff --git a/test/pr-e2e-gate.test.ts b/test/pr-e2e-gate.test.ts index 548d5929a8c..85d99c96f52 100644 --- a/test/pr-e2e-gate.test.ts +++ b/test/pr-e2e-gate.test.ts @@ -62,6 +62,7 @@ const BROAD_FILES = [ const BROAD_JOBS = [ "cloud-inference", "cloud-onboard", + "managed-image-multiarch-startup", "security-posture", "channels-add-remove", "channels-stop-start", @@ -1408,7 +1409,7 @@ describe("PR E2E controller", () => { expect(checkUpdates[1]?.body).toMatchObject({ status: "in_progress", output: { - title: "Running 13 E2E checks", + title: "Running 14 E2E checks", summary: expect.stringContaining("rebuild-openclaw"), }, }); diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index 2c1eaf1ae60..120208f0474 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -24,6 +24,7 @@ const HERMES_SANDBOX_BOUNDARY_JOBS = [ "full-e2e", "hermes-e2e", "hermes-inference-switch", + "managed-image-multiarch-startup", "security-posture", ]; const HERMES_CLI_ADAPTER_JOBS = ["channels-stop-start", "mcp-bridge"]; @@ -79,7 +80,7 @@ describe("deterministic PR risk plan", () => { const second = plan("src/lib/onboard.ts", "src/lib/state/registry.ts"); expect(first).toEqual(second); - expect(first.version).toBe(13); + expect(first.version).toBe(14); expect(first.headSha).toBe(HEAD_SHA); expect(first.planHash).toMatch(/^[a-f0-9]{64}$/u); expect(first.changedFiles).toEqual(["src/lib/onboard.ts", "src/lib/state/registry.ts"]); @@ -223,6 +224,7 @@ describe("deterministic PR risk plan", () => { "full-e2e", "hermes-e2e", "hermes-inference-switch", + "managed-image-multiarch-startup", "security-posture", ]); }); @@ -317,29 +319,91 @@ describe("deterministic PR risk plan", () => { ]); }); - it("keeps the protected managed-image lane dormant until its trusted activation marker (#7744)", () => { + it("activates protected multiarch qualification for every managed-image build input (#7744)", () => { const activation = "ci/protected-managed-image-multiarch-activation-v1.json"; - const result = plan(activation); - const preActivationPaths = [ + const managedImageInputs = [ + activation, + ".github/workflows/managed-images.yaml", + "Dockerfile", + "agents/hermes/Dockerfile", + "agents/langchain-deepagents-code/Dockerfile", "scripts/checks/run-managed-image-direct-e2e.ts", - "scripts/checks/build-protected-managed-images.sh", - "scripts/checks/protected-managed-image-contract.ts", - "test/e2e/live/managed-image-multiarch-startup.test.ts", + "src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.99.json", + "src/lib/onboard/managed-startup/image-runtime.ts", ]; + const result = plan(...managedImageInputs); + const adjacentOnboardChange = plan("src/lib/onboard/provider-selection.ts"); expect(result.families).toContainEqual( expect.objectContaining({ id: "managed-image-multiarch", - matchedFiles: [activation], + matchedFiles: [...managedImageInputs].sort((left, right) => left.localeCompare(right)), requiredJobs: ["managed-image-multiarch-startup"], }), ); - expect(riskPlanRequiredJobIds(result)).toEqual(["managed-image-multiarch-startup"]); - for (const file of preActivationPaths) { - expect(plan(file).families.map((family) => family.id)).not.toContain( - "managed-image-multiarch", - ); - } + expect(riskPlanRequiredJobIds(result)).toContain("managed-image-multiarch-startup"); + expect(riskPlanRequiredJobIds(plan(activation))).toEqual(["managed-image-multiarch-startup"]); + expect( + adjacentOnboardChange.families.some((family) => family.id === "managed-image-multiarch"), + ).toBe(false); + }); + + it.each([ + ".github/workflows/managed-images.yaml", + ".dockerignore", + "Dockerfile", + "agents/hermes/Dockerfile", + "ci/npm-audit-exceptions.json", + "nemoclaw/src/index.ts", + "nemoclaw-blueprint/blueprint.yaml", + "scripts/checks/build-protected-managed-images.sh", + "src/lib/actions/sandbox/openshell-child-visible-credentials.v0.0.99.json", + "src/lib/core/json-types.ts", + "src/lib/core/ports.ts", + "src/lib/messaging/runtime.ts", + "src/lib/onboard/managed-bootstrap/envelope.ts", + "src/lib/onboard/managed-startup/image-runtime.ts", + "src/lib/security/credential-hash.ts", + "src/lib/state/paths.ts", + "src/lib/state/state-root.ts", + "src/lib/tool-disclosure.ts", + "tools/mcp-tool-discovery-runtime/index.ts", + "tsconfig.runtime-preloads.json", + ])("selects protected multiarch qualification for managed-image input %s (#7744)", (file) => { + expect(riskPlanRequiredJobIds(plan(file))).toContain("managed-image-multiarch-startup"); + }); + + it("does not select protected multiarch qualification for adjacent changes (#7744)", () => { + expect( + plan( + ".github/workflows/e2e.yaml", + "docs/get-started/quickstart.mdx", + "src/lib/onboard/provider-selection.ts", + ).families.some((family) => family.id === "managed-image-multiarch"), + ).toBe(false); + }); + + it("keeps protected GPU and local-inference qualification activation-only until trusted (#7744)", () => { + const activation = "ci/protected-managed-image-runtime-activation-v1.json"; + const result = plan(activation); + const dormantImplementation = plan( + "scripts/checks/run-managed-image-openshell-e2e.ts", + "test/e2e/live/managed-image-protected-runtime.test.ts", + ); + + expect(result.families).toContainEqual( + expect.objectContaining({ + id: "managed-image-protected-runtime", + matchedFiles: [activation], + requiredJobs: ["managed-image-protected-runtime"], + }), + ); + expect(riskPlanRequiredJobIds(result)).toEqual(["managed-image-protected-runtime"]); + expect( + dormantImplementation.families.some( + (family) => family.id === "managed-image-protected-runtime", + ), + ).toBe(false); }); it("loads protected multiarch identifiers through the workflow node loader (#7744)", () => { @@ -475,7 +539,8 @@ describe("deterministic PR risk plan", () => { matchedFiles: ["agents/langchain-deepagents-code/patch-managed-deepagents-code.py"], }), ]); - expect(result.tier).toBe(2); + expect(result.tier).toBe(3); + expect(riskPlanRequiredJobIds(result)).toContain("managed-image-multiarch-startup"); expect(riskPlanRequiredTargetIds(docsAndTestsOnly)).toEqual([]); }); @@ -582,8 +647,13 @@ describe("deterministic PR risk plan", () => { expect(rootImage.families.map((family) => family.id)).toEqual([ "platform-install", "openclaw-image", + "managed-image-multiarch", + ]); + expect(riskPlanRequiredJobIds(rootImage)).toEqual([ + "cloud-onboard", + "full-e2e", + "managed-image-multiarch-startup", ]); - expect(riskPlanRequiredJobIds(rootImage)).toEqual(["cloud-onboard", "full-e2e"]); expect(adjacentImage.families.map((family) => family.id)).toEqual(["platform-install"]); expect(riskPlanRequiredJobIds(adjacentImage)).toEqual(["cloud-onboard"]); }); @@ -694,6 +764,7 @@ describe("deterministic PR risk plan", () => { expect(riskPlanRequiredJobIds(result)).toEqual([ "cloud-inference", "cloud-onboard", + "managed-image-multiarch-startup", "security-posture", "channels-add-remove", "channels-stop-start", diff --git a/test/protected-managed-image-build-script.test.ts b/test/protected-managed-image-build-script.test.ts new file mode 100644 index 00000000000..5dafc61e0fe --- /dev/null +++ b/test/protected-managed-image-build-script.test.ts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +const REPO_ROOT = fileURLToPath(new URL("..", import.meta.url)); +const SCRIPT = path.join(REPO_ROOT, "scripts/checks/build-protected-managed-images.sh"); +const REVISION = "a".repeat(40); +const DIGEST = "b".repeat(64); + +let testRoot = ""; +let stubBin = ""; +let dockerLog = ""; + +function writeExecutable(name: string, source: string): void { + const target = path.join(stubBin, name); + writeFileSync(target, source, "utf8"); + chmodSync(target, 0o755); +} + +function runBuild(sourceRoot: string) { + const output = path.join(testRoot, "contracts.json"); + return spawnSync( + "bash", + [ + SCRIPT, + "--output", + output, + "--revision", + REVISION, + "--cohort", + "protected-1-1", + "--platform", + "linux/amd64", + "--openclaw-base", + `ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:${DIGEST}`, + "--hermes-base", + `ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@sha256:${DIGEST}`, + "--dcode-base", + `ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base@sha256:${DIGEST}`, + "--source-root", + sourceRoot, + ], + { + cwd: REPO_ROOT, + encoding: "utf8", + env: { + ...process.env, + NEMOCLAW_TEST_DOCKER_LOG: dockerLog, + PATH: `${stubBin}:${process.env.PATH ?? ""}`, + RUNNER_TEMP: testRoot, + }, + }, + ); +} + +beforeEach(() => { + testRoot = mkdtempSync(path.join(os.tmpdir(), "nemoclaw-protected-build-")); + stubBin = path.join(testRoot, "bin"); + dockerLog = path.join(testRoot, "docker.log"); + mkdirSync(stubBin); + writeExecutable( + "docker", + '#!/usr/bin/env bash\nprintf "%s\\n" "$*" >> "$NEMOCLAW_TEST_DOCKER_LOG"\nexit 88\n', + ); + writeExecutable("jq", "#!/usr/bin/env bash\nexit 89\n"); + writeExecutable("sha256sum", "#!/usr/bin/env bash\nexit 90\n"); +}); + +afterEach(() => { + rmSync(testRoot, { force: true, recursive: true }); +}); + +describe("protected managed-image source-root boundary", () => { + it("accepts one absolute non-symlink source root before invoking Docker", () => { + const sourceRoot = path.join(testRoot, "candidate"); + mkdirSync(sourceRoot); + + const result = runBuild(sourceRoot); + + expect(result.status, result.stderr).toBe(88); + expect(readFileSync(dockerLog, "utf8")).toContain("buildx imagetools inspect"); + }); + + it.each([ + ["relative", () => "."], + ["newline-bearing", () => `${testRoot}/candidate\n`], + ["missing", () => path.join(testRoot, "missing")], + [ + "symlink", + () => { + const target = path.join(testRoot, "candidate"); + const link = path.join(testRoot, "candidate-link"); + mkdirSync(target); + symlinkSync(target, link, "dir"); + return link; + }, + ], + ])("rejects a %s source root before invoking Docker", (_case, sourceRoot) => { + const result = runBuild(sourceRoot()); + + expect(result.status, result.stderr).toBe(2); + expect(existsSync(dockerLog)).toBe(false); + }); +}); diff --git a/test/protected-managed-image-contract.test.ts b/test/protected-managed-image-contract.test.ts index 5b7cb001708..b96b9a1ec5d 100644 --- a/test/protected-managed-image-contract.test.ts +++ b/test/protected-managed-image-contract.test.ts @@ -1,9 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { readFileSync } from "node:fs"; + import { describe, expect, it } from "vitest"; import { + PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, PROTECTED_MANAGED_IMAGE_AGENTS, PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID, PROTECTED_MANAGED_IMAGE_PLATFORMS, @@ -73,7 +76,7 @@ function evidenceIdentity(platform: ProtectedManagedImagePlatform) { } describe("protected managed-image build contract", () => { - it("accepts only the dormant all-agent multiarch activation contract (#7744)", () => { + it("accepts only the all-agent multiarch activation contract (#7744)", () => { const activation = { agents: PROTECTED_MANAGED_IMAGE_AGENTS, contractVersion: 1, @@ -87,6 +90,19 @@ describe("protected managed-image build contract", () => { ).toThrow("activation contract is invalid"); }); + it("ships the exact activation contract consumed by the trusted lane (#7744)", () => { + const activation = JSON.parse( + readFileSync(PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, "utf8"), + ) as unknown; + + expect(parseProtectedManagedImageActivation(activation)).toEqual({ + agents: PROTECTED_MANAGED_IMAGE_AGENTS, + contractVersion: 1, + jobId: PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID, + platforms: PROTECTED_MANAGED_IMAGE_PLATFORMS, + }); + }); + it.each( PROTECTED_MANAGED_IMAGE_PLATFORMS, )("accepts one unique immutable image for every shipped agent on %s (#7744)", (platform) => { diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 6cb5411bc31..31ebbbbb576 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -18,7 +18,7 @@ const protectedManagedImageContract = ( const { PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, PROTECTED_MANAGED_IMAGE_MULTIARCH_JOB_ID } = protectedManagedImageContract; -export const RISK_PLAN_VERSION = 13 as const; +export const RISK_PLAN_VERSION = 14 as const; export const PR_E2E_TYPED_TARGET_IDS = [ "ubuntu-repo-cloud-langchain-deepagents-code", @@ -65,6 +65,39 @@ const HERMES_MANAGED_POLICY_FILES = new Set([ "agents/hermes/start.sh", "src/lib/hermes-managed-route.ts", ]); +const MANAGED_IMAGE_PROTECTED_RUNTIME_ACTIVATION = + "ci/protected-managed-image-runtime-activation-v1.json"; +const MANAGED_IMAGE_PROTECTED_RUNTIME_JOB_ID = "managed-image-protected-runtime" as const; +// The activation-only phase is complete. Any input that can change bytes or +// startup policy in a shipped managed image must requalify the exact all-agent +// amd64/arm64 cohort; the positive and adjacent-path cases in +// test/pr-risk-plan.test.ts keep this inventory intentional and bounded. +const MANAGED_IMAGE_MULTIARCH_INPUTS = new Set([ + PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, + ".dockerignore", + ".github/workflows/managed-images.yaml", + "Dockerfile", + "ci/npm-audit-exceptions.json", + "src/lib/core/json-types.ts", + "src/lib/core/ports.ts", + "src/lib/onboard/managed-bootstrap/envelope.ts", + "src/lib/security/credential-hash.ts", + "src/lib/state/paths.ts", + "src/lib/state/state-root.ts", + "src/lib/tool-disclosure.ts", + "tsconfig.runtime-preloads.json", +]); +const MANAGED_IMAGE_MULTIARCH_CHILD_CREDENTIALS = + /^src\/lib\/actions\/sandbox\/openshell-child-visible-credentials[.]v[^/]+[.]json$/u; +const MANAGED_IMAGE_MULTIARCH_INPUT_PREFIXES = [ + "agents/", + "nemoclaw/", + "nemoclaw-blueprint/", + "scripts/", + "src/lib/messaging/", + "src/lib/onboard/managed-startup/", + "tools/mcp-tool-discovery-runtime/", +] as const; export type RiskTier = 0 | 1 | 2 | 3; export type RiskFamilyId = @@ -78,6 +111,7 @@ export type RiskFamilyId = | "credentials-security" | "e2e-control-plane" | "managed-image-multiarch" + | typeof MANAGED_IMAGE_PROTECTED_RUNTIME_JOB_ID | "sandbox-boundary" | "focused-e2e"; @@ -417,12 +451,31 @@ export const RISK_RULES: readonly RiskRule[] = [ "amd64 and arm64 shards emit exact head, base, platform, cohort, image, and direct-start evidence before cleanup", "the isolated registry is removed before a shard can publish passing risk evidence", ], - // Bootstrap contract: this first trusted-controller slice recognizes only - // the activation marker. The follow-on candidate adds that marker and - // broadens the runtime paths after this job exists on trusted main, which - // lets the follow-on prove its own exact head without loading PR-authored - // workflow structure into the controller. - matches: (file) => file === PROTECTED_MANAGED_IMAGE_ACTIVATION_PATH, + // Keep this source boundary synchronized with the managed-image workflow's + // path filter. The preceding trusted-controller slice intentionally matched + // only the activation marker; after that lane lands, this candidate can + // select and prove its own exact head before broadening future qualification. + matches: (file) => + MANAGED_IMAGE_MULTIARCH_INPUTS.has(file) || + MANAGED_IMAGE_MULTIARCH_CHILD_CREDENTIALS.test(file) || + MANAGED_IMAGE_MULTIARCH_INPUT_PREFIXES.some((prefix) => file.startsWith(prefix)), + }, + { + id: MANAGED_IMAGE_PROTECTED_RUNTIME_JOB_ID, + summary: + "Protected managed-image runtime qualification must retain real GPU access, host-local Ollama, NVIDIA NIM, vLLM, transactional rollback, and exact cleanup for every shipped agent.", + tier: 3, + requiredJobs: [MANAGED_IMAGE_PROTECTED_RUNTIME_JOB_ID], + invariants: [ + "OpenClaw, Hermes, and Deep Agents Code run from exact PR image digests through the production managed-bootstrap path", + "real NVIDIA GPU access and host-local Ollama, NVIDIA NIM, and vLLM inference.local completions are all required", + "bootstrap completion failure removes the exact failed sandbox, container, network, and transaction state for every agent", + "NGC credentials remain host-scoped and never enter a managed sandbox or persisted artifact", + ], + // The trusted workflow and validator land before activation. The follow-on + // activation slice broadens this boundary to runtime inputs after the + // protected job exists on main and can safely qualify candidate code. + matches: (file) => file === MANAGED_IMAGE_PROTECTED_RUNTIME_ACTIVATION, }, { id: "sandbox-boundary", diff --git a/tools/e2e/managed-image-multiarch-workflow-boundary.mts b/tools/e2e/managed-image-multiarch-workflow-boundary.mts index 151506d6f7a..4a17e307308 100644 --- a/tools/e2e/managed-image-multiarch-workflow-boundary.mts +++ b/tools/e2e/managed-image-multiarch-workflow-boundary.mts @@ -115,6 +115,9 @@ export function validateManagedImageMultiarchWorkflow(workflow: WorkflowRecord): if (record(job.permissions).contents !== "read") { errors.push(`${JOB_ID} permissions must be contents: read`); } + if (job["continue-on-error"] !== undefined) { + errors.push(`${JOB_ID} must not weaken failures with continue-on-error`); + } const expectedStrategy = { "fail-fast": false, diff --git a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts new file mode 100644 index 00000000000..7521d09c0d1 --- /dev/null +++ b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts @@ -0,0 +1,329 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +type WorkflowRecord = Record; +type WorkflowStep = WorkflowRecord & { + env?: WorkflowRecord; + name?: string; + run?: string; + uses?: string; + with?: WorkflowRecord; +}; + +const JOB_ID = "managed-image-protected-runtime"; +const SELECTOR = + "${{ contains(format(',{0},', inputs.jobs), ',managed-image-protected-runtime,') || contains(format(',{0},', inputs.targets), ',managed-image-protected-runtime,') }}"; +const ACTIVATION_PATH = "ci/protected-managed-image-runtime-activation-v1.json"; +const LIVE_TEST_PATH = "test/e2e/live/managed-image-protected-runtime.test.ts"; +const REGISTRY_IMAGE = + "docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"; + +// Keep lane-specific trust assertions explicit: the multiarch lane executes +// candidate code directly, while this GPU lane keeps secrets in trusted code +// and isolates candidate source. The workflow-boundary aggregate runs both +// validators, which fail closed on the common job invariants without weakening +// either boundary into a generic lowest-common-denominator validator. + +function record(value: unknown): WorkflowRecord { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as WorkflowRecord) + : {}; +} + +function steps(value: unknown): WorkflowStep[] { + return Array.isArray(value) ? (value as WorkflowStep[]) : []; +} + +function text(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +function requireStep( + errors: string[], + workflowSteps: readonly WorkflowStep[], + name: string, +): WorkflowStep | undefined { + const matches = workflowSteps.filter((step) => step.name === name); + if (matches.length !== 1) errors.push(`${JOB_ID} must define exactly one '${name}' step`); + return matches[0]; +} + +function requireValues( + errors: string[], + subject: string, + actual: WorkflowRecord, + expected: Readonly>, +): void { + for (const [key, value] of Object.entries(expected)) { + if (actual[key] !== value) errors.push(`${subject} must bind ${key} to ${String(value)}`); + } +} + +function requireFragments( + errors: string[], + step: WorkflowStep | undefined, + fragments: readonly string[], +): void { + const run = text(step?.run); + for (const fragment of fragments) { + if (!run.includes(fragment)) { + errors.push(`${JOB_ID} step '${step?.name ?? "missing"}' must include ${fragment}`); + } + } +} + +function requireOrderedSteps( + errors: string[], + workflowSteps: readonly WorkflowStep[], + names: readonly string[], +): void { + const indexes = names.map((name) => workflowSteps.findIndex((step) => step.name === name)); + if (indexes.some((index) => index < 0)) return; + if (indexes.some((index, offset) => offset > 0 && index <= indexes[offset - 1])) { + errors.push(`${JOB_ID} protected qualification and cleanup steps drifted`); + } +} + +export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowRecord): string[] { + const errors: string[] = []; + const job = record(record(workflow.jobs)[JOB_ID]); + if (Object.keys(job).length === 0) return [`workflow missing ${JOB_ID} job`]; + + if (job.needs !== "generate-matrix") errors.push(`${JOB_ID} must depend on generate-matrix`); + if (job.if !== SELECTOR) errors.push(`${JOB_ID} must remain explicit-only and selector-bound`); + if (job["runs-on"] !== "linux-amd64-gpu-rtxpro6000-latest-1") { + errors.push(`${JOB_ID} must run on the protected amd64 GPU runner`); + } + if (job["timeout-minutes"] !== 300) errors.push(`${JOB_ID} must keep the 300 minute timeout`); + if (record(job.permissions).contents !== "read") { + errors.push(`${JOB_ID} permissions must be contents: read`); + } + if (job["continue-on-error"] !== undefined) { + errors.push(`${JOB_ID} must not weaken failures with continue-on-error`); + } + + const jobEnv = record(job.env); + requireValues(errors, `${JOB_ID} env`, jobEnv, { + E2E_ARTIFACT_DIR: "${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime", + E2E_DEFAULT_ENABLED: "0", + E2E_JOB: "1", + E2E_TARGET_ID: JOB_ID, + RELEASE_E2E_ACTIVATION_PATH: ACTIVATION_PATH, + NEMOCLAW_E2E_SHARD: "linux-amd64-gpu", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: "${{ inputs.base_sha }}", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: + "protected-${{ github.run_id }}-${{ github.run_attempt }}", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT: + "${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime/contracts.json", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM: "linux/amd64", + NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA: "${{ inputs.workflow_sha }}", + NEMOCLAW_PROTECTED_REGISTRY_NAME: + "nemoclaw-managed-runtime-${{ github.run_id }}-${{ github.run_attempt }}", + NEMOCLAW_RUN_LIVE_E2E: "1", + }); + if (jobEnv.NVIDIA_API_KEY !== undefined) { + errors.push(`${JOB_ID} must not expose NVIDIA_API_KEY at job scope`); + } + + const workflowSteps = steps(job.steps); + const guard = requireStep( + errors, + workflowSteps, + "Validate protected runtime exact-head dispatch", + ); + requireValues(errors, `${JOB_ID} exact-head guard env`, record(guard?.env), { + ACTOR: "${{ github.actor }}", + BASE_SHA: "${{ inputs.base_sha }}", + CHECKOUT_SHA: "${{ inputs.checkout_sha }}", + EVENT_NAME: "${{ github.event_name }}", + EXPECTED_WORKFLOW_SHA: "${{ inputs.workflow_sha }}", + REF: "${{ github.ref }}", + REPOSITORY: "${{ github.repository }}", + RUNNER_ARCH_KIND: "${{ runner.arch }}", + WORKFLOW_SHA: "${{ github.workflow_sha }}", + }); + requireFragments(errors, guard, [ + '"NVIDIA/NemoClaw"', + '"refs/heads/main"', + '"workflow_dispatch"', + '"github-actions[bot]"', + '[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]]', + '"$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA"', + '"$RUNNER_ARCH_KIND" == "X64"', + ]); + + const checkouts = workflowSteps.filter((step) => text(step.uses).startsWith("actions/checkout@")); + if (checkouts.length !== 2) { + errors.push(`${JOB_ID} must define one trusted checkout and one isolated candidate checkout`); + } + const trustedCheckout = requireStep( + errors, + workflowSteps, + "Checkout trusted protected runtime qualification", + ); + const candidateCheckout = requireStep( + errors, + workflowSteps, + "Checkout exact protected runtime candidate source", + ); + const checkoutAction = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; + if (trustedCheckout?.uses !== checkoutAction || candidateCheckout?.uses !== checkoutAction) { + errors.push(`${JOB_ID} must pin both trusted and candidate checkouts`); + } + requireValues(errors, `${JOB_ID} trusted checkout`, record(trustedCheckout?.with), { + repository: "${{ github.repository }}", + ref: "${{ inputs.workflow_sha }}", + "fetch-depth": 0, + "persist-credentials": false, + }); + requireValues(errors, `${JOB_ID} candidate checkout`, record(candidateCheckout?.with), { + repository: "${{ inputs.checkout_repository || github.repository }}", + ref: "${{ inputs.checkout_sha || github.sha }}", + path: ".candidate-runtime", + "fetch-depth": 0, + "persist-credentials": false, + }); + + const buildx = requireStep(errors, workflowSteps, "Set up protected runtime Buildx"); + if (buildx?.uses !== "docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c") { + errors.push(`${JOB_ID} must pin the reviewed Buildx setup action`); + } + requireValues(errors, `${JOB_ID} Buildx setup`, record(buildx?.with), { + "driver-opts": "network=host", + "buildkitd-config-inline": '[registry."localhost:5000"]\n http = true\n', + }); + + const prepare = requireStep(errors, workflowSteps, "Prepare E2E workspace"); + if ( + prepare?.uses !== + "NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75" + ) { + errors.push(`${JOB_ID} must pin the trusted E2E preparation action`); + } + if (prepare?.with !== undefined) { + errors.push(`${JOB_ID} must use the default CLI build`); + } + + const activation = requireStep( + errors, + workflowSteps, + "Validate protected runtime activation contract", + ); + requireFragments(errors, activation, [ + 'candidate_root=".candidate-runtime"', + `activation="$candidate_root/${ACTIVATION_PATH}"`, + '[[ "$(git -C "$candidate_root" rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]]', + '[[ -f "$activation" && ! -L "$activation" ]]', + '(keys | sort) == ["agents", "contractVersion", "jobId", "platform", "providers"]', + '.agents == ["openclaw", "hermes", "langchain-deepagents-code"]', + '.platform == "linux/amd64"', + '.providers == ["ollama", "nim", "vllm"]', + ]); + + const bases = requireStep(errors, workflowSteps, "Resolve exact amd64 runtime base images"); + requireFragments(errors, bases, [ + 'docker buildx imagetools inspect "$alias" --raw', + '.platform.os == "linux" and .platform.architecture == "amd64"', + 'reference="${repository}@${digest}"', + '"sha256:$(sha256sum "$exact_raw" | awk \'{print $1}\')" == "$digest"', + "ghcr.io/nvidia/nemoclaw/sandbox-base:latest", + "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest", + "ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:latest", + ]); + + const registry = requireStep(errors, workflowSteps, "Start isolated protected runtime registry"); + requireFragments(errors, registry, [ + 'docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME"', + "http://127.0.0.1:5000/v2/", + "io.nvidia.nemoclaw.e2e-owner=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT}", + "--publish 127.0.0.1:5000:5000", + REGISTRY_IMAGE, + ]); + + const build = requireStep( + errors, + workflowSteps, + "Build exact all-agent protected runtime images", + ); + requireFragments(errors, build, [ + "scripts/checks/build-protected-managed-images.sh", + '--revision "$CHECKOUT_SHA"', + '--cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT"', + "--platform linux/amd64", + '--source-root "$GITHUB_WORKSPACE/.candidate-runtime"', + '--openclaw-base "$BASE_OPENCLAW"', + '--hermes-base "$BASE_HERMES"', + '--dcode-base "$BASE_DCODE"', + ]); + + const install = requireStep(errors, workflowSteps, "Install OpenShell CLI"); + requireFragments(errors, install, [ + "env -u DOCKER_CONFIG", + "-u NVIDIA_API_KEY", + "-u NVIDIA_INFERENCE_API_KEY", + "bash scripts/install-openshell.sh", + ]); + + const qualification = requireStep( + errors, + workflowSteps, + "Run all-agent GPU, local inference, rollback, and cleanup qualification", + ); + requireValues(errors, `${JOB_ID} qualification env`, record(qualification?.env), { + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + }); + const secretBearingSteps = workflowSteps.filter( + (step) => record(step.env).NVIDIA_API_KEY !== undefined, + ); + if (secretBearingSteps.length !== 1 || secretBearingSteps[0] !== qualification) { + errors.push(`${JOB_ID} must expose NVIDIA_API_KEY only to trusted qualification code`); + } + requireFragments(errors, qualification, [ + '[[ "$(git rev-parse --verify HEAD)" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA" ]]', + 'export OPENSHELL_BIN="$(command -v openshell)"', + "tools/e2e/live-vitest-invocation.mts run", + `--test-path ${LIVE_TEST_PATH}`, + ]); + if (text(qualification?.run).includes(".candidate-runtime")) { + errors.push(`${JOB_ID} trusted qualification must not execute candidate checkout paths`); + } + + const cleanup = requireStep(errors, workflowSteps, "Remove isolated protected runtime registry"); + if (cleanup?.if !== "always()") errors.push(`${JOB_ID} registry cleanup must always run`); + requireFragments(errors, cleanup, [ + "io.nvidia.nemoclaw.e2e-owner", + '[[ "$owner" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" ]]', + 'docker rm -f "$NEMOCLAW_PROTECTED_REGISTRY_NAME"', + "http://127.0.0.1:5000/v2/", + ]); + + const upload = requireStep( + errors, + workflowSteps, + "Upload protected managed-image runtime artifacts", + ); + if (upload?.if !== "always()") errors.push(`${JOB_ID} artifact upload must always run`); + requireValues(errors, `${JOB_ID} artifact upload`, record(upload?.with), { + name: "e2e-managed-image-protected-runtime", + path: "e2e-artifacts/live/managed-image-protected-runtime/", + }); + requireStep(errors, workflowSteps, "Clean up Docker auth"); + requireOrderedSteps(errors, workflowSteps, [ + "Validate protected runtime exact-head dispatch", + "Checkout trusted protected runtime qualification", + "Checkout exact protected runtime candidate source", + "Prepare E2E workspace", + "Validate protected runtime activation contract", + "Resolve exact amd64 runtime base images", + "Start isolated protected runtime registry", + "Build exact all-agent protected runtime images", + "Install OpenShell CLI", + "Run all-agent GPU, local inference, rollback, and cleanup qualification", + "Remove isolated protected runtime registry", + "Upload protected managed-image runtime artifacts", + "Clean up Docker auth", + ]); + + return errors; +} diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 33615918ce6..6421505dea2 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -419,11 +419,17 @@ function validatePrGateDispatch(errors: string[], workflow: OperationsWorkflow): step.name === "Check out trusted E2E workflow" && step.if === PUBLICATION_REQUIRED_CONDITION && step.with?.ref === "${{ github.sha }}"; + const trustedManagedImageRuntimeCheckout = + jobName === "managed-image-protected-runtime" && + step.name === "Checkout trusted protected runtime qualification" && + step.with?.repository === "${{ github.repository }}" && + step.with?.ref === "${{ inputs.workflow_sha }}"; const trustedCheckout = trustedHermesFixtureCheckout || trustedReportHelperCheckout || trustedLaunchableLaneCheckout || - trustedPublicationCheckout; + trustedPublicationCheckout || + trustedManagedImageRuntimeCheckout; if ( step.uses?.startsWith("actions/checkout@") && step.with?.ref !== "${{ inputs.checkout_sha || github.sha }}" && diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 481196cad25..774a40b4cef 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -150,6 +150,13 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ path: "e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/", }, ], + [ + "managed-image-protected-runtime", + { + name: "e2e-managed-image-protected-runtime", + path: "e2e-artifacts/live/managed-image-protected-runtime/", + }, + ], [ "network-policy", { diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 02aa66417c1..aff98099001 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -27,6 +27,7 @@ import { validateInferenceSwitchWorkflow, } from "./inference-switch-workflow-boundary.mts"; import { validateManagedImageMultiarchWorkflow } from "./managed-image-multiarch-workflow-boundary.mts"; +import { validateManagedImageProtectedRuntimeWorkflow } from "./managed-image-protected-runtime-workflow-boundary.mts"; import { type OpenClawPluginRuntimeExdevWorkflow, validateOpenClawPluginRuntimeExdevWorkflow, @@ -2493,7 +2494,9 @@ function validateDockerHubAuthBoundary(errors: string[], jobs: WorkflowRecord): requireCanonicalDockerHubCleanupRun(errors, jobName, cleanup); const checkoutIndex = steps.findIndex((step) => - stringValue(step.uses).startsWith("actions/checkout@"), + jobName === "managed-image-protected-runtime" + ? step.name === "Checkout exact protected runtime candidate source" + : stringValue(step.uses).startsWith("actions/checkout@"), ); const authIndex = steps.indexOf(auth); const cleanupIndex = steps.indexOf(cleanup); @@ -4239,6 +4242,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { errors.push(...validateHermesGpuStartupWorkflow(workflow)); errors.push(...validateInferenceSwitchWorkflow(workflow as unknown as InferenceSwitchWorkflow)); errors.push(...validateManagedImageMultiarchWorkflow(workflow)); + errors.push(...validateManagedImageProtectedRuntimeWorkflow(workflow)); errors.push( ...validateOpenClawPluginRuntimeExdevWorkflow( workflow as unknown as OpenClawPluginRuntimeExdevWorkflow,