diff --git a/src/lib/onboard/inference-selection-validation.test.ts b/src/lib/onboard/inference-selection-validation.test.ts index 03ca4718c35..47060d5cd53 100644 --- a/src/lib/onboard/inference-selection-validation.test.ts +++ b/src/lib/onboard/inference-selection-validation.test.ts @@ -269,67 +269,66 @@ describe("inference selection validation", () => { expectedEndpointUrl: "https://anthropic.corp.example/v1", expectedProbeOptions: { calibrateTimeouts: true, skipResponsesProbe: true }, }, - ])("probes an exactly allowlisted private Anthropic endpoint on its $runtimeSurface surface (#7037)", async ({ - intendedApi, - expectedEndpointUrl, - expectedProbeOptions, - }) => { - vi.stubEnv("NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS", "anthropic.corp.example"); - vi.stubEnv("NEMOCLAW_REASONING", "false"); - const probeEndpoint = vi.fn(() => ({ - ok: true, - api: intendedApi, - label: "Compatible API", - })); - const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); - const log = vi.spyOn(console, "log").mockImplementation(() => {}); - const helpers = createInferenceSelectionValidationHelpers({ - isNonInteractive: () => false, - agentProductName: () => "NemoClaw agent", - getCredential: () => "test-key", - probeAnthropicEndpoint: probeEndpoint, - probeOpenAiLikeEndpoint: probeEndpoint, - promptValidationRecovery: vi.fn(async () => "selection" as const), - resolveEndpointHost: async () => [{ address: "10.0.0.8", family: 4 }], - }); - - try { - const result = await helpers.validateCustomAnthropicSelection( - "Custom Anthropic endpoint", - "https://anthropic.corp.example", - "model-a", - "COMPATIBLE_ANTHROPIC_API_KEY", - null, - { intendedApi }, - ); - - expect(result).toMatchObject({ + ])( + "probes an exactly allowlisted private Anthropic endpoint on its $runtimeSurface surface (#7037)", + async ({ intendedApi, expectedEndpointUrl, expectedProbeOptions }) => { + vi.stubEnv("NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS", "anthropic.corp.example"); + vi.stubEnv("NEMOCLAW_REASONING", "false"); + const probeEndpoint = vi.fn(() => ({ ok: true, api: intendedApi, - pinnedAddresses: ["10.0.0.8"], - trustedPrivateCapability: { - host: "anthropic.corp.example", - addresses: ["10.0.0.8"], - }, + label: "Compatible API", + })); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const helpers = createInferenceSelectionValidationHelpers({ + isNonInteractive: () => false, + agentProductName: () => "NemoClaw agent", + getCredential: () => "test-key", + probeAnthropicEndpoint: probeEndpoint, + probeOpenAiLikeEndpoint: probeEndpoint, + promptValidationRecovery: vi.fn(async () => "selection" as const), + resolveEndpointHost: async () => [{ address: "10.0.0.8", family: 4 }], }); - expect(probeEndpoint).toHaveBeenCalledOnce(); - expect(probeEndpoint).toHaveBeenCalledWith( - expectedEndpointUrl, - "model-a", - "test-key", - expect.objectContaining({ - ...expectedProbeOptions, + + try { + const result = await helpers.validateCustomAnthropicSelection( + "Custom Anthropic endpoint", + "https://anthropic.corp.example", + "model-a", + "COMPATIBLE_ANTHROPIC_API_KEY", + null, + { intendedApi }, + ); + + expect(result).toMatchObject({ + ok: true, + api: intendedApi, pinnedAddresses: ["10.0.0.8"], - trustedPrivateCapability: expect.objectContaining({ addresses: ["10.0.0.8"] }), - }), - ); - expect(warn).toHaveBeenCalledWith(expect.stringContaining("operator-trusted private")); - } finally { - log.mockRestore(); - warn.mockRestore(); - vi.unstubAllEnvs(); - } - }); + trustedPrivateCapability: { + host: "anthropic.corp.example", + addresses: ["10.0.0.8"], + }, + }); + expect(probeEndpoint).toHaveBeenCalledOnce(); + expect(probeEndpoint).toHaveBeenCalledWith( + expectedEndpointUrl, + "model-a", + "test-key", + expect.objectContaining({ + ...expectedProbeOptions, + pinnedAddresses: ["10.0.0.8"], + trustedPrivateCapability: expect.objectContaining({ addresses: ["10.0.0.8"] }), + }), + ); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("operator-trusted private")); + } finally { + log.mockRestore(); + warn.mockRestore(); + vi.unstubAllEnvs(); + } + }, + ); it("honors an exactly allowlisted private endpoint during non-interactive validation (#6861)", async () => { vi.stubEnv("NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS", "llm.corp.example"); @@ -403,47 +402,46 @@ describe("inference selection validation", () => { } }); - it.each([ - "http://127.0.0.1:8000/v1", - "https://inference.local/v1", - "https://93.184.216.34/v1", - ])("carries the approved no-pin capability to probes for %s (#6293)", async (endpointUrl) => { - const probeOpenAiLikeEndpoint = vi.fn(() => ({ ok: true, api: "openai-completions" })); - const resolveEndpointHost = vi.fn(async () => [{ address: "10.0.0.8", family: 4 }]); - const log = vi.spyOn(console, "log").mockImplementation(() => {}); - const helpers = createInferenceSelectionValidationHelpers({ - isNonInteractive: () => false, - agentProductName: () => "OpenClaw", - getCredential: () => "test-key", - probeOpenAiLikeEndpoint, - promptValidationRecovery: vi.fn(async () => "selection" as const), - resolveEndpointHost, - }); + it.each(["http://127.0.0.1:8000/v1", "https://inference.local/v1", "https://93.184.216.34/v1"])( + "carries the approved no-pin capability to probes for %s (#6293)", + async (endpointUrl) => { + const probeOpenAiLikeEndpoint = vi.fn(() => ({ ok: true, api: "openai-completions" })); + const resolveEndpointHost = vi.fn(async () => [{ address: "10.0.0.8", family: 4 }]); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const helpers = createInferenceSelectionValidationHelpers({ + isNonInteractive: () => false, + agentProductName: () => "OpenClaw", + getCredential: () => "test-key", + probeOpenAiLikeEndpoint, + promptValidationRecovery: vi.fn(async () => "selection" as const), + resolveEndpointHost, + }); - try { - await expect( - helpers.validateCustomOpenAiLikeSelection( - "Custom endpoint", + try { + await expect( + helpers.validateCustomOpenAiLikeSelection( + "Custom endpoint", + endpointUrl, + "model-a", + "COMPATIBLE_API_KEY", + ), + ).resolves.toEqual({ + ok: true, + api: "openai-completions", + pinnedAddresses: [], + }); + expect(probeOpenAiLikeEndpoint).toHaveBeenCalledWith( endpointUrl, "model-a", - "COMPATIBLE_API_KEY", - ), - ).resolves.toEqual({ - ok: true, - api: "openai-completions", - pinnedAddresses: [], - }); - expect(probeOpenAiLikeEndpoint).toHaveBeenCalledWith( - endpointUrl, - "model-a", - "test-key", - expect.objectContaining({ pinnedAddresses: [] }), - ); - expect(resolveEndpointHost).not.toHaveBeenCalled(); - } finally { - log.mockRestore(); - } - }); + "test-key", + expect.objectContaining({ pinnedAddresses: [] }), + ); + expect(resolveEndpointHost).not.toHaveBeenCalled(); + } finally { + log.mockRestore(); + } + }, + ); it("exits non-interactively when a custom Anthropic endpoint resolves to link-local metadata, without probing (#6293)", async () => { const originalExitCode = process.exitCode; diff --git a/test/e2e/live/onboard-interactive-pty.ts b/test/e2e/live/onboard-interactive-pty.ts new file mode 100644 index 00000000000..604d11ed425 --- /dev/null +++ b/test/e2e/live/onboard-interactive-pty.ts @@ -0,0 +1,243 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ChildProcessProgress } from "../fixtures/observed-child-process.ts"; +import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; + +// Drives an interactive CLI through a real PTY, the same technique +// `test/helpers/installer-express-prompt-pty-harness.ts` uses for the +// installer's express prompt. The real onboard wizard behaves differently +// under a piped, non-TTY stdin than under a real terminal (raw-mode +// keypress selectors, `isTTY`-gated prompts), so a faithful regression test +// for interactive-only behavior must drive a real PTY rather than pipe +// stdin. +// +// Rules fire independently and out of order: a rule whose trigger never +// appears (for example, a first-run license notice already accepted on a +// prior run) must not block a later rule from firing when its own trigger +// appears. +// +// The child process itself is launched through the shared +// `spawnObservedChild` boundary (the suite's single audited asynchronous +// child-process call) so it still tracks a content-free progress activity +// and canonical lifecycle checkpoints; this module attaches its own +// listeners on top to capture output and match rule triggers. + +export interface InteractiveCommandRule { + readonly trigger: string; + readonly response: string; +} + +export interface InteractiveCommandResult { + readonly exitCode: number; + readonly output: string; + readonly firedTriggers: readonly string[]; + readonly timedOut: boolean; +} + +export interface DriveInteractiveCommandOptions { + readonly activityLabel: string; + readonly cmd: readonly [string, ...string[]]; + readonly cwd?: string; + readonly env: NodeJS.ProcessEnv; + readonly progress: ChildProcessProgress; + readonly rules: readonly InteractiveCommandRule[]; + readonly timeoutMs: number; +} + +// Runs inside a Python child so it can fork a real pseudo-terminal; +// Node has no built-in PTY primitive and this repo does not depend on +// node-pty. The child's own deadline is generous; the Node-side timer +// below is the enforced hard bound and SIGKILLs the whole process tree. +const PTY_DRIVER_SCRIPT = ` +import json, os, pty, select, signal, sys, time + +# Read from stdin, not argv: the payload embeds every scripted response, +# including any credential a rule supplies (e.g. an onboard API key), and a +# process argument stays visible to anything that can list the command +# line for as long as the child runs. +payload = json.loads(sys.stdin.read()) +cmd = payload["cmd"] +rules = payload["rules"] +timeout_s = payload["timeoutSeconds"] + +pid, fd = pty.fork() +if pid == 0: + os.execvp(cmd[0], cmd) + +# pty.fork() makes the command the leader of a separate session/process +# group. Tell the Node parent which group it must terminate on its hard +# timeout; killing only this Python driver's group cannot reach that child. +sys.stderr.write("PTY_CHILD_PID\\t" + str(pid) + "\\n") +sys.stderr.flush() + +def terminate_pty_child(): + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + +def handle_driver_signal(_signum, _frame): + terminate_pty_child() + try: + os.waitpid(pid, 0) + except ChildProcessError: + pass + sys.exit(124) + +signal.signal(signal.SIGTERM, handle_driver_signal) +signal.signal(signal.SIGINT, handle_driver_signal) + +output = bytearray() +os.set_blocking(fd, False) +deadline = time.monotonic() + timeout_s +fired = [False] * len(rules) +exit_code = None +while time.monotonic() < deadline: + ready, _, _ = select.select([fd], [], [], 0.2) + if ready: + try: + chunk = os.read(fd, 65536) + except OSError: + chunk = b"" + if not chunk: + # PTY close (EOF, or EIO on Linux) after the child has already + # exited normally must not fall through to the timeout branch + # below: reap it here and record its real exit code so a + # successful run is never misreported as DRIVER_TIMEOUT. + _, status = os.waitpid(pid, 0) + exit_code = os.waitstatus_to_exitcode(status) + break + output.extend(chunk) + sys.stdout.buffer.write(chunk) + sys.stdout.flush() + text = output.decode("utf-8", errors="ignore") + for i, rule in enumerate(rules): + if fired[i]: + continue + if rule["trigger"] in text: + os.write(fd, rule["response"].encode()) + sys.stderr.write("FIRED\\t" + rule["trigger"] + "\\n") + fired[i] = True + waited = os.waitpid(pid, os.WNOHANG) + if waited[0] == pid: + exit_code = os.waitstatus_to_exitcode(waited[1]) + break +if exit_code is None: + terminate_pty_child() + try: + os.waitpid(pid, 0) + except ChildProcessError: + pass + sys.stderr.write("DRIVER_TIMEOUT\\n") + sys.exit(124) +sys.exit(exit_code) +`; + +function resolvePython(): string { + return process.env.NEMOCLAW_E2E_PYTHON3_BIN || "python3"; +} + +export function driveInteractiveCommand( + options: DriveInteractiveCommandOptions, +): Promise { + const payload = JSON.stringify({ + cmd: options.cmd, + rules: options.rules.map((rule) => ({ trigger: rule.trigger, response: rule.response })), + // Comfortably longer than the Node-side hard timeout below so the + // driver's own bookkeeping never races the enforced bound. + timeoutSeconds: Math.ceil(options.timeoutMs / 1000) + 30, + }); + // Kept directly in this function's body, not inside the Promise executor + // below, so the sole audited async child-process boundary stays attached + // to a named, reviewed callsite. `detached: true` makes the Python driver + // its process-group leader, which gives fallback cleanup a stable target. + // pty.fork() creates a separate child session, whose process-group id is + // reported over stderr below. + const child = spawnObservedChild(resolvePython(), ["-c", PTY_DRIVER_SCRIPT], { + activityLabel: options.activityLabel, + progress: options.progress, + spawn: { cwd: options.cwd, env: options.env, detached: true }, + }); + // Written to stdin rather than passed as a process argument: the payload + // carries every scripted response, including any credential a rule + // supplies (see PTY_DRIVER_SCRIPT's matching comment). + child.stdin?.end(payload); + + return new Promise((resolve, reject) => { + let output = ""; + const firedTriggers: string[] = []; + let stderrRest = ""; + let timedOut = false; + let settled = false; + let ptyChildPid: number | null = null; + let forceKillTimer: ReturnType | null = null; + + const timer = setTimeout(() => { + timedOut = true; + // pty.fork() puts the onboard command in its own session. Kill that + // exact process group, then let the driver reap it through its SIGTERM + // handler. The delayed driver-group SIGKILL is only a hard fallback. + try { + if (ptyChildPid) process.kill(-ptyChildPid, "SIGKILL"); + } catch { + // The PTY child may already have exited between the deadline and the + // signal. The driver still receives SIGTERM and reaps its status. + } + child.kill("SIGTERM"); + forceKillTimer = setTimeout(() => { + try { + if (ptyChildPid) process.kill(-ptyChildPid, "SIGKILL"); + } catch { + // Already gone. + } + try { + if (child.pid) process.kill(-child.pid, "SIGKILL"); + } catch { + child.kill("SIGKILL"); + } + }, 1_000); + }, options.timeoutMs); + + // Additional listeners alongside spawnObservedChild's own content-free + // observer; this module needs the real transcript to match rule + // triggers and to report ordered step evidence. + child.stdout?.on("data", (chunk: Buffer) => { + output += chunk.toString("utf-8"); + }); + child.stderr?.on("data", (chunk: Buffer) => { + const lines = (stderrRest + chunk.toString("utf-8")).split("\n"); + stderrRest = lines.pop() ?? ""; + for (const line of lines) { + const childPid = line.match(/^PTY_CHILD_PID\t([1-9]\d*)$/); + if (childPid) { + ptyChildPid = Number(childPid[1]); + continue; + } + const fired = line.match(/^FIRED\t(.*)$/); + if (fired) firedTriggers.push(fired[1]); + } + }); + child.once("error", (error) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (forceKillTimer) clearTimeout(forceKillTimer); + reject(error); + }); + child.once("close", (code) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (forceKillTimer) clearTimeout(forceKillTimer); + const fired = stderrRest.match(/^FIRED\t(.*)$/); + if (fired) firedTriggers.push(fired[1]); + resolve({ + exitCode: timedOut ? 124 : (code ?? 1), + output, + firedTriggers, + timedOut, + }); + }); + }); +} diff --git a/test/e2e/live/onboard-policy-preset-sequencing.test.ts b/test/e2e/live/onboard-policy-preset-sequencing.test.ts new file mode 100644 index 00000000000..9a6c014af82 --- /dev/null +++ b/test/e2e/live/onboard-policy-preset-sequencing.test.ts @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { randomBytes } from "node:crypto"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; +import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; +import { redactString } from "../fixtures/redaction.ts"; +import { driveInteractiveCommand } from "./onboard-interactive-pty.ts"; + +// Regression coverage for #6042: "interactive onboard wizard skips Policy +// Presets TUI step". Three independent investigations could not reproduce a +// skip — the onboard state machine has no transition from any earlier state +// directly to `complete`, every path passes through `policies` — but no +// checked-in test drove the real interactive TUI through a PTY to prove it. +// This test is that proof: it answers every interactive prompt in the +// compatible-endpoint journey through a real pseudo-terminal (piped stdin +// does not reproduce the raw-mode selectors this wizard uses) and asserts +// the ordered step markers appear in order, ending with `[8/8] Policy +// presets`, before the wizard can report completion. +// +// This is a hermetic, mock-provider variant of the reporter's journey +// (`nemoclaw onboard` with "Other OpenAI-compatible endpoint"), so it needs +// no NVIDIA credential and runs in ordinary CI, not just a live-inference +// lane. + +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-policy-order"; +validateSandboxName(SANDBOX_NAME); +const ONBOARD_TIMEOUT_MS = 40 * 60_000; +const MODEL = "test-model"; +// A Docker network namespace cannot reach host loopback directly; bind the +// fake endpoint on all interfaces and advertise the OpenShell host alias so +// both the host-side onboard validation and the sandbox's own inference +// route can reach it (matches the shared E2E inference adapter's mock mode). +const SANDBOX_HOST_ALIAS = "host.openshell.internal"; + +// The ordered, observable step headers the real interactive wizard prints. +// Each must appear strictly after the previous one; `[8/8] Policy presets` +// is the step the issue claims gets skipped. +const ORDERED_STEP_MARKERS = [ + "[1/8] Preflight checks", + "[2/8] Starting OpenShell gateway", + "[3/8] Configuring inference provider", + "[4/8] Setting up inference provider", + "[5/8] Messaging channels", + "[6/8] Creating sandbox", + "[7/8] Setting up OpenClaw inside sandbox", + "[8/8] Policy presets", +] as const; + +test( + "interactive onboard wizard reaches Policy presets in step order (#6042)", + { + timeout: ONBOARD_TIMEOUT_MS, + meta: { + e2ePhases: [ + "start the local compatible-endpoint fake server", + "drive the interactive onboard wizard through a real PTY", + "confirm every ordered onboarding step appears in order", + "confirm Policy presets is reached before completion", + ], + }, + }, + async ({ artifacts, cleanup, docker, host, progress }) => { + await docker.requireDocker(); + + progress.phase("start the local compatible-endpoint fake server"); + const apiKey = `e2e-6042-${randomBytes(16).toString("hex")}`; + const fake = await startFakeOpenAiCompatibleServer({ + apiKey, + chatContent: "PONG", + host: "0.0.0.0", + model: MODEL, + progress, + publicHost: SANDBOX_HOST_ALIAS, + requireAuth: true, + responseText: "PONG", + }); + artifacts.addRedactionValues([apiKey]); + cleanup.trackDisposable("close fake compatible-endpoint server", () => fake.close()); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-onboard-policy-order", + env: buildAvailabilityProbeEnv(), + redactionValues: [apiKey], + timeoutMs: 120_000, + }); + + progress.phase("drive the interactive onboard wizard through a real PTY"); + const result = await driveInteractiveCommand({ + activityLabel: "command: onboard-interactive-pty", + progress, + cmd: [ + process.execPath, + CLI_ENTRYPOINT, + "onboard", + "--fresh", + "--agent", + "openclaw", + "--name", + SANDBOX_NAME, + "--yes-i-accept-third-party-software", + ], + cwd: REPO_ROOT, + env: buildAvailabilityProbeEnv(), + rules: [ + // A reused host can already have accepted this first-run license + // notice, so this rule might not fire. + { trigger: "Type 'yes' to accept", response: "yes\n" }, + // Only appears when the preflight resource check warns; skipped on + // an adequately provisioned CI runner. + { trigger: "Continue with onboarding?", response: "y\n" }, + // "Other OpenAI-compatible endpoint" — position depends on + // src/lib/onboard/providers.ts's provider list for the openclaw agent. + { trigger: "Select your inference provider:", response: "4\n" }, + { trigger: "Other OpenAI-compatible endpoint", response: "" }, + { trigger: "OpenAI-compatible base URL", response: `${fake.baseUrl}\n` }, + { trigger: "Other OpenAI-compatible endpoint API key:", response: `${apiKey}\n` }, + { trigger: "endpoint model", response: `${MODEL}\n` }, + { trigger: "Apply this configuration?", response: "y\n" }, + { trigger: "Enable web search", response: "1\n" }, + // Raw-mode messaging-channel selector; Enter with none toggled skips. + { trigger: "Press 1-7 to toggle", response: "\r" }, + { trigger: "Resource profiles:", response: "6\n" }, + // Raw-mode Policy tier selector; Enter confirms the pre-selected + // default (Balanced). This is the exact prompt the issue claims the + // wizard never reaches. + { trigger: "Policy tier", response: "\r" }, + // A second raw-mode selector follows immediately: individual preset + // inclusion/rw toggles, pre-populated from the chosen tier. Enter + // confirms the Balanced defaults. + { trigger: "Presets (", response: "\r" }, + ], + timeoutMs: ONBOARD_TIMEOUT_MS - 5 * 60_000, + }); + await artifacts.writeText("onboard-transcript.txt", result.output); + + const redactedTranscript = redactString(result.output, [apiKey]); + expect( + result.timedOut, + `onboard command timed out; see onboard-transcript.txt:\n${redactedTranscript}`, + ).toBe(false); + expect( + result.exitCode, + `onboard command exited non-zero; see onboard-transcript.txt:\n${redactedTranscript}`, + ).toBe(0); + expect(result.firedTriggers).toContain("Other OpenAI-compatible endpoint"); + expect(result.firedTriggers).toContain("Policy tier"); + + progress.phase("confirm every ordered onboarding step appears in order"); + let searchFrom = 0; + for (const marker of ORDERED_STEP_MARKERS) { + const index = result.output.indexOf(marker, searchFrom); + expect( + index, + `expected step marker ${JSON.stringify(marker)} after offset ${searchFrom} in the transcript; see onboard-transcript.txt`, + ).toBeGreaterThanOrEqual(searchFrom); + searchFrom = index + marker.length; + } + + progress.phase("confirm Policy presets is reached before completion"); + const policyIndex = result.output.indexOf("[8/8] Policy presets"); + const abortedIndex = result.output.search(/Onboarding did not finish/i); + expect(policyIndex, "Policy presets step must be observed").toBeGreaterThanOrEqual(0); + expect( + abortedIndex, + `onboarding must not abort after reaching Policy presets; see onboard-transcript.txt:\n${redactedTranscript}`, + ).not.toBeGreaterThanOrEqual(0); + }, +); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 3260cec4e68..ef1a3d68e77 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -220,6 +220,12 @@ "test/e2e/support/e2e-clients.test.ts" ] }, + { + "live": "test/e2e/live/onboard-policy-preset-sequencing.test.ts", + "fast": [ + "test/e2e/support/onboard-interactive-pty.test.ts" + ] + }, { "live": "test/e2e/live/snapshot-commands.test.ts", "fast": [ diff --git a/test/e2e/support/onboard-interactive-pty.test.ts b/test/e2e/support/onboard-interactive-pty.test.ts new file mode 100644 index 00000000000..f85976c3933 --- /dev/null +++ b/test/e2e/support/onboard-interactive-pty.test.ts @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; + +const { spawnMock } = vi.hoisted(() => ({ + spawnMock: vi.fn(), +})); + +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + spawnMock.mockImplementation((...args: Parameters) => actual.spawn(...args)); + return { ...actual, spawn: spawnMock }; +}); + +import { startTestProgress, type TestProgress } from "../fixtures/progress.ts"; +import { driveInteractiveCommand } from "../live/onboard-interactive-pty.ts"; + +function observedProgress(scenario: string): TestProgress { + return startTestProgress(scenario, ["drive the child", "observe its result"], { + logLine: () => undefined, + }); +} + +describe("interactive PTY driver", () => { + it("reports the real exit code when the child exits normally instead of a false timeout", async () => { + const progress = observedProgress("onboard-interactive-pty clean exit"); + try { + const result = await driveInteractiveCommand({ + activityLabel: "command: onboard-interactive-pty-clean-exit", + cmd: ["python3", "-c", "print('hello')"], + env: process.env, + progress, + rules: [], + timeoutMs: 10_000, + }); + expect(result.timedOut).toBe(false); + expect(result.exitCode).toBe(0); + } finally { + progress.stop(); + } + }); + + it("keeps every scripted response, including a secret, out of the spawned process arguments", async () => { + spawnMock.mockClear(); + const progress = observedProgress("onboard-interactive-pty argv secrecy"); + const secret = "test-secret-abc123"; + try { + const result = await driveInteractiveCommand({ + activityLabel: "command: onboard-interactive-pty-argv-secret", + cmd: ["python3", "-c", "import sys; print('prompt:'); sys.stdout.flush(); print(input())"], + env: process.env, + progress, + rules: [{ trigger: "prompt:", response: `${secret}\n` }], + timeoutMs: 10_000, + }); + expect(result.exitCode).toBe(0); + // Confirms the response was actually delivered to the child, not just + // that it never got sent. + expect(result.output).toContain(secret); + + const call = spawnMock.mock.calls.at(-1); + expect(call, "expected driveInteractiveCommand to spawn the driver process").toBeTruthy(); + const [, args] = call as [unknown, readonly string[]]; + expect(args.join(" ")).not.toContain(secret); + } finally { + progress.stop(); + } + }); + + it("terminates the driver and its forked PTY child together on timeout", async () => { + const progress = observedProgress("onboard-interactive-pty timeout cleanup"); + const pidDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pty-timeout-")); + const pidFile = path.join(pidDir, "child.pid"); + try { + const result = await driveInteractiveCommand({ + activityLabel: "command: onboard-interactive-pty-timeout-cleanup", + cmd: [ + "python3", + "-c", + "import os, signal, sys, time\nsignal.signal(signal.SIGHUP, signal.SIG_IGN)\nopen(sys.argv[1], 'w').write(str(os.getpid()))\ntime.sleep(30)", + pidFile, + ], + env: process.env, + progress, + rules: [], + timeoutMs: 500, + }); + expect(result.timedOut).toBe(true); + + // Brief grace period for the OS to finish reaping the killed group. + await new Promise((resolve) => setTimeout(resolve, 300)); + const childPid = Number(fs.readFileSync(pidFile, "utf8").trim()); + expect(() => process.kill(childPid, 0)).toThrow(); + } finally { + progress.stop(); + fs.rmSync(pidDir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/e2e/support/onboard-policy-preset-sequencing-workflow-boundary.test.ts b/test/e2e/support/onboard-policy-preset-sequencing-workflow-boundary.test.ts new file mode 100644 index 00000000000..07b9a948e5c --- /dev/null +++ b/test/e2e/support/onboard-policy-preset-sequencing-workflow-boundary.test.ts @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + catalogueTarget, + validateE2eTargetCatalogue, +} from "../../../tools/e2e/target-catalogue.mts"; +import { buildE2eWorkflowPlan } from "../../../tools/e2e/workflow-plan.mts"; + +// #6042: onboard-policy-preset-sequencing.test.ts drives the real +// interactive onboard wizard through a PTY; forcing non-interactive mode +// (as cloud-onboard and double-onboard both do) would defeat the whole +// regression. Prove the catalogue selects the right test file today, and +// that reintroducing NEMOCLAW_NON_INTERACTIVE on this target is caught. +describe("onboard-policy-preset-sequencing workflow boundary", () => { + it("selects onboard-policy-preset-sequencing.test.ts with interactive mode enabled", () => { + const target = catalogueTarget("onboard-policy-preset-sequencing"); + const plan = buildE2eWorkflowPlan({ jobs: target.id }); + + expect(target.testFile).toBe("test/e2e/live/onboard-policy-preset-sequencing.test.ts"); + expect(target.installNonInteractive).toBe(false); + expect(target.environment.NEMOCLAW_NON_INTERACTIVE).toBeUndefined(); + expect(plan.catalogueMatrices.standard).toEqual([ + expect.objectContaining({ + id: target.id, + install_non_interactive: false, + test_file: target.testFile, + }), + ]); + }); + + it("rejects onboard-policy-preset-sequencing forced back into non-interactive mode", () => { + const target = catalogueTarget("onboard-policy-preset-sequencing"); + + expect(() => + validateE2eTargetCatalogue([ + { + ...target, + installNonInteractive: true, + environment: { ...target.environment, NEMOCLAW_NON_INTERACTIVE: "1" }, + }, + ]), + ).toThrow( + "E2E target onboard-policy-preset-sequencing requires interactive installation and execution", + ); + }); +}); diff --git a/tools/e2e/check-semantic-phases.mts b/tools/e2e/check-semantic-phases.mts index 27945c655c1..51863863d93 100644 --- a/tools/e2e/check-semantic-phases.mts +++ b/tools/e2e/check-semantic-phases.mts @@ -403,6 +403,10 @@ const OBSERVED_CHILD_PROGRESS_POLICIES = new Map