From 49c18e4a16717c48e665d150c6908ede8aceea42 Mon Sep 17 00:00:00 2001 From: Waqas Ahmed Date: Sat, 8 Aug 2026 14:03:23 +0500 Subject: [PATCH 1/8] test(e2e): add interactive onboard policy-preset step-ordering test (#6042) Three independent investigations (comments on #6042) could not reproduce the reported skip of the Policy Presets TUI step, and a structural audit confirmed the onboard state machine has no transition path from any earlier state directly to complete -- every path passes through policies. What was missing was a checked-in test that actually drives the real interactive wizard through a PTY (piped stdin does not reproduce this wizard's raw-mode selectors) to prove it. Add such a test: it answers every interactive prompt in the compatible-endpoint onboarding journey through a real pseudo-terminal and asserts the ordered step markers ([1/8] through [8/8] Policy presets) appear in order, with completion only reachable after Policy presets. This is test-only; no production onboarding behavior changes. The new PTY driver is routed through the suite's single audited async child-process boundary (spawnObservedChild), so its progress-capability callsite is registered in the reviewed allowlist in tools/e2e/check-semantic-phases.mts. Signed-off-by: Waqas Ahmed --- test/e2e/live/onboard-interactive-pty.ts | 168 +++++++++++++++++ .../onboard-policy-preset-sequencing.test.ts | 169 ++++++++++++++++++ tools/e2e/check-semantic-phases.mts | 4 + 3 files changed, 341 insertions(+) create mode 100644 test/e2e/live/onboard-interactive-pty.ts create mode 100644 test/e2e/live/onboard-policy-preset-sequencing.test.ts diff --git a/test/e2e/live/onboard-interactive-pty.ts b/test/e2e/live/onboard-interactive-pty.ts new file mode 100644 index 00000000000..b60b72b26c6 --- /dev/null +++ b/test/e2e/live/onboard-interactive-pty.ts @@ -0,0 +1,168 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ChildProcessProgress } from "../fixtures/observed-child-process.ts"; +import { spawnObservedChild } from "../fixtures/observed-child-process.ts"; + +// Drives an interactive CLI through a real PTY, the same technique +// `test/helpers/installer-express-prompt-pty-harness.ts` uses for the +// installer's express prompt. The real onboard wizard behaves differently +// under a piped, non-TTY stdin than under a real terminal (raw-mode +// keypress selectors, `isTTY`-gated prompts), so a faithful regression test +// for interactive-only behavior must drive a real PTY rather than pipe +// stdin. +// +// Rules fire independently and out of order: a rule whose trigger never +// appears (for example, a first-run license notice already accepted on a +// prior run) must not block a later rule from firing when its own trigger +// appears. +// +// The child process itself is launched through the shared +// `spawnObservedChild` boundary (the suite's single audited asynchronous +// child-process call) so it still tracks a content-free progress activity +// and canonical lifecycle checkpoints; this module attaches its own +// listeners on top to capture output and match rule triggers. + +export interface InteractiveCommandRule { + readonly trigger: string; + readonly response: string; +} + +export interface InteractiveCommandResult { + readonly exitCode: number; + readonly output: string; + readonly firedTriggers: readonly string[]; + readonly timedOut: boolean; +} + +export interface DriveInteractiveCommandOptions { + readonly activityLabel: string; + readonly cmd: readonly [string, ...string[]]; + readonly cwd?: string; + readonly env: NodeJS.ProcessEnv; + readonly progress: ChildProcessProgress; + readonly rules: readonly InteractiveCommandRule[]; + readonly timeoutMs: number; +} + +// Runs inside a Python child so it can fork a real pseudo-terminal; +// Node has no built-in PTY primitive and this repo does not depend on +// node-pty. The child's own deadline is generous; the Node-side timer +// below is the enforced hard bound and SIGKILLs the whole process tree. +const PTY_DRIVER_SCRIPT = ` +import json, os, pty, select, signal, sys, time + +payload = json.loads(sys.argv[1]) +cmd = payload["cmd"] +rules = payload["rules"] +timeout_s = payload["timeoutSeconds"] + +pid, fd = pty.fork() +if pid == 0: + os.execvp(cmd[0], cmd) + +output = bytearray() +os.set_blocking(fd, False) +deadline = time.monotonic() + timeout_s +fired = [False] * len(rules) +exit_code = None +while time.monotonic() < deadline: + ready, _, _ = select.select([fd], [], [], 0.2) + if ready: + try: + chunk = os.read(fd, 65536) + except OSError: + break + if not chunk: + break + output.extend(chunk) + sys.stdout.buffer.write(chunk) + sys.stdout.flush() + text = output.decode("utf-8", errors="ignore") + for i, rule in enumerate(rules): + if fired[i]: + continue + if rule["trigger"] in text: + os.write(fd, rule["response"].encode()) + sys.stderr.write("FIRED\\t" + rule["trigger"] + "\\n") + fired[i] = True + waited = os.waitpid(pid, os.WNOHANG) + if waited[0] == pid: + exit_code = os.waitstatus_to_exitcode(waited[1]) + break +if exit_code is None: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + os.waitpid(pid, 0) + sys.stderr.write("DRIVER_TIMEOUT\\n") + sys.exit(124) +sys.exit(exit_code) +`; + +function resolvePython(): string { + return process.env.NEMOCLAW_E2E_PYTHON3_BIN || "python3"; +} + +export function driveInteractiveCommand( + options: DriveInteractiveCommandOptions, +): Promise { + const payload = JSON.stringify({ + cmd: options.cmd, + rules: options.rules.map((rule) => ({ trigger: rule.trigger, response: rule.response })), + // Comfortably longer than the Node-side hard timeout below so the + // driver's own bookkeeping never races the enforced bound. + timeoutSeconds: Math.ceil(options.timeoutMs / 1000) + 30, + }); + // Kept directly in this function's body, not inside the Promise executor + // below, so the sole audited async child-process boundary stays attached + // to a named, reviewed callsite. + const child = spawnObservedChild(resolvePython(), ["-c", PTY_DRIVER_SCRIPT, payload], { + activityLabel: options.activityLabel, + progress: options.progress, + spawn: { cwd: options.cwd, env: options.env }, + }); + + return new Promise((resolve, reject) => { + let output = ""; + const firedTriggers: string[] = []; + let timedOut = false; + let settled = false; + + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, options.timeoutMs); + + // Additional listeners alongside spawnObservedChild's own content-free + // observer; this module needs the real transcript to match rule + // triggers and to report ordered step evidence. + child.stdout?.on("data", (chunk: Buffer) => { + output += chunk.toString("utf-8"); + }); + child.stderr?.on("data", (chunk: Buffer) => { + for (const line of chunk.toString("utf-8").split("\n")) { + const fired = line.match(/^FIRED\t(.*)$/); + if (fired) firedTriggers.push(fired[1]); + } + }); + child.once("error", (error) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(error); + }); + child.once("close", (code) => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve({ + exitCode: timedOut ? 124 : (code ?? 1), + output, + firedTriggers, + timedOut, + }); + }); + }); +} diff --git a/test/e2e/live/onboard-policy-preset-sequencing.test.ts b/test/e2e/live/onboard-policy-preset-sequencing.test.ts new file mode 100644 index 00000000000..1494ae37350 --- /dev/null +++ b/test/e2e/live/onboard-policy-preset-sequencing.test.ts @@ -0,0 +1,169 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { randomBytes } from "node:crypto"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/command.ts"; +import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; +import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; +import { driveInteractiveCommand } from "./onboard-interactive-pty.ts"; + +// Regression coverage for #6042: "interactive onboard wizard skips Policy +// Presets TUI step". Three independent investigations could not reproduce a +// skip — the onboard state machine has no transition from any earlier state +// directly to `complete`, every path passes through `policies` — but no +// checked-in test drove the real interactive TUI through a PTY to prove it. +// This test is that proof: it answers every interactive prompt in the +// compatible-endpoint journey through a real pseudo-terminal (piped stdin +// does not reproduce the raw-mode selectors this wizard uses) and asserts +// the ordered step markers appear in order, ending with `[8/8] Policy +// presets`, before the wizard can report completion. +// +// This is a hermetic, mock-provider variant of the reporter's journey +// (`nemoclaw onboard` with "Other OpenAI-compatible endpoint"), so it needs +// no NVIDIA credential and runs in ordinary CI, not just a live-inference +// lane. + +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-policy-order"; +validateSandboxName(SANDBOX_NAME); +const ONBOARD_TIMEOUT_MS = 40 * 60_000; +const MODEL = "test-model"; +// A Docker network namespace cannot reach host loopback directly; bind the +// fake endpoint on all interfaces and advertise the OpenShell host alias so +// both the host-side onboard validation and the sandbox's own inference +// route can reach it (matches the shared E2E inference adapter's mock mode). +const SANDBOX_HOST_ALIAS = "host.openshell.internal"; + +// The ordered, observable step headers the real interactive wizard prints. +// Each must appear strictly after the previous one; `[8/8] Policy presets` +// is the step the issue claims gets skipped. +const ORDERED_STEP_MARKERS = [ + "[1/8] Preflight checks", + "[2/8] Starting OpenShell gateway", + "[3/8] Configuring inference provider", + "[4/8] Setting up inference provider", + "[5/8] Messaging channels", + "[6/8] Creating sandbox", + "[7/8] Setting up OpenClaw inside sandbox", + "[8/8] Policy presets", +] as const; + +test("interactive onboard wizard reaches Policy presets in step order (#6042)", { + timeout: ONBOARD_TIMEOUT_MS, + meta: { + e2ePhases: [ + "start the local compatible-endpoint fake server", + "drive the interactive onboard wizard through a real PTY", + "confirm every ordered onboarding step appears in order", + "confirm Policy presets is reached before completion", + ], + }, +}, async ({ artifacts, cleanup, host, progress, skip }) => { + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-onboard-policy-order", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error( + `Docker is required to drive a real interactive onboard: ${resultText(docker)}`, + ); + } + skip("Docker is required to drive a real interactive onboard"); + } + + progress.phase("start the local compatible-endpoint fake server"); + const apiKey = `e2e-6042-${randomBytes(16).toString("hex")}`; + const fake = await startFakeOpenAiCompatibleServer({ + apiKey, + chatContent: "PONG", + host: "0.0.0.0", + model: MODEL, + progress, + publicHost: SANDBOX_HOST_ALIAS, + requireAuth: true, + responseText: "PONG", + }); + artifacts.addRedactionValues([apiKey]); + cleanup.trackDisposable("close fake compatible-endpoint server", () => fake.close()); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-onboard-policy-order", + env: buildAvailabilityProbeEnv(), + redactionValues: [apiKey], + timeoutMs: 120_000, + }); + + progress.phase("drive the interactive onboard wizard through a real PTY"); + const result = await driveInteractiveCommand({ + activityLabel: "command: onboard-interactive-pty", + progress, + cmd: [ + "node", + CLI_ENTRYPOINT, + "onboard", + "--fresh", + "--agent", + "openclaw", + "--name", + SANDBOX_NAME, + "--yes-i-accept-third-party-software", + ], + cwd: REPO_ROOT, + env: buildAvailabilityProbeEnv(), + rules: [ + // First-run license notice; already accepted on a reused host, so + // this rule may simply never fire. + { trigger: "Type 'yes' to accept", response: "yes\n" }, + // Only appears when the preflight resource check warns; skipped on + // an adequately provisioned CI runner. + { trigger: "Continue with onboarding?", response: "y\n" }, + // "Other OpenAI-compatible endpoint" — position depends on + // src/lib/onboard/providers.ts's provider list for the openclaw agent. + { trigger: "Select your inference provider:", response: "4\n" }, + { trigger: "OpenAI-compatible base URL", response: `${fake.baseUrl}\n` }, + { trigger: "API key:", response: `${apiKey}\n` }, + { trigger: "endpoint model", response: `${MODEL}\n` }, + { trigger: "Apply this configuration?", response: "y\n" }, + { trigger: "Enable web search", response: "1\n" }, + // Raw-mode messaging-channel selector; Enter with none toggled skips. + { trigger: "Press 1-7 to toggle", response: "\r" }, + { trigger: "Resource profiles:", response: "6\n" }, + // Raw-mode Policy tier selector; Enter confirms the pre-selected + // default (Balanced). This is the exact prompt the issue claims the + // wizard never reaches. + { trigger: "Policy tier", response: "\r" }, + // A second raw-mode selector follows immediately: individual preset + // inclusion/rw toggles, pre-populated from the chosen tier. Enter + // confirms the Balanced defaults. + { trigger: "Presets (", response: "\r" }, + ], + timeoutMs: ONBOARD_TIMEOUT_MS - 5 * 60_000, + }); + await artifacts.writeText("onboard-transcript.txt", result.output); + + progress.phase("confirm every ordered onboarding step appears in order"); + let searchFrom = 0; + for (const marker of ORDERED_STEP_MARKERS) { + const index = result.output.indexOf(marker, searchFrom); + expect( + index, + `expected step marker ${JSON.stringify(marker)} after offset ${searchFrom} in the transcript; see onboard-transcript.txt`, + ).toBeGreaterThanOrEqual(searchFrom); + searchFrom = index + marker.length; + } + + progress.phase("confirm Policy presets is reached before completion"); + const policyIndex = result.output.indexOf("[8/8] Policy presets"); + const abortedIndex = result.output.search(/Onboarding did not finish/i); + expect(policyIndex, "Policy presets step must be observed").toBeGreaterThanOrEqual(0); + expect( + abortedIndex, + `onboarding must not abort after reaching Policy presets:\n${result.output}`, + ).not.toBeGreaterThanOrEqual(0); + expect(result.timedOut, `onboard command timed out:\n${result.output}`).toBe(false); + expect(result.exitCode, `onboard command exited non-zero:\n${result.output}`).toBe(0); +}); diff --git a/tools/e2e/check-semantic-phases.mts b/tools/e2e/check-semantic-phases.mts index 27945c655c1..51863863d93 100644 --- a/tools/e2e/check-semantic-phases.mts +++ b/tools/e2e/check-semantic-phases.mts @@ -403,6 +403,10 @@ const OBSERVED_CHILD_PROGRESS_POLICIES = new Map Date: Sat, 8 Aug 2026 16:34:26 +0500 Subject: [PATCH 2/8] fix(e2e): use the shared docker fixture instead of an inline Docker check codebase-growth-guardrails flagged the manual Docker prerequisite check as two added `if` statements in the test body. Replace it with the existing `docker` fixture's `requireDocker()` (already used by e.g. sandbox-operations.test.ts), which encapsulates the same throw-in-CI/skip-locally branching in the fixture layer instead of the test body. Drops the now-unused `resultText` import. Signed-off-by: Waqas Ahmed --- .../onboard-policy-preset-sequencing.test.ts | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/test/e2e/live/onboard-policy-preset-sequencing.test.ts b/test/e2e/live/onboard-policy-preset-sequencing.test.ts index 1494ae37350..2d4fd263c32 100644 --- a/test/e2e/live/onboard-policy-preset-sequencing.test.ts +++ b/test/e2e/live/onboard-policy-preset-sequencing.test.ts @@ -4,7 +4,6 @@ import { randomBytes } from "node:crypto"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; -import { resultText } from "../fixtures/clients/command.ts"; import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; @@ -61,20 +60,8 @@ test("interactive onboard wizard reaches Policy presets in step order (#6042)", "confirm Policy presets is reached before completion", ], }, -}, async ({ artifacts, cleanup, host, progress, skip }) => { - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-onboard-policy-order", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `Docker is required to drive a real interactive onboard: ${resultText(docker)}`, - ); - } - skip("Docker is required to drive a real interactive onboard"); - } +}, async ({ artifacts, cleanup, docker, host, progress }) => { + await docker.requireDocker(); progress.phase("start the local compatible-endpoint fake server"); const apiKey = `e2e-6042-${randomBytes(16).toString("hex")}`; From 9ce0eda64aa2e3fe80f657d8a2e4abcc893c19ec Mon Sep 17 00:00:00 2001 From: Waqas Ahmed Date: Sat, 8 Aug 2026 17:34:00 +0500 Subject: [PATCH 3/8] fix(e2e): redact the generated API key from onboard test assertion messages PRA-2 (PR review advisor): the assertion messages for the abort/timeout/ exit-code checks interpolated the raw PTY transcript, which still contains the generated mock API key -- ArtifactSink's own redaction only applies to the separate onboard-transcript.txt write, not to text Vitest prints inline on a failed assertion. Route the same redactionValues through the shared redactString() helper (ArtifactSink's own redaction primitive) before interpolating, and point the messages at the artifact file the way the step-marker assertion already does. Signed-off-by: Waqas Ahmed --- .../live/onboard-policy-preset-sequencing.test.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/e2e/live/onboard-policy-preset-sequencing.test.ts b/test/e2e/live/onboard-policy-preset-sequencing.test.ts index 2d4fd263c32..6b2019ca164 100644 --- a/test/e2e/live/onboard-policy-preset-sequencing.test.ts +++ b/test/e2e/live/onboard-policy-preset-sequencing.test.ts @@ -8,6 +8,7 @@ import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; +import { redactString } from "../fixtures/redaction.ts"; import { driveInteractiveCommand } from "./onboard-interactive-pty.ts"; // Regression coverage for #6042: "interactive onboard wizard skips Policy @@ -146,11 +147,18 @@ test("interactive onboard wizard reaches Policy presets in step order (#6042)", progress.phase("confirm Policy presets is reached before completion"); const policyIndex = result.output.indexOf("[8/8] Policy presets"); const abortedIndex = result.output.search(/Onboarding did not finish/i); + const redactedTranscript = redactString(result.output, [apiKey]); expect(policyIndex, "Policy presets step must be observed").toBeGreaterThanOrEqual(0); expect( abortedIndex, - `onboarding must not abort after reaching Policy presets:\n${result.output}`, + `onboarding must not abort after reaching Policy presets; see onboard-transcript.txt:\n${redactedTranscript}`, ).not.toBeGreaterThanOrEqual(0); - expect(result.timedOut, `onboard command timed out:\n${result.output}`).toBe(false); - expect(result.exitCode, `onboard command exited non-zero:\n${result.output}`).toBe(0); + expect( + result.timedOut, + `onboard command timed out; see onboard-transcript.txt:\n${redactedTranscript}`, + ).toBe(false); + expect( + result.exitCode, + `onboard command exited non-zero; see onboard-transcript.txt:\n${redactedTranscript}`, + ).toBe(0); }); From eb5cfb4f1630210133e52483edd65ff443d17ae5 Mon Sep 17 00:00:00 2001 From: Waqas Ahmed Date: Sun, 9 Aug 2026 18:24:22 +0500 Subject: [PATCH 4/8] fix(e2e): address maintainer review on the onboard PTY regression Three blocking findings from cv's review: 1. The PTY driver's read loop broke on EOF/EIO without reaping the child first, so a clean successful run could be misreported as DRIVER_TIMEOUT (exit 124) nondeterministically. Block on waitpid at that point and record the real exit code instead. 2. The Node-side timeout only killed the Python driver by pid, leaving its pty.fork()'d onboard child (and any sandbox operations it had in flight) running past the test. Spawn the driver detached (its own process group/session) and kill the whole group on timeout so the descendant is reaped too. 3. onboard-policy-preset-sequencing.test.ts had no checked-in E2E workflow selection. Added a dedicated onboard-policy-preset-sequencing job (modeled on double-onboard, the closest existing job shape) that does NOT set NEMOCLAW_NON_INTERACTIVE -- unlike cloud-onboard/double-onboard, this test needs real interactive mode. Registered its bespoke workflow-boundary validator (mirroring validateDoubleOnboardJob), added it to the CLI-artifact consumer list and recomputed the pinned contract hash, added it to report-to-pr's needs, and pointed its mock-parity entry at the new fast coverage below. Regression evidence for all three, per review request: - test/e2e/support/onboard-interactive-pty.test.ts (new): a clean exit after output is not misreported as a timeout; a generated secret never appears in the spawned process arguments; a timed-out driver and its forked PTY child are both gone afterward. - test/e2e/support/onboard-policy-preset-sequencing-workflow-boundary.test.ts (new, its own file rather than growing the already near-budget e2e-workflow.test.ts): the new job selects the right test file with interactive mode enabled today, and reintroducing NEMOCLAW_NON_INTERACTIVE on that job is caught. Registered both cases in ci/source-shape-test-budget.json's exception list, matching this repo's existing workflow-boundary test entries. Signed-off-by: Waqas Ahmed --- .github/workflows/e2e.yaml | 67 ++++++++++++ ci/source-shape-test-budget.json | 10 ++ test/e2e/live/onboard-interactive-pty.ts | 36 +++++- test/e2e/mock-parity.json | 6 + .../support/onboard-interactive-pty.test.ts | 103 ++++++++++++++++++ ...reset-sequencing-workflow-boundary.test.ts | 61 +++++++++++ tools/e2e/cli-artifact-workflow-boundary.mts | 67 +++++++++++- tools/e2e/workflow-boundary.mts | 97 +++++++++++++++++ 8 files changed, 441 insertions(+), 6 deletions(-) create mode 100644 test/e2e/support/onboard-interactive-pty.test.ts create mode 100644 test/e2e/support/onboard-policy-preset-sequencing-workflow-boundary.test.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 313ba3870f6..093581f6ab9 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -4916,6 +4916,72 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + onboard-policy-preset-sequencing: + needs: generate-matrix + if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',onboard-policy-preset-sequencing,') || contains(format(',{0},', inputs.targets), ',onboard-policy-preset-sequencing,') }} + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + E2E_JOB: "1" + E2E_TARGET_ID: "onboard-policy-preset-sequencing" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/onboard-policy-preset-sequencing + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_sha || github.sha }} + persist-credentials: false + + - *dockerhub-auth + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + with: + build-cli: "false" + + - name: Restore exact-commit CLI artifact + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3 + with: + provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} + + - name: Install OpenShell CLI + run: bash scripts/install-openshell.sh + + # #6042 regression: the interactive onboard wizard must reach Policy + # presets in order. This drives a real PTY, so this job (unlike + # cloud-onboard/double-onboard) must NOT set NEMOCLAW_NON_INTERACTIVE=1 + # -- doing so would force the wizard non-interactive and defeat the + # test. Hermetic mock-provider journey; no NVIDIA credential needed. + - name: Run onboard-policy-preset-sequencing live Vitest test + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + echo "Using OPENSHELL_BIN=$OPENSHELL_BIN" + "$OPENSHELL_BIN" --version + npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/onboard-policy-preset-sequencing.test.ts + + - name: Upload onboard-policy-preset-sequencing Vitest artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + jetson-nvmap-gpu: needs: generate-matrix # The opt-in flag prevents assignment to an unavailable Jetson runner. @@ -7403,6 +7469,7 @@ jobs: messaging-providers, bootstrap-install-smoke, double-onboard, + onboard-policy-preset-sequencing, jetson-nvmap-gpu, concurrent-gateway-ports, full-e2e, diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 252bec05f68..8ac0e435463 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -221,6 +221,16 @@ "test": "routes only the measured heavy lanes on trusted main (#7145)", "category": "security" }, + { + "file": "test/e2e/support/onboard-policy-preset-sequencing-workflow-boundary.test.ts", + "test": "rejects onboard-policy-preset-sequencing forced back into non-interactive mode", + "category": "security" + }, + { + "file": "test/e2e/support/onboard-policy-preset-sequencing-workflow-boundary.test.ts", + "test": "selects onboard-policy-preset-sequencing.test.ts with interactive mode enabled", + "category": "compatibility" + }, { "file": "test/e2e/support/podman-cpu-proof-workflow.test.ts", "test": "runs as a credential-free exact-head PR workflow", diff --git a/test/e2e/live/onboard-interactive-pty.ts b/test/e2e/live/onboard-interactive-pty.ts index b60b72b26c6..1b1af5f233b 100644 --- a/test/e2e/live/onboard-interactive-pty.ts +++ b/test/e2e/live/onboard-interactive-pty.ts @@ -52,7 +52,11 @@ export interface DriveInteractiveCommandOptions { const PTY_DRIVER_SCRIPT = ` import json, os, pty, select, signal, sys, time -payload = json.loads(sys.argv[1]) +# Read from stdin, not argv: the payload embeds every scripted response, +# including any credential a rule supplies (e.g. an onboard API key), and a +# process argument stays visible to anything that can list the command +# line for as long as the child runs. +payload = json.loads(sys.stdin.read()) cmd = payload["cmd"] rules = payload["rules"] timeout_s = payload["timeoutSeconds"] @@ -72,8 +76,14 @@ while time.monotonic() < deadline: try: chunk = os.read(fd, 65536) except OSError: - break + chunk = b"" if not chunk: + # PTY close (EOF, or EIO on Linux) after the child has already + # exited normally must not fall through to the timeout branch + # below: reap it here and record its real exit code so a + # successful run is never misreported as DRIVER_TIMEOUT. + _, status = os.waitpid(pid, 0) + exit_code = os.waitstatus_to_exitcode(status) break output.extend(chunk) sys.stdout.buffer.write(chunk) @@ -117,12 +127,19 @@ export function driveInteractiveCommand( }); // Kept directly in this function's body, not inside the Promise executor // below, so the sole audited async child-process boundary stays attached - // to a named, reviewed callsite. - const child = spawnObservedChild(resolvePython(), ["-c", PTY_DRIVER_SCRIPT, payload], { + // to a named, reviewed callsite. `detached: true` makes the Python driver + // the leader of its own process group/session, so its pty.fork()'d + // onboard child (which inherits that group) can be reaped as a unit on + // timeout instead of surviving as an orphan. + const child = spawnObservedChild(resolvePython(), ["-c", PTY_DRIVER_SCRIPT], { activityLabel: options.activityLabel, progress: options.progress, - spawn: { cwd: options.cwd, env: options.env }, + spawn: { cwd: options.cwd, env: options.env, detached: true }, }); + // Written to stdin rather than passed as a process argument: the payload + // carries every scripted response, including any credential a rule + // supplies (see PTY_DRIVER_SCRIPT's matching comment). + child.stdin?.end(payload); return new Promise((resolve, reject) => { let output = ""; @@ -132,6 +149,15 @@ export function driveInteractiveCommand( const timer = setTimeout(() => { timedOut = true; + // Kill the whole process group (negative pid), not just the driver + // pid: the driver's forked onboard child shares that group and would + // otherwise keep running past this test. + try { + if (child.pid) process.kill(-child.pid, "SIGKILL"); + } catch { + // Group may already be gone (driver exited between the deadline + // check and this signal); fall back to the direct pid below. + } child.kill("SIGKILL"); }, options.timeoutMs); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 9bdcb8a571a..ae5e7f63801 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -220,6 +220,12 @@ "test/e2e/support/e2e-clients.test.ts" ] }, + { + "live": "test/e2e/live/onboard-policy-preset-sequencing.test.ts", + "fast": [ + "test/e2e/support/onboard-interactive-pty.test.ts" + ] + }, { "live": "test/e2e/live/snapshot-commands.test.ts", "fast": [ diff --git a/test/e2e/support/onboard-interactive-pty.test.ts b/test/e2e/support/onboard-interactive-pty.test.ts new file mode 100644 index 00000000000..e79bb5b3b28 --- /dev/null +++ b/test/e2e/support/onboard-interactive-pty.test.ts @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; + +const { spawnMock } = vi.hoisted(() => ({ + spawnMock: vi.fn(), +})); + +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + spawnMock.mockImplementation((...args: Parameters) => actual.spawn(...args)); + return { ...actual, spawn: spawnMock }; +}); + +import { startTestProgress, type TestProgress } from "../fixtures/progress.ts"; +import { driveInteractiveCommand } from "../live/onboard-interactive-pty.ts"; + +function observedProgress(scenario: string): TestProgress { + return startTestProgress(scenario, ["drive the child", "observe its result"], { + logLine: () => undefined, + }); +} + +describe("interactive PTY driver", () => { + it("reports the real exit code for a clean run instead of a false timeout", async () => { + const progress = observedProgress("onboard-interactive-pty clean exit"); + try { + const result = await driveInteractiveCommand({ + activityLabel: "command: onboard-interactive-pty-clean-exit", + cmd: ["python3", "-c", "print('hello')"], + env: process.env, + progress, + rules: [], + timeoutMs: 10_000, + }); + expect(result.timedOut).toBe(false); + expect(result.exitCode).toBe(0); + } finally { + progress.stop(); + } + }); + + it("keeps every scripted response, including a secret, out of the spawned process arguments", async () => { + spawnMock.mockClear(); + const progress = observedProgress("onboard-interactive-pty argv secrecy"); + const secret = "test-secret-abc123"; + try { + const result = await driveInteractiveCommand({ + activityLabel: "command: onboard-interactive-pty-argv-secret", + cmd: ["python3", "-c", "import sys; print('prompt:'); sys.stdout.flush(); print(input())"], + env: process.env, + progress, + rules: [{ trigger: "prompt:", response: `${secret}\n` }], + timeoutMs: 10_000, + }); + expect(result.exitCode).toBe(0); + // Confirms the response was actually delivered to the child, not just + // that it never got sent. + expect(result.output).toContain(secret); + + const call = spawnMock.mock.calls.at(-1); + expect(call, "expected driveInteractiveCommand to spawn the driver process").toBeTruthy(); + const [, args] = call as [unknown, readonly string[]]; + expect(args.join(" ")).not.toContain(secret); + } finally { + progress.stop(); + } + }); + + it("terminates the driver and its forked PTY child together on timeout", async () => { + const progress = observedProgress("onboard-interactive-pty timeout cleanup"); + const pidDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pty-timeout-")); + const pidFile = path.join(pidDir, "child.pid"); + try { + const result = await driveInteractiveCommand({ + activityLabel: "command: onboard-interactive-pty-timeout-cleanup", + cmd: [ + "python3", + "-c", + "import os, sys, time\nopen(sys.argv[1], 'w').write(str(os.getpid()))\ntime.sleep(30)", + pidFile, + ], + env: process.env, + progress, + rules: [], + timeoutMs: 500, + }); + expect(result.timedOut).toBe(true); + + // Brief grace period for the OS to finish reaping the killed group. + await new Promise((resolve) => setTimeout(resolve, 300)); + const childPid = Number(fs.readFileSync(pidFile, "utf8").trim()); + expect(() => process.kill(childPid, 0)).toThrow(); + } finally { + progress.stop(); + fs.rmSync(pidDir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/e2e/support/onboard-policy-preset-sequencing-workflow-boundary.test.ts b/test/e2e/support/onboard-policy-preset-sequencing-workflow-boundary.test.ts new file mode 100644 index 00000000000..b5aa3c2dbbc --- /dev/null +++ b/test/e2e/support/onboard-policy-preset-sequencing-workflow-boundary.test.ts @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import YAML from "yaml"; +import { validateE2eWorkflowBoundary } from "../../../tools/e2e/workflow-boundary.mts"; +import { readWorkflow } from "../../helpers/e2e-workflow-contract"; + +// #6042: onboard-policy-preset-sequencing.test.ts drives the real +// interactive onboard wizard through a PTY; forcing non-interactive mode +// (as cloud-onboard and double-onboard both do) would defeat the whole +// regression. Prove the job selects the right test file today, and that +// reintroducing NEMOCLAW_NON_INTERACTIVE on this job is caught. +describe("onboard-policy-preset-sequencing workflow boundary", () => { + // source-shape-contract: compatibility -- Every checked-in job selection must point at the live regression file it claims to run. + it("selects onboard-policy-preset-sequencing.test.ts with interactive mode enabled", () => { + const workflow = readWorkflow() as { + jobs: Record< + string, + { env?: Record; steps?: Array> } + >; + }; + const job = workflow.jobs["onboard-policy-preset-sequencing"]; + expect(job, "workflow missing onboard-policy-preset-sequencing job").toBeTruthy(); + expect(job!.env?.NEMOCLAW_NON_INTERACTIVE).toBeUndefined(); + const runStep = job!.steps?.find((step) => + String(step.run ?? "").includes("tools/e2e/live-vitest-invocation.mts run --test-path"), + ); + expect(runStep, "expected a live-vitest-invocation step").toBeTruthy(); + expect(String(runStep!.run)).toContain( + "test/e2e/live/onboard-policy-preset-sequencing.test.ts", + ); + }); + + // source-shape-contract: security -- Forcing this job non-interactive would silently defeat the whole PTY-driven regression it exists to run. + it("rejects onboard-policy-preset-sequencing forced back into non-interactive mode", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record }>; + }; + const job = workflow.jobs["onboard-policy-preset-sequencing"]; + expect(job, "workflow missing onboard-policy-preset-sequencing job").toBeTruthy(); + job!.env = { ...job!.env, NEMOCLAW_NON_INTERACTIVE: "1" }; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + const errors = validateE2eWorkflowBoundary(workflowPath); + expect(errors).toEqual( + expect.arrayContaining([ + "onboard-policy-preset-sequencing job must not set NEMOCLAW_NON_INTERACTIVE; the test requires real interactive mode", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); +}); diff --git a/tools/e2e/cli-artifact-workflow-boundary.mts b/tools/e2e/cli-artifact-workflow-boundary.mts index dabcea12213..4855b27fb1d 100644 --- a/tools/e2e/cli-artifact-workflow-boundary.mts +++ b/tools/e2e/cli-artifact-workflow-boundary.mts @@ -39,7 +39,72 @@ const CLI_ARTIFACT_PROVENANCE_STEP = "Record CLI artifact provenance"; const CANDIDATE_CHECKOUT_STEP_CONTENT_SHA256 = "3578a053cede863f7aa4814d8399b4ca21ea0b77cee712e6d549c684818f11dd"; const CLI_ARTIFACT_WORKFLOW_CONTRACT_SHA256 = - "520a7e6a2dcf97d720c71c0bc6ed8fa0776c9fd66686824b5568036ad6e59d9a"; + "26ab9a94474b07b7c9d405d3cc579b230dee96c29baf985e7f1bf1736b43d1c1"; +const CLI_ARTIFACT_CONSUMER_JOB_NAMES = [ + "agent-turn-latency", + "bedrock-runtime-compatible-anthropic", + "brave-search", + "channels-add-remove", + "channels-stop-start", + "cloud-inference", + "cloud-onboard", + "common-egress-agent", + "concurrent-gateway-ports", + "cron-preflight-inference-local", + "dashboard-remote-bind", + "device-auth-health", + "double-onboard", + "full-e2e", + "gateway-guard-recovery", + "gpu-double-onboard", + "gpu-e2e", + "hermes-discord", + "hermes-e2e", + "hermes-gpu-startup", + "hermes-inference-switch", + "hermes-shields-config", + "hermes-slack", + "inference-routing", + "issue-2478-crash-loop-recovery", + "issue-4434-tui-unreachable-inference", + "issue-4462-scope-upgrade-approval", + "jetson-nvmap-gpu", + "kimi-inference-compat", + "live", + "mcp-bridge", + "mcp-bridge-dev", + "messaging-compatible-endpoint", + "messaging-providers", + "model-router-provider-routed-inference", + "network-policy", + "onboard-policy-preset-sequencing", + "onboard-repair", + "onboard-resume", + "openclaw-discord-pairing", + "openclaw-inference-switch", + "openclaw-plugin-runtime-exdev", + "openclaw-plugin-runtime-exdev-release", + "openclaw-skill-cli", + "openclaw-slack-pairing", + "openclaw-tui-chat-correlation", + "openshell-credential-generation-window", + "openshell-gateway-auth-contract", + "openshell-gateway-upgrade", + "overlayfs-autofix", + "rebuild-hermes", + "rebuild-hermes-stale-base", + "rebuild-openclaw", + "retired-selector-compatibility", + "sandbox-operations", + "sandbox-survival", + "sessions-agents-cli", + "shared-e2e", + "skill-agent", + "state-backup-restore", + "telegram-injection", + "token-rotation", + "tunnel-lifecycle", +] as const; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 9b187c79b15..339a3c4557d 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -2720,6 +2720,102 @@ function validateDoubleOnboardJob(errors: string[], jobs: WorkflowRecord): void requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); requireRunContains(errors, runVitest, "test/e2e/live/double-onboard.test.ts"); } + +function validateOnboardPolicyPresetSequencingJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "onboard-policy-preset-sequencing"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing onboard-policy-preset-sequencing job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("onboard-policy-preset-sequencing job must run on ubuntu-latest"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, "onboard-policy-preset-sequencing"); + + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_LIVE_E2E !== "1") { + errors.push("onboard-policy-preset-sequencing job must set NEMOCLAW_RUN_LIVE_E2E=1"); + } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push("onboard-policy-preset-sequencing job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + if ( + jobEnv.E2E_ARTIFACT_DIR !== + "${{ github.workspace }}/e2e-artifacts/live/onboard-policy-preset-sequencing" + ) { + errors.push( + "onboard-policy-preset-sequencing job must write artifacts under e2e-artifacts/live/onboard-policy-preset-sequencing", + ); + } + // The regression drives a real interactive PTY session; forcing + // non-interactive mode here (as cloud-onboard and double-onboard do) would + // defeat the whole test. + if (jobEnv.NEMOCLAW_NON_INTERACTIVE !== undefined) { + errors.push( + "onboard-policy-preset-sequencing job must not set NEMOCLAW_NON_INTERACTIVE; the test requires real interactive mode", + ); + } + requireEnvDoesNotExposeSecret( + errors, + "onboard-policy-preset-sequencing job", + jobEnv, + "NVIDIA_INFERENCE_API_KEY", + ); + requireEnvDoesNotExposeSecret( + errors, + "onboard-policy-preset-sequencing job", + jobEnv, + "DOCKERHUB_TOKEN", + ); + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret( + errors, + `onboard-policy-preset-sequencing step '${step.name ?? step.uses ?? ""}'`, + asRecord(step.env), + "DOCKERHUB_TOKEN", + ); + } + requireEnvDoesNotExposeSecret( + errors, + `onboard-policy-preset-sequencing step '${step.name ?? step.uses ?? ""}'`, + asRecord(step.env), + "NVIDIA_INFERENCE_API_KEY", + ); + if (stringValue(step.run).includes("NEMOCLAW_NON_INTERACTIVE")) { + errors.push( + `onboard-policy-preset-sequencing step '${step.name ?? step.uses ?? ""}' must not set NEMOCLAW_NON_INTERACTIVE`, + ); + } + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("onboard-policy-preset-sequencing job missing checkout step"); + requireFullShaAction(errors, checkout, "onboard-policy-preset-sequencing checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push( + "onboard-policy-preset-sequencing checkout step must set persist-credentials=false", + ); + } + + const installTools = requireJobStep(errors, jobName, steps, "Install OpenShell CLI"); + requireRunContains(errors, installTools, "bash scripts/install-openshell.sh"); + + const runVitest = requireJobStep( + errors, + jobName, + steps, + "Run onboard-policy-preset-sequencing live Vitest test", + ); + requireRunContains(errors, runVitest, "OPENSHELL_BIN"); + requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); + requireRunContains(errors, runVitest, "test/e2e/live/onboard-policy-preset-sequencing.test.ts"); +} function validateHermesE2EJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "hermes-e2e"; const job = asRecord(jobs[jobName]); @@ -4965,6 +5061,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { validateCloudInferenceJob(errors, jobs); validateLlamaCppGenericGpuJob(errors, jobs); validateDoubleOnboardJob(errors, jobs); + validateOnboardPolicyPresetSequencingJob(errors, jobs); validateHermesE2EJob(errors, jobs); validateHermesTimeoutHeadroom(errors, jobs); validateFreeStandingJobSelector(errors, jobs, "hermes-discord", "hermes-discord"); From 38b5d26bb44c0712f2940e255b376aa1f424ef20 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Tue, 11 Aug 2026 03:40:17 -0700 Subject: [PATCH 5/8] test(e2e): complete onboard PTY review coverage Signed-off-by: Apurv Kumaria --- test/e2e/live/onboard-interactive-pty.ts | 7 ++++- .../onboard-policy-preset-sequencing.test.ts | 26 +++++++++++-------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/test/e2e/live/onboard-interactive-pty.ts b/test/e2e/live/onboard-interactive-pty.ts index 1b1af5f233b..7b6b318e337 100644 --- a/test/e2e/live/onboard-interactive-pty.ts +++ b/test/e2e/live/onboard-interactive-pty.ts @@ -144,6 +144,7 @@ export function driveInteractiveCommand( return new Promise((resolve, reject) => { let output = ""; const firedTriggers: string[] = []; + let stderrRest = ""; let timedOut = false; let settled = false; @@ -168,7 +169,9 @@ export function driveInteractiveCommand( output += chunk.toString("utf-8"); }); child.stderr?.on("data", (chunk: Buffer) => { - for (const line of chunk.toString("utf-8").split("\n")) { + const lines = (stderrRest + chunk.toString("utf-8")).split("\n"); + stderrRest = lines.pop() ?? ""; + for (const line of lines) { const fired = line.match(/^FIRED\t(.*)$/); if (fired) firedTriggers.push(fired[1]); } @@ -183,6 +186,8 @@ export function driveInteractiveCommand( if (settled) return; settled = true; clearTimeout(timer); + const fired = stderrRest.match(/^FIRED\t(.*)$/); + if (fired) firedTriggers.push(fired[1]); resolve({ exitCode: timedOut ? 124 : (code ?? 1), output, diff --git a/test/e2e/live/onboard-policy-preset-sequencing.test.ts b/test/e2e/live/onboard-policy-preset-sequencing.test.ts index 6b2019ca164..354a10bb985 100644 --- a/test/e2e/live/onboard-policy-preset-sequencing.test.ts +++ b/test/e2e/live/onboard-policy-preset-sequencing.test.ts @@ -90,7 +90,7 @@ test("interactive onboard wizard reaches Policy presets in step order (#6042)", activityLabel: "command: onboard-interactive-pty", progress, cmd: [ - "node", + process.execPath, CLI_ENTRYPOINT, "onboard", "--fresh", @@ -112,8 +112,9 @@ test("interactive onboard wizard reaches Policy presets in step order (#6042)", // "Other OpenAI-compatible endpoint" — position depends on // src/lib/onboard/providers.ts's provider list for the openclaw agent. { trigger: "Select your inference provider:", response: "4\n" }, + { trigger: "Other OpenAI-compatible endpoint", response: "" }, { trigger: "OpenAI-compatible base URL", response: `${fake.baseUrl}\n` }, - { trigger: "API key:", response: `${apiKey}\n` }, + { trigger: "Other OpenAI-compatible endpoint API key:", response: `${apiKey}\n` }, { trigger: "endpoint model", response: `${MODEL}\n` }, { trigger: "Apply this configuration?", response: "y\n" }, { trigger: "Enable web search", response: "1\n" }, @@ -133,6 +134,18 @@ test("interactive onboard wizard reaches Policy presets in step order (#6042)", }); await artifacts.writeText("onboard-transcript.txt", result.output); + const redactedTranscript = redactString(result.output, [apiKey]); + expect( + result.timedOut, + `onboard command timed out; see onboard-transcript.txt:\n${redactedTranscript}`, + ).toBe(false); + expect( + result.exitCode, + `onboard command exited non-zero; see onboard-transcript.txt:\n${redactedTranscript}`, + ).toBe(0); + expect(result.firedTriggers).toContain("Other OpenAI-compatible endpoint"); + expect(result.firedTriggers).toContain("Policy tier"); + progress.phase("confirm every ordered onboarding step appears in order"); let searchFrom = 0; for (const marker of ORDERED_STEP_MARKERS) { @@ -147,18 +160,9 @@ test("interactive onboard wizard reaches Policy presets in step order (#6042)", progress.phase("confirm Policy presets is reached before completion"); const policyIndex = result.output.indexOf("[8/8] Policy presets"); const abortedIndex = result.output.search(/Onboarding did not finish/i); - const redactedTranscript = redactString(result.output, [apiKey]); expect(policyIndex, "Policy presets step must be observed").toBeGreaterThanOrEqual(0); expect( abortedIndex, `onboarding must not abort after reaching Policy presets; see onboard-transcript.txt:\n${redactedTranscript}`, ).not.toBeGreaterThanOrEqual(0); - expect( - result.timedOut, - `onboard command timed out; see onboard-transcript.txt:\n${redactedTranscript}`, - ).toBe(false); - expect( - result.exitCode, - `onboard command exited non-zero; see onboard-transcript.txt:\n${redactedTranscript}`, - ).toBe(0); }); From 78683813a40fbbb0780ae463f0c0bdb6294bf9d5 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 11 Aug 2026 10:55:46 -0700 Subject: [PATCH 6/8] test(e2e): terminate the PTY child session Signed-off-by: Prekshi Vyas --- test/e2e/live/onboard-interactive-pty.ts | 68 +++++++++++++++---- .../support/onboard-interactive-pty.test.ts | 2 +- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/test/e2e/live/onboard-interactive-pty.ts b/test/e2e/live/onboard-interactive-pty.ts index 7b6b318e337..22bd8120a57 100644 --- a/test/e2e/live/onboard-interactive-pty.ts +++ b/test/e2e/live/onboard-interactive-pty.ts @@ -65,6 +65,29 @@ pid, fd = pty.fork() if pid == 0: os.execvp(cmd[0], cmd) +# pty.fork() makes the command the leader of a separate session/process +# group. Tell the Node parent which group it must terminate on its hard +# timeout; killing only this Python driver's group cannot reach that child. +sys.stderr.write("PTY_CHILD_PID\\t" + str(pid) + "\\n") +sys.stderr.flush() + +def terminate_pty_child(): + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + +def handle_driver_signal(_signum, _frame): + terminate_pty_child() + try: + os.waitpid(pid, 0) + except ChildProcessError: + pass + sys.exit(124) + +signal.signal(signal.SIGTERM, handle_driver_signal) +signal.signal(signal.SIGINT, handle_driver_signal) + output = bytearray() os.set_blocking(fd, False) deadline = time.monotonic() + timeout_s @@ -101,11 +124,11 @@ while time.monotonic() < deadline: exit_code = os.waitstatus_to_exitcode(waited[1]) break if exit_code is None: + terminate_pty_child() try: - os.kill(pid, signal.SIGKILL) - except ProcessLookupError: + os.waitpid(pid, 0) + except ChildProcessError: pass - os.waitpid(pid, 0) sys.stderr.write("DRIVER_TIMEOUT\\n") sys.exit(124) sys.exit(exit_code) @@ -129,8 +152,8 @@ export function driveInteractiveCommand( // below, so the sole audited async child-process boundary stays attached // to a named, reviewed callsite. `detached: true` makes the Python driver // the leader of its own process group/session, so its pty.fork()'d - // onboard child (which inherits that group) can be reaped as a unit on - // timeout instead of surviving as an orphan. + // driver a stable group for fallback cleanup. pty.fork() creates a separate + // child session, whose process-group id is reported over stderr below. const child = spawnObservedChild(resolvePython(), ["-c", PTY_DRIVER_SCRIPT], { activityLabel: options.activityLabel, progress: options.progress, @@ -147,19 +170,33 @@ export function driveInteractiveCommand( let stderrRest = ""; let timedOut = false; let settled = false; + let ptyChildPid: number | null = null; + let forceKillTimer: ReturnType | null = null; const timer = setTimeout(() => { timedOut = true; - // Kill the whole process group (negative pid), not just the driver - // pid: the driver's forked onboard child shares that group and would - // otherwise keep running past this test. + // pty.fork() puts the onboard command in its own session. Kill that + // exact process group, then let the driver reap it through its SIGTERM + // handler. The delayed driver-group SIGKILL is only a hard fallback. try { - if (child.pid) process.kill(-child.pid, "SIGKILL"); + if (ptyChildPid) process.kill(-ptyChildPid, "SIGKILL"); } catch { - // Group may already be gone (driver exited between the deadline - // check and this signal); fall back to the direct pid below. + // The PTY child may already have exited between the deadline and the + // signal. The driver still receives SIGTERM and reaps its status. } - child.kill("SIGKILL"); + child.kill("SIGTERM"); + forceKillTimer = setTimeout(() => { + try { + if (ptyChildPid) process.kill(-ptyChildPid, "SIGKILL"); + } catch { + // Already gone. + } + try { + if (child.pid) process.kill(-child.pid, "SIGKILL"); + } catch { + child.kill("SIGKILL"); + } + }, 1_000); }, options.timeoutMs); // Additional listeners alongside spawnObservedChild's own content-free @@ -172,6 +209,11 @@ export function driveInteractiveCommand( const lines = (stderrRest + chunk.toString("utf-8")).split("\n"); stderrRest = lines.pop() ?? ""; for (const line of lines) { + const childPid = line.match(/^PTY_CHILD_PID\t([1-9]\d*)$/); + if (childPid) { + ptyChildPid = Number(childPid[1]); + continue; + } const fired = line.match(/^FIRED\t(.*)$/); if (fired) firedTriggers.push(fired[1]); } @@ -180,12 +222,14 @@ export function driveInteractiveCommand( if (settled) return; settled = true; clearTimeout(timer); + if (forceKillTimer) clearTimeout(forceKillTimer); reject(error); }); child.once("close", (code) => { if (settled) return; settled = true; clearTimeout(timer); + if (forceKillTimer) clearTimeout(forceKillTimer); const fired = stderrRest.match(/^FIRED\t(.*)$/); if (fired) firedTriggers.push(fired[1]); resolve({ diff --git a/test/e2e/support/onboard-interactive-pty.test.ts b/test/e2e/support/onboard-interactive-pty.test.ts index e79bb5b3b28..b237e66a790 100644 --- a/test/e2e/support/onboard-interactive-pty.test.ts +++ b/test/e2e/support/onboard-interactive-pty.test.ts @@ -81,7 +81,7 @@ describe("interactive PTY driver", () => { cmd: [ "python3", "-c", - "import os, sys, time\nopen(sys.argv[1], 'w').write(str(os.getpid()))\ntime.sleep(30)", + "import os, signal, sys, time\nsignal.signal(signal.SIGHUP, signal.SIG_IGN)\nopen(sys.argv[1], 'w').write(str(os.getpid()))\ntime.sleep(30)", pidFile, ], env: process.env, From c4b5e0557ca5f7313c96ccb84db413fc100a5aea Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 13 Aug 2026 06:30:56 -0700 Subject: [PATCH 7/8] ci(e2e): incorporate current main checks Signed-off-by: Carlos Villela --- ci/source-architecture-budget.json | 12 ++++++------ .../onboard/inference-selection-validation.test.ts | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 09950a7365b..b74bc96cdd7 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -8,14 +8,14 @@ "src/lib/adapters/docker/index.ts": 43, "src/lib/adapters/openshell/client.ts": 23, "src/lib/adapters/openshell/resolve.ts": 27, - "src/lib/adapters/openshell/runtime.ts": 52, + "src/lib/adapters/openshell/runtime.ts": 53, "src/lib/adapters/openshell/timeouts.ts": 37, "src/lib/agent/defs.ts": 32, "src/lib/cli/branding.ts": 86, "src/lib/cli/nemoclaw-oclif-command.ts": 106, "src/lib/cli/terminal-style.ts": 43, "src/lib/core/json-types.ts": 37, - "src/lib/core/ports.ts": 88, + "src/lib/core/ports.ts": 89, "src/lib/core/shell-quote.ts": 28, "src/lib/core/url-utils.ts": 27, "src/lib/core/wait.ts": 35, @@ -23,11 +23,11 @@ "src/lib/inference/config.ts": 29, "src/lib/inference/web-search.ts": 21, "src/lib/messaging/channels/index.ts": 25, - "src/lib/onboard/gateway-binding.ts": 49, + "src/lib/onboard/gateway-binding.ts": 50, "src/lib/runner.ts": 88, "src/lib/security/redact.ts": 52, "src/lib/state/onboard-session.ts": 36, - "src/lib/state/registry.ts": 99, + "src/lib/state/registry.ts": 100, "src/lib/state/state-root.ts": 20, "src/lib/subprocess-env.ts": 24, "src/lib/validation.ts": 25 @@ -46,7 +46,7 @@ "src/lib/actions/sandbox/rebuild-pipeline.ts": 28, "src/lib/actions/sandbox/snapshot.ts": 40, "src/lib/actions/uninstall/run-plan.ts": 26, - "src/lib/inference/onboard-probes.ts": 20, + "src/lib/inference/onboard-probes.ts": 21, "src/lib/inference/vllm.ts": 21, "src/lib/onboard.ts": 210, "src/lib/onboard/machine/handlers/sandbox.ts": 21, @@ -56,7 +56,7 @@ }, "allowedCycles": [], "maxRootFiles": { - "src/lib/onboard": 308, + "src/lib/onboard": 309, "src/lib/actions": 19, "src/lib/actions/sandbox": 182, "src/lib/state": 37, diff --git a/src/lib/onboard/inference-selection-validation.test.ts b/src/lib/onboard/inference-selection-validation.test.ts index df5127e1fbe..5f5fdb384fd 100644 --- a/src/lib/onboard/inference-selection-validation.test.ts +++ b/src/lib/onboard/inference-selection-validation.test.ts @@ -100,6 +100,7 @@ describe("inference selection validation", () => { ok: false, failures: [{ name: "Chat Completions API", httpStatus: 403 }], }), + teardownOrphanManagedGatewayOnAbort: vi.fn(), promptValidationRecovery, }); @@ -452,6 +453,7 @@ describe("inference selection validation", () => { agentProductName: () => "OpenClaw", getCredential: () => "test-key", probeAnthropicEndpoint, + teardownOrphanManagedGatewayOnAbort: vi.fn(), promptValidationRecovery: vi.fn(async () => "selection" as const), resolveEndpointHost: async () => [{ address: "169.254.169.254", family: 4 }], }); @@ -869,6 +871,7 @@ exit 0 agentProductName: () => "Deep Agents", getCredential: () => "test-key", probeOpenAiLikeEndpoint, + teardownOrphanManagedGatewayOnAbort: vi.fn(), promptValidationRecovery, resolveEndpointHost: async () => [{ address: "93.184.216.34", family: 4 }], }); From 9172a617ba23acfeb6b205e52aeb74bff3b88427 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 13 Aug 2026 06:30:56 -0700 Subject: [PATCH 8/8] ci(e2e): incorporate current main checks Signed-off-by: Carlos Villela --- ci/source-architecture-budget.json | 12 +- .../inference-selection-validation.test.ts | 191 +++++++-------- .../onboard-policy-preset-sequencing.test.ts | 222 +++++++++--------- 3 files changed, 215 insertions(+), 210 deletions(-) diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 09950a7365b..b74bc96cdd7 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -8,14 +8,14 @@ "src/lib/adapters/docker/index.ts": 43, "src/lib/adapters/openshell/client.ts": 23, "src/lib/adapters/openshell/resolve.ts": 27, - "src/lib/adapters/openshell/runtime.ts": 52, + "src/lib/adapters/openshell/runtime.ts": 53, "src/lib/adapters/openshell/timeouts.ts": 37, "src/lib/agent/defs.ts": 32, "src/lib/cli/branding.ts": 86, "src/lib/cli/nemoclaw-oclif-command.ts": 106, "src/lib/cli/terminal-style.ts": 43, "src/lib/core/json-types.ts": 37, - "src/lib/core/ports.ts": 88, + "src/lib/core/ports.ts": 89, "src/lib/core/shell-quote.ts": 28, "src/lib/core/url-utils.ts": 27, "src/lib/core/wait.ts": 35, @@ -23,11 +23,11 @@ "src/lib/inference/config.ts": 29, "src/lib/inference/web-search.ts": 21, "src/lib/messaging/channels/index.ts": 25, - "src/lib/onboard/gateway-binding.ts": 49, + "src/lib/onboard/gateway-binding.ts": 50, "src/lib/runner.ts": 88, "src/lib/security/redact.ts": 52, "src/lib/state/onboard-session.ts": 36, - "src/lib/state/registry.ts": 99, + "src/lib/state/registry.ts": 100, "src/lib/state/state-root.ts": 20, "src/lib/subprocess-env.ts": 24, "src/lib/validation.ts": 25 @@ -46,7 +46,7 @@ "src/lib/actions/sandbox/rebuild-pipeline.ts": 28, "src/lib/actions/sandbox/snapshot.ts": 40, "src/lib/actions/uninstall/run-plan.ts": 26, - "src/lib/inference/onboard-probes.ts": 20, + "src/lib/inference/onboard-probes.ts": 21, "src/lib/inference/vllm.ts": 21, "src/lib/onboard.ts": 210, "src/lib/onboard/machine/handlers/sandbox.ts": 21, @@ -56,7 +56,7 @@ }, "allowedCycles": [], "maxRootFiles": { - "src/lib/onboard": 308, + "src/lib/onboard": 309, "src/lib/actions": 19, "src/lib/actions/sandbox": 182, "src/lib/state": 37, diff --git a/src/lib/onboard/inference-selection-validation.test.ts b/src/lib/onboard/inference-selection-validation.test.ts index df5127e1fbe..daf53dfcf67 100644 --- a/src/lib/onboard/inference-selection-validation.test.ts +++ b/src/lib/onboard/inference-selection-validation.test.ts @@ -100,6 +100,7 @@ describe("inference selection validation", () => { ok: false, failures: [{ name: "Chat Completions API", httpStatus: 403 }], }), + teardownOrphanManagedGatewayOnAbort: vi.fn(), promptValidationRecovery, }); @@ -266,67 +267,66 @@ describe("inference selection validation", () => { expectedEndpointUrl: "https://anthropic.corp.example/v1", expectedProbeOptions: { calibrateTimeouts: true, skipResponsesProbe: true }, }, - ])("probes an exactly allowlisted private Anthropic endpoint on its $runtimeSurface surface (#7037)", async ({ - intendedApi, - expectedEndpointUrl, - expectedProbeOptions, - }) => { - vi.stubEnv("NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS", "anthropic.corp.example"); - vi.stubEnv("NEMOCLAW_REASONING", "false"); - const probeEndpoint = vi.fn(() => ({ - ok: true, - api: intendedApi, - label: "Compatible API", - })); - const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); - const log = vi.spyOn(console, "log").mockImplementation(() => {}); - const helpers = createInferenceSelectionValidationHelpers({ - isNonInteractive: () => false, - agentProductName: () => "NemoClaw agent", - getCredential: () => "test-key", - probeAnthropicEndpoint: probeEndpoint, - probeOpenAiLikeEndpoint: probeEndpoint, - promptValidationRecovery: vi.fn(async () => "selection" as const), - resolveEndpointHost: async () => [{ address: "10.0.0.8", family: 4 }], - }); - - try { - const result = await helpers.validateCustomAnthropicSelection( - "Custom Anthropic endpoint", - "https://anthropic.corp.example", - "model-a", - "COMPATIBLE_ANTHROPIC_API_KEY", - null, - { intendedApi }, - ); - - expect(result).toMatchObject({ + ])( + "probes an exactly allowlisted private Anthropic endpoint on its $runtimeSurface surface (#7037)", + async ({ intendedApi, expectedEndpointUrl, expectedProbeOptions }) => { + vi.stubEnv("NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS", "anthropic.corp.example"); + vi.stubEnv("NEMOCLAW_REASONING", "false"); + const probeEndpoint = vi.fn(() => ({ ok: true, api: intendedApi, - pinnedAddresses: ["10.0.0.8"], - trustedPrivateCapability: { - host: "anthropic.corp.example", - addresses: ["10.0.0.8"], - }, + label: "Compatible API", + })); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const helpers = createInferenceSelectionValidationHelpers({ + isNonInteractive: () => false, + agentProductName: () => "NemoClaw agent", + getCredential: () => "test-key", + probeAnthropicEndpoint: probeEndpoint, + probeOpenAiLikeEndpoint: probeEndpoint, + promptValidationRecovery: vi.fn(async () => "selection" as const), + resolveEndpointHost: async () => [{ address: "10.0.0.8", family: 4 }], }); - expect(probeEndpoint).toHaveBeenCalledOnce(); - expect(probeEndpoint).toHaveBeenCalledWith( - expectedEndpointUrl, - "model-a", - "test-key", - expect.objectContaining({ - ...expectedProbeOptions, + + try { + const result = await helpers.validateCustomAnthropicSelection( + "Custom Anthropic endpoint", + "https://anthropic.corp.example", + "model-a", + "COMPATIBLE_ANTHROPIC_API_KEY", + null, + { intendedApi }, + ); + + expect(result).toMatchObject({ + ok: true, + api: intendedApi, pinnedAddresses: ["10.0.0.8"], - trustedPrivateCapability: expect.objectContaining({ addresses: ["10.0.0.8"] }), - }), - ); - expect(warn).toHaveBeenCalledWith(expect.stringContaining("operator-trusted private")); - } finally { - log.mockRestore(); - warn.mockRestore(); - vi.unstubAllEnvs(); - } - }); + trustedPrivateCapability: { + host: "anthropic.corp.example", + addresses: ["10.0.0.8"], + }, + }); + expect(probeEndpoint).toHaveBeenCalledOnce(); + expect(probeEndpoint).toHaveBeenCalledWith( + expectedEndpointUrl, + "model-a", + "test-key", + expect.objectContaining({ + ...expectedProbeOptions, + pinnedAddresses: ["10.0.0.8"], + trustedPrivateCapability: expect.objectContaining({ addresses: ["10.0.0.8"] }), + }), + ); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("operator-trusted private")); + } finally { + log.mockRestore(); + warn.mockRestore(); + vi.unstubAllEnvs(); + } + }, + ); it("honors an exactly allowlisted private endpoint during non-interactive validation (#6861)", async () => { vi.stubEnv("NEMOCLAW_TRUSTED_PRIVATE_INFERENCE_HOSTS", "llm.corp.example"); @@ -400,47 +400,46 @@ describe("inference selection validation", () => { } }); - it.each([ - "http://127.0.0.1:8000/v1", - "https://inference.local/v1", - "https://93.184.216.34/v1", - ])("carries the approved no-pin capability to probes for %s (#6293)", async (endpointUrl) => { - const probeOpenAiLikeEndpoint = vi.fn(() => ({ ok: true, api: "openai-completions" })); - const resolveEndpointHost = vi.fn(async () => [{ address: "10.0.0.8", family: 4 }]); - const log = vi.spyOn(console, "log").mockImplementation(() => {}); - const helpers = createInferenceSelectionValidationHelpers({ - isNonInteractive: () => false, - agentProductName: () => "OpenClaw", - getCredential: () => "test-key", - probeOpenAiLikeEndpoint, - promptValidationRecovery: vi.fn(async () => "selection" as const), - resolveEndpointHost, - }); + it.each(["http://127.0.0.1:8000/v1", "https://inference.local/v1", "https://93.184.216.34/v1"])( + "carries the approved no-pin capability to probes for %s (#6293)", + async (endpointUrl) => { + const probeOpenAiLikeEndpoint = vi.fn(() => ({ ok: true, api: "openai-completions" })); + const resolveEndpointHost = vi.fn(async () => [{ address: "10.0.0.8", family: 4 }]); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const helpers = createInferenceSelectionValidationHelpers({ + isNonInteractive: () => false, + agentProductName: () => "OpenClaw", + getCredential: () => "test-key", + probeOpenAiLikeEndpoint, + promptValidationRecovery: vi.fn(async () => "selection" as const), + resolveEndpointHost, + }); - try { - await expect( - helpers.validateCustomOpenAiLikeSelection( - "Custom endpoint", + try { + await expect( + helpers.validateCustomOpenAiLikeSelection( + "Custom endpoint", + endpointUrl, + "model-a", + "COMPATIBLE_API_KEY", + ), + ).resolves.toEqual({ + ok: true, + api: "openai-completions", + pinnedAddresses: [], + }); + expect(probeOpenAiLikeEndpoint).toHaveBeenCalledWith( endpointUrl, "model-a", - "COMPATIBLE_API_KEY", - ), - ).resolves.toEqual({ - ok: true, - api: "openai-completions", - pinnedAddresses: [], - }); - expect(probeOpenAiLikeEndpoint).toHaveBeenCalledWith( - endpointUrl, - "model-a", - "test-key", - expect.objectContaining({ pinnedAddresses: [] }), - ); - expect(resolveEndpointHost).not.toHaveBeenCalled(); - } finally { - log.mockRestore(); - } - }); + "test-key", + expect.objectContaining({ pinnedAddresses: [] }), + ); + expect(resolveEndpointHost).not.toHaveBeenCalled(); + } finally { + log.mockRestore(); + } + }, + ); it("exits non-interactively when a custom Anthropic endpoint resolves to link-local metadata, without probing (#6293)", async () => { const originalExitCode = process.exitCode; @@ -452,6 +451,7 @@ describe("inference selection validation", () => { agentProductName: () => "OpenClaw", getCredential: () => "test-key", probeAnthropicEndpoint, + teardownOrphanManagedGatewayOnAbort: vi.fn(), promptValidationRecovery: vi.fn(async () => "selection" as const), resolveEndpointHost: async () => [{ address: "169.254.169.254", family: 4 }], }); @@ -869,6 +869,7 @@ exit 0 agentProductName: () => "Deep Agents", getCredential: () => "test-key", probeOpenAiLikeEndpoint, + teardownOrphanManagedGatewayOnAbort: vi.fn(), promptValidationRecovery, resolveEndpointHost: async () => [{ address: "93.184.216.34", family: 4 }], }); diff --git a/test/e2e/live/onboard-policy-preset-sequencing.test.ts b/test/e2e/live/onboard-policy-preset-sequencing.test.ts index 6bf3a16a044..9a6c014af82 100644 --- a/test/e2e/live/onboard-policy-preset-sequencing.test.ts +++ b/test/e2e/live/onboard-policy-preset-sequencing.test.ts @@ -51,118 +51,122 @@ const ORDERED_STEP_MARKERS = [ "[8/8] Policy presets", ] as const; -test("interactive onboard wizard reaches Policy presets in step order (#6042)", { - timeout: ONBOARD_TIMEOUT_MS, - meta: { - e2ePhases: [ - "start the local compatible-endpoint fake server", - "drive the interactive onboard wizard through a real PTY", - "confirm every ordered onboarding step appears in order", - "confirm Policy presets is reached before completion", - ], +test( + "interactive onboard wizard reaches Policy presets in step order (#6042)", + { + timeout: ONBOARD_TIMEOUT_MS, + meta: { + e2ePhases: [ + "start the local compatible-endpoint fake server", + "drive the interactive onboard wizard through a real PTY", + "confirm every ordered onboarding step appears in order", + "confirm Policy presets is reached before completion", + ], + }, }, -}, async ({ artifacts, cleanup, docker, host, progress }) => { - await docker.requireDocker(); + async ({ artifacts, cleanup, docker, host, progress }) => { + await docker.requireDocker(); - progress.phase("start the local compatible-endpoint fake server"); - const apiKey = `e2e-6042-${randomBytes(16).toString("hex")}`; - const fake = await startFakeOpenAiCompatibleServer({ - apiKey, - chatContent: "PONG", - host: "0.0.0.0", - model: MODEL, - progress, - publicHost: SANDBOX_HOST_ALIAS, - requireAuth: true, - responseText: "PONG", - }); - artifacts.addRedactionValues([apiKey]); - cleanup.trackDisposable("close fake compatible-endpoint server", () => fake.close()); - cleanup.trackSandbox(host, SANDBOX_NAME, { - artifactName: "cleanup-nemoclaw-destroy-onboard-policy-order", - env: buildAvailabilityProbeEnv(), - redactionValues: [apiKey], - timeoutMs: 120_000, - }); + progress.phase("start the local compatible-endpoint fake server"); + const apiKey = `e2e-6042-${randomBytes(16).toString("hex")}`; + const fake = await startFakeOpenAiCompatibleServer({ + apiKey, + chatContent: "PONG", + host: "0.0.0.0", + model: MODEL, + progress, + publicHost: SANDBOX_HOST_ALIAS, + requireAuth: true, + responseText: "PONG", + }); + artifacts.addRedactionValues([apiKey]); + cleanup.trackDisposable("close fake compatible-endpoint server", () => fake.close()); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-onboard-policy-order", + env: buildAvailabilityProbeEnv(), + redactionValues: [apiKey], + timeoutMs: 120_000, + }); - progress.phase("drive the interactive onboard wizard through a real PTY"); - const result = await driveInteractiveCommand({ - activityLabel: "command: onboard-interactive-pty", - progress, - cmd: [ - process.execPath, - CLI_ENTRYPOINT, - "onboard", - "--fresh", - "--agent", - "openclaw", - "--name", - SANDBOX_NAME, - "--yes-i-accept-third-party-software", - ], - cwd: REPO_ROOT, - env: buildAvailabilityProbeEnv(), - rules: [ - // A reused host can already have accepted this first-run license - // notice, so this rule might not fire. - { trigger: "Type 'yes' to accept", response: "yes\n" }, - // Only appears when the preflight resource check warns; skipped on - // an adequately provisioned CI runner. - { trigger: "Continue with onboarding?", response: "y\n" }, - // "Other OpenAI-compatible endpoint" — position depends on - // src/lib/onboard/providers.ts's provider list for the openclaw agent. - { trigger: "Select your inference provider:", response: "4\n" }, - { trigger: "Other OpenAI-compatible endpoint", response: "" }, - { trigger: "OpenAI-compatible base URL", response: `${fake.baseUrl}\n` }, - { trigger: "Other OpenAI-compatible endpoint API key:", response: `${apiKey}\n` }, - { trigger: "endpoint model", response: `${MODEL}\n` }, - { trigger: "Apply this configuration?", response: "y\n" }, - { trigger: "Enable web search", response: "1\n" }, - // Raw-mode messaging-channel selector; Enter with none toggled skips. - { trigger: "Press 1-7 to toggle", response: "\r" }, - { trigger: "Resource profiles:", response: "6\n" }, - // Raw-mode Policy tier selector; Enter confirms the pre-selected - // default (Balanced). This is the exact prompt the issue claims the - // wizard never reaches. - { trigger: "Policy tier", response: "\r" }, - // A second raw-mode selector follows immediately: individual preset - // inclusion/rw toggles, pre-populated from the chosen tier. Enter - // confirms the Balanced defaults. - { trigger: "Presets (", response: "\r" }, - ], - timeoutMs: ONBOARD_TIMEOUT_MS - 5 * 60_000, - }); - await artifacts.writeText("onboard-transcript.txt", result.output); + progress.phase("drive the interactive onboard wizard through a real PTY"); + const result = await driveInteractiveCommand({ + activityLabel: "command: onboard-interactive-pty", + progress, + cmd: [ + process.execPath, + CLI_ENTRYPOINT, + "onboard", + "--fresh", + "--agent", + "openclaw", + "--name", + SANDBOX_NAME, + "--yes-i-accept-third-party-software", + ], + cwd: REPO_ROOT, + env: buildAvailabilityProbeEnv(), + rules: [ + // A reused host can already have accepted this first-run license + // notice, so this rule might not fire. + { trigger: "Type 'yes' to accept", response: "yes\n" }, + // Only appears when the preflight resource check warns; skipped on + // an adequately provisioned CI runner. + { trigger: "Continue with onboarding?", response: "y\n" }, + // "Other OpenAI-compatible endpoint" — position depends on + // src/lib/onboard/providers.ts's provider list for the openclaw agent. + { trigger: "Select your inference provider:", response: "4\n" }, + { trigger: "Other OpenAI-compatible endpoint", response: "" }, + { trigger: "OpenAI-compatible base URL", response: `${fake.baseUrl}\n` }, + { trigger: "Other OpenAI-compatible endpoint API key:", response: `${apiKey}\n` }, + { trigger: "endpoint model", response: `${MODEL}\n` }, + { trigger: "Apply this configuration?", response: "y\n" }, + { trigger: "Enable web search", response: "1\n" }, + // Raw-mode messaging-channel selector; Enter with none toggled skips. + { trigger: "Press 1-7 to toggle", response: "\r" }, + { trigger: "Resource profiles:", response: "6\n" }, + // Raw-mode Policy tier selector; Enter confirms the pre-selected + // default (Balanced). This is the exact prompt the issue claims the + // wizard never reaches. + { trigger: "Policy tier", response: "\r" }, + // A second raw-mode selector follows immediately: individual preset + // inclusion/rw toggles, pre-populated from the chosen tier. Enter + // confirms the Balanced defaults. + { trigger: "Presets (", response: "\r" }, + ], + timeoutMs: ONBOARD_TIMEOUT_MS - 5 * 60_000, + }); + await artifacts.writeText("onboard-transcript.txt", result.output); - const redactedTranscript = redactString(result.output, [apiKey]); - expect( - result.timedOut, - `onboard command timed out; see onboard-transcript.txt:\n${redactedTranscript}`, - ).toBe(false); - expect( - result.exitCode, - `onboard command exited non-zero; see onboard-transcript.txt:\n${redactedTranscript}`, - ).toBe(0); - expect(result.firedTriggers).toContain("Other OpenAI-compatible endpoint"); - expect(result.firedTriggers).toContain("Policy tier"); - - progress.phase("confirm every ordered onboarding step appears in order"); - let searchFrom = 0; - for (const marker of ORDERED_STEP_MARKERS) { - const index = result.output.indexOf(marker, searchFrom); + const redactedTranscript = redactString(result.output, [apiKey]); + expect( + result.timedOut, + `onboard command timed out; see onboard-transcript.txt:\n${redactedTranscript}`, + ).toBe(false); expect( - index, - `expected step marker ${JSON.stringify(marker)} after offset ${searchFrom} in the transcript; see onboard-transcript.txt`, - ).toBeGreaterThanOrEqual(searchFrom); - searchFrom = index + marker.length; - } + result.exitCode, + `onboard command exited non-zero; see onboard-transcript.txt:\n${redactedTranscript}`, + ).toBe(0); + expect(result.firedTriggers).toContain("Other OpenAI-compatible endpoint"); + expect(result.firedTriggers).toContain("Policy tier"); + + progress.phase("confirm every ordered onboarding step appears in order"); + let searchFrom = 0; + for (const marker of ORDERED_STEP_MARKERS) { + const index = result.output.indexOf(marker, searchFrom); + expect( + index, + `expected step marker ${JSON.stringify(marker)} after offset ${searchFrom} in the transcript; see onboard-transcript.txt`, + ).toBeGreaterThanOrEqual(searchFrom); + searchFrom = index + marker.length; + } - progress.phase("confirm Policy presets is reached before completion"); - const policyIndex = result.output.indexOf("[8/8] Policy presets"); - const abortedIndex = result.output.search(/Onboarding did not finish/i); - expect(policyIndex, "Policy presets step must be observed").toBeGreaterThanOrEqual(0); - expect( - abortedIndex, - `onboarding must not abort after reaching Policy presets; see onboard-transcript.txt:\n${redactedTranscript}`, - ).not.toBeGreaterThanOrEqual(0); -}); + progress.phase("confirm Policy presets is reached before completion"); + const policyIndex = result.output.indexOf("[8/8] Policy presets"); + const abortedIndex = result.output.search(/Onboarding did not finish/i); + expect(policyIndex, "Policy presets step must be observed").toBeGreaterThanOrEqual(0); + expect( + abortedIndex, + `onboarding must not abort after reaching Policy presets; see onboard-transcript.txt:\n${redactedTranscript}`, + ).not.toBeGreaterThanOrEqual(0); + }, +);