Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
183 changes: 183 additions & 0 deletions test/pr-review-advisor-diff.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { execFileSync, spawnSync } from "node:child_process";
import fs from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { getDiff } from "../tools/advisors/git.mts";

const ROOT = path.resolve(import.meta.dirname, "..");

describe("PR review advisor diff", () => {
it("keeps content after 160,000 characters", () => {
const tmp = fs.mkdtempSync(path.join(tmpdir(), "nemoclaw-pr-advisor-diff-"));
const previousCwd = process.cwd();
let diff = "";

try {
execFileSync("git", ["init", "--quiet"], { cwd: tmp });
fs.writeFileSync(path.join(tmp, "review.txt"), "base\n");
execFileSync("git", ["add", "review.txt"], { cwd: tmp });
execFileSync(
"git",
[
"-c",
"user.name=NemoClaw Test",
"-c",
"user.email=nemoclaw-test@example.com",
"-c",
"commit.gpgsign=false",
"commit",
"--quiet",
"-m",
"base",
],
{ cwd: tmp },
);
const base = execFileSync("git", ["rev-parse", "HEAD"], {
cwd: tmp,
encoding: "utf8",
}).trim();

fs.writeFileSync(
path.join(tmp, "review.txt"),
`${"x".repeat(170_000)}\ncomplete-diff-tail\n`,
);
execFileSync("git", ["add", "review.txt"], { cwd: tmp });
execFileSync(
"git",
[
"-c",
"user.name=NemoClaw Test",
"-c",
"user.email=nemoclaw-test@example.com",
"-c",
"commit.gpgsign=false",
"commit",
"--quiet",
"-m",
"head",
],
{ cwd: tmp },
);

process.chdir(tmp);
diff = getDiff(base, "HEAD");
} finally {
process.chdir(previousCwd);
fs.rmSync(tmp, { recursive: true, force: true });
}

expect(diff).toContain("complete-diff-tail");
expect(diff).not.toContain("<diff truncated");
});

it("falls back to a two-dot diff when the refs have no merge base", () => {
const tmp = fs.mkdtempSync(path.join(tmpdir(), "nemoclaw-pr-advisor-diff-"));
const previousCwd = process.cwd();
let diff = "";

try {
execFileSync("git", ["init", "--quiet"], { cwd: tmp });
fs.writeFileSync(path.join(tmp, "base.txt"), "base\n");
execFileSync("git", ["add", "base.txt"], { cwd: tmp });
commit(tmp, "base");
const base = execFileSync("git", ["rev-parse", "HEAD"], {
cwd: tmp,
encoding: "utf8",
}).trim();

execFileSync("git", ["checkout", "--orphan", "unrelated", "--quiet"], { cwd: tmp });
execFileSync("git", ["rm", "-rf", ".", "--quiet"], { cwd: tmp });
fs.writeFileSync(path.join(tmp, "head.txt"), "fallback-tail\n");
execFileSync("git", ["add", "head.txt"], { cwd: tmp });
commit(tmp, "head");

process.chdir(tmp);
diff = getDiff(base, "HEAD");
} finally {
process.chdir(previousCwd);
fs.rmSync(tmp, { recursive: true, force: true });
}

expect(diff).toContain("fallback-tail");
});

it("fails when neither diff form can resolve the requested ref", () => {
const tmp = fs.mkdtempSync(path.join(tmpdir(), "nemoclaw-pr-advisor-diff-"));
const previousCwd = process.cwd();

try {
execFileSync("git", ["init", "--quiet"], { cwd: tmp });
fs.writeFileSync(path.join(tmp, "review.txt"), "base\n");
execFileSync("git", ["add", "review.txt"], { cwd: tmp });
commit(tmp, "base");
const base = execFileSync("git", ["rev-parse", "HEAD"], {
cwd: tmp,
encoding: "utf8",
}).trim();

process.chdir(tmp);
expect(() => getDiff(base, "missing-ref")).toThrow("failed to read complete diff");
} finally {
process.chdir(previousCwd);
fs.rmSync(tmp, { recursive: true, force: true });
}
});

it("writes failure artifacts when trusted Git inputs are unavailable", () => {
const tmp = fs.mkdtempSync(path.join(tmpdir(), "nemoclaw-pr-advisor-diff-"));
const result = spawnSync(
process.execPath,
[
"--experimental-strip-types",
path.join(ROOT, "tools/pr-review-advisor/analyze.mts"),
"--base",
"missing-ref",
"--head",
"HEAD",
"--schema",
path.join(ROOT, "tools/pr-review-advisor/schema.json"),
"--out-dir",
tmp,
],
{ cwd: ROOT, encoding: "utf8" },
);

try {
expect(result.status).toBe(1);
expect(
JSON.parse(fs.readFileSync(path.join(tmp, "pr-review-advisor-result.json"), "utf8")),
).toMatchObject({ failed: true });
expect(
JSON.parse(fs.readFileSync(path.join(tmp, "pr-review-advisor-final-result.json"), "utf8")),
).toMatchObject({
headSha: expect.stringMatching(/^[0-9a-f]{40}$/u),
reviewCompleteness: { requiresHumanReview: true },
});
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});
});

function commit(cwd: string, message: string): void {
execFileSync(
"git",
[
"-c",
"user.name=NemoClaw Test",
"-c",
"user.email=nemoclaw-test@example.com",
"-c",
"commit.gpgsign=false",
"commit",
"--quiet",
"-m",
message,
],
{ cwd },
);
}
70 changes: 61 additions & 9 deletions test/pr-review-advisor-workflow-boundary.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -893,17 +893,69 @@ process.exitCode = valid ? 0 : 1;`,
}
});

it("fails the supported advisor lane when analyze exits non-zero", () => {
const input = advisorAnalysisInput();
it("writes failure artifacts when analysis exits before producing artifacts", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "pr-review-advisor-failure-"));
const input = advisorAnalysisInput({ outDir: path.join(tmp, "artifacts") });
const analyzePath = path.join(input.advisorDir, "tools", "pr-review-advisor", "analyze.mts");

expect(() =>
runPrReviewAdvisorAnalysis(input, {
fileExists: (file) => file === analyzePath,
readText: supportedAdvisorReadText(input),
runNode: () => 17,
}),
).toThrow("analyze.mts exited with status 17");
try {
expect(() =>
runPrReviewAdvisorAnalysis(input, {
fileExists: (file) => file === analyzePath,
readText: supportedAdvisorReadText(input),
runGit: () => HEAD_SHA,
runNode: () => 17,
}),
).toThrow("analyze.mts exited with status 17");

const result = JSON.parse(
fs.readFileSync(path.join(input.outDir, "pr-review-advisor-result.json"), "utf8"),
);
const finalResult = JSON.parse(
fs.readFileSync(path.join(input.outDir, "pr-review-advisor-final-result.json"), "utf8"),
);
expect(result).toMatchObject({
failed: true,
reason: "analyze.mts exited with status 17",
});
expect(finalResult).toMatchObject({
headSha: HEAD_SHA,
summary: { recommendation: "info_only", confidence: "low" },
reviewCompleteness: { requiresHumanReview: true },
});
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});

it("preserves partial artifacts while completing an early analysis failure", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "pr-review-advisor-failure-"));
const input = advisorAnalysisInput({ outDir: path.join(tmp, "artifacts") });
const analyzePath = path.join(input.advisorDir, "tools", "pr-review-advisor", "analyze.mts");
const resultPath = path.join(input.outDir, "pr-review-advisor-result.json");
const partialResult = '{"failed":true,"partial":true}\n';

try {
fs.mkdirSync(input.outDir, { recursive: true });
fs.writeFileSync(resultPath, partialResult, { flag: "wx", mode: 0o600 });

expect(() =>
runPrReviewAdvisorAnalysis(input, {
fileExists: (file) => file === analyzePath || fs.existsSync(file),
readText: supportedAdvisorReadText(input),
runGit: () => HEAD_SHA,
runNode: () => 17,
}),
).toThrow("analyze.mts exited with status 17");

expect(fs.readFileSync(resultPath, "utf8")).toBe(partialResult);
expect(fs.existsSync(path.join(input.outDir, "pr-review-advisor-final-result.json"))).toBe(
true,
);
expect(fs.existsSync(path.join(input.outDir, "pr-review-advisor-summary.md"))).toBe(true);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});

it("runs analyze in unavailable-result mode when the trusted checkout lacks model support", () => {
Expand Down
14 changes: 6 additions & 8 deletions tools/advisors/git.mts
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,18 @@ export function getChangedFiles(base: string, head: string): string[] {
.sort();
}

export function getDiff(base: string, head: string, maxChars: number): string {
export function getDiff(base: string, head: string): string {
const stdout = gitOutput(
[
["diff", "--find-renames", "--find-copies", "--unified=80", `${base}...${head}`],
["diff", "--find-renames", "--find-copies", "--unified=80", `${base}..${head}`],
],
20 * 1024 * 1024,
Number.POSITIVE_INFINITY,
);
return stdout === undefined ? "" : truncate(stdout, maxChars);
if (stdout === undefined) {
throw new Error(`failed to read complete diff ${base}..${head}; ensure both refs are fetched`);
}
return stdout;
}

export function getDiffStat(base: string, head: string): string {
Expand Down Expand Up @@ -66,8 +69,3 @@ export function gitOutput(commands: string[][], maxBuffer: number): string | und
}
return undefined;
}

export function truncate(text: string, maxChars: number): string {
if (text.length <= maxChars) return text;
return `${text.slice(0, maxChars)}\n\n<diff truncated at ${maxChars} characters>`;
}
4 changes: 2 additions & 2 deletions tools/pr-review-advisor/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,15 +155,15 @@ instead of failing closed without artifacts.

- `prompts/00-system.md` — system prompt sent to the advisor.
- `prompts/01-scope-risk-map-analysis.md` through `prompts/16-validate-synthesis-json.md` — seven alternating analysis/commit pairs followed by draft and validation synthesis turns in the same session, in execution order.
- `prompts/*.tool-results/` — bounded deterministic, domain-specific context payloads exposed as real tools after the matching user turn. The untrusted truncated diff appears only in the first turn, and repeated risk-plan projections use capped path samples.
- `prompts/*.tool-results/` — deterministic, domain-specific context payloads exposed as real tools after the matching user turn. The complete untrusted diff appears only in the first turn, and repeated risk-plan projections use capped path samples.
- `turns/01-scope-risk-map-analysis.txt` through `turns/16-validate-synthesis-json.txt` — assistant output and completed/failed/timed-out status written as each turn settles.
- `context/drift-context.json` — deterministic drift and overlap context.
- `context/security-context.json` — deterministic security-risk context and the risk plan for the
PR SHA.
- `context/validation-context.json` — deterministic acceptance, source-of-truth, static
test-inventory, simplification-signal, and risk plan for the PR SHA, including the
regression invariants reviewed for the PR.
- `context/pr.diff` — truncated PR diff used by the advisor.
- `context/pr.diff` — complete PR diff used by the advisor.
- `pr-review-advisor-raw-output.txt` — raw multi-turn advisor transcript and diagnostics.
- `pr-review-advisor-result.json` — normalized advisor result with findings projected from the canonical open ledger records, or execution metadata when analysis is unavailable.
- `pr-review-advisor-final-result.json` — normalized canonical result used for comments.
Expand Down
Loading
Loading