Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions reviewer.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import { describe, it, expect } from "vitest";
import { parseVerdict, cleanReviewText, isLgtmResult } from "./reviewer";

describe("reviewer verdict parsing", () => {
it("parseVerdict extracts LGTM", () => {
expect(parseVerdict("some text <verdict>LGTM</verdict> more")).toBe("lgtm");
});

it("parseVerdict extracts ISSUES_FOUND", () => {
expect(parseVerdict("<verdict>ISSUES_FOUND</verdict>")).toBe("issues");
});

it("parseVerdict returns null when no tag", () => {
expect(parseVerdict("no verdict here")).toBeNull();
});

it("isLgtmResult returns true for empty text", () => {
expect(isLgtmResult("")).toBe(true);
expect(isLgtmResult(" ")).toBe(true);
});

it("isLgtmResult returns false when severity markers present", () => {
expect(isLgtmResult("- **High:** something bad")).toBe(false);
expect(isLgtmResult("- **Medium:** fix this")).toBe(false);
});

it("isLgtmResult returns true for explicit LGTM text", () => {
expect(isLgtmResult("LGTM — looks good")).toBe(true);
});

it("cleanReviewText strips verdict tags", () => {
const raw = "Some findings\n<verdict>ISSUES_FOUND</verdict>";
expect(cleanReviewText(raw)).toBe("Some findings");
});

it("empty cleanedText with ISSUES_FOUND verdict should be treated as LGTM", () => {
// This tests the bug where model returns <verdict>ISSUES_FOUND</verdict>
// with no actual findings text — resulting in confusing "found potential issues:" + nothing
const raw = "<verdict>ISSUES_FOUND</verdict>";
const cleaned = cleanReviewText(raw);
expect(cleaned).toBe("");
// The fix: verdict=issues + empty cleaned = treat as LGTM
// (tested at integration level in reviewer.ts line ~395)
expect(isLgtmResult(cleaned)).toBe(true);
});
});
3 changes: 2 additions & 1 deletion reviewer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,8 @@ export async function runReviewSession(prompt: string, opts: ReviewOptions): Pro
}

const cleanedText = cleanReviewText(reviewText);
const isLgtm = verdict === "lgtm";
// If model said ISSUES_FOUND but produced no actual findings text, treat as LGTM
const isLgtm = verdict === "lgtm" || (verdict === "issues" && !cleanedText.trim());
Comment on lines +395 to +396

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Honor ISSUES_FOUND verdict even when body is empty

Do not convert an explicit ISSUES_FOUND verdict into LGTM based on cleanedText being empty. In runReviewSession, if the reviewer outputs only <verdict>ISSUES_FOUND</verdict> (which can happen with truncated/minimal responses), this branch now marks the result as LGTM and the orchestrator will skip the remediation loop, silently dropping a reported failure. This change weakens the explicit machine-readable contract from parseVerdict; keep verdict === "issues" as non-LGTM and handle empty-body UX separately in message formatting.

Useful? React with 👍 / 👎.

const durationMs = Date.now() - startTime;

rlog(
Expand Down
Loading