Varin/benchmark changes

.factory/droids/file-group-reviewer.md

@@ -0,0 +1,74 @@
+---
+name: file-group-reviewer
+description: Reviews an assigned subset of PR files for bugs, security issues, and correctness problems. Spawned in parallel by the main review agent to ensure thorough coverage.
+model: inherit
+tools: ["Read", "Grep", "Glob", "LS"]
+---
+
+You are a senior staff software engineer and expert code reviewer.
+
+Your task: Review the assigned files from the PR and generate a JSON array of **high-confidence, actionable** review comments that pinpoint genuine issues.
+
+<review_guidelines>
+- You are currently checked out to the PR branch.
+- Review ALL files assigned to you thoroughly.
+- Focus on: functional correctness, syntax errors, logic bugs, broken dependencies/contracts/tests, security issues, and performance problems.
+- High-signal bug patterns to actively check for (only comment when evidenced in the diff):
+  - Null/undefined/Optional dereferences; missing-key errors on untrusted/external dict/JSON payloads
+  - Resource leaks (unclosed files/streams/connections; missing cleanup on error paths)
+  - Injection vulnerabilities (SQL injection, XSS, command/template injection) and auth/security invariant violations
+  - OAuth/CSRF invariants: state must be per-flow unpredictable and validated; avoid deterministic/predictable state or missing state checks
+  - Concurrency/race/atomicity hazards (TOCTOU, lost updates, unsafe shared state, process/thread lifecycle bugs)
+  - Missing error handling for critical operations (network, persistence, auth, migrations, external APIs)
+  - Wrong-variable/shadowing mistakes; contract mismatches (serializer/validated_data, interfaces/abstract methods)
+  - Type-assumption bugs (e.g., numeric ops on datetime/strings, ordering key type mismatches)
+  - Offset/cursor/pagination semantic mismatches (off-by-one, prev/next behavior, commit semantics)
+- Only flag issues you are confident about—avoid speculative or stylistic nitpicks.
+</review_guidelines>
+
+<workflow>
+1. Read each assigned file in full to understand the context
+2. Read the relevant diff sections provided in the prompt
+3. Read related files as needed to fully understand the changes:
+   - Imported modules and dependencies
+   - Interfaces, base classes, and type definitions
+   - Related tests to understand expected behavior
+   - Callers/callees of modified functions
+   - Configuration files if behavior depends on them
+4. Analyze the changes for issues matching the bug patterns above
+5. For each issue found, verify it against the actual code and related context before including it
+</workflow>
+
+<output_format>
+Return your findings as a JSON array (no wrapper object, just the array):
+
+```json
+[
+  {
+    "path": "src/index.ts",
+    "body": "[P1] Title\n\n1 paragraph explanation.",
+    "line": 42,
+    "startLine": null,
+    "side": "RIGHT"
+  }
+]
+```
+
+If no issues found, return an empty array: `[]`
+
+Field definitions:
+- `path`: Relative file path (must match exactly as provided in your assignment)
+- `body`: Comment text starting with priority tag [P0|P1|P2], then title, then 1 paragraph explanation
+  - P0: Critical bugs (crashes, security vulnerabilities, data loss)
+  - P1: Important bugs (incorrect behavior, logic errors)
+  - P2: Minor bugs (edge cases, non-critical issues)
+- `line`: Target line number (single-line) or end line number (multi-line). Must be ≥ 0.
+- `startLine`: `null` for single-line comments, or start line number for multi-line comments
+- `side`: "RIGHT" for new/modified code (default), "LEFT" only for commenting on removed code
+</output_format>
+
+<constraints>
+- Output ONLY the JSON array—no additional commentary or markdown formatting around it.
+- Do not include `commit_id` in your output—the parent agent will add this.
+- Do not attempt to post comments to GitHub—just return the JSON array.
+</constraints>

action.yml

@@ -67,6 +67,18 @@ inputs:
     description: "Override reasoning effort for review flows (passed to Droid Exec as --reasoning-effort). If empty and review_model is also empty, the action defaults internally to gpt-5.2 at high reasoning."
     required: false
     default: ""
+  review_use_validator:
+    description: "Enable two-pass review: generate candidate comments to JSON, then validate and post only approved ones."
+    required: false
+    default: "true"
+  review_candidates_path:
+    description: "Path to write review candidates JSON (run #1 when review_use_validator=true)."
+    required: false
+    default: "${{ runner.temp }}/droid-prompts/review_candidates.json"
+  review_validated_path:
+    description: "Path to write review validated JSON (run #2 when review_use_validator=true)."
+    required: false
+    default: "${{ runner.temp }}/droid-prompts/review_validated.json"
   fill_model:
     description: "Override the model used for PR description fill (e.g., 'claude-sonnet-4-5-20250929', 'gpt-5.1-codex'). Only applies to fill flows."
     required: false
@@ -137,6 +149,9 @@ runs:
         AUTOMATIC_REVIEW: ${{ inputs.automatic_review }}
         REVIEW_MODEL: ${{ inputs.review_model }}
         REASONING_EFFORT: ${{ inputs.reasoning_effort }}
+        REVIEW_USE_VALIDATOR: ${{ inputs.review_use_validator }}
+        REVIEW_CANDIDATES_PATH: ${{ inputs.review_candidates_path }}
+        REVIEW_VALIDATED_PATH: ${{ inputs.review_validated_path }}
         FILL_MODEL: ${{ inputs.fill_model }}
         ADDITIONAL_PERMISSIONS: ${{ inputs.additional_permissions }}
         DROID_ARGS: ${{ inputs.droid_args }}
@@ -169,6 +184,20 @@ runs:
         DROID_DIR=$(dirname "${{ inputs.path_to_droid_executable }}")
         echo "$DROID_DIR" >> "$GITHUB_PATH"
 
+    - name: Setup Custom Droids
+      if: steps.prepare.outputs.contains_trigger == 'true'
+      shell: bash
+      run: |
+        echo "Setting up custom droids..."
+        mkdir -p ~/.factory/droids
+        if [ -d "${GITHUB_ACTION_PATH}/.factory/droids" ]; then
+          cp -r ${GITHUB_ACTION_PATH}/.factory/droids/* ~/.factory/droids/
+          echo "Copied custom droids to ~/.factory/droids/"
+          ls -la ~/.factory/droids/
+        else
+          echo "No custom droids found in action"
+        fi
+
     - name: Setup Network Restrictions
       if: steps.prepare.outputs.contains_trigger == 'true' && inputs.experimental_allowed_domains != ''
       shell: bash
@@ -178,18 +207,6 @@ runs:
       env:
         EXPERIMENTAL_ALLOWED_DOMAINS: ${{ inputs.experimental_allowed_domains }}
 
-    - name: Checkout PR branch for review
-      if: steps.prepare.outputs.contains_trigger == 'true' && steps.prepare.outputs.review_pr_number != ''
-      shell: bash
-      run: |
-        echo "Checking out PR #${{ steps.prepare.outputs.review_pr_number }} branch for full file access..."
-        # Reset any local changes from the merge commit to allow clean checkout
-        git reset --hard HEAD
-        gh pr checkout ${{ steps.prepare.outputs.review_pr_number }}
-        echo "Successfully checked out PR branch: $(git rev-parse --abbrev-ref HEAD)"
-      env:
-        GH_TOKEN: ${{ steps.prepare.outputs.github_token }}
-
     - name: Run Droid Exec
       id: droid
       if: steps.prepare.outputs.contains_trigger == 'true'
@@ -216,6 +233,46 @@ runs:
         DETAILED_PERMISSION_MESSAGES: "1"
         FACTORY_API_KEY: ${{ inputs.factory_api_key }}
 
+    - name: Prepare validator
+      id: prepare_validator
+      if: steps.prepare.outputs.contains_trigger == 'true' && inputs.review_use_validator == 'true'
+      shell: bash
+      run: |
+        bun run ${GITHUB_ACTION_PATH}/src/entrypoints/prepare-validator.ts
+      env:
+        GITHUB_TOKEN: ${{ steps.prepare.outputs.github_token }}
+        REVIEW_USE_VALIDATOR: ${{ inputs.review_use_validator }}
+        REVIEW_VALIDATED_PATH: ${{ inputs.review_validated_path }}
+        REVIEW_CANDIDATES_PATH: ${{ inputs.review_candidates_path }}
+        DROID_COMMENT_ID: ${{ steps.prepare.outputs.droid_comment_id }}
+
+    - name: Run Droid Exec (validator)
+      id: droid_validator
+      if: steps.prepare.outputs.contains_trigger == 'true' && inputs.review_use_validator == 'true'
+      shell: bash
+      run: |
+
+        # Run the base-action
+        bun run ${GITHUB_ACTION_PATH}/base-action/src/index.ts
+      env:
+        # Base-action inputs
+        INPUT_PROMPT_FILE: ${{ runner.temp }}/droid-prompts/droid-prompt.txt
+        INPUT_SETTINGS: ${{ inputs.settings }}
+        INPUT_DROID_ARGS: ${{ steps.prepare_validator.outputs.droid_args }}
+        INPUT_MCP_TOOLS: ${{ steps.prepare_validator.outputs.mcp_tools }}
+        INPUT_EXPERIMENTAL_SLASH_COMMANDS_DIR: ${{ github.action_path }}/slash-commands
+        INPUT_ACTION_INPUTS_PRESENT: ${{ steps.prepare.outputs.action_inputs_present }}
+        INPUT_PATH_TO_DROID_EXECUTABLE: ${{ inputs.path_to_droid_executable }}
+        INPUT_PATH_TO_BUN_EXECUTABLE: ${{ inputs.path_to_bun_executable }}
+        INPUT_SHOW_FULL_OUTPUT: ${{ inputs.show_full_output }}
+
+        # Model configuration
+        GITHUB_TOKEN: ${{ steps.prepare.outputs.GITHUB_TOKEN }}
+        NODE_VERSION: ${{ env.NODE_VERSION }}
+        DETAILED_PERMISSION_MESSAGES: "1"
+        FACTORY_API_KEY: ${{ inputs.factory_api_key }}
+
+
     - name: Update comment with job link
       if: steps.prepare.outputs.contains_trigger == 'true' && steps.prepare.outputs.droid_comment_id && always()
       shell: bash
@@ -230,7 +287,7 @@ runs:
         GITHUB_EVENT_NAME: ${{ github.event_name }}
         TRIGGER_COMMENT_ID: ${{ github.event.comment.id }}
         IS_PR: ${{ github.event.issue.pull_request != null || github.event_name == 'pull_request_target' || github.event_name == 'pull_request_review_comment' }}
-        DROID_SUCCESS: ${{ steps.droid.outputs.conclusion == 'success' }}
+        DROID_SUCCESS: ${{ (inputs.review_use_validator == 'true' && steps.droid_validator.outputs.conclusion == 'success') || (inputs.review_use_validator != 'true' && steps.droid.outputs.conclusion == 'success') }}
         TRIGGER_USERNAME: ${{ github.event.comment.user.login || github.event.issue.user.login || github.event.pull_request.user.login || github.event.sender.login || github.triggering_actor || github.actor || '' }}
         PREPARE_SUCCESS: ${{ steps.prepare.outcome == 'success' }}
         PREPARE_ERROR: ${{ steps.prepare.outputs.prepare_error || '' }}
@@ -247,16 +304,7 @@ runs:
           ~/.factory/logs/droid-log-single.log
           ~/.factory/logs/console.log
           ~/.factory/sessions/*
+          ~/.factory/droids/*
+          ${{ runner.temp }}/droid-prompts/**
         if-no-files-found: ignore
         retention-days: 7
-
-    - name: Revoke app token
-      if: always() && inputs.github_token == '' && steps.prepare.outputs.skipped_due_to_workflow_validation_mismatch != 'true'
-      shell: bash
-      run: |
-        curl -L \
-          -X DELETE \
-          -H "Accept: application/vnd.github+json" \
-          -H "Authorization: Bearer ${{ steps.prepare.outputs.GITHUB_TOKEN }}" \
-          -H "X-GitHub-Api-Version: 2022-11-28" \
-          ${GITHUB_API_URL:-https://api.github.com}/installation/token

base-action/test/run-droid-mcp.test.ts

@@ -90,6 +90,7 @@ mock.module("child_process", () => ({
     });
   },
   spawn: mockSpawn,
+  execSync: (_cmd: string) => "",
 }));
 
 type RunDroidModule = typeof import("../src/run-droid");

src/create-prompt/index.ts

@@ -15,14 +15,20 @@ import {
   isPullRequestReviewCommentEvent,
 } from "../github/context";
 import type { ParsedGitHubContext } from "../github/context";
-import type { CommonFields, PreparedContext, EventData } from "./types";
+import type {
+  CommonFields,
+  PreparedContext,
+  EventData,
+  ReviewArtifacts,
+} from "./types";
 
-export type { CommonFields, PreparedContext } from "./types";
+export type { CommonFields, PreparedContext, ReviewArtifacts } from "./types";
 
 const BASE_ALLOWED_TOOLS = [
   "Execute",
   "Edit",
   "Create",
+  "ApplyPatch",
   "Read",
   "Glob",
   "Grep",
@@ -70,6 +76,7 @@ export function prepareContext(
   baseBranch?: string,
   droidBranch?: string,
   prBranchData?: { headRefName: string; headRefOid: string },
+  reviewArtifacts?: ReviewArtifacts,
 ): PreparedContext {
   const repository = context.repository.full_name;
   const triggerPhrase = context.inputs.triggerPhrase || "@droid";
@@ -108,15 +115,12 @@ export function prepareContext(
     commonFields.droidBranch = droidBranch;
   }
 
-  const eventData = buildEventData(
-    context,
-    {
-      commentId,
-      commentBody,
-      baseBranch,
-      droidBranch,
-    },
-  );
+  const eventData = buildEventData(context, {
+    commentId,
+    commentBody,
+    baseBranch,
+    droidBranch,
+  });
 
   const result: PreparedContext = {
     ...commonFields,
@@ -128,6 +132,10 @@ export function prepareContext(
     result.prBranchData = prBranchData;
   }
 
+  if (reviewArtifacts) {
+    result.reviewArtifacts = reviewArtifacts;
+  }
+
   return result;
 }
 
@@ -282,9 +290,7 @@ function buildEventData(
   }
 }
 
-export type PromptGenerator = (
-  context: PreparedContext,
-) => string;
+export type PromptGenerator = (context: PreparedContext) => string;
 
 export type PromptCreationOptions = {
   githubContext: ParsedGitHubContext;
@@ -296,6 +302,7 @@ export type PromptCreationOptions = {
   allowedTools?: string[];
   disallowedTools?: string[];
   includeActionsTools?: boolean;
+  reviewArtifacts?: ReviewArtifacts;
 };
 
 export async function createPrompt({
@@ -308,6 +315,7 @@ export async function createPrompt({
   allowedTools = [],
   disallowedTools = [],
   includeActionsTools = false,
+  reviewArtifacts,
 }: PromptCreationOptions) {
   try {
     const droidCommentId = commentId.toString();
@@ -317,6 +325,7 @@ export async function createPrompt({
       baseBranch,
       droidBranch,
       prBranchData,
+      reviewArtifacts,
     );
 
     await mkdir(`${process.env.RUNNER_TEMP || "/tmp"}/droid-prompts`, {