diff --git a/.github/ISSUE_TEMPLATE/40_blocking_clean_ci.yml b/.github/ISSUE_TEMPLATE/50_failing_test.yml similarity index 50% rename from .github/ISSUE_TEMPLATE/40_blocking_clean_ci.yml rename to .github/ISSUE_TEMPLATE/50_failing_test.yml index ce1370aa92c..0bb64fa7922 100644 --- a/.github/ISSUE_TEMPLATE/40_blocking_clean_ci.yml +++ b/.github/ISSUE_TEMPLATE/50_failing_test.yml @@ -1,17 +1,17 @@ -name: Failing test on CI -description: Create a report about a failing test +name: Failing test +description: Create a report about a failing test. title: "[Failing test]: " -labels: [ 'blocking-clean-ci' ] +labels: [ "failing-test" ] body: - type: markdown attributes: value: | - Test(s) failing on an Azure DevOps CI run. This issue will track the failures on Azure DevOps runs using the [Known Issues infrastructure](https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/KnownIssues.md). - Fill in the `Build information` block. And `Error message template` needs to be filled in by using either `ErrorMessage` or `ErrorPattern` to enable automatic tracking of failing builds. + Test(s) failing on a GitHub Actions CI run. This issue tracks the failure using the known-issues style format. + Fill in the `Build information` block. `Fill in the error message template` should use either `ErrorMessage` or `ErrorPattern` so the issue stays actionable. - type: checkboxes attributes: label: Is there an existing issue for this? - description: Please search to see if an issue already exists for the failure you encountered ([aspire/issues](https://github.com/microsoft/aspire/issues)). More information on our issue management policies is available [here](https://aka.ms/aspnet/issue-policies). + description: Please search [aspire/issues](https://github.com/microsoft/aspire/issues) before filing a new failing-test issue. options: - label: I have searched the existing issues required: true @@ -19,9 +19,8 @@ body: attributes: label: Build information value: | - Build: - Build error leg or test failing: - Pull Request: N/A + Build: + Build error leg or test failing: validations: required: true - type: textarea @@ -47,7 +46,30 @@ body: required: false - type: textarea attributes: - label: Other info - description: Any other useful information about the failure + label: Error details + description: Include the failing error message and stack trace in the same style as existing failing-test issues. + value: | + ```yml + Error Message: + Stack Trace: + ``` validations: required: false + - type: textarea + attributes: + label: Standard output + description: Paste the relevant standard output inside the existing collapsible details block format. + value: | +
+ Standard Output + + ```yml + ``` + +
+ validations: + required: false + - type: markdown + attributes: + value: | + diff --git a/.github/skills/ci-test-failures/SKILL.md b/.github/skills/ci-test-failures/SKILL.md index b7f5144bb5f..526544a09a4 100644 --- a/.github/skills/ci-test-failures/SKILL.md +++ b/.github/skills/ci-test-failures/SKILL.md @@ -1,17 +1,127 @@ --- name: ci-test-failures -description: Guide for diagnosing and fixing CI test failures using the DownloadFailingJobLogs tool. Use this when asked to investigate GitHub Actions test failures, download failure logs, or debug CI issues. +description: Guide for diagnosing GitHub Actions test failures, extracting failed tests from runs, and creating or updating failing-test issues. Use this when asked to investigate GitHub Actions test failures, download failure logs, create failing-test issues, or debug CI issues. --- -# CI Test Failure Diagnosis +# CI Test Failure Diagnosis and Issue Filing -This skill provides patterns and practices for diagnosing GitHub Actions test failures in the Aspire repository using the `DownloadFailingJobLogs` tool. +## Recipe: Create an Issue for a Test Failure -## Overview +When the user asks to create an issue for a failing test, follow these steps. Always redirect full output to a log file (not `tail`) so you can inspect it if the command fails. -When CI tests fail, use the `DownloadFailingJobLogs.cs` tool to automatically download logs and artifacts for failed jobs. This tool eliminates manual log hunting and provides structured access to test failures. +### Step 1: List failed tests (if user didn't specify one) -**Location**: `tools/scripts/DownloadFailingJobLogs.cs` +Omit `--test` to discover all failures. Redirect output to a log file: + +```bash +dotnet run --project tools/CreateFailingTestIssue -- \ + --url "" \ + --output /tmp/cfti-result.json \ + > /tmp/cfti-list.log 2>&1 +``` + +```powershell +dotnet run --project tools/CreateFailingTestIssue -- ` + --url "" ` + --output $env:TEMP/cfti-result.json ` + > $env:TEMP/cfti-list.log 2>&1 +``` + +Then read the result with `jq`: + +```bash +jq '{ success, availableFailedTests: .diagnostics.availableFailedTests, errorMessage: .errorMessage }' /tmp/cfti-result.json +``` + +```powershell +Get-Content $env:TEMP/cfti-result.json | ConvertFrom-Json | Select-Object success, errorMessage, @{N='availableFailedTests';E={$_.diagnostics.availableFailedTests}} +``` + +If `success` is `false`, inspect the full log: `cat /tmp/cfti-list.log` (bash) or `Get-Content $env:TEMP/cfti-list.log` (PowerShell). + +Ask the user which test to file for, then proceed to Step 2. + +### Step 2: Create the issue + +```bash +dotnet run --project tools/CreateFailingTestIssue -- \ + --url "" \ + --test "" \ + --create \ + --output /tmp/cfti-result.json \ + > /tmp/cfti-create.log 2>&1 +``` + +```powershell +dotnet run --project tools/CreateFailingTestIssue -- ` + --url "" ` + --test "" ` + --create ` + --output $env:TEMP/cfti-result.json ` + > $env:TEMP/cfti-create.log 2>&1 +``` + +Then read the result: + +```bash +jq '{ success, issue: .issue.createdIssue, errorMessage: .errorMessage }' /tmp/cfti-result.json +``` + +```powershell +Get-Content $env:TEMP/cfti-result.json | ConvertFrom-Json | Select-Object success, errorMessage, @{N='issue';E={$_.issue.createdIssue}} +``` + +If `success` is `false`, inspect the full log: `cat /tmp/cfti-create.log` (bash) or `Get-Content $env:TEMP/cfti-create.log` (PowerShell). + +That's it — do not add analysis comments, do not use `--dry-run` unless the user explicitly asks for a preview. + +**Rules:** +- Always use `--output ` to keep JSON clean. Do NOT try to parse JSON from stdout — it is interleaved with dotnet build progress output. +- Use `jq` to extract fields from the output file. Key paths: + - `.success` — whether the operation succeeded + - `.issue.createdIssue.number` and `.issue.createdIssue.url` — the created/updated issue + - `.diagnostics.availableFailedTests[]` — test names when `--test` is omitted + - `.errorMessage` — error details when `.success` is false +- Do NOT create issues manually with `gh issue create`. The tool handles everything: resolving the run, finding the test, generating a template-compliant body, and creating the issue. +- Do NOT invent your own issue markdown. The tool generates content that matches `.github/ISSUE_TEMPLATE/50_failing_test.yml`. +- If the tool fails, report the error from `diagnostics.log` and the JSON output. Do not fall back to manual issue creation. + +## Recipe: Investigate a Failing Run (No Issue Creation) + +To download and inspect failure artifacts without creating an issue: + +```bash +cd tools/scripts +dotnet run DownloadFailingJobLogs.cs -- +``` + +```powershell +Set-Location tools/scripts +dotnet run DownloadFailingJobLogs.cs -- +``` + +Then search the downloaded logs and `.trx` files for errors. + +--- + +## Reference Documentation + +Everything below is reference material for edge cases and deeper investigation. + +### Overview + +Use this skill in two phases: + +1. **Investigate the run** with `DownloadFailingJobLogs.cs` to fetch failed job logs and artifacts. +2. **Create or update a failing-test issue** with `tools/CreateFailingTestIssue --create`. + +### Tools covered + +| Tool | Purpose | Location | +|------|---------|----------| +| `DownloadFailingJobLogs.cs` | Download failed job logs and test artifacts from a GitHub Actions run | `tools/scripts/DownloadFailingJobLogs.cs` | +| `CreateFailingTestIssue` | Resolve a failing test from PR/run/job URLs and create/update issues | `tools/CreateFailingTestIssue` | +| `/create-issue` workflow | Create, reopen, or comment on failing-test issues from issue/PR comments | `.github/workflows/create-failing-test-issue.yml` | ## Quick Start @@ -31,6 +141,18 @@ gh run list --repo microsoft/aspire --branch --limit 1 --json data gh pr checks --repo microsoft/aspire ``` +```powershell +# From URL: https://github.com/microsoft/aspire/actions/runs/19846215629 +# ^^^^^^^^^^ +# run ID + +# Or find the latest run on a branch +gh run list --repo microsoft/aspire --branch --limit 1 --json databaseId --jq '.[0].databaseId' + +# Or for a PR +gh pr checks --repo microsoft/aspire +``` + ### Step 2: Run the Tool ```bash @@ -38,6 +160,11 @@ cd tools/scripts dotnet run DownloadFailingJobLogs.cs -- ``` +```powershell +Set-Location tools/scripts +dotnet run DownloadFailingJobLogs.cs -- +``` + **Example:** ```bash dotnet run DownloadFailingJobLogs.cs -- 19846215629 @@ -62,6 +189,133 @@ The tool creates files in your current directory: 5. **Downloads test artifacts** containing .trx files and test logs 6. **Extracts artifacts** to local directories for inspection +## Creating or Updating Failing-Test Issues + +After you know which test failed, use the branch automation to create a failing-test issue in the known-issues format. + +### Preferred path: `/create-issue` from a PR or issue comment + +Comment on the PR or issue with: + +```text +/create-issue --test "" [--url ] [--workflow ] [--force-new] +``` + +Examples: + +```text +/create-issue --test "Tests.Namespace.Type.Method(input: 1)" +/create-issue --test "Tests.Namespace.Type.Method(input: 1)" --url https://github.com/microsoft/aspire/actions/runs/123 +/create-issue "Tests.Namespace.Type.Method(input: 1)" https://github.com/microsoft/aspire/actions/runs/123/job/456 +/create-issue --test "Tests.Namespace.Type.Method(input: 1)" --url https://github.com/microsoft/aspire/actions/runs/123/attempts/2/job/456?pr=321 --force-new +``` + +Notes: + +- When the command is posted on a PR and no `--url` is supplied, the workflow defaults to that PR URL. +- `--workflow` defaults to `ci`. +- `--force-new` bypasses issue reuse and always requests a fresh issue. +- The workflow requires write or admin access to the repository before it will create or update issues. + +### Supported source URLs + +The resolver accepts: + +- Pull request URLs: `https://github.com///pull/` +- Workflow run URLs: `https://github.com///actions/runs/` +- Attempt URLs: `https://github.com///actions/runs//attempts/` +- Job URLs: `https://github.com///actions/runs//job/` +- Attempt job URLs with query strings + +### Local path: run the resolver directly + +Always use `--output` to write results to a file so JSON is not interleaved with build output: + +To generate the JSON result locally without creating an issue (dry run): + +```bash +dotnet run --project tools/CreateFailingTestIssue -- \ + --url "https://github.com/microsoft/aspire/actions/runs/123" \ + --test "" \ + --repo "microsoft/aspire" \ + --output /tmp/cfti-result.json +``` + +```powershell +dotnet run --project tools/CreateFailingTestIssue -- ` + --url "https://github.com/microsoft/aspire/actions/runs/123" ` + --test "" ` + --repo "microsoft/aspire" ` + --output $env:TEMP/cfti-result.json +``` + +To resolve the failure **and create the issue on GitHub** in one step: + +```bash +dotnet run --project tools/CreateFailingTestIssue -- \ + --url "https://github.com/microsoft/aspire/actions/runs/123" \ + --test "" \ + --repo "microsoft/aspire" \ + --create \ + --output /tmp/cfti-result.json +``` + +```powershell +dotnet run --project tools/CreateFailingTestIssue -- ` + --url "https://github.com/microsoft/aspire/actions/runs/123" ` + --test "" ` + --repo "microsoft/aspire" ` + --create ` + --output $env:TEMP/cfti-result.json +``` + +Read the result with `jq`: + +```bash +jq '{ success, issue: .issue, availableFailedTests: .diagnostics.availableFailedTests }' /tmp/cfti-result.json +``` + +```powershell +Get-Content $env:TEMP/cfti-result.json | ConvertFrom-Json | Select-Object success, @{N='issue';E={$_.issue}}, @{N='availableFailedTests';E={$_.diagnostics.availableFailedTests}} +``` + +If `--test` is omitted, the tool emits structured JSON for all failing tests it found in the run (useful for picking which test to file). + +The command writes a `diagnostics.log` file in the current directory. The JSON output (written to the `--output` file or stdout) contains: + +- the resolved run and job URLs +- either the matched canonical and display test names plus generated issue content, or a per-test list of all failures in the run +- primary failure details (error, stack trace, stdout) +- when `--create` is set, the created issue number and URL +- warnings and alternate failed-test names if the match fails + +### What the resolver does + +`CreateFailingTestIssue`: + +1. Resolves the workflow selector and source URL. +2. Finds the workflow run and failed jobs, including attempt URLs. +3. Downloads failed test occurrences from `.trx` artifacts. +4. Falls back to failed job logs when artifacts are missing or the run is still active. +5. Matches the requested test using canonical or display names. +6. Generates issue content that matches `.github/ISSUE_TEMPLATE/50_failing_test.yml`. The error details code block is wrapped in a collapsible `
` element when it exceeds 30 lines. +7. Reuses an open issue with the same stable signature, reopens a closed one, or creates a new issue. + +### Finding the failed tests to file + +For a run with multiple failures, first extract the candidate test names, then issue one `/create-issue` command per test: + +```powershell +Get-ChildItem -Path "artifact_*" -Recurse -Filter "*.trx" | ForEach-Object { + [xml]$xml = Get-Content $_.FullName + $xml.TestRun.Results.UnitTestResult | + Where-Object { $_.outcome -eq "Failed" } | + Select-Object -ExpandProperty testName +} +``` + +If the resolver cannot match the requested test exactly, it returns `availableFailedTests` so you can retry with one of the discovered names. + ## Example Workflow ```bash @@ -83,6 +337,9 @@ Get-ChildItem -Recurse -Filter "*.trx" | ForEach-Object { [xml]$xml = Get-Content $_.FullName $xml.TestRun.Results.UnitTestResult | Where-Object { $_.outcome -eq "Failed" } } + +# 6. Create or update the failing-test issue from the PR or issue thread +/create-issue --test "Tests.Namespace.Type.Method(input: 1)" --url https://github.com/microsoft/aspire/actions/runs/$runId ``` ## Understanding Job Log Output @@ -265,6 +522,11 @@ cd tools/scripts dotnet run DownloadFailingJobLogs.cs -- ``` +```powershell +Set-Location tools/scripts +dotnet run DownloadFailingJobLogs.cs -- +``` + ### Don't Commit Log Files The downloaded log files can be large. Don't commit them to the repository: @@ -276,6 +538,13 @@ rm tools/scripts/*.zip rm -rf tools/scripts/artifact_* ``` +```powershell +# Before committing +Remove-Item tools/scripts/*.log -Force -ErrorAction SilentlyContinue +Remove-Item tools/scripts/*.zip -Force -ErrorAction SilentlyContinue +Remove-Item tools/scripts/artifact_* -Recurse -Force -ErrorAction SilentlyContinue +``` + ## Prerequisites - .NET 10 SDK or later diff --git a/.github/workflows/apply-test-attributes.yml b/.github/workflows/apply-test-attributes.yml index edeea8c014e..5ed51aefe31 100644 --- a/.github/workflows/apply-test-attributes.yml +++ b/.github/workflows/apply-test-attributes.yml @@ -27,6 +27,13 @@ jobs: issues: write pull-requests: write steps: + - name: Checkout workflow helpers + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: .github/workflows/workflow-command-helpers.js + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Extract command and arguments uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 id: extract-command @@ -69,19 +76,34 @@ jobs: const args = match[2].trim(); const matched = commandConfig[commandName]; - // Parse arguments - extract --target-pr option first + // Parse arguments using shared quote-aware tokenizer + const { tokenizeArguments } = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/workflow-command-helpers.js`); + + let allTokens; + try { + allTokens = tokenizeArguments(args); + } + catch (error) { + failWithError(error.message); + return; + } + + // Extract --target-pr option first let targetPrUrl = ''; - let remainingArgs = args; - - // Extract --target-pr if present - const targetPrPattern = /--target-pr\s+(\S+)/; - const targetPrMatch = targetPrPattern.exec(args); - if (targetPrMatch) { - targetPrUrl = targetPrMatch[1]; - remainingArgs = args.replace(targetPrPattern, '').trim(); + const filteredTokens = []; + for (let i = 0; i < allTokens.length; i++) { + if (allTokens[i] === '--target-pr') { + if (i + 1 >= allTokens.length) { + failWithError('Missing value for --target-pr.'); + return; + } + targetPrUrl = allTokens[++i]; + } else { + filteredTokens.push(allTokens[i]); + } } - const parts = remainingArgs.split(/\s+/).filter(p => p.length > 0); + const parts = filteredTokens; // Check for unknown arguments (anything starting with - or --) const unknownArgs = parts.filter(p => p.startsWith('-')); diff --git a/.github/workflows/create-failing-test-issue.js b/.github/workflows/create-failing-test-issue.js new file mode 100644 index 00000000000..ec98e014aa4 --- /dev/null +++ b/.github/workflows/create-failing-test-issue.js @@ -0,0 +1,116 @@ +function parseCommand(body, defaultSourceUrl = null) { + const match = /^\/create-issue(?:\s+(?.+))?$/m.exec(body ?? ''); + if (!match) { + return { success: false, errorMessage: 'No /create-issue command was found in the comment.' }; + } + + const { tokenizeArguments } = require('./workflow-command-helpers.js'); + let tokens; + try { + tokens = tokenizeArguments(match.groups?.args ?? ''); + } + catch (error) { + return { success: false, errorMessage: error.message }; + } + + const result = { + success: true, + testQuery: '', + sourceUrl: defaultSourceUrl, + workflow: 'ci', + forceNew: false, + listOnly: false, + }; + + const hasFlags = tokens.some(token => token.startsWith('--')); + if (hasFlags) { + for (let index = 0; index < tokens.length; index++) { + const token = tokens[index]; + + switch (token) { + case '--test': + if (index + 1 >= tokens.length) { + return { success: false, errorMessage: 'Missing value for --test.' }; + } + + result.testQuery = tokens[++index]; + break; + + case '--url': + if (index + 1 >= tokens.length) { + return { success: false, errorMessage: 'Missing value for --url.' }; + } + + result.sourceUrl = tokens[++index]; + break; + + case '--workflow': + if (index + 1 >= tokens.length) { + return { success: false, errorMessage: 'Missing value for --workflow.' }; + } + + result.workflow = tokens[++index]; + break; + + case '--force-new': + result.forceNew = true; + break; + + default: + return { + success: false, + errorMessage: `Unknown argument '${token}'. Supported arguments are --test, --url, --workflow, and --force-new.`, + }; + } + } + + if (!result.testQuery) { + result.listOnly = true; + } + + return result; + } + + if (tokens.length === 0) { + result.listOnly = true; + return result; + } + + if (tokens.length === 1) { + result.testQuery = tokens[0]; + return result; + } + + const candidateUrl = tokens[tokens.length - 1]; + if (isSupportedSourceUrl(candidateUrl)) { + result.sourceUrl = candidateUrl; + result.testQuery = tokens.slice(0, -1).join(' '); + return result; + } + + return { + success: false, + errorMessage: 'Positional input is ambiguous. Use /create-issue --test "" [--url ] [--workflow ] [--force-new].', + }; +} + +function buildIssueSearchQuery(owner, repo, metadataMarker) { + const escapedMarker = String(metadataMarker ?? '').replaceAll('"', '\\"'); + return `repo:${owner}/${repo} is:issue label:failing-test in:body "${escapedMarker}"`; +} + +function isSupportedSourceUrl(value) { + if (typeof value !== 'string') { + return false; + } + + return /^https:\/\/github\.com\/[^/]+\/[^/]+\/pull\/\d+(?:\/.*)?$/i.test(value) + || /^https:\/\/github\.com\/[^/]+\/[^/]+\/actions\/runs\/\d+(?:\/attempts\/\d+)?(?:\/.*)?$/i.test(value) + || /^https:\/\/github\.com\/[^/]+\/[^/]+\/actions\/runs\/\d+\/job\/\d+(?:\/.*)?$/i.test(value); +} + +module.exports = { + buildIssueSearchQuery, + isSupportedSourceUrl, + parseCommand, +}; diff --git a/.github/workflows/create-failing-test-issue.yml b/.github/workflows/create-failing-test-issue.yml new file mode 100644 index 00000000000..77c95b5de70 --- /dev/null +++ b/.github/workflows/create-failing-test-issue.yml @@ -0,0 +1,305 @@ +name: Create Failing-Test Issue + +on: + issue_comment: + types: [created] + +permissions: {} + +concurrency: + group: create-failing-test-issue-${{ github.event.issue.pull_request && 'pr' || 'issue' }}-${{ github.event.issue.number }} + cancel-in-progress: false + +jobs: + create_failing_test_issue: + name: Create failing-test issue + if: >- + github.repository == 'microsoft/aspire' && + startsWith(github.event.comment.body, '/create-issue') + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read # checkout repository + issues: write # create/reopen/comment on issues + pull-requests: write # comment on PRs + actions: read # list workflow runs and download artifacts + steps: + - name: Verify user has write access + id: verify-permission + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const commentUser = context.payload.comment.user.login; + const { data: permission } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: commentUser + }); + + const writePermissions = ['admin', 'maintain', 'write']; + if (!writePermissions.includes(permission.permission)) { + const message = `@${commentUser} The \`/create-issue\` command requires write, maintain, or admin access to this repository.`; + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: message + }); + core.setOutput('error_message', message); + core.setFailed(message); + return; + } + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: success() + with: + persist-credentials: false + + - name: Extract command + if: success() + id: extract-command + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const helper = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/create-failing-test-issue.js`); + const defaultSourceUrl = context.payload.issue.pull_request + ? `https://github.com/${context.repo.owner}/${context.repo.repo}/pull/${context.issue.number}` + : null; + + const parsed = helper.parseCommand(context.payload.comment.body, defaultSourceUrl); + if (!parsed.success) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: `@${context.payload.comment.user.login} ❌ ${parsed.errorMessage}` + }); + core.setOutput('error_message', parsed.errorMessage); + core.setFailed(parsed.errorMessage); + return; + } + + core.setOutput('test_query', parsed.testQuery); + core.setOutput('source_url', parsed.sourceUrl ?? ''); + core.setOutput('workflow', parsed.workflow); + core.setOutput('force_new', parsed.forceNew ? 'true' : 'false'); + core.setOutput('list_only', parsed.listOnly ? 'true' : 'false'); + + - name: Add processing reaction + if: success() + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'eyes' + }); + + - name: Restore SDK + if: success() + run: ./restore.sh + + - name: Resolve failing test details + if: success() + id: resolve-failure + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + TEST_QUERY: ${{ steps.extract-command.outputs.test_query }} + SOURCE_URL: ${{ steps.extract-command.outputs.source_url }} + WORKFLOW_SELECTOR: ${{ steps.extract-command.outputs.workflow }} + FORCE_NEW: ${{ steps.extract-command.outputs.force_new }} + LIST_ONLY: ${{ steps.extract-command.outputs.list_only }} + run: | + args=(--workflow "$WORKFLOW_SELECTOR" --repo "${GITHUB_REPOSITORY}") + if [ "$LIST_ONLY" != "true" ]; then + args+=(--test "$TEST_QUERY") + fi + if [ -n "$SOURCE_URL" ]; then + args+=(--url "$SOURCE_URL") + fi + if [ "$FORCE_NEW" = "true" ]; then + args+=(--force-new) + fi + + dotnet build tools/CreateFailingTestIssue --no-restore -v:q + dotnet run --no-build --project tools/CreateFailingTestIssue -- "${args[@]}" --output "$RUNNER_TEMP/failing-test-result.json" + + - name: Create or update failing-test issue + if: steps.resolve-failure.outcome != 'skipped' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + RESULT_PATH: ${{ runner.temp }}/failing-test-result.json + FORCE_NEW: ${{ steps.extract-command.outputs.force_new }} + LIST_ONLY: ${{ steps.extract-command.outputs.list_only }} + with: + script: | + const fs = require('fs'); + const helper = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/create-failing-test-issue.js`); + + const helpBlock = [ + '📋 **`/create-issue` — Usage**', + '', + 'Creates or updates a failing-test issue from CI failures.', + '', + '```', + '/create-issue ', + '/create-issue ', + '/create-issue --test ""', + '/create-issue --test "" --url ', + '/create-issue --test "" --force-new', + '```', + ].join('\n'); + + // List-only mode: show available failures + help when no --test was given + if (process.env.LIST_ONLY === 'true') { + let body = ''; + + if (fs.existsSync(process.env.RESULT_PATH) && fs.statSync(process.env.RESULT_PATH).size > 0) { + const result = JSON.parse(fs.readFileSync(process.env.RESULT_PATH, 'utf8')); + const tests = result.allFailures?.tests?.map(t => t.canonicalTestName ?? t.displayTestName) + ?? result.diagnostics?.availableFailedTests + ?? []; + + if (tests.length > 0) { + body = '**Failed tests found on this PR:**\n\n' + + tests.map(name => `- \`/create-issue ${name}\``).join('\n') + + '\n\n'; + } + } + + if (!body) { + body = 'No test failures were found. Use `--url` to point to a specific workflow run.\n\n'; + } + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: `${body}${helpBlock}` + }); + return; + } + + if (!fs.existsSync(process.env.RESULT_PATH) || fs.statSync(process.env.RESULT_PATH).size === 0) { + const message = `@${context.payload.comment.user.login} ❌ The failing-test resolver did not produce a JSON result. See the workflow run for details.`; + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: message + }); + core.setFailed(message); + return; + } + + const result = JSON.parse(fs.readFileSync(process.env.RESULT_PATH, 'utf8')); + if (!result.success) { + const candidates = result.diagnostics?.availableFailedTests?.length + ? `\n\n**Available failed tests:**\n${result.diagnostics.availableFailedTests.map(name => `- \`${name}\``).join('\n')}` + : ''; + const message = `@${context.payload.comment.user.login} ❌ ${result.errorMessage ?? 'The failing-test resolver could not create an issue.'}${candidates}`; + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: message + }); + core.setFailed(result.errorMessage ?? 'The failing-test resolver failed.'); + return; + } + + let targetIssue = null; + if (process.env.FORCE_NEW !== 'true') { + const query = helper.buildIssueSearchQuery(context.repo.owner, context.repo.repo, result.issue.metadataMarker); + const { data: search } = await github.rest.search.issuesAndPullRequests({ + q: query, + per_page: 20 + }); + + const issues = search.items.filter(item => !item.pull_request); + targetIssue = issues.find(item => item.state === 'open') ?? issues.find(item => item.state === 'closed') ?? null; + } + + let action; + let issueNumber; + let issueUrl; + + if (targetIssue && targetIssue.state === 'open') { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: targetIssue.number, + body: result.issue.commentBody + }); + + action = 'updated'; + issueNumber = targetIssue.number; + issueUrl = targetIssue.html_url; + } else if (targetIssue && targetIssue.state === 'closed') { + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: targetIssue.number, + state: 'open' + }); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: targetIssue.number, + body: result.issue.commentBody + }); + + action = 'reopened'; + issueNumber = targetIssue.number; + issueUrl = targetIssue.html_url; + } else { + const { data: issue } = await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: result.issue.title, + body: result.issue.body, + labels: result.issue.labels + }); + + action = 'created'; + issueNumber = issue.number; + issueUrl = issue.html_url; + } + + const testName = result.match?.canonicalTestName ?? result.match?.displayTestName ?? ''; + let disableHint = ''; + if (testName) { + disableHint = `\n\nTo disable this test on your PR, comment:\n\`\`\`\n/disable-test ${testName} ${issueUrl}\n\`\`\``; + } + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: `✅ ${action[0].toUpperCase()}${action.slice(1)} failing-test issue #${issueNumber}: ${issueUrl}${disableHint}` + }); + + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'rocket' + }); + + - name: Post failure comment on unexpected error + if: failure() && steps.extract-command.outcome == 'success' && steps.verify-permission.outcome == 'success' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const message = `@${context.payload.comment.user.login} ❌ The \`/create-issue\` command failed. See the [workflow run](${runUrl}) for details.`; + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: message + }); diff --git a/.github/workflows/pr-docs-check.lock.yml b/.github/workflows/pr-docs-check.lock.yml index 5123f013760..45bc326a61d 100644 --- a/.github/workflows/pr-docs-check.lock.yml +++ b/.github/workflows/pr-docs-check.lock.yml @@ -918,6 +918,7 @@ jobs: if: always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true') runs-on: ubuntu-slim permissions: + discussions: write issues: write concurrency: group: "gh-aw-conclusion-pr-docs-check" @@ -946,6 +947,7 @@ jobs: aspire.dev aspire github-api-url: ${{ github.api_url }} + permission-discussions: write permission-issues: write - name: Download agent output artifact id: download-agent-output @@ -1237,6 +1239,7 @@ jobs: runs-on: ubuntu-slim permissions: contents: write + discussions: write issues: write pull-requests: write timeout-minutes: 15 @@ -1299,6 +1302,7 @@ jobs: aspire github-api-url: ${{ github.api_url }} permission-contents: write + permission-discussions: write permission-issues: write permission-pull-requests: write - name: Checkout repository diff --git a/.github/workflows/workflow-command-helpers.js b/.github/workflows/workflow-command-helpers.js new file mode 100644 index 00000000000..fe97c161baf --- /dev/null +++ b/.github/workflows/workflow-command-helpers.js @@ -0,0 +1,77 @@ +/** + * Shared helpers for GitHub Actions workflow command parsing. + * + * Used by: + * - create-failing-test-issue.js + * - apply-test-attributes.yml (inline github-script) + * + * Tested via: + * - tests/Infrastructure.Tests/WorkflowScripts/CreateFailingTestIssueWorkflowTests.cs + */ + +/** + * Tokenizes a command argument string, respecting single and double quotes + * with backslash escaping inside quoted segments. + * + * @param {string} input - Raw argument string (e.g., '--test "My Test" --url https://...') + * @returns {string[]} Array of tokens + * @throws {Error} If a quoted segment is not closed + */ +function tokenizeArguments(input) { + const tokens = []; + let current = ''; + let quote = null; + + for (let index = 0; index < input.length; index++) { + const character = input[index]; + + if (quote) { + if (character === '\\' && index + 1 < input.length) { + const next = input[index + 1]; + if (next === quote || next === '\\') { + current += next; + index++; + continue; + } + } + + if (character === quote) { + quote = null; + continue; + } + + current += character; + continue; + } + + if (character === '"' || character === '\'') { + quote = character; + continue; + } + + if (/\s/.test(character)) { + if (current.length > 0) { + tokens.push(current); + current = ''; + } + + continue; + } + + current += character; + } + + if (quote) { + throw new Error(`Unterminated ${quote} quote in command arguments.`); + } + + if (current.length > 0) { + tokens.push(current); + } + + return tokens; +} + +module.exports = { + tokenizeArguments, +}; diff --git a/.gitignore b/.gitignore index 1d148c62241..704438abb53 100644 --- a/.gitignore +++ b/.gitignore @@ -202,3 +202,4 @@ extension/package.nls.*.json # Azure Functions local settings (may contain encrypted secrets) local.settings.json +diagnostics.log diff --git a/AGENTS.md b/AGENTS.md index 3792943fe50..32c610ae911 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -396,6 +396,7 @@ For most development tasks, following these instructions should be sufficient to The following specialized skills are available in `.github/skills/`: - **cli-e2e-testing**: Guide for writing Aspire CLI end-to-end tests using Hex1b terminal automation +- **ci-test-failures**: Diagnoses GitHub Actions test failures, extracts failed tests from runs, and creates or updates failing-test issues - **code-review**: Reviews a GitHub pull request for problems (bugs, security, correctness, convention violations). Use this when asked to review a PR or do a code review. - **fix-flaky-test**: Reproduces and fixes flaky/quarantined tests using the CI reproduce workflow (`reproduce-flaky-tests.yml`). Use this when investigating, reproducing, or fixing a flaky or quarantined test. - **dashboard-testing**: Guide for writing tests for the Aspire Dashboard using xUnit and bUnit diff --git a/Aspire.slnx b/Aspire.slnx index df9a7ed337f..56db3bc54b8 100644 --- a/Aspire.slnx +++ b/Aspire.slnx @@ -520,6 +520,8 @@ + + diff --git a/tests/Infrastructure.Tests/CreateFailingTestIssue/CreateFailingTestIssueFixture.cs b/tests/Infrastructure.Tests/CreateFailingTestIssue/CreateFailingTestIssueFixture.cs new file mode 100644 index 00000000000..9b522157069 --- /dev/null +++ b/tests/Infrastructure.Tests/CreateFailingTestIssue/CreateFailingTestIssueFixture.cs @@ -0,0 +1,89 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using Xunit; + +namespace Infrastructure.Tests; + +/// +/// Builds the CreateFailingTestIssue tool once before tests run. +/// +public sealed class CreateFailingTestIssueFixture : IAsyncLifetime +{ + public string RepoRoot { get; private set; } = string.Empty; + + public string ToolProjectPath { get; private set; } = string.Empty; + + public async ValueTask InitializeAsync() + { + RepoRoot = FindRepoRoot(); + ToolProjectPath = Path.Combine(RepoRoot, "tools", "CreateFailingTestIssue", "CreateFailingTestIssue.csproj"); + + if (!File.Exists(ToolProjectPath)) + { + throw new InvalidOperationException($"CreateFailingTestIssue project not found at {ToolProjectPath}."); + } + + await BuildToolAsync(ToolProjectPath).ConfigureAwait(false); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + private static async Task BuildToolAsync(string toolProjectPath) + { + ProcessStartInfo processStartInfo = new() + { + FileName = "dotnet", + Arguments = $"build \"{toolProjectPath}\" --restore", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var process = new Process { StartInfo = processStartInfo }; + process.Start(); + + var stdoutTask = process.StandardOutput.ReadToEndAsync(); + var stderrTask = process.StandardError.ReadToEndAsync(); + + using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + try + { + await process.WaitForExitAsync(cancellationTokenSource.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + process.Kill(entireProcessTree: true); + throw; + } + + var stdout = await stdoutTask.ConfigureAwait(false); + var stderr = await stderrTask.ConfigureAwait(false); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException( + $"Failed to build CreateFailingTestIssue tool. Exit code: {process.ExitCode}{Environment.NewLine}" + + $"stdout:{Environment.NewLine}{stdout}{Environment.NewLine}" + + $"stderr:{Environment.NewLine}{stderr}"); + } + } + + private static string FindRepoRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + if (File.Exists(Path.Combine(directory.FullName, "Aspire.slnx"))) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + throw new InvalidOperationException("Could not find repository root (looking for Aspire.slnx)."); + } +} diff --git a/tests/Infrastructure.Tests/CreateFailingTestIssue/CreateFailingTestIssueToolTests.cs b/tests/Infrastructure.Tests/CreateFailingTestIssue/CreateFailingTestIssueToolTests.cs new file mode 100644 index 00000000000..7be53b1e9fa --- /dev/null +++ b/tests/Infrastructure.Tests/CreateFailingTestIssue/CreateFailingTestIssueToolTests.cs @@ -0,0 +1,1033 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Globalization; +using System.Text.Json; +using Xunit; + +namespace Infrastructure.Tests; + +/// +/// End-to-end tests for the CreateFailingTestIssue tool. +/// +public sealed class CreateFailingTestIssueToolTests : IClassFixture, IDisposable +{ + private readonly TestTempDirectory _tempDirectory = new(); + private readonly CreateFailingTestIssueFixture _fixture; + private readonly ITestOutputHelper _output; + + public CreateFailingTestIssueToolTests(CreateFailingTestIssueFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + } + + public void Dispose() => _tempDirectory.Dispose(); + + [Fact] + public async Task ResolvesFailureFromFixtureArtifacts() + { + var fixtureDirectory = CreateFixtureDirectory(); + + var result = await RunToolAsync( + fixtureDirectory, + "--test", "Tests.Namespace.Type.Method", + "--url", "https://github.com/microsoft/aspire/actions/runs/123", + "--workflow", "ci", + "--repo", "microsoft/aspire"); + + Assert.Equal(0, result.ExitCode); + + var response = JsonSerializer.Deserialize(result.Output, JsonOptions); + Assert.NotNull(response); + Assert.True(response!.Success); + Assert.NotNull(response.Match); + Assert.Equal("exactCanonical", response.Match!.Strategy); + Assert.Equal("Tests.Namespace.Type.Method", response.Match.CanonicalTestName); + Assert.Equal("Tests.Namespace.Type.Method(input: 1)", response.Match.DisplayTestName); + Assert.NotNull(response.Failure); + Assert.Equal("Expected 1 but found 2.", response.Failure!.ErrorMessage); + Assert.Equal("at Tests.Namespace.Type.Method() in TestFile.cs:line 42", response.Failure.StackTrace); + Assert.Equal("stdout line 1", response.Failure.Stdout); + Assert.NotNull(response.Issue); + Assert.Contains("### Build information", response.Issue!.Body, StringComparison.Ordinal); + Assert.Contains("Build: https://github.com/microsoft/aspire/actions/runs/123", response.Issue.Body, StringComparison.Ordinal); + Assert.Contains("Build error leg or test failing: Tests.Namespace.Type.Method", response.Issue.Body, StringComparison.Ordinal); + Assert.Contains("### Fill in the error message template", response.Issue.Body, StringComparison.Ordinal); + Assert.Contains(@"""ErrorMessage"": ""Expected 1 but found 2.""", response.Issue.Body, StringComparison.Ordinal); + Assert.Contains("### Error details", response.Issue.Body, StringComparison.Ordinal); + Assert.Contains("Error Message: Expected 1 but found 2.", response.Issue.Body, StringComparison.Ordinal); + Assert.Contains("Stack Trace:", response.Issue.Body, StringComparison.Ordinal); + Assert.Contains("Standard Output", response.Issue.Body, StringComparison.Ordinal); + Assert.Contains("stdout line 1", response.Issue.Body, StringComparison.Ordinal); + Assert.DoesNotContain("### Other info", response.Issue.Body, StringComparison.Ordinal); + Assert.DoesNotContain("Pull Request:", response.Issue.Body, StringComparison.Ordinal); + Assert.StartsWith("" + }); + + Assert.Equal("repo:microsoft/aspire is:issue label:failing-test in:body \"\"", query); + } + + private async Task InvokeHarnessAsync(string operation, object payload) + { + var requestPath = Path.Combine(_tempDirectory.Path, $"{Guid.NewGuid():N}.json"); + await File.WriteAllTextAsync(requestPath, JsonSerializer.Serialize(new { operation, payload }, s_jsonOptions)); + + using var command = new NodeCommand(_output, "create-failing-test-issue"); + command.WithWorkingDirectory(_repoRoot); + + var result = await command.ExecuteScriptAsync(_harnessPath, requestPath); + Assert.Equal(0, result.ExitCode); + + var response = JsonSerializer.Deserialize>(result.Output, s_jsonOptions); + Assert.NotNull(response); + return response!.Result; + } + + private static string FindRepoRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + + while (directory is not null) + { + var gitPath = Path.Combine(directory.FullName, ".git"); + if (Directory.Exists(gitPath) || File.Exists(gitPath)) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + throw new InvalidOperationException("Could not locate the repository root."); + } + + private sealed record HarnessResponse(T Result); + + private sealed record ParseCommandResult(bool Success, string TestQuery, string? SourceUrl, string Workflow, bool ForceNew, bool ListOnly, string? ErrorMessage); +} diff --git a/tests/Infrastructure.Tests/WorkflowScripts/create-failing-test-issue.harness.js b/tests/Infrastructure.Tests/WorkflowScripts/create-failing-test-issue.harness.js new file mode 100644 index 00000000000..9d8c6483047 --- /dev/null +++ b/tests/Infrastructure.Tests/WorkflowScripts/create-failing-test-issue.harness.js @@ -0,0 +1,31 @@ +const fs = require('node:fs/promises'); +const helper = require('../../../.github/workflows/create-failing-test-issue.js'); + +async function main() { + const inputPath = process.argv[2]; + if (!inputPath) { + throw new Error('Expected the input payload file path as the first argument.'); + } + + const request = JSON.parse(await fs.readFile(inputPath, 'utf8')); + const result = await dispatch(request.operation, request.payload ?? {}); + process.stdout.write(JSON.stringify({ result })); +} + +async function dispatch(operation, payload) { + switch (operation) { + case 'parseCommand': + return helper.parseCommand(payload.body, payload.defaultSourceUrl ?? null); + + case 'buildIssueSearchQuery': + return helper.buildIssueSearchQuery(payload.owner ?? 'microsoft', payload.repo ?? 'aspire', payload.metadataMarker); + + default: + throw new Error(`Unsupported operation '${operation}'.`); + } +} + +main().catch(error => { + process.stderr.write(`${error.stack ?? error}\n`); + process.exitCode = 1; +}); diff --git a/tools/Aspire.TestTools/Aspire.TestTools.csproj b/tools/Aspire.TestTools/Aspire.TestTools.csproj new file mode 100644 index 00000000000..9ed914b5ba4 --- /dev/null +++ b/tools/Aspire.TestTools/Aspire.TestTools.csproj @@ -0,0 +1,9 @@ + + + + net10.0 + enable + enable + + + diff --git a/tools/Aspire.TestTools/GitHubActionsApi.cs b/tools/Aspire.TestTools/GitHubActionsApi.cs new file mode 100644 index 00000000000..fcd8d2a9391 --- /dev/null +++ b/tools/Aspire.TestTools/GitHubActionsApi.cs @@ -0,0 +1,92 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Aspire.TestTools; + +public sealed record GitHubActionsJob( + long Id, + string Name, + string HtmlUrl, + string? Conclusion, + string? Status); + +public sealed record GitHubActionsArtifact( + long Id, + string Name); + +public static class GitHubActionsApi +{ + public static async Task> ListJobsAsync(string repository, long runId, int? runAttempt, CancellationToken cancellationToken) + { + List jobs = []; + + for (var page = 1; ; page++) + { + var endpoint = runAttempt is int attempt + ? $"repos/{repository}/actions/runs/{runId}/attempts/{attempt}/jobs?per_page=100&page={page}" + : $"repos/{repository}/actions/runs/{runId}/jobs?per_page=100&page={page}"; + + using var jobsDocument = await GitHubCli.GetJsonAsync(endpoint, cancellationToken).ConfigureAwait(false); + var jobsArray = jobsDocument.RootElement.GetProperty("jobs"); + if (jobsArray.GetArrayLength() == 0) + { + break; + } + + foreach (var job in jobsArray.EnumerateArray()) + { + jobs.Add(new GitHubActionsJob( + Id: job.GetProperty("id").GetInt64(), + Name: job.GetProperty("name").GetString() ?? $"job-{job.GetProperty("id").GetInt64()}", + HtmlUrl: job.TryGetProperty("html_url", out var htmlUrlElement) ? htmlUrlElement.GetString() ?? string.Empty : string.Empty, + Conclusion: job.TryGetProperty("conclusion", out var conclusionElement) ? conclusionElement.GetString() : null, + Status: job.TryGetProperty("status", out var statusElement) ? statusElement.GetString() : null)); + } + + if (jobsArray.GetArrayLength() < 100) + { + break; + } + } + + return jobs; + } + + public static async Task> ListArtifactsAsync(string repository, long runId, CancellationToken cancellationToken) + { + List artifacts = []; + + for (var page = 1; ; page++) + { + using var artifactsDocument = await GitHubCli.GetJsonAsync( + $"repos/{repository}/actions/runs/{runId}/artifacts?per_page=100&page={page}", + cancellationToken).ConfigureAwait(false); + + var artifactsArray = artifactsDocument.RootElement.GetProperty("artifacts"); + if (artifactsArray.GetArrayLength() == 0) + { + break; + } + + foreach (var artifact in artifactsArray.EnumerateArray()) + { + artifacts.Add(new GitHubActionsArtifact( + Id: artifact.GetProperty("id").GetInt64(), + Name: artifact.GetProperty("name").GetString() ?? $"artifact-{artifact.GetProperty("id").GetInt64()}")); + } + + if (artifactsArray.GetArrayLength() < 100) + { + break; + } + } + + return artifacts; + } + + public static Task DownloadJobLogAsync(string repository, long jobId, CancellationToken cancellationToken) + => GitHubCli.GetStringAsync($"repos/{repository}/actions/jobs/{jobId}/logs", cancellationToken); + + public static Task DownloadArtifactZipAsync(string repository, long artifactId, string outputPath, CancellationToken cancellationToken) + => GitHubCli.DownloadFileAsync($"repos/{repository}/actions/artifacts/{artifactId}/zip", outputPath, cancellationToken); +} diff --git a/tools/Aspire.TestTools/GitHubCli.cs b/tools/Aspire.TestTools/GitHubCli.cs new file mode 100644 index 00000000000..63bad8a74ef --- /dev/null +++ b/tools/Aspire.TestTools/GitHubCli.cs @@ -0,0 +1,354 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace Aspire.TestTools; + +public static class GitHubCli +{ + private const string FixtureDirectoryEnvironmentVariable = "ASPIRE_FAILING_TEST_ISSUE_FIXTURE_DIR"; + + public static async Task GetJsonAsync(string endpoint, CancellationToken cancellationToken) + { + var stdout = await GetStringAsync(endpoint, cancellationToken).ConfigureAwait(false); + return JsonDocument.Parse(stdout); + } + + public static Task GetStringAsync(string endpoint, CancellationToken cancellationToken) + { + if (TryGetFixturePath(endpoint, ".json", out var fixturePath)) + { + return File.ReadAllTextAsync(fixturePath, cancellationToken); + } + + if (TryGetFixturePath(endpoint, ".txt", out fixturePath)) + { + return File.ReadAllTextAsync(fixturePath, cancellationToken); + } + + if (TryGetFixturePath(endpoint, ".err", out fixturePath)) + { + return Task.FromException(new InvalidOperationException(File.ReadAllText(fixturePath))); + } + + return RunGhAsync(["api", "-H", "Accept: application/vnd.github+json", endpoint], cancellationToken); + } + + public static async Task DownloadFileAsync(string endpoint, string outputPath, CancellationToken cancellationToken) + { + var outputExtension = Path.GetExtension(outputPath); + if (TryGetFixturePath(endpoint, outputExtension, out var fixturePath) || TryGetFixturePath(endpoint, ".bin", out fixturePath)) + { + File.Copy(fixturePath, outputPath, overwrite: true); + return; + } + + await RunGhToFileAsync( + ["api", "-H", "Accept: application/vnd.github+json", endpoint], + outputPath, + cancellationToken).ConfigureAwait(false); + } + + public static async Task<(int Number, string Url)> CreateIssueAsync( + string repository, + string title, + string body, + IReadOnlyList labels, + CancellationToken cancellationToken) + { + if (TryGetFixtureContent("create-issue", ".json", out var fixtureContent)) + { + using var document = JsonDocument.Parse(fixtureContent); + var root = document.RootElement; + return (root.GetProperty("number").GetInt32(), root.GetProperty("url").GetString()!); + } + + List arguments = ["issue", "create", "--repo", repository, "--title", title, "--body", body]; + + foreach (var label in labels) + { + arguments.Add("--label"); + arguments.Add(label); + } + + var stdout = await RunGhAsync(arguments, cancellationToken).ConfigureAwait(false); + + // gh issue create outputs the issue URL on stdout + var issueUrl = stdout.Trim(); + var issueNumber = 0; + + // Extract issue number from URL: https://github.com/owner/repo/issues/123 + var lastSlash = issueUrl.LastIndexOf('/'); + if (lastSlash >= 0 && int.TryParse(issueUrl[(lastSlash + 1)..], out var parsed)) + { + issueNumber = parsed; + } + + return (issueNumber, issueUrl); + } + + /// + /// Searches for an existing failing-test issue by metadata marker. + /// Returns the first matching issue (prefers open, then closed), or null if none found. + /// + public static async Task<(int Number, string Url, string State)?> SearchExistingIssueAsync( + string repository, + string metadataMarker, + CancellationToken cancellationToken) + { + if (TryGetFixtureContent("search-issue", ".json", out var fixtureContent)) + { + using var document = JsonDocument.Parse(fixtureContent); + var root = document.RootElement; + if (root.TryGetProperty("number", out var numberElement)) + { + return (numberElement.GetInt32(), root.GetProperty("url").GetString()!, root.GetProperty("state").GetString()!); + } + + return null; + } + + var escapedMarker = metadataMarker.Replace("\"", "\\\""); + var query = $"repo:{repository} is:issue label:failing-test in:body \"{escapedMarker}\""; + + using var searchDocument = await GetJsonAsync( + $"search/issues?q={Uri.EscapeDataString(query)}&per_page=20", + cancellationToken).ConfigureAwait(false); + + var items = searchDocument.RootElement.GetProperty("items"); + + // Prefer open issues, then closed issues (matching workflow JS logic). + JsonElement? openMatch = null; + JsonElement? closedMatch = null; + + foreach (var item in items.EnumerateArray()) + { + // Skip pull requests + if (item.TryGetProperty("pull_request", out _)) + { + continue; + } + + var state = item.GetProperty("state").GetString(); + if (state is "open" && openMatch is null) + { + openMatch = item; + break; // Open match is highest priority + } + else if (state is "closed" && closedMatch is null) + { + closedMatch = item; + } + } + + var match = openMatch ?? closedMatch; + if (match is null) + { + return null; + } + + return ( + match.Value.GetProperty("number").GetInt32(), + match.Value.GetProperty("html_url").GetString()!, + match.Value.GetProperty("state").GetString()!); + } + + /// + /// Reopens a closed issue. + /// + public static async Task ReopenIssueAsync(string repository, int issueNumber, CancellationToken cancellationToken) + { + if (TryGetFixtureContent("reopen-issue", ".json", out _)) + { + return; + } + + var issueNumberString = issueNumber.ToString(CultureInfo.InvariantCulture); + + // Use the API to set the state to open + await RunGhAsync( + ["api", "-X", "PATCH", $"repos/{repository}/issues/{issueNumberString}", "-f", "state=open"], + cancellationToken).ConfigureAwait(false); + } + + /// + /// Adds a comment to an issue. + /// + public static async Task AddIssueCommentAsync(string repository, int issueNumber, string body, CancellationToken cancellationToken) + { + if (TryGetFixtureContent("add-issue-comment", ".json", out _)) + { + return; + } + + await RunGhAsync( + ["issue", "comment", issueNumber.ToString(CultureInfo.InvariantCulture), "--repo", repository, "--body", body], + cancellationToken).ConfigureAwait(false); + } + + private static bool TryGetFixtureContent(string name, string extension, out string content) + { + var fixtureDirectory = Environment.GetEnvironmentVariable(FixtureDirectoryEnvironmentVariable); + if (!string.IsNullOrWhiteSpace(fixtureDirectory)) + { + var fixturePath = Path.Combine(fixtureDirectory, $"{name}{extension}"); + if (File.Exists(fixturePath)) + { + content = File.ReadAllText(fixturePath); + return true; + } + } + + content = string.Empty; + return false; + } + + private static readonly TimeSpan s_defaultProcessTimeout = TimeSpan.FromMinutes(5); + + private static async Task RunGhAsync(IReadOnlyList arguments, CancellationToken cancellationToken) + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(s_defaultProcessTimeout); + + ProcessStartInfo processStartInfo = new() + { + FileName = "gh", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + foreach (var argument in arguments) + { + processStartInfo.ArgumentList.Add(argument); + } + + using var process = new Process + { + StartInfo = processStartInfo + }; + + process.Start(); + + var stdoutTask = process.StandardOutput.ReadToEndAsync(cts.Token); + var stderrTask = process.StandardError.ReadToEndAsync(cts.Token); + + try + { + await process.WaitForExitAsync(cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + process.Kill(entireProcessTree: true); + throw; + } + + var stdout = await stdoutTask.ConfigureAwait(false); + var stderr = await stderrTask.ConfigureAwait(false); + + if (process.ExitCode != 0) + { + var message = string.IsNullOrWhiteSpace(stderr) ? stdout : stderr; + throw new InvalidOperationException($"gh {BuildDisplayArguments(arguments)} failed: {message.Trim()}"); + } + + return stdout; + } + + private static async Task RunGhToFileAsync(IReadOnlyList arguments, string outputPath, CancellationToken cancellationToken) + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(s_defaultProcessTimeout); + + ProcessStartInfo processStartInfo = new() + { + FileName = "gh", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + foreach (var argument in arguments) + { + processStartInfo.ArgumentList.Add(argument); + } + + using var process = new Process + { + StartInfo = processStartInfo + }; + + process.Start(); + + string stderr; + { + using var outputStream = File.Create(outputPath); + var stdoutTask = process.StandardOutput.BaseStream.CopyToAsync(outputStream, cts.Token); + var stderrTask = process.StandardError.ReadToEndAsync(cts.Token); + + try + { + await Task.WhenAll(process.WaitForExitAsync(cts.Token), stdoutTask, stderrTask).ConfigureAwait(false); + stderr = await stderrTask.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + process.Kill(entireProcessTree: true); + throw; + } + catch + { + try { stderr = await stderrTask.ConfigureAwait(false); } catch { stderr = string.Empty; } + throw; + } + } + + // Stream is disposed above so file handle is released before delete (required on Windows). + if (process.ExitCode != 0) + { + if (File.Exists(outputPath)) + { + File.Delete(outputPath); + } + + throw new InvalidOperationException($"gh {BuildDisplayArguments(arguments)} failed: {stderr.Trim()}"); + } + } + + private static string BuildDisplayArguments(IReadOnlyList arguments) + { + StringBuilder builder = new(); + + for (var i = 0; i < arguments.Count; i++) + { + if (i > 0) + { + builder.Append(' '); + } + + var argument = arguments[i]; + builder.Append(argument.Contains(' ') ? $"\"{argument}\"" : argument); + } + + return builder.ToString(); + } + + private static bool TryGetFixturePath(string endpoint, string extension, out string fixturePath) + { + var fixtureDirectory = Environment.GetEnvironmentVariable(FixtureDirectoryEnvironmentVariable); + if (string.IsNullOrWhiteSpace(fixtureDirectory)) + { + fixturePath = string.Empty; + return false; + } + + var fileName = Regex.Replace(endpoint, @"[^A-Za-z0-9._-]+", "_").Trim('_'); + fixturePath = Path.Combine(fixtureDirectory, $"{fileName}{extension}"); + return File.Exists(fixturePath); + } +} diff --git a/tools/GenerateTestSummary/TrxReader.cs b/tools/Aspire.TestTools/TrxReader.cs similarity index 63% rename from tools/GenerateTestSummary/TrxReader.cs rename to tools/Aspire.TestTools/TrxReader.cs index 0ade12971cf..3f7a7738d47 100644 --- a/tools/GenerateTestSummary/TrxReader.cs +++ b/tools/Aspire.TestTools/TrxReader.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Globalization; -using System.Xml; using System.Xml.Serialization; namespace Aspire.TestTools; @@ -49,6 +48,47 @@ public static IList GetTestResultsFromTrx(string filepath, Func GetDetailedTestResultsFromTrx(string filepath, Func? testFilter = null) + { + var testRun = DeserializeTrxFile(filepath); + if (testRun?.Results?.UnitTestResults is null) + { + return Array.Empty(); + } + + var testDefinitions = testRun.TestDefinitions?.UnitTests? + .Where(static testDefinition => !string.IsNullOrWhiteSpace(testDefinition.Id)) + .ToDictionary(static testDefinition => testDefinition.Id!, StringComparer.OrdinalIgnoreCase); + + var testResults = new List(); + + foreach (var unitTestResult in testRun.Results.UnitTestResults) + { + if (string.IsNullOrWhiteSpace(unitTestResult.TestName) || string.IsNullOrWhiteSpace(unitTestResult.Outcome)) + { + continue; + } + + var canonicalName = GetCanonicalTestName(unitTestResult, testDefinitions); + var detailedTestResult = new DetailedTestResult( + CanonicalName: canonicalName, + DisplayName: unitTestResult.TestName, + Outcome: unitTestResult.Outcome, + ErrorMessage: unitTestResult.Output?.ErrorMessage, + StackTrace: unitTestResult.Output?.StackTrace, + Stdout: unitTestResult.Output?.StdOut); + + if (testFilter is not null && !testFilter(detailedTestResult)) + { + continue; + } + + testResults.Add(detailedTestResult); + } + + return testResults; + } + public static TestRun? DeserializeTrxFile(string filePath) { if (string.IsNullOrWhiteSpace(filePath)) @@ -111,6 +151,24 @@ public static double GetTestRunDurationInMinutes(TestRun testRun) return (latestEndTime.Value - earliestStartTime.Value).TotalMinutes; } + + private static string GetCanonicalTestName(UnitTestResult unitTestResult, IReadOnlyDictionary? testDefinitions) + { + if (!string.IsNullOrWhiteSpace(unitTestResult.TestId) + && testDefinitions is not null + && testDefinitions.TryGetValue(unitTestResult.TestId, out var testDefinition) + && testDefinition.TestMethod is { ClassName: { Length: > 0 } className, Name: { Length: > 0 } methodName }) + { + if (methodName.StartsWith($"{className}.", StringComparison.Ordinal)) + { + return methodName; + } + + return $"{className}.{methodName}"; + } + + return unitTestResult.TestName ?? string.Empty; + } } [XmlRoot("TestRun", Namespace = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010")] @@ -121,6 +179,8 @@ public class TestRun public ResultSummary? ResultSummary { get; set; } public Times? Times { get; set; } + + public TestDefinitions? TestDefinitions { get; set; } } public class Results @@ -146,6 +206,9 @@ public class Times public class UnitTestResult { + [XmlAttribute("testId")] + public string? TestId { get; set; } + [XmlAttribute("testName")] public string? TestName { get; set; } @@ -166,12 +229,26 @@ public class UnitTestResult public class Output { - [XmlAnyElement] - public XmlElement? ErrorInfo { get; set; } + public ErrorInfo? ErrorInfo { get; set; } public string? StdOut { get; set; } [XmlIgnore] - public string ErrorInfoString => ErrorInfo?.InnerText ?? string.Empty; + public string ErrorInfoString => string.Join( + Environment.NewLine, + new[] { ErrorMessage, StackTrace }.Where(static value => !string.IsNullOrWhiteSpace(value))); + + [XmlIgnore] + public string? ErrorMessage => ErrorInfo?.Message; + + [XmlIgnore] + public string? StackTrace => ErrorInfo?.StackTrace; +} + +public class ErrorInfo +{ + public string? Message { get; set; } + + public string? StackTrace { get; set; } } public class ResultSummary @@ -217,3 +294,31 @@ public class Counters } public record TestResult(string Name, string Outcome, TimeSpan StartTime, TimeSpan EndTime, string? ErrorMessage = null, string? Stdout = null); + +public record DetailedTestResult(string CanonicalName, string DisplayName, string Outcome, string? ErrorMessage = null, string? StackTrace = null, string? Stdout = null); + +public class TestDefinitions +{ + [XmlElement("UnitTest")] + public List? UnitTests { get; set; } +} + +public class UnitTestDefinition +{ + [XmlAttribute("id")] + public string? Id { get; set; } + + [XmlAttribute("name")] + public string? Name { get; set; } + + public TestMethodDefinition? TestMethod { get; set; } +} + +public class TestMethodDefinition +{ + [XmlAttribute("className")] + public string? ClassName { get; set; } + + [XmlAttribute("name")] + public string? Name { get; set; } +} diff --git a/tools/CreateFailingTestIssue/CreateFailingTestIssue.csproj b/tools/CreateFailingTestIssue/CreateFailingTestIssue.csproj new file mode 100644 index 00000000000..7b37a13df28 --- /dev/null +++ b/tools/CreateFailingTestIssue/CreateFailingTestIssue.csproj @@ -0,0 +1,19 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/tools/CreateFailingTestIssue/FailingTestIssueCommand.cs b/tools/CreateFailingTestIssue/FailingTestIssueCommand.cs new file mode 100644 index 00000000000..4303103504f --- /dev/null +++ b/tools/CreateFailingTestIssue/FailingTestIssueCommand.cs @@ -0,0 +1,1746 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Globalization; +using System.IO.Compression; +using System.IO.Hashing; +using System.Net; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using Aspire.TestTools; + +namespace CreateFailingTestIssue; + +internal static class FailingTestIssueCommand +{ + private static readonly HashSet s_failedOutcomes = new(StringComparer.OrdinalIgnoreCase) + { + "Failed", + "Error", + "Timeout", + "Aborted" + }; + + private static readonly HashSet s_failedConclusions = new(StringComparer.OrdinalIgnoreCase) + { + "failure", + "timed_out", + "cancelled", + "startup_failure" + }; + + private static readonly Regex s_failedTestPattern = new(@"(?:^|\r?\n)\s*failed\s+(?.+?)\s*(?:\[|\()", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex s_logPrefixPattern = new(@"^\d{4}-\d{2}-\d{2}T[^\s]+\s+", RegexOptions.Compiled); + private static readonly Regex s_logSummaryPattern = new(@"
.*?(?.+?)", RegexOptions.Compiled); + private static readonly Regex s_directFailedLinePattern = new(@"^\s*failed\s+(?.+?)\s*\(", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex s_inlineStackPattern = new(@"^(?.*?)(?:\s{2,}|\t+)(?at .+)$", RegexOptions.Compiled); + + public static async Task ExecuteAsync(CommandInput input, CancellationToken cancellationToken) + { + input = input with { TestQuery = string.IsNullOrWhiteSpace(input.TestQuery) ? null : input.TestQuery.Trim() }; + List executionLog = []; + List warnings = []; + WorkflowSection? workflowSection = null; + ResolutionSection? resolutionSection = null; + void Log(string message) => executionLog.Add(message); + + try + { + var workflowSelectorResolution = FailingTestIssueLogic.ResolveWorkflowSelector(input.WorkflowSelector); + if (!workflowSelectorResolution.Success || workflowSelectorResolution.ResolvedWorkflowFile is null) + { + var errorMessage = workflowSelectorResolution.ErrorMessage ?? "Unable to resolve the workflow selector."; + Log(errorMessage); + return CreateFailureResult(input, workflowSection, resolutionSection, errorMessage, executionLog, warnings, []); + } + + Log($"Resolved workflow selector '{workflowSelectorResolution.Requested}' to '{workflowSelectorResolution.ResolvedWorkflowFile}'."); + var workflowMetadata = await ResolveWorkflowAsync(input.Repository, workflowSelectorResolution.ResolvedWorkflowFile, cancellationToken).ConfigureAwait(false); + Log($"Resolved workflow metadata: '{workflowMetadata.Name}' ({workflowMetadata.Path})."); + workflowSection = new WorkflowSection( + Requested: workflowSelectorResolution.Requested, + ResolvedWorkflowFile: workflowMetadata.Path, + ResolvedWorkflowName: workflowMetadata.Name); + + var sourceResolution = FailingTestIssueLogic.ParseSourceUrl(input.SourceUrl, input.Repository); + if (!sourceResolution.Success || sourceResolution.SourceKind is null) + { + var errorMessage = sourceResolution.ErrorMessage ?? "Unable to resolve the source URL."; + Log(errorMessage); + return CreateFailureResult(input, workflowSection, resolutionSection, errorMessage, executionLog, warnings, []); + } + + Log($"Parsed source URL as '{sourceResolution.SourceKind}'."); + var resolvedRun = await ResolveRunAsync(input.Repository, workflowMetadata, sourceResolution, cancellationToken).ConfigureAwait(false); + Log($"Resolved workflow run {resolvedRun.RunId} attempt {resolvedRun.RunAttempt} with status '{resolvedRun.Status}'."); + resolutionSection = new ResolutionSection( + SourceKind: sourceResolution.SourceKind, + RunId: resolvedRun.RunId, + RunUrl: resolvedRun.RunUrl, + RunAttempt: resolvedRun.RunAttempt, + PullRequestNumber: resolvedRun.PullRequestNumber, + JobUrls: []); + + var isCompletedRun = string.Equals(resolvedRun.Status, "completed", StringComparison.OrdinalIgnoreCase); + if (!isCompletedRun) + { + warnings.Add($"The selected workflow run is not completed yet (status: {resolvedRun.Status}). Results may be incomplete."); + Log("Run is still active; results may be incomplete."); + } + + var failedJobs = await ListFailedJobsAsync(input.Repository, resolvedRun, cancellationToken).ConfigureAwait(false); + Log($"Found {failedJobs.Count} failed job(s)."); + resolutionSection = resolutionSection with { JobUrls = failedJobs.Select(job => job.HtmlUrl).Where(url => !string.IsNullOrWhiteSpace(url)).Cast().ToArray() }; + + var jobLogs = failedJobs.Count == 0 + ? [] + : await DownloadFailedTestsFromLogsAsync(input.Repository, failedJobs, warnings, Log, cancellationToken).ConfigureAwait(false); + var failedTestsByJobUrl = jobLogs + .Where(static jobLog => !string.IsNullOrWhiteSpace(jobLog.Job.HtmlUrl)) + .ToDictionary(static jobLog => jobLog.Job.HtmlUrl, static jobLog => jobLog.FailedTests, StringComparer.OrdinalIgnoreCase); + var trxOccurrences = await DownloadFailedTestOccurrencesAsync(input.Repository, resolvedRun.RunId, allowArtifactFallback: isCompletedRun, Log, cancellationToken).ConfigureAwait(false); + var logOccurrences = jobLogs.SelectMany(static jobLog => jobLog.Occurrences).ToList(); + Log($"Collected {trxOccurrences.Count} artifact-derived occurrence(s) and {logOccurrences.Count} log-derived occurrence(s)."); + var occurrences = trxOccurrences.Count > 0 ? trxOccurrences : logOccurrences; + var availableFailedTests = GetAvailableFailedTests(occurrences); + + if (occurrences.Count == 0) + { + var message = isCompletedRun + ? "The workflow run did not contain any failed test results in downloadable .trx artifacts." + : "The workflow run has not produced any failed test results in downloadable .trx artifacts yet."; + Log(message); + return CreateFailureResult(input, workflowSection, resolutionSection, message, executionLog, warnings, availableFailedTests); + } + + if (trxOccurrences.Count == 0) + { + warnings.Add("No downloadable .trx artifacts were available. Falling back to failed job logs."); + Log("Using log-derived occurrences because no downloadable .trx artifacts were available."); + } + + if (input.TestQuery is null) + { + if (input.ForceNew) + { + warnings.Add("Ignoring --force-new because issue generation requires a specific test query."); + Log("Ignoring --force-new because no test query was supplied."); + } + + var uniqueFailedTestCount = occurrences + .Select(static occurrence => (occurrence.CanonicalTestName, occurrence.DisplayTestName)) + .Distinct() + .Count(); + Log($"No test query was supplied; returning all {uniqueFailedTestCount} failing test(s)."); + return CreateAllFailuresResult( + input, + workflowSection!, + resolutionSection!, + occurrences, + failedJobs, + failedTestsByJobUrl, + executionLog, + warnings, + availableFailedTests); + } + + var matchResult = FailingTestIssueLogic.MatchTestFailures(input.TestQuery, occurrences); + if (!matchResult.Success || matchResult.CanonicalTestName is null || matchResult.DisplayTestName is null) + { + var errorMessage = matchResult.ErrorMessage ?? "The requested test could not be matched."; + Log(errorMessage); + return CreateFailureResult(input, workflowSection, resolutionSection, errorMessage, executionLog, warnings, matchResult.CandidateNames.Count > 0 ? matchResult.CandidateNames : availableFailedTests); + } + + Log($"Matched query using '{matchResult.Strategy}'."); + var matchingJobs = GetMatchingJobs(matchResult, failedJobs, failedTestsByJobUrl); + Log($"Matched {matchingJobs.Count} job(s) to the resolved test."); + GitHubActionsJob? preferredJob = null; + + if (resolvedRun.PreferredJobId is long preferredJobId) + { + preferredJob = failedJobs.FirstOrDefault(job => job.Id == preferredJobId); + } + + var primaryJob = preferredJob ?? matchingJobs.FirstOrDefault(); + if (matchingJobs.Count > 1) + { + warnings.Add("The matched test appeared in multiple failed jobs. The primary job was selected from the explicit source job or the first matching job."); + } + else if (matchingJobs.Count == 0) + { + warnings.Add("The matched test was found in .trx artifacts, but no failed job log mentioned the same test name. Job context is omitted from the occurrence list."); + } + + var resolvedOccurrences = AttachSingleJobContext(matchResult.Occurrences, matchingJobs); + var primaryOccurrence = SelectPrimaryOccurrence(resolvedOccurrences, primaryJob); + + if (primaryOccurrence is null) + { + const string errorMessage = "The matched test did not produce any resolvable failure occurrences."; + Log(errorMessage); + return CreateFailureResult(input, workflowSection, resolutionSection, errorMessage, executionLog, warnings, availableFailedTests); + } + + Log($"Selected primary occurrence from '{primaryOccurrence.ArtifactName}'."); + var issueArtifacts = FailingTestIssueLogic.CreateIssueArtifacts( + workflow: workflowSection, + resolution: resolutionSection, + matchResult: matchResult, + occurrences: resolvedOccurrences, + primaryOccurrence: primaryOccurrence, + matchingJobs: matchingJobs.Select(job => (job.Name, job.HtmlUrl)).ToArray(), + warnings: warnings); + + CreatedIssueInfo? createdIssue = null; + ExistingIssueInfo? existingIssue = null; + if (input.Create) + { + if (!input.ForceNew) + { + Log("--create flag set. Searching for existing issue..."); + var existing = await GitHubCli.SearchExistingIssueAsync( + input.Repository, + issueArtifacts.MetadataMarker, + cancellationToken).ConfigureAwait(false); + + if (existing is not null) + { + var (existingNumber, existingUrl, existingState) = existing.Value; + existingIssue = new ExistingIssueInfo(existingNumber, existingUrl); + + if (existingState is "closed") + { + Log($"Found closed issue #{existingNumber}. Reopening..."); + await GitHubCli.ReopenIssueAsync(input.Repository, existingNumber, cancellationToken).ConfigureAwait(false); + } + else + { + Log($"Found open issue #{existingNumber}. Adding comment..."); + } + + await GitHubCli.AddIssueCommentAsync( + input.Repository, + existingNumber, + issueArtifacts.CommentBody, + cancellationToken).ConfigureAwait(false); + + Log($"Updated existing issue #{existingNumber}: {existingUrl}"); + } + } + + if (existingIssue is null) + { + Log(input.ForceNew ? "--force-new flag set. Creating new issue on GitHub..." : "No existing issue found. Creating new issue on GitHub..."); + var (issueNumber, issueUrl) = await GitHubCli.CreateIssueAsync( + input.Repository, + issueArtifacts.Title, + issueArtifacts.Body, + issueArtifacts.Labels, + cancellationToken).ConfigureAwait(false); + createdIssue = new CreatedIssueInfo(issueNumber, issueUrl); + Log($"Created issue #{issueNumber}: {issueUrl}"); + } + } + + return new CreateFailingTestIssueResult + { + Success = true, + Input = new InputSection( + TestQuery: input.TestQuery, + SourceUrl: input.SourceUrl, + Workflow: workflowSection, + ForceNew: input.ForceNew), + Resolution = resolutionSection, + Match = new MatchSection( + Query: input.TestQuery, + Strategy: matchResult.Strategy!, + CanonicalTestName: matchResult.CanonicalTestName, + DisplayTestName: matchResult.DisplayTestName, + AllMatchingOccurrences: resolvedOccurrences + .Select(ToMatchedOccurrence) + .ToArray()), + Failure = ToFailureSection(primaryOccurrence, primaryJob), + Matrix = new MatrixSection( + MatchedOccurrences: resolvedOccurrences.Count, + Summary: resolvedOccurrences + .Select(occurrence => new MatrixSummaryEntry( + ArtifactName: occurrence.ArtifactName, + ArtifactUrl: occurrence.ArtifactUrl, + TrxPath: occurrence.TrxPath, + JobName: occurrence.JobName, + JobUrl: occurrence.JobUrl)) + .ToArray()), + Issue = new IssueSection( + Title: issueArtifacts.Title, + Labels: issueArtifacts.Labels, + StableSignature: issueArtifacts.StableSignature, + MetadataMarker: issueArtifacts.MetadataMarker, + Body: issueArtifacts.Body, + CommentBody: issueArtifacts.CommentBody, + ExistingIssue: existingIssue, + CreatedIssue: createdIssue), + Diagnostics = new DiagnosticsSection( + Log: executionLog.ToArray(), + LogFile: "diagnostics.log", + Warnings: warnings.ToArray(), + AvailableFailedTests: availableFailedTests) + }; + } + catch (Exception ex) + { + warnings.Add(ex.Message); + Log($"Unhandled exception: {ex.Message}"); + return CreateFailureResult(input, workflowSection, resolutionSection, ex.Message, executionLog, warnings, []); + } + } + + private static CreateFailingTestIssueResult CreateAllFailuresResult( + CommandInput input, + WorkflowSection workflow, + ResolutionSection resolution, + IReadOnlyList occurrences, + IReadOnlyList failedJobs, + IReadOnlyDictionary> failedTestsByJobUrl, + IReadOnlyList executionLog, + IReadOnlyList warnings, + IReadOnlyList availableFailedTests) + { + var tests = occurrences + .GroupBy( + static occurrence => (occurrence.CanonicalTestName, occurrence.DisplayTestName), + (key, groupedOccurrences) => + { + var groupedOccurrenceList = groupedOccurrences.ToArray(); + var matchingJobs = GetMatchingJobs(key.CanonicalTestName, key.DisplayTestName, failedJobs, failedTestsByJobUrl); + var resolvedOccurrences = AttachSingleJobContext(groupedOccurrenceList, matchingJobs); + var primaryJob = matchingJobs.Count == 1 ? matchingJobs[0] : null; + var primaryOccurrence = SelectPrimaryOccurrence(resolvedOccurrences, primaryJob) ?? resolvedOccurrences[0]; + + return new FailingTestEntry( + CanonicalTestName: key.CanonicalTestName, + DisplayTestName: key.DisplayTestName, + OccurrenceCount: resolvedOccurrences.Count, + Occurrences: resolvedOccurrences.Select(ToMatchedOccurrence).ToArray(), + PrimaryFailure: ToFailureSection(primaryOccurrence, primaryJob)); + }) + .OrderBy(static test => test.CanonicalTestName, StringComparer.OrdinalIgnoreCase) + .ThenBy(static test => test.DisplayTestName, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + return new CreateFailingTestIssueResult + { + Success = true, + Input = new InputSection( + TestQuery: input.TestQuery, + SourceUrl: input.SourceUrl, + Workflow: workflow, + ForceNew: input.ForceNew), + Resolution = resolution, + AllFailures = new AllFailuresSection( + FailedTests: tests.Length, + Tests: tests), + Diagnostics = new DiagnosticsSection( + Log: executionLog.ToArray(), + LogFile: "diagnostics.log", + Warnings: warnings.ToArray(), + AvailableFailedTests: availableFailedTests) + }; + } + + private static CreateFailingTestIssueResult CreateFailureResult( + CommandInput input, + WorkflowSection? workflow, + ResolutionSection? resolution, + string errorMessage, + IReadOnlyList executionLog, + IReadOnlyList warnings, + IReadOnlyList availableFailedTests) + { + return new CreateFailingTestIssueResult + { + Success = false, + ErrorMessage = errorMessage, + Input = new InputSection( + TestQuery: input.TestQuery, + SourceUrl: input.SourceUrl, + Workflow: workflow ?? new WorkflowSection( + Requested: input.WorkflowSelector, + ResolvedWorkflowFile: string.Empty, + ResolvedWorkflowName: string.Empty), + ForceNew: input.ForceNew), + Resolution = resolution, + Diagnostics = new DiagnosticsSection( + Log: executionLog.ToArray(), + LogFile: "diagnostics.log", + Warnings: warnings.ToArray(), + AvailableFailedTests: availableFailedTests) + }; + } + + private static async Task ResolveWorkflowAsync(string repository, string workflowFile, CancellationToken cancellationToken) + { + using var workflowDocument = await GitHubCli.GetJsonAsync( + $"repos/{repository}/actions/workflows/{Path.GetFileName(workflowFile)}", + cancellationToken).ConfigureAwait(false); + + var root = workflowDocument.RootElement; + return new WorkflowMetadata( + Id: root.GetProperty("id").GetInt64(), + Name: root.GetProperty("name").GetString() ?? Path.GetFileName(workflowFile), + Path: root.GetProperty("path").GetString() ?? workflowFile); + } + + private static async Task ResolveRunAsync( + string repository, + WorkflowMetadata workflow, + SourceUrlResolution sourceResolution, + CancellationToken cancellationToken) + { + return sourceResolution.SourceKind switch + { + "pull_request" => await ResolveRunFromPullRequestAsync(repository, workflow, sourceResolution.PullRequestNumber!.Value, cancellationToken).ConfigureAwait(false), + "run" => await ResolveRunFromRunIdAsync(repository, workflow, sourceResolution.RunId!.Value, sourceResolution.RunAttempt, null, cancellationToken).ConfigureAwait(false), + "job" => await ResolveRunFromJobIdAsync(repository, workflow, sourceResolution.JobId!.Value, sourceResolution.RunId, sourceResolution.RunAttempt, cancellationToken).ConfigureAwait(false), + _ => throw new InvalidOperationException($"Unsupported source kind '{sourceResolution.SourceKind}'.") + }; + } + + private static async Task ResolveRunFromPullRequestAsync(string repository, WorkflowMetadata workflow, int pullRequestNumber, CancellationToken cancellationToken) + { + using var pullRequestDocument = await GitHubCli.GetJsonAsync( + $"repos/{repository}/pulls/{pullRequestNumber}", + cancellationToken).ConfigureAwait(false); + + var root = pullRequestDocument.RootElement; + var headSha = root.GetProperty("head").GetProperty("sha").GetString(); + if (string.IsNullOrWhiteSpace(headSha)) + { + throw new InvalidOperationException($"Pull request #{pullRequestNumber} does not have a resolvable head SHA."); + } + + // Search all runs (not just completed) to find in-progress reruns and the latest matching run. + // Prefer completed runs, but fall back to in-progress/queued runs if no completed run is found. + JsonElement? bestNonCompletedRun = null; + + for (var page = 1; ; page++) + { + using var runsDocument = await GitHubCli.GetJsonAsync( + $"repos/{repository}/actions/workflows/{Path.GetFileName(workflow.Path)}/runs?per_page=100&page={page}", + cancellationToken).ConfigureAwait(false); + + var runs = runsDocument.RootElement.GetProperty("workflow_runs"); + if (runs.GetArrayLength() == 0) + { + break; + } + + foreach (var run in runs.EnumerateArray()) + { + if (run.GetProperty("workflow_id").GetInt64() != workflow.Id) + { + continue; + } + + var runPullRequestNumbers = GetPullRequestNumbers(run); + var runHeadSha = run.TryGetProperty("head_sha", out var headShaElement) ? headShaElement.GetString() : null; + if (!runPullRequestNumbers.Contains(pullRequestNumber) && !string.Equals(runHeadSha, headSha, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var status = run.TryGetProperty("status", out var statusElement) ? statusElement.GetString() : null; + if (status is "completed") + { + return CreateResolvedRun(run, pullRequestNumber, preferredJobId: null, requestedRunAttempt: null); + } + + // Remember the first non-completed match as a fallback + bestNonCompletedRun ??= run.Clone(); + } + + if (runs.GetArrayLength() < 100) + { + break; + } + } + + if (bestNonCompletedRun is not null) + { + return CreateResolvedRun(bestNonCompletedRun.Value, pullRequestNumber, preferredJobId: null, requestedRunAttempt: null); + } + + throw new InvalidOperationException($"No workflow run for '{workflow.Name}' was found for pull request #{pullRequestNumber}."); + } + + private static async Task ResolveRunFromRunIdAsync( + string repository, + WorkflowMetadata workflow, + long runId, + int? requestedRunAttempt, + long? preferredJobId, + CancellationToken cancellationToken) + { + using var runDocument = await GitHubCli.GetJsonAsync( + $"repos/{repository}/actions/runs/{runId}", + cancellationToken).ConfigureAwait(false); + + var root = runDocument.RootElement; + if (root.GetProperty("workflow_id").GetInt64() != workflow.Id) + { + var actualWorkflowName = root.TryGetProperty("name", out var nameElement) ? nameElement.GetString() : null; + throw new InvalidOperationException( + $"Run {runId} belongs to workflow '{actualWorkflowName ?? "unknown"}', not '{workflow.Name}'."); + } + + var pullRequestNumbers = GetPullRequestNumbers(root); + return CreateResolvedRun( + root, + pullRequestNumbers.FirstOrDefault(defaultValue: 0) is var number && number > 0 ? number : null, + preferredJobId, + requestedRunAttempt); + } + + private static async Task ResolveRunFromJobIdAsync( + string repository, + WorkflowMetadata workflow, + long jobId, + long? runIdHint, + int? requestedRunAttempt, + CancellationToken cancellationToken) + { + using var jobDocument = await GitHubCli.GetJsonAsync( + $"repos/{repository}/actions/jobs/{jobId}", + cancellationToken).ConfigureAwait(false); + + var root = jobDocument.RootElement; + var runId = root.TryGetProperty("run_id", out var runIdProperty) + ? runIdProperty.GetInt64() + : runIdHint ?? throw new InvalidOperationException($"Job {jobId} does not include a run id."); + + return await ResolveRunFromRunIdAsync(repository, workflow, runId, requestedRunAttempt, jobId, cancellationToken).ConfigureAwait(false); + } + + private static ResolvedRun CreateResolvedRun(JsonElement run, int? pullRequestNumber, long? preferredJobId, int? requestedRunAttempt) + { + var actualRunAttempt = run.TryGetProperty("run_attempt", out var runAttemptElement) ? runAttemptElement.GetInt32() : 1; + var effectiveRunAttempt = requestedRunAttempt ?? actualRunAttempt; + if (effectiveRunAttempt <= 0) + { + throw new InvalidOperationException($"Workflow run {run.GetProperty("id").GetInt64()} has an invalid attempt number '{effectiveRunAttempt}'."); + } + + if (requestedRunAttempt is not null && requestedRunAttempt > actualRunAttempt) + { + throw new InvalidOperationException( + $"Workflow run {run.GetProperty("id").GetInt64()} does not have attempt {requestedRunAttempt.Value}."); + } + + return new ResolvedRun( + RunId: run.GetProperty("id").GetInt64(), + RunUrl: BuildRunUrl(run.GetProperty("html_url").GetString() ?? string.Empty, requestedRunAttempt), + RunAttempt: effectiveRunAttempt, + Status: run.TryGetProperty("status", out var statusElement) ? statusElement.GetString() ?? string.Empty : string.Empty, + PullRequestNumber: pullRequestNumber, + PreferredJobId: preferredJobId, + IsSpecificAttempt: requestedRunAttempt is not null); + } + + private static async Task> ListFailedJobsAsync(string repository, ResolvedRun resolvedRun, CancellationToken cancellationToken) + { + var jobs = await GitHubActionsApi.ListJobsAsync( + repository, + resolvedRun.RunId, + resolvedRun.IsSpecificAttempt ? resolvedRun.RunAttempt : null, + cancellationToken).ConfigureAwait(false); + + return jobs + .Where(job => !string.IsNullOrWhiteSpace(job.Conclusion) && s_failedConclusions.Contains(job.Conclusion)) + .ToList(); + } + + private static string BuildRunUrl(string runUrl, int? requestedRunAttempt) + { + if (string.IsNullOrWhiteSpace(runUrl) || requestedRunAttempt is null) + { + return runUrl; + } + + return runUrl.TrimEnd('/') + $"/attempts/{requestedRunAttempt.Value.ToString(CultureInfo.InvariantCulture)}"; + } + + private static async Task> DownloadFailedTestsFromLogsAsync( + string repository, + IReadOnlyList jobs, + List warnings, + Action? log, + CancellationToken cancellationToken) + { + List results = []; + + foreach (var job in jobs) + { + string logs; + try + { + logs = await GitHubActionsApi.DownloadJobLogAsync(repository, job.Id, cancellationToken).ConfigureAwait(false); + } + catch (InvalidOperationException ex) when (IsMissingJobLog(ex)) + { + log?.Invoke($"Skipping missing log for failed job '{job.Name}' ({job.Id})."); + warnings?.Add($"Failed job log was unavailable for '{job.Name}' ({job.Id}); continuing without it."); + continue; + } + + var occurrences = ParseFailedTestOccurrencesFromLog(job, logs); + var failedTests = occurrences + .SelectMany(static occurrence => new[] { occurrence.CanonicalTestName, occurrence.DisplayTestName }) + .Where(static name => !string.IsNullOrWhiteSpace(name)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var name in s_failedTestPattern.Matches(logs) + .Select(match => match.Groups["name"].Value.Trim()) + .Where(name => !string.IsNullOrWhiteSpace(name))) + { + failedTests.Add(name); + } + + results.Add(new JobLogData(job, failedTests, occurrences)); + log?.Invoke($"Loaded failed job log '{job.Name}' and found {failedTests.Count} failed test name(s) with {occurrences.Count} detailed occurrence(s)."); + } + + return results; + } + + private static bool IsMissingJobLog(InvalidOperationException ex) + => ex.Message.Contains("/logs", StringComparison.OrdinalIgnoreCase) + && ex.Message.Contains("HTTP 404", StringComparison.OrdinalIgnoreCase); + + private static List ParseFailedTestOccurrencesFromLog(GitHubActionsJob job, string logs) + { + var normalizedLines = logs + .Split('\n') + .Select(static line => s_logPrefixPattern.Replace(line.TrimEnd('\r'), string.Empty)) + .ToArray(); + + var summaryOccurrences = ParseSummaryOccurrences(job, normalizedLines); + if (summaryOccurrences.Count > 0) + { + return summaryOccurrences; + } + + return ParseDirectFailureOccurrences(job, normalizedLines); + } + + private static List ParseSummaryOccurrences(GitHubActionsJob job, IReadOnlyList lines) + { + List occurrences = []; + + for (var i = 0; i < lines.Count; i++) + { + var match = s_logSummaryPattern.Match(lines[i]); + if (!match.Success) + { + continue; + } + + var displayName = WebUtility.HtmlDecode(match.Groups["name"].Value.Trim()); + if (string.IsNullOrWhiteSpace(displayName)) + { + continue; + } + + var fenceStart = FindNextFenceLine(lines, i + 1); + if (fenceStart < 0) + { + continue; + } + + var fenceEnd = FindNextFenceLine(lines, fenceStart + 1); + if (fenceEnd < 0) + { + continue; + } + + var details = lines + .Skip(fenceStart + 1) + .Take(fenceEnd - fenceStart - 1) + .ToArray(); + + var (errorMessage, stackTrace) = SplitErrorAndStack(details); + occurrences.Add(CreateLogOccurrence(job, displayName, errorMessage, stackTrace)); + i = fenceEnd; + } + + return occurrences; + } + + private static List ParseDirectFailureOccurrences(GitHubActionsJob job, IReadOnlyList lines) + { + List occurrences = []; + + for (var i = 0; i < lines.Count; i++) + { + var match = s_directFailedLinePattern.Match(lines[i]); + if (!match.Success) + { + continue; + } + + var displayName = match.Groups["name"].Value.Trim(); + if (string.IsNullOrWhiteSpace(displayName)) + { + continue; + } + + List details = []; + for (var detailIndex = i + 1; detailIndex < lines.Count; detailIndex++) + { + var detailLine = lines[detailIndex]; + var trimmedDetailLine = detailLine.Trim(); + + if (string.IsNullOrWhiteSpace(trimmedDetailLine)) + { + if (details.Count > 0) + { + break; + } + + continue; + } + + if (s_directFailedLinePattern.IsMatch(detailLine) || s_logSummaryPattern.IsMatch(detailLine)) + { + break; + } + + if (!detailLine.StartsWith(" ", StringComparison.Ordinal) && !IsFailureDetailLine(trimmedDetailLine)) + { + if (details.Count > 0) + { + break; + } + + continue; + } + + details.Add(trimmedDetailLine); + } + + var (errorMessage, stackTrace) = SplitErrorAndStack(details); + occurrences.Add(CreateLogOccurrence(job, displayName, errorMessage, stackTrace)); + } + + return occurrences; + } + + private static FailedTestOccurrence CreateLogOccurrence(GitHubActionsJob job, string displayName, string errorMessage, string stackTrace) + => new( + CanonicalTestName: GetCanonicalTestName(displayName), + DisplayTestName: displayName, + ArtifactName: "job log", + TrxPath: "n/a", + ErrorMessage: errorMessage, + StackTrace: stackTrace, + Stdout: string.Empty, + JobName: job.Name, + JobUrl: job.HtmlUrl); + + private static int FindNextFenceLine(IReadOnlyList lines, int startIndex) + { + for (var i = startIndex; i < lines.Count; i++) + { + if (lines[i].TrimStart().StartsWith("```", StringComparison.Ordinal)) + { + return i; + } + } + + return -1; + } + + private static (string ErrorMessage, string StackTrace) SplitErrorAndStack(IReadOnlyList lines) + { + List errorLines = []; + List stackLines = []; + + foreach (var rawLine in lines) + { + var line = rawLine.TrimEnd(); + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var inlineStackMatch = s_inlineStackPattern.Match(line); + if (inlineStackMatch.Success) + { + var error = inlineStackMatch.Groups["error"].Value.TrimEnd(); + if (!string.IsNullOrWhiteSpace(error)) + { + errorLines.Add(error); + } + + stackLines.Add(inlineStackMatch.Groups["stack"].Value.Trim()); + continue; + } + + if (IsStackTraceLine(line)) + { + stackLines.Add(line.Trim()); + continue; + } + + if (stackLines.Count == 0) + { + errorLines.Add(line.TrimEnd()); + } + else + { + stackLines.Add(line.Trim()); + } + } + + return (string.Join(Environment.NewLine, errorLines).Trim(), string.Join(Environment.NewLine, stackLines).Trim()); + } + + private static bool IsStackTraceLine(string line) + { + var trimmed = line.TrimStart(); + return trimmed.StartsWith("at ", StringComparison.Ordinal) + || trimmed.StartsWith("--- End of stack trace", StringComparison.Ordinal); + } + + private static bool IsFailureDetailLine(string line) + => line.StartsWith("Assert.", StringComparison.Ordinal) + || line.StartsWith("Expected:", StringComparison.Ordinal) + || line.StartsWith("Actual:", StringComparison.Ordinal) + || line.StartsWith("Error Message:", StringComparison.Ordinal) + || IsStackTraceLine(line); + + private static string GetCanonicalTestName(string displayName) + { + var parameterStart = displayName.IndexOf('('); + return parameterStart > 0 ? displayName[..parameterStart] : displayName; + } + + private static async Task> DownloadFailedTestOccurrencesAsync( + string repository, + long runId, + bool allowArtifactFallback, + Action? log, + CancellationToken cancellationToken) + { + var artifacts = await GitHubActionsApi.ListArtifactsAsync(repository, runId, cancellationToken).ConfigureAwait(false); + + log?.Invoke($"Enumerated {artifacts.Count} artifact(s) for workflow run {runId}."); + + if (artifacts.Count == 0) + { + return []; + } + + var candidateArtifacts = artifacts.Where(static artifact => MayContainTrxFiles(artifact.Name)).ToList(); + log?.Invoke($"Identified {candidateArtifacts.Count} artifact(s) whose names suggest test results."); + if (candidateArtifacts.Count == 0) + { + if (!allowArtifactFallback) + { + log?.Invoke("Skipping fallback to all artifacts because the run is still active and no likely test-results artifacts were found."); + return []; + } + + log?.Invoke("Falling back to all artifacts because no artifact names matched likely test-results patterns."); + candidateArtifacts = artifacts; + } + + var tempRoot = Directory.CreateTempSubdirectory("aspire-failing-test-issue").FullName; + + try + { + List occurrences = []; + + foreach (var artifact in candidateArtifacts) + { + log?.Invoke($"Downloading artifact '{artifact.Name}' ({artifact.Id})."); + var zipPath = Path.Combine(tempRoot, $"{artifact.Id}.zip"); + await GitHubActionsApi.DownloadArtifactZipAsync( + repository, + artifact.Id, + zipPath, + cancellationToken).ConfigureAwait(false); + + var extractDirectory = Path.Combine(tempRoot, artifact.Id.ToString(CultureInfo.InvariantCulture)); + ValidateZipEntries(zipPath, extractDirectory); + ZipFile.ExtractToDirectory(zipPath, extractDirectory, overwriteFiles: true); + + foreach (var trxPath in Directory.EnumerateFiles(extractDirectory, "*.trx", SearchOption.AllDirectories)) + { + var relativeTrxPath = Path.GetRelativePath(extractDirectory, trxPath).Replace('\\', '/'); + var results = TrxReader.GetDetailedTestResultsFromTrx( + trxPath, + static result => s_failedOutcomes.Contains(result.Outcome)); + + occurrences.AddRange(results.Select(result => new FailedTestOccurrence( + CanonicalTestName: result.CanonicalName, + DisplayTestName: result.DisplayName, + ArtifactName: artifact.Name, + TrxPath: relativeTrxPath, + ErrorMessage: result.ErrorMessage ?? string.Empty, + StackTrace: result.StackTrace ?? string.Empty, + Stdout: result.Stdout ?? string.Empty, + ArtifactUrl: $"https://github.com/{repository}/actions/runs/{runId}/artifacts/{artifact.Id}"))); + } + } + + log?.Invoke($"Extracted {occurrences.Count} failed test occurrence(s) from downloaded artifacts."); + + return occurrences; + } + finally + { + if (Directory.Exists(tempRoot)) + { + Directory.Delete(tempRoot, recursive: true); + } + } + } + + private static bool MayContainTrxFiles(string artifactName) + => artifactName.Contains("TestResults", StringComparison.OrdinalIgnoreCase); + + private static List GetAvailableFailedTests(IReadOnlyCollection occurrences) + { + return occurrences + .Select(static occurrence => FailingTestIssueLogic.FormatCandidateName(occurrence.CanonicalTestName, occurrence.DisplayTestName)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static name => name, StringComparer.OrdinalIgnoreCase) + .Take(50) + .ToList(); + } + + private static List GetMatchingJobs( + FailureMatchResult matchResult, + IReadOnlyList failedJobs, + IReadOnlyDictionary> failedTestsByJobUrl) + => GetMatchingJobs(matchResult.CanonicalTestName, matchResult.DisplayTestName, failedJobs, failedTestsByJobUrl); + + private static List GetMatchingJobs( + string? canonicalTestName, + string? displayTestName, + IReadOnlyList failedJobs, + IReadOnlyDictionary> failedTestsByJobUrl) + { + List jobs = []; + + foreach (var job in failedJobs) + { + if (string.IsNullOrWhiteSpace(job.HtmlUrl)) + { + continue; + } + + if (!failedTestsByJobUrl.TryGetValue(job.HtmlUrl, out var failedTests)) + { + continue; + } + + if (failedTests.Any(test => string.Equals(test, canonicalTestName, StringComparison.OrdinalIgnoreCase) + || string.Equals(test, displayTestName, StringComparison.OrdinalIgnoreCase))) + { + jobs.Add(job); + } + } + + return jobs; + } + + private static IReadOnlyList AttachSingleJobContext( + IReadOnlyList occurrences, + IReadOnlyList matchingJobs) + { + if (matchingJobs.Count != 1) + { + return occurrences; + } + + var job = matchingJobs[0]; + return occurrences + .Select(occurrence => occurrence with + { + JobName = job.Name, + JobUrl = job.HtmlUrl + }) + .ToArray(); + } + + private static FailedTestOccurrence? SelectPrimaryOccurrence( + IReadOnlyList occurrences, + GitHubActionsJob? primaryJob) + { + if (occurrences.Count == 0) + { + return null; + } + + if (primaryJob is not null) + { + var byJobUrl = occurrences.FirstOrDefault(occurrence => string.Equals(occurrence.JobUrl, primaryJob.HtmlUrl, StringComparison.OrdinalIgnoreCase)); + if (byJobUrl is not null) + { + return byJobUrl; + } + } + + return occurrences[0]; + } + + private static MatchedOccurrence ToMatchedOccurrence(FailedTestOccurrence occurrence) + => new( + CanonicalTestName: occurrence.CanonicalTestName, + DisplayTestName: occurrence.DisplayTestName, + ArtifactName: occurrence.ArtifactName, + ArtifactUrl: occurrence.ArtifactUrl, + TrxPath: occurrence.TrxPath, + JobName: occurrence.JobName, + JobUrl: occurrence.JobUrl); + + private static FailureSection ToFailureSection(FailedTestOccurrence occurrence, GitHubActionsJob? primaryJob) + => new( + ErrorMessage: occurrence.ErrorMessage, + StackTrace: occurrence.StackTrace, + Stdout: occurrence.Stdout, + ArtifactName: occurrence.ArtifactName, + ArtifactUrl: occurrence.ArtifactUrl, + TrxPath: occurrence.TrxPath, + JobName: primaryJob?.Name ?? occurrence.JobName, + JobUrl: primaryJob?.HtmlUrl ?? occurrence.JobUrl); + + private static List GetPullRequestNumbers(JsonElement run) + { + if (!run.TryGetProperty("pull_requests", out var pullRequestsElement) || pullRequestsElement.ValueKind != JsonValueKind.Array) + { + return []; + } + + return pullRequestsElement.EnumerateArray() + .Where(static pullRequest => pullRequest.TryGetProperty("number", out _)) + .Select(static pullRequest => pullRequest.GetProperty("number").GetInt32()) + .Distinct() + .ToList(); + } + + private sealed record WorkflowMetadata( + long Id, + string Name, + string Path); + + private sealed record ResolvedRun( + long RunId, + string RunUrl, + int RunAttempt, + string Status, + int? PullRequestNumber, + long? PreferredJobId, + bool IsSpecificAttempt); + + private sealed record JobLogData( + GitHubActionsJob Job, + HashSet FailedTests, + IReadOnlyList Occurrences); + + private static void ValidateZipEntries(string zipPath, string extractDirectory) + { + var fullExtractPath = Path.GetFullPath(extractDirectory) + Path.DirectorySeparatorChar; + using var archive = ZipFile.OpenRead(zipPath); + foreach (var entry in archive.Entries) + { + var destinationPath = Path.GetFullPath(Path.Combine(extractDirectory, entry.FullName)); + if (!destinationPath.StartsWith(fullExtractPath, StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Zip entry '{entry.FullName}' would extract outside the target directory."); + } + } + } +} + +public static class FailingTestIssueLogic +{ + private const string DefaultWorkflowSelector = "ci"; + private const int ErrorMessageCommentBudget = 4_000; + private const int IssueBodyMaxLength = 58 * 1024; + private const int MinimumErrorSectionLength = 512; + private const int MinimumStackSectionLength = 512; + private const int MinimumStdoutSectionLength = 512; + private const int ErrorDetailsCollapsibleLineThreshold = 30; + private const string IssueSizeTruncationNote = "Snipped in the middle to keep the issue body under GitHub's 64 KB limit."; + private static readonly IReadOnlyDictionary s_workflowAliases = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["ci"] = ".github/workflows/ci.yml" + }; + + private static readonly Regex s_pullRequestUrlPattern = new(@"^https://github\.com/(?[^/]+)/(?[^/]+)/pull/(?\d+)(?:/.*)?(?:\?.*)?(?:#.*)?$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex s_jobUrlPattern = new(@"^https://github\.com/(?[^/]+)/(?[^/]+)/actions/runs/(?\d+)(?:/attempts/(?\d+))?/job/(?\d+)/?(?:\?.*)?(?:#.*)?$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex s_runUrlPattern = new(@"^https://github\.com/(?[^/]+)/(?[^/]+)/actions/runs/(?\d+)(?:/attempts/(?\d+))?/?(?:\?.*)?(?:#.*)?$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex s_rawWorkflowPathPattern = new(@"^\.github/workflows/[^/\\]+\.(?:yml|yaml)$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + + public static WorkflowSelectorResolution ResolveWorkflowSelector(string? selector) + { + var requestedSelector = string.IsNullOrWhiteSpace(selector) ? DefaultWorkflowSelector : selector.Trim(); + + if (s_workflowAliases.TryGetValue(requestedSelector, out var aliasedWorkflow)) + { + return new WorkflowSelectorResolution(true, requestedSelector, aliasedWorkflow, null); + } + + if (s_rawWorkflowPathPattern.IsMatch(requestedSelector.Replace('\\', '/'))) + { + return new WorkflowSelectorResolution(true, requestedSelector, requestedSelector.Replace('\\', '/'), null); + } + + return new WorkflowSelectorResolution( + Success: false, + Requested: requestedSelector, + ResolvedWorkflowFile: null, + ErrorMessage: $"Unknown workflow selector '{requestedSelector}'. Use the 'ci' alias or a .github/workflows/*.yml path."); + } + + public static SourceUrlResolution ParseSourceUrl(string? sourceUrl, string repository) + { + if (string.IsNullOrWhiteSpace(sourceUrl)) + { + return new SourceUrlResolution(false, null, null, null, null, null, "A PR, workflow run, or workflow job URL is required."); + } + + var value = sourceUrl.Trim(); + + var jobMatch = s_jobUrlPattern.Match(value); + if (jobMatch.Success) + { + if (!MatchesRepository(repository, jobMatch.Groups["owner"].Value, jobMatch.Groups["repo"].Value)) + { + return new SourceUrlResolution(false, null, null, null, null, null, $"The source URL repository does not match '{repository}'."); + } + + return new SourceUrlResolution( + Success: true, + SourceKind: "job", + PullRequestNumber: null, + RunId: long.Parse(jobMatch.Groups["runId"].Value, CultureInfo.InvariantCulture), + RunAttempt: ParseOptionalInt(jobMatch.Groups["attempt"].Value), + JobId: long.Parse(jobMatch.Groups["jobId"].Value, CultureInfo.InvariantCulture), + ErrorMessage: null); + } + + var runMatch = s_runUrlPattern.Match(value); + if (runMatch.Success) + { + if (!MatchesRepository(repository, runMatch.Groups["owner"].Value, runMatch.Groups["repo"].Value)) + { + return new SourceUrlResolution(false, null, null, null, null, null, $"The source URL repository does not match '{repository}'."); + } + + return new SourceUrlResolution( + Success: true, + SourceKind: "run", + PullRequestNumber: null, + RunId: long.Parse(runMatch.Groups["runId"].Value, CultureInfo.InvariantCulture), + RunAttempt: ParseOptionalInt(runMatch.Groups["attempt"].Value), + JobId: null, + ErrorMessage: null); + } + + var pullRequestMatch = s_pullRequestUrlPattern.Match(value); + if (pullRequestMatch.Success) + { + if (!MatchesRepository(repository, pullRequestMatch.Groups["owner"].Value, pullRequestMatch.Groups["repo"].Value)) + { + return new SourceUrlResolution(false, null, null, null, null, null, $"The source URL repository does not match '{repository}'."); + } + + return new SourceUrlResolution( + Success: true, + SourceKind: "pull_request", + PullRequestNumber: int.Parse(pullRequestMatch.Groups["number"].Value, CultureInfo.InvariantCulture), + RunId: null, + RunAttempt: null, + JobId: null, + ErrorMessage: null); + } + + return new SourceUrlResolution(false, null, null, null, null, null, "The source URL must be a GitHub pull request, workflow run, or workflow job URL."); + } + + public static FailureMatchResult MatchTestFailures(string query, IReadOnlyCollection occurrences) + { + var normalizedQuery = NormalizeName(query); + if (string.IsNullOrWhiteSpace(normalizedQuery)) + { + return new FailureMatchResult(false, "A non-empty test query is required.", null, null, null, [], []); + } + + var groups = occurrences + .GroupBy(static occurrence => new MatchGroupKey( + CanonicalName: NormalizeName(occurrence.CanonicalTestName), + DisplayName: NormalizeName(occurrence.DisplayTestName))) + .Select(static group => new MatchGroup(group.Key, group.ToArray())) + .ToArray(); + + var exactCanonicalGroups = groups.Where(group => string.Equals(group.Key.CanonicalName, normalizedQuery, StringComparison.Ordinal)).ToArray(); + if (exactCanonicalGroups.Length == 1) + { + return CreateSuccessfulMatch("exactCanonical", exactCanonicalGroups[0].Occurrences); + } + + if (exactCanonicalGroups.Length > 1) + { + return CreateFailedMatch($"The query '{query}' matched multiple canonical test names.", exactCanonicalGroups); + } + + var exactDisplayGroups = groups.Where(group => string.Equals(group.Key.DisplayName, normalizedQuery, StringComparison.Ordinal)).ToArray(); + if (exactDisplayGroups.Length == 1) + { + return CreateSuccessfulMatch("exactDisplay", exactDisplayGroups[0].Occurrences); + } + + if (exactDisplayGroups.Length > 1) + { + return CreateFailedMatch($"The query '{query}' matched multiple display test names.", exactDisplayGroups); + } + + var partialGroups = groups.Where(group => + group.Key.CanonicalName.Contains(normalizedQuery, StringComparison.Ordinal) + || group.Key.DisplayName.Contains(normalizedQuery, StringComparison.Ordinal)) + .ToArray(); + + if (partialGroups.Length == 1) + { + return CreateSuccessfulMatch("uniqueCaseInsensitiveContains", partialGroups[0].Occurrences); + } + + if (partialGroups.Length > 1) + { + return CreateFailedMatch($"The query '{query}' is ambiguous. Use a more specific canonical or display test name.", partialGroups); + } + + var candidates = occurrences + .Select(static occurrence => FormatCandidateName(occurrence.CanonicalTestName, occurrence.DisplayTestName)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static name => name, StringComparer.OrdinalIgnoreCase) + .Take(20) + .ToArray(); + + return new FailureMatchResult( + Success: false, + ErrorMessage: $"The query '{query}' did not match any failed tests in the available .trx artifacts.", + Strategy: null, + CanonicalTestName: null, + DisplayTestName: null, + Occurrences: [], + CandidateNames: candidates); + } + + public static string ComputeStableSignature(string canonicalTestName, string workflowFile) + { + var normalizedTestName = NormalizeName(canonicalTestName); + var normalizedWorkflowFile = workflowFile.Trim().Replace('\\', '/').ToLowerInvariant(); + var payload = $"{normalizedTestName}|{normalizedWorkflowFile}"; + var hash = XxHash3.Hash(Encoding.UTF8.GetBytes(payload)); + return Convert.ToHexStringLower(hash); + } + + public static TruncationResult TruncateContent(string? content, int maxChars, TruncationPreference preference) + { + if (string.IsNullOrEmpty(content)) + { + return new TruncationResult(string.Empty, false, null); + } + + if (maxChars <= 0 || content.Length <= maxChars) + { + return new TruncationResult(content, false, null); + } + + return preference switch + { + TruncationPreference.Start => new TruncationResult(content[..maxChars], true, $"Truncated to the first {maxChars:N0} characters."), + TruncationPreference.Middle => new TruncationResult( + SnipMiddle(content, maxChars), + true, + $"Snipped in the middle to {maxChars:N0} characters."), + TruncationPreference.End => new TruncationResult(content[^maxChars..], true, $"Truncated to the last {maxChars:N0} characters."), + _ => throw new ArgumentOutOfRangeException(nameof(preference)) + }; + } + + public static IssueArtifacts CreateIssueArtifacts( + WorkflowSection workflow, + ResolutionSection resolution, + FailureMatchResult matchResult, + IReadOnlyList occurrences, + FailedTestOccurrence primaryOccurrence, + IReadOnlyList<(string Name, string Url)> matchingJobs, + IReadOnlyList warnings) + { + var stableSignature = ComputeStableSignature(matchResult.CanonicalTestName!, workflow.ResolvedWorkflowFile); + var metadataMarker = $""; + var title = $"[Failing test]: {EscapeMarkdownInline(matchResult.DisplayTestName)}"; + var testFailingLine = EscapeMarkdownInline(matchResult.CanonicalTestName ?? matchResult.DisplayTestName!); + + var errorTemplateMessage = CreateKnownIssueErrorMessage(primaryOccurrence.ErrorMessage); + + var errorResult = new TruncationResult(primaryOccurrence.ErrorMessage, false, null); + var stackTraceResult = new TruncationResult(primaryOccurrence.StackTrace, false, null); + var stdoutResult = new TruncationResult(primaryOccurrence.Stdout, false, null); + + var body = BuildIssueBody( + resolution, + metadataMarker, + testFailingLine, + primaryOccurrence, + errorTemplateMessage, + errorResult, + stackTraceResult, + stdoutResult); + + if (body.Length > IssueBodyMaxLength) + { + ShrinkSectionUntilWithinLimit(ref stdoutResult, MinimumStdoutSectionLength, () => BuildIssueBody( + resolution, + metadataMarker, + testFailingLine, + primaryOccurrence, + errorTemplateMessage, + errorResult, + stackTraceResult, + stdoutResult)); + + body = BuildIssueBody( + resolution, + metadataMarker, + testFailingLine, + primaryOccurrence, + errorTemplateMessage, + errorResult, + stackTraceResult, + stdoutResult); + } + + if (body.Length > IssueBodyMaxLength) + { + ShrinkSectionUntilWithinLimit(ref stackTraceResult, MinimumStackSectionLength, () => BuildIssueBody( + resolution, + metadataMarker, + testFailingLine, + primaryOccurrence, + errorTemplateMessage, + errorResult, + stackTraceResult, + stdoutResult)); + + body = BuildIssueBody( + resolution, + metadataMarker, + testFailingLine, + primaryOccurrence, + errorTemplateMessage, + errorResult, + stackTraceResult, + stdoutResult); + } + + if (body.Length > IssueBodyMaxLength) + { + ShrinkSectionUntilWithinLimit(ref errorResult, MinimumErrorSectionLength, () => BuildIssueBody( + resolution, + metadataMarker, + testFailingLine, + primaryOccurrence, + errorTemplateMessage, + errorResult, + stackTraceResult, + stdoutResult)); + + body = BuildIssueBody( + resolution, + metadataMarker, + testFailingLine, + primaryOccurrence, + errorTemplateMessage, + errorResult, + stackTraceResult, + stdoutResult); + } + + StringBuilder commentBuilder = new(); + AppendInvariantLine(commentBuilder, $"Another failure for `{EscapeBacktickContent(matchResult.DisplayTestName)}` was resolved from {ToMarkdownLink(resolution.RunUrl)}."); + commentBuilder.AppendLine(); + AppendInvariantLine(commentBuilder, $"- Artifact: `{EscapeBacktickContent(primaryOccurrence.ArtifactName)}`"); + AppendInvariantLine(commentBuilder, $"- `.trx`: `{EscapeBacktickContent(primaryOccurrence.TrxPath)}`"); + if (!string.IsNullOrWhiteSpace(primaryOccurrence.JobName)) + { + AppendInvariantLine(commentBuilder, $"- Job: {ToMarkdownLink(primaryOccurrence.JobUrl, primaryOccurrence.JobName)}"); + } + + commentBuilder.AppendLine(); + var commentErrorResult = TruncateContent(primaryOccurrence.ErrorMessage, ErrorMessageCommentBudget, TruncationPreference.Start); + AppendCodeSection(commentBuilder, "Error message", commentErrorResult.Content, commentErrorResult.Note); + + return new IssueArtifacts( + Title: title, + Labels: ["failing-test"], + StableSignature: stableSignature, + MetadataMarker: metadataMarker, + Body: body.TrimEnd(), + CommentBody: commentBuilder.ToString().TrimEnd()); + } + + public static string FormatCandidateName(string canonicalTestName, string displayTestName) + { + if (string.Equals(canonicalTestName, displayTestName, StringComparison.OrdinalIgnoreCase)) + { + return canonicalTestName; + } + + return $"{canonicalTestName} | {displayTestName}"; + } + + private static string NormalizeName(string value) + => Regex.Replace(value.Trim().ToLowerInvariant(), @"\s+", " "); + + private static bool MatchesRepository(string repository, string owner, string repo) + { + var split = repository.Split('/', count: 2); + return split.Length == 2 + && string.Equals(split[0], owner, StringComparison.OrdinalIgnoreCase) + && string.Equals(split[1], repo, StringComparison.OrdinalIgnoreCase); + } + + private static int? ParseOptionalInt(string value) + => int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var parsedValue) ? parsedValue : null; + + private static FailureMatchResult CreateSuccessfulMatch(string strategy, IReadOnlyList occurrences) + { + var first = occurrences[0]; + return new FailureMatchResult( + Success: true, + ErrorMessage: null, + Strategy: strategy, + CanonicalTestName: first.CanonicalTestName, + DisplayTestName: first.DisplayTestName, + Occurrences: occurrences, + CandidateNames: []); + } + + private static FailureMatchResult CreateFailedMatch(string errorMessage, IReadOnlyCollection groups) + { + var candidates = groups + .Select(static group => FormatCandidateName(group.Occurrences[0].CanonicalTestName, group.Occurrences[0].DisplayTestName)) + .OrderBy(static name => name, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + return new FailureMatchResult( + Success: false, + ErrorMessage: errorMessage, + Strategy: null, + CanonicalTestName: null, + DisplayTestName: null, + Occurrences: [], + CandidateNames: candidates); + } + + private static void AppendCodeSection(StringBuilder builder, string heading, string content, string? truncationNote) + { + AppendInvariantLine(builder, $"#### {heading}"); + builder.AppendLine(); + + if (string.IsNullOrEmpty(content)) + { + builder.AppendLine("_No content captured._"); + builder.AppendLine(); + return; + } + + if (!string.IsNullOrWhiteSpace(truncationNote)) + { + AppendInvariantLine(builder, $"_{truncationNote}_"); + builder.AppendLine(); + } + + var fence = GetMarkdownFence(content); + AppendInvariantLine(builder, $"{fence}text"); + builder.AppendLine(content); + builder.AppendLine(fence); + builder.AppendLine(); + } + + private static string BuildIssueBody( + ResolutionSection resolution, + string metadataMarker, + string testFailingLine, + FailedTestOccurrence primaryOccurrence, + string errorTemplateMessage, + TruncationResult errorResult, + TruncationResult stackTraceResult, + TruncationResult stdoutResult) + { + StringBuilder bodyBuilder = new(); + + bodyBuilder.AppendLine("### Build information"); + bodyBuilder.AppendLine(); + AppendInvariantLine(bodyBuilder, $"Build: {resolution.RunUrl}"); + bodyBuilder.Append("Build error leg or test failing: "); + bodyBuilder.AppendLine(testFailingLine); + if (!string.IsNullOrWhiteSpace(primaryOccurrence.JobUrl)) + { + AppendInvariantLine(bodyBuilder, $"Logs: {ToMarkdownLink(primaryOccurrence.JobUrl, primaryOccurrence.JobName ?? "Job logs")}"); + } + if (!string.IsNullOrWhiteSpace(primaryOccurrence.ArtifactUrl)) + { + AppendInvariantLine(bodyBuilder, $"Artifact: {ToMarkdownLink(primaryOccurrence.ArtifactUrl, primaryOccurrence.ArtifactName)}"); + } + bodyBuilder.AppendLine(); + + bodyBuilder.AppendLine("### Fill in the error message template"); + bodyBuilder.AppendLine(); + bodyBuilder.AppendLine(""); + bodyBuilder.AppendLine("## Error Message"); + bodyBuilder.AppendLine(); + bodyBuilder.AppendLine("Fill the error message using [step by step known issues guidance](https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/KnownIssueJsonStepByStep.md)."); + bodyBuilder.AppendLine(); + bodyBuilder.AppendLine(""); + bodyBuilder.AppendLine(); + var jsonBlock = new StringBuilder(); + jsonBlock.AppendLine("{"); + AppendInvariantLine(jsonBlock, $$""" "ErrorMessage": {{JsonSerializer.Serialize(errorTemplateMessage)}},"""); + jsonBlock.AppendLine(""" "ErrorPattern": "", """.TrimEnd()); + jsonBlock.AppendLine(""" "BuildRetry": false,"""); + jsonBlock.AppendLine(""" "ExcludeConsoleLog": false"""); + jsonBlock.Append('}'); + var jsonContent = jsonBlock.ToString(); + var jsonFence = GetMarkdownFence(jsonContent); + AppendInvariantLine(bodyBuilder, $"{jsonFence}json"); + bodyBuilder.AppendLine(jsonContent); + bodyBuilder.AppendLine(jsonFence); + bodyBuilder.AppendLine(); + + AppendErrorDetailsSection(bodyBuilder, errorResult, stackTraceResult); + AppendStandardOutputSection(bodyBuilder, stdoutResult); + + bodyBuilder.AppendLine(metadataMarker); + + return bodyBuilder.ToString(); + } + + private static string CreateKnownIssueErrorMessage(string? rawErrorMessage) + { + if (string.IsNullOrWhiteSpace(rawErrorMessage)) + { + return string.Empty; + } + + foreach (var line in rawErrorMessage.Split('\n')) + { + var trimmedLine = line.Trim(); + if (!string.IsNullOrWhiteSpace(trimmedLine)) + { + return trimmedLine; + } + } + + return string.Empty; + } + + private static void AppendErrorDetailsSection( + StringBuilder builder, + TruncationResult errorResult, + TruncationResult stackTraceResult) + { + builder.AppendLine("### Error details"); + builder.AppendLine(); + + var notes = new[] { errorResult.Note, stackTraceResult.Note } + .Where(static note => !string.IsNullOrWhiteSpace(note)) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + foreach (var note in notes) + { + AppendInvariantLine(builder, $"_{note}_"); + } + + if (notes.Length > 0) + { + builder.AppendLine(); + } + + var detailsBuilder = new StringBuilder(); + detailsBuilder.Append("Error Message: "); + detailsBuilder.AppendLine(string.IsNullOrWhiteSpace(errorResult.Content) ? "n/a" : errorResult.Content.TrimEnd()); + detailsBuilder.AppendLine("Stack Trace:"); + detailsBuilder.AppendLine(string.IsNullOrWhiteSpace(stackTraceResult.Content) ? "n/a" : stackTraceResult.Content.TrimEnd()); + + var detailsContent = detailsBuilder.ToString().TrimEnd(); + var lineCount = detailsContent.AsSpan().Count('\n') + 1; + var fence = GetMarkdownFence(detailsContent); + var collapsible = lineCount > ErrorDetailsCollapsibleLineThreshold; + + if (collapsible) + { + builder.AppendLine("
"); + AppendInvariantLine(builder, $"Error details ({lineCount} lines)"); + builder.AppendLine(); + } + + AppendInvariantLine(builder, $"{fence}yml"); + builder.AppendLine(detailsContent); + builder.AppendLine(fence); + builder.AppendLine(); + + if (collapsible) + { + builder.AppendLine("
"); + builder.AppendLine(); + } + } + + private static void AppendStandardOutputSection(StringBuilder builder, TruncationResult stdoutResult) + { + builder.AppendLine("
"); + builder.AppendLine("Standard Output"); + builder.AppendLine(); + + if (string.IsNullOrEmpty(stdoutResult.Content)) + { + builder.AppendLine("_No content captured._"); + builder.AppendLine(); + builder.AppendLine("
"); + builder.AppendLine(); + return; + } + + if (!string.IsNullOrWhiteSpace(stdoutResult.Note)) + { + AppendInvariantLine(builder, $"_{stdoutResult.Note}_"); + builder.AppendLine(); + } + + var fence = GetMarkdownFence(stdoutResult.Content); + AppendInvariantLine(builder, $"{fence}yml"); + builder.AppendLine(stdoutResult.Content); + builder.AppendLine(fence); + builder.AppendLine(); + builder.AppendLine("
"); + builder.AppendLine(); + } + + private static void ShrinkSectionUntilWithinLimit(ref TruncationResult section, int minimumLength, Func renderBody) + { + var body = renderBody(); + while (body.Length > IssueBodyMaxLength && section.Content.Length > minimumLength) + { + var excess = body.Length - IssueBodyMaxLength; + var nextLength = Math.Max(minimumLength, section.Content.Length - excess - 256); + if (nextLength >= section.Content.Length) + { + break; + } + + section = TruncateContent(section.Content, nextLength, TruncationPreference.Middle) with + { + Note = IssueSizeTruncationNote + }; + body = renderBody(); + } + } + + private static string SnipMiddle(string content, int maxChars) + { + if (string.IsNullOrEmpty(content) || content.Length <= maxChars) + { + return content; + } + + const string marker = "\n\n... [snipped middle content] ...\n\n"; + if (maxChars <= marker.Length) + { + return content[..maxChars]; + } + + var remaining = maxChars - marker.Length; + var prefixLength = remaining / 2; + var suffixLength = remaining - prefixLength; + + return string.Concat( + content.AsSpan(0, prefixLength), + marker, + content.AsSpan(content.Length - suffixLength)); + } + + private static void AppendInvariantLine(StringBuilder builder, FormattableString value) + { + builder.Append(value.ToString(CultureInfo.InvariantCulture)); + builder.AppendLine(); + } + + private static string EscapeMarkdownInline(string? text) + { + if (string.IsNullOrEmpty(text)) + { + return text ?? string.Empty; + } + + // Escape characters that have special meaning in GitHub-flavoured Markdown. + // We deliberately leave # and > alone because they only trigger at line-start + // and the values we escape are always mid-line. + return Regex.Replace(text, @"([\\`*_\[\]\(\)~])", @"\$1"); + } + + private static string EscapeBacktickContent(string? text) + { + if (string.IsNullOrEmpty(text)) + { + return text ?? string.Empty; + } + + // Replace backticks so they cannot break out of an inline code span. + return text.Replace("`", "'"); + } + + private static string ToMarkdownLink(string? url, string? text = null) + { + if (string.IsNullOrWhiteSpace(url)) + { + return text ?? "n/a"; + } + + var displayText = string.IsNullOrWhiteSpace(text) ? url : text; + // Escape brackets in display text and parentheses in URL to prevent injection. + var safeDisplay = EscapeMarkdownInline(displayText); + var safeUrl = url.Replace("(", "%28").Replace(")", "%29"); + return $"[{safeDisplay}]({safeUrl})"; + } + + private static string GetMarkdownFence(string content) + { + var maxBackticks = 2; + foreach (Match match in Regex.Matches(content, @"`+")) + { + if (match.Length > maxBackticks) + { + maxBackticks = match.Length; + } + } + + return new string('`', maxBackticks + 1); + } + + private sealed record MatchGroupKey( + string CanonicalName, + string DisplayName); + + private sealed record MatchGroup( + MatchGroupKey Key, + IReadOnlyList Occurrences); +} diff --git a/tools/CreateFailingTestIssue/Models.cs b/tools/CreateFailingTestIssue/Models.cs new file mode 100644 index 00000000000..fa827427a0b --- /dev/null +++ b/tools/CreateFailingTestIssue/Models.cs @@ -0,0 +1,185 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Serialization; + +namespace CreateFailingTestIssue; + +public sealed record CommandInput( + string? TestQuery, + string? SourceUrl, + string WorkflowSelector, + string Repository, + bool ForceNew, + bool Create); + +public sealed record CreateFailingTestIssueResult +{ + public required bool Success { get; init; } + + public string? ErrorMessage { get; init; } + + public required InputSection Input { get; init; } + + public ResolutionSection? Resolution { get; init; } + + public MatchSection? Match { get; init; } + + public AllFailuresSection? AllFailures { get; init; } + + public FailureSection? Failure { get; init; } + + public MatrixSection? Matrix { get; init; } + + public IssueSection? Issue { get; init; } + + public required DiagnosticsSection Diagnostics { get; init; } +} + +public sealed record InputSection( + string? TestQuery, + string? SourceUrl, + WorkflowSection Workflow, + bool ForceNew); + +public sealed record WorkflowSection( + string Requested, + string ResolvedWorkflowFile, + string ResolvedWorkflowName); + +public sealed record ResolutionSection( + string SourceKind, + long RunId, + string RunUrl, + int RunAttempt, + int? PullRequestNumber, + IReadOnlyList JobUrls); + +public sealed record MatchSection( + string Query, + string Strategy, + string CanonicalTestName, + string DisplayTestName, + IReadOnlyList AllMatchingOccurrences); + +public sealed record AllFailuresSection( + int FailedTests, + IReadOnlyList Tests); + +public sealed record FailingTestEntry( + string CanonicalTestName, + string DisplayTestName, + int OccurrenceCount, + IReadOnlyList Occurrences, + FailureSection PrimaryFailure); + +public sealed record MatchedOccurrence( + string CanonicalTestName, + string DisplayTestName, + string ArtifactName, + string? ArtifactUrl, + string TrxPath, + string? JobName, + string? JobUrl); + +public sealed record FailureSection( + string ErrorMessage, + string StackTrace, + string Stdout, + string ArtifactName, + string? ArtifactUrl, + string TrxPath, + string? JobName, + string? JobUrl); + +public sealed record MatrixSection( + int MatchedOccurrences, + IReadOnlyList Summary); + +public sealed record MatrixSummaryEntry( + string ArtifactName, + string? ArtifactUrl, + string TrxPath, + string? JobName, + string? JobUrl); + +public sealed record IssueSection( + string Title, + IReadOnlyList Labels, + string StableSignature, + string MetadataMarker, + string Body, + string CommentBody, + ExistingIssueInfo? ExistingIssue, + CreatedIssueInfo? CreatedIssue); + +public sealed record CreatedIssueInfo( + int Number, + string Url); + +public sealed record ExistingIssueInfo( + int Number, + string Url); + +public sealed record DiagnosticsSection( + [property: JsonIgnore] IReadOnlyList Log, + string LogFile, + IReadOnlyList Warnings, + IReadOnlyList AvailableFailedTests); + +public sealed record WorkflowSelectorResolution( + bool Success, + string Requested, + string? ResolvedWorkflowFile, + string? ErrorMessage); + +public sealed record SourceUrlResolution( + bool Success, + string? SourceKind, + int? PullRequestNumber, + long? RunId, + int? RunAttempt, + long? JobId, + string? ErrorMessage); + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum TruncationPreference +{ + Start, + Middle, + End +} + +public sealed record TruncationResult( + string Content, + bool WasTruncated, + string? Note); + +public sealed record FailedTestOccurrence( + string CanonicalTestName, + string DisplayTestName, + string ArtifactName, + string TrxPath, + string ErrorMessage, + string StackTrace, + string Stdout, + string? ArtifactUrl = null, + string? JobName = null, + string? JobUrl = null); + +public sealed record FailureMatchResult( + bool Success, + string? ErrorMessage, + string? Strategy, + string? CanonicalTestName, + string? DisplayTestName, + IReadOnlyList Occurrences, + IReadOnlyList CandidateNames); + +public sealed record IssueArtifacts( + string Title, + IReadOnlyList Labels, + string StableSignature, + string MetadataMarker, + string Body, + string CommentBody); diff --git a/tools/CreateFailingTestIssue/Program.cs b/tools/CreateFailingTestIssue/Program.cs new file mode 100644 index 00000000000..2c093c0a14e --- /dev/null +++ b/tools/CreateFailingTestIssue/Program.cs @@ -0,0 +1,167 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.CommandLine; +using System.Text.Json; + +namespace CreateFailingTestIssue; + +public static class Program +{ + private const string DiagnosticsLogFileName = "diagnostics.log"; + + public static Task Main(string[] args) + { + var testOption = new Option("--test", "-t") + { + Description = "Canonical or display test name to resolve from .trx artifacts. Omit to emit all failing tests." + }; + + var urlOption = new Option("--url", "-u") + { + Description = "Source URL: PR, workflow run, or workflow job." + }; + + var workflowOption = new Option("--workflow", "-w") + { + Description = "Workflow selector alias or workflow file path.", + DefaultValueFactory = _ => "ci" + }; + + var repositoryOption = new Option("--repo", "-r") + { + Description = "GitHub repository in owner/repo form.", + DefaultValueFactory = _ => Environment.GetEnvironmentVariable("GITHUB_REPOSITORY") ?? "microsoft/aspire" + }; + + var forceNewOption = new Option("--force-new") + { + Description = "Bypass idempotent issue reuse and request a fresh issue." + }; + + var createOption = new Option("--create") + { + Description = "Create the issue on GitHub after generating content. Without this flag, the tool only generates the issue body and outputs JSON." + }; + + var dryRunOption = new Option("--dry-run") + { + Description = "Print the raw issue body markdown to stdout and exit. Does not create the issue or emit JSON." + }; + + var outputOption = new Option("--output", "-o") + { + Description = "Write JSON result to a file instead of stdout. Keeps output clean when dotnet build noise is interleaved." + }; + + var rootCommand = new RootCommand("Resolve one failing test, or emit all failing tests, from GitHub Actions artifacts."); + rootCommand.Options.Add(testOption); + rootCommand.Options.Add(urlOption); + rootCommand.Options.Add(workflowOption); + rootCommand.Options.Add(repositoryOption); + rootCommand.Options.Add(forceNewOption); + rootCommand.Options.Add(createOption); + rootCommand.Options.Add(dryRunOption); + rootCommand.Options.Add(outputOption); + + rootCommand.SetAction(async (parseResult, cancellationToken) => + { + var commandInput = new CommandInput( + TestQuery: parseResult.GetValue(testOption), + SourceUrl: parseResult.GetValue(urlOption), + WorkflowSelector: parseResult.GetValue(workflowOption)!, + Repository: parseResult.GetValue(repositoryOption)!, + ForceNew: parseResult.GetValue(forceNewOption), + Create: parseResult.GetValue(createOption)); + + var result = await ExecuteAsync(commandInput, cancellationToken).ConfigureAwait(false); + result = await WriteDiagnosticsLogAsync(result, cancellationToken).ConfigureAwait(false); + + if (parseResult.GetValue(dryRunOption)) + { + var body = result.Issue?.Body; + if (!string.IsNullOrEmpty(body)) + { + Console.WriteLine(body); + return 0; + } + + Console.Error.WriteLine(result.ErrorMessage ?? "No issue body was generated. Ensure --test is provided and matches a failing test."); + return 1; + } + + var json = JsonSerializer.Serialize(result, JsonSerializerOptions); + + var outputPath = parseResult.GetValue(outputOption); + if (!string.IsNullOrEmpty(outputPath)) + { + await File.WriteAllTextAsync(outputPath, json, cancellationToken).ConfigureAwait(false); + } + else + { + Console.WriteLine(json); + } + + return result.Success ? 0 : 1; + }); + + return rootCommand.Parse(args).InvokeAsync(); + } + + private static async Task ExecuteAsync(CommandInput input, CancellationToken cancellationToken) + { + try + { + return await FailingTestIssueCommand.ExecuteAsync(input, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + return new CreateFailingTestIssueResult + { + Success = false, + ErrorMessage = ex.Message, + Input = new InputSection( + TestQuery: input.TestQuery, + SourceUrl: input.SourceUrl, + Workflow: new WorkflowSection( + Requested: input.WorkflowSelector, + ResolvedWorkflowFile: string.Empty, + ResolvedWorkflowName: string.Empty), + ForceNew: input.ForceNew), + Diagnostics = new DiagnosticsSection( + Log: + [ + "Unhandled exception during command execution.", + ex.ToString() + ], + LogFile: DiagnosticsLogFileName, + Warnings: [ex.ToString()], + AvailableFailedTests: []) + }; + } + } + + private static async Task WriteDiagnosticsLogAsync(CreateFailingTestIssueResult result, CancellationToken cancellationToken) + { + try + { + await File.WriteAllLinesAsync(DiagnosticsLogFileName, result.Diagnostics.Log, cancellationToken).ConfigureAwait(false); + return result; + } + catch (Exception ex) + { + return result with + { + Diagnostics = result.Diagnostics with + { + Warnings = result.Diagnostics.Warnings.Concat([$"Failed to write {DiagnosticsLogFileName}: {ex.Message}"]).ToArray() + } + }; + } + } + + private static JsonSerializerOptions JsonSerializerOptions { get; } = new(JsonSerializerDefaults.Web) + { + WriteIndented = true + }; +} diff --git a/tools/GenerateTestSummary/GenerateTestSummary.csproj b/tools/GenerateTestSummary/GenerateTestSummary.csproj index adf1a87a3bd..32a12e2d3af 100644 --- a/tools/GenerateTestSummary/GenerateTestSummary.csproj +++ b/tools/GenerateTestSummary/GenerateTestSummary.csproj @@ -11,4 +11,8 @@ + + + + diff --git a/tools/GenerateTestSummary/TestSummaryGenerator.cs b/tools/GenerateTestSummary/TestSummaryGenerator.cs index 7ca87f3bdc2..5e55d1cfad5 100644 --- a/tools/GenerateTestSummary/TestSummaryGenerator.cs +++ b/tools/GenerateTestSummary/TestSummaryGenerator.cs @@ -270,7 +270,7 @@ public static void CreateSingleTestSummaryReport(string trxFilePath, StringBuild reportBuilder.AppendLine(); reportBuilder.AppendLine("```yml"); - reportBuilder.AppendLine(test.Output?.ErrorInfo?.InnerText); + reportBuilder.AppendLine(test.Output?.ErrorInfoString); if (test.Output?.StdOut is not null) { const int halfLength = 5_000; diff --git a/tools/scripts/DownloadFailingJobLogs.cs b/tools/scripts/DownloadFailingJobLogs.cs index e216eddcc18..f693eb9b33f 100644 --- a/tools/scripts/DownloadFailingJobLogs.cs +++ b/tools/scripts/DownloadFailingJobLogs.cs @@ -1,10 +1,12 @@ +#:project ../Aspire.TestTools/Aspire.TestTools.csproj + // Downloads logs and artifacts for failed jobs from a GitHub Actions workflow run. // Usage: dotnet run DownloadFailingJobLogs.cs using System.Globalization; using System.IO.Compression; -using System.Text.Json; using System.Text.RegularExpressions; +using Aspire.TestTools; if (args.Length == 0) { @@ -13,61 +15,23 @@ return; } -var runId = args[0]; -var repo = "microsoft/aspire"; - -Console.WriteLine($"Finding failed jobs for run {runId}..."); - -// Fetch all jobs for this run (paginated, 100 per page) -var jobs = new List(); -int jobsPage = 1; -bool hasMoreJobPages = true; - -while (hasMoreJobPages) +if (!long.TryParse(args[0], NumberStyles.None, CultureInfo.InvariantCulture, out var runId)) { - var (success, jobsJson) = await RunGhAsync($"api repos/{repo}/actions/runs/{runId}/jobs?page={jobsPage}&per_page=100"); - if (!success) - { - Console.WriteLine($"Error getting jobs page {jobsPage}: {jobsJson}"); - return; - } - - using var jobsDoc = JsonDocument.Parse(jobsJson); - if (jobsDoc.RootElement.TryGetProperty("jobs", out var jobsArray)) - { - var jobsOnPage = jobsArray.GetArrayLength(); - if (jobsOnPage == 0) - { - hasMoreJobPages = false; - } - - foreach (var job in jobsArray.EnumerateArray()) - { - jobs.Add(job.Clone()); - } + Console.WriteLine($"Invalid run id '{args[0]}'."); + return; +} - // If we got fewer than 100 jobs, this is the last page - if (jobsOnPage < 100) - { - hasMoreJobPages = false; - } - } - else - { - hasMoreJobPages = false; - } +var repo = "microsoft/aspire"; +var cancellationToken = CancellationToken.None; - jobsPage++; -} +Console.WriteLine($"Finding failed jobs for run {runId}..."); +var jobs = await GitHubActionsApi.ListJobsAsync(repo, runId, runAttempt: null, cancellationToken).ConfigureAwait(false); Console.WriteLine($"Found {jobs.Count} total jobs"); -// Filter failed jobs -var failedJobs = jobs.Where(j => -{ - var conclusion = j.GetProperty("conclusion").GetString(); - return conclusion == "failure"; -}).ToList(); +var failedJobs = jobs + .Where(static job => string.Equals(job.Conclusion, "failure", StringComparison.OrdinalIgnoreCase)) + .ToList(); Console.WriteLine($"Found {failedJobs.Count} failed jobs"); @@ -77,54 +41,49 @@ return; } -// Download logs and artifacts for each failed job -int counter = 0; -foreach (var job in failedJobs) +var artifacts = await GitHubActionsApi.ListArtifactsAsync(repo, runId, cancellationToken).ConfigureAwait(false); +Console.WriteLine($"Found {artifacts.Count} total artifacts"); + +var logsDownloaded = 0; +for (var counter = 0; counter < failedJobs.Count; counter++) { - var jobId = job.GetProperty("id").GetInt64(); - var jobName = job.GetProperty("name").GetString(); - var jobUrl = job.GetProperty("html_url").GetString(); + var job = failedJobs[counter]; Console.WriteLine($"\n=== Failed Job {counter + 1}/{failedJobs.Count} ==="); - Console.WriteLine($"Name: {jobName}"); - Console.WriteLine($"ID: {jobId}"); - Console.WriteLine($"URL: {jobUrl}"); + Console.WriteLine($"Name: {job.Name}"); + Console.WriteLine($"ID: {job.Id}"); + Console.WriteLine($"URL: {job.HtmlUrl}"); - // Download job logs Console.WriteLine("Downloading job logs..."); - var (logsSuccess, logs) = await RunGhAsync($"api repos/{repo}/actions/jobs/{jobId}/logs"); - - if (logsSuccess) + try { - // Save logs to file - var safeName = Regex.Replace(jobName ?? $"job_{jobId}", @"[^a-zA-Z0-9_-]", "_"); + var logs = await GitHubActionsApi.DownloadJobLogAsync(repo, job.Id, cancellationToken).ConfigureAwait(false); + var safeName = Regex.Replace(job.Name ?? $"job_{job.Id}", @"[^a-zA-Z0-9_-]", "_"); var filename = $"failed_job_{counter}_{safeName}.log"; - File.WriteAllText(filename, logs); + await File.WriteAllTextAsync(filename, logs, cancellationToken).ConfigureAwait(false); Console.WriteLine($"Saved job logs to: {filename} ({logs.Length} characters)"); + logsDownloaded++; - // Parse logs for test failures and exceptions Console.WriteLine("\nSearching for test failures in job logs..."); var failedTestPattern = @"Failed\s+(.+?)\s*\["; var errorPattern = @"Error Message:\s*(.+?)(?:\r?\n|$)"; var exceptionPattern = @"(System\.\w+Exception:.+?)(?:\r?\n at|\r?\n\r?\n|$)"; - var failedTests = new HashSet(); - var errors = new List(); - - foreach (Match match in Regex.Matches(logs, failedTestPattern)) - { - failedTests.Add(match.Groups[1].Value.Trim()); - } + var failedTests = Regex.Matches(logs, failedTestPattern) + .Select(match => match.Groups[1].Value.Trim()) + .Where(static name => !string.IsNullOrWhiteSpace(name)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); - foreach (Match match in Regex.Matches(logs, errorPattern)) - { - errors.Add(match.Groups[1].Value.Trim()); - } + var errors = Regex.Matches(logs, errorPattern) + .Select(match => match.Groups[1].Value.Trim()) + .Where(static error => !string.IsNullOrWhiteSpace(error)) + .ToList(); foreach (Match match in Regex.Matches(logs, exceptionPattern, RegexOptions.Singleline)) { var exception = match.Groups[1].Value.Trim(); - if (!errors.Contains(exception)) + if (!errors.Contains(exception, StringComparer.Ordinal)) { errors.Add(exception); } @@ -157,186 +116,92 @@ } } } - else + catch (Exception ex) { - Console.WriteLine($"Error downloading job logs: {logs}"); + Console.WriteLine($"Error downloading job logs: {ex.Message}"); } - // Try to find and download the test artifact based on job name pattern - // e.g. "Tests / Integrations macos (Hosting.Azure) / Hosting.Azure (macos-latest)" -> "logs-Hosting.Azure-macos-latest" - var artifactMatch = Regex.Match(jobName ?? "", @".*\(([^)]+)\)\s*/\s*\S+\s+\(([^)]+)\)"); - if (artifactMatch.Success) + var artifactMatch = Regex.Match(job.Name ?? string.Empty, @".*\(([^)]+)\)\s*/\s*\S+\s+\(([^)]+)\)"); + if (!artifactMatch.Success) { - var testShortName = artifactMatch.Groups[1].Value.Trim(); - var os = artifactMatch.Groups[2].Value.Trim(); - var artifactName = $"logs-{testShortName}-{os}"; - - Console.WriteLine($"\nAttempting to download artifact: {artifactName}"); + Console.WriteLine("\nCould not parse job name to determine artifact name."); + continue; + } - // Search for matching artifact (paginated, 100 per page) - string? artifactId = null; - var allArtifactNames = new List(); - int page = 1; - bool hasMorePages = true; + var testShortName = artifactMatch.Groups[1].Value.Trim(); + var os = artifactMatch.Groups[2].Value.Trim(); + var artifactName = $"logs-{testShortName}-{os}"; - while (hasMorePages) - { - var (artifactsSuccess, artifactsJson) = await RunGhAsync($"api repos/{repo}/actions/runs/{runId}/artifacts?page={page}&per_page=100"); - if (!artifactsSuccess) - { - Console.WriteLine($"Error getting artifacts page {page}: {artifactsJson}"); - break; - } + Console.WriteLine($"\nAttempting to download artifact: {artifactName}"); - using var artifactsDoc = JsonDocument.Parse(artifactsJson); - if (artifactsDoc.RootElement.TryGetProperty("artifacts", out var artifactsArray)) - { - var artifactsOnPage = artifactsArray.GetArrayLength(); - if (artifactsOnPage == 0) - { - hasMorePages = false; - } + var artifact = artifacts.FirstOrDefault(candidate => string.Equals(candidate.Name, artifactName, StringComparison.Ordinal)); + if (artifact is null) + { + Console.WriteLine($"Artifact '{artifactName}' not found for this run."); + continue; + } - foreach (var artifact in artifactsArray.EnumerateArray()) - { - if (artifact.TryGetProperty("name", out var nameProperty)) - { - var name = nameProperty.GetString(); - allArtifactNames.Add(name ?? "null"); + Console.WriteLine($"Found artifact ID: {artifact.Id}"); - if (name == artifactName && artifact.TryGetProperty("id", out var idProperty)) - { - artifactId = idProperty.GetInt64().ToString(CultureInfo.InvariantCulture); - } - } - } + var artifactZip = $"artifact_{counter}_{testShortName}_{os}.zip"; + try + { + await GitHubActionsApi.DownloadArtifactZipAsync(repo, artifact.Id, artifactZip, cancellationToken).ConfigureAwait(false); + Console.WriteLine($"Downloaded artifact to: {artifactZip}"); - // If we got fewer than 100 artifacts, this is the last page - if (artifactsOnPage < 100) - { - hasMorePages = false; - } - } - else + var extractDir = $"artifact_{counter}_{testShortName}_{os}"; + try + { + if (Directory.Exists(extractDir)) { - hasMorePages = false; + Directory.Delete(extractDir, recursive: true); } - page++; - } - - Console.WriteLine($"Found {allArtifactNames.Count} total artifacts"); - - if (artifactId != null) - { - Console.WriteLine($"Found artifact ID: {artifactId}"); - - // Download artifact - var artifactZip = $"artifact_{counter}_{testShortName}_{os}.zip"; - var downloadSuccess = await DownloadGhFileAsync($"api repos/{repo}/actions/artifacts/{artifactId}/zip", artifactZip); + ValidateZipEntries(artifactZip, extractDir); + ZipFile.ExtractToDirectory(artifactZip, extractDir, overwriteFiles: true); + Console.WriteLine($"Extracted artifact to: {extractDir}"); - if (downloadSuccess) + var trxFiles = Directory.GetFiles(extractDir, "*.trx", SearchOption.AllDirectories); + if (trxFiles.Length > 0) { - Console.WriteLine($"Downloaded artifact to: {artifactZip}"); - - // Extract and look for .trx test result files - var extractDir = $"artifact_{counter}_{testShortName}_{os}"; - try - { - if (Directory.Exists(extractDir)) - { - Directory.Delete(extractDir, true); - } - - ZipFile.ExtractToDirectory(artifactZip, extractDir); - Console.WriteLine($"Extracted artifact to: {extractDir}"); - - var trxFiles = Directory.GetFiles(extractDir, "*.trx", SearchOption.AllDirectories); - if (trxFiles.Length > 0) - { - Console.WriteLine($"\nFound {trxFiles.Length} .trx file(s):"); - foreach (var trxFile in trxFiles) - { - Console.WriteLine($" - {trxFile}"); - } - } - else - { - Console.WriteLine("\nNo .trx files found in artifact."); - } - } - catch (Exception ex) + Console.WriteLine($"\nFound {trxFiles.Length} .trx file(s):"); + foreach (var trxFile in trxFiles) { - Console.WriteLine($"Error extracting artifact: {ex.Message}"); + Console.WriteLine($" - {trxFile}"); } } else { - Console.WriteLine("Error downloading artifact"); + Console.WriteLine("\nNo .trx files found in artifact."); } } - else + catch (Exception ex) { - Console.WriteLine($"Artifact '{artifactName}' not found for this run."); + Console.WriteLine($"Error extracting artifact: {ex.Message}"); } } - else + catch (Exception ex) { - Console.WriteLine("\nCould not parse job name to determine artifact name."); + Console.WriteLine($"Error downloading artifact: {ex.Message}"); } - - counter++; } -Console.WriteLine($"\n=== Summary ==="); +Console.WriteLine("\n=== Summary ==="); Console.WriteLine($"Total jobs: {jobs.Count}"); Console.WriteLine($"Failed jobs: {failedJobs.Count}"); -Console.WriteLine($"Logs downloaded: {counter}"); -Console.WriteLine($"\nAll logs saved in current directory with pattern: failed_job_*.log"); - -// Runs gh CLI and returns the text output -async Task<(bool Success, string Output)> RunGhAsync(string arguments) -{ - using var process = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo - { - FileName = "gh", - Arguments = arguments, - RedirectStandardOutput = true, - UseShellExecute = false - }); - - if (process is null) - { - return (false, "Failed to start process"); - } - - var output = await process.StandardOutput.ReadToEndAsync().ConfigureAwait(false); - await process.WaitForExitAsync().ConfigureAwait(false); +Console.WriteLine($"Logs downloaded: {logsDownloaded}"); +Console.WriteLine("\nAll logs saved in current directory with pattern: failed_job_*.log"); - return (process.ExitCode == 0, output); -} - -// Runs gh CLI and streams binary output to a file -async Task DownloadGhFileAsync(string arguments, string outputPath) +static void ValidateZipEntries(string zipPath, string extractDirectory) { - using var process = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo - { - FileName = "gh", - Arguments = arguments, - RedirectStandardOutput = true, - UseShellExecute = false - }); - - if (process is null) - { - return false; - } - - using (var fileStream = File.Create(outputPath)) + var fullExtractPath = Path.GetFullPath(extractDirectory) + Path.DirectorySeparatorChar; + using var archive = ZipFile.OpenRead(zipPath); + foreach (var entry in archive.Entries) { - await process.StandardOutput.BaseStream.CopyToAsync(fileStream).ConfigureAwait(false); + var destinationPath = Path.GetFullPath(Path.Combine(extractDirectory, entry.FullName)); + if (!destinationPath.StartsWith(fullExtractPath, StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Zip entry '{entry.FullName}' would extract outside the target directory."); + } } - await process.WaitForExitAsync().ConfigureAwait(false); - - return process.ExitCode == 0; }