diff --git a/CLAUDE.md b/CLAUDE.md index ef031db..bb2a78b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,23 +43,24 @@ The upload flow has two stages handled by two classes, with a shared `MarkerPars - Also exports a standalone `formatMarker()` function used by parsers 2. **`ResultUploadCommandHandler`** — Orchestrates the overall flow: - - Parses report inputs using the appropriate parser (JUnit XML file, Playwright JSON file, or Allure results directory). File-based parsers receive file contents; directory-based parsers (Allure) receive the path — controlled by the module-level `directoryInputTypes` Set + - Parses report inputs using the appropriate parser (JUnit XML file, Playwright JSON file, or Allure results directory), which return `ParseResult` objects containing both `testCaseResults` and `runFailureLogs`. File-based parsers receive file contents; directory-based parsers (Allure) receive the path — controlled by the module-level `directoryInputTypes` Set - `ParserOptions` includes `allowPartialParse` (set from `--force`) to skip invalid files instead of aborting - Detects project code from test case names via `MarkerParser` (or from `--run-url`) - Creates a new test run (or reuses an existing one if title conflicts) - - Delegates actual result uploading to `ResultUploader` + - Collects run-level logs from all parsed files and passes them to `ResultUploader` 3. **`ResultUploader`** — Handles the upload-to-run mechanics: - Fetches test cases from the run, maps parsed results to them via `MarkerParser` matching - Validates unmatched/missing test cases (respects `--force`, `--ignore-unmatched`) + - If run-level log is present, uploads it via `createRunLog` API before uploading test case results - Uploads file attachments concurrently (max 10 parallel), then creates results in batches (max 50 per request) ### Report Parsers -- `junitXmlParser.ts` — Parses JUnit XML via `xml2js` + Zod validation. Extracts attachments from `[[ATTACHMENT|path]]` markers in system-out/failure/error/skipped elements. -- `playwrightJsonParser.ts` — Parses Playwright JSON report. Supports two test case linking methods: (1) test annotations with `type: "test case"` and URL description, (2) marker in test name. Handles nested suites recursively. +- `junitXmlParser.ts` — Parses JUnit XML via `xml2js` + Zod validation. Extracts attachments from `[[ATTACHMENT|path]]` markers in system-out/failure/error/skipped elements. Extracts suite-level `` and empty-name `` errors as run level error logs. +- `playwrightJsonParser.ts` — Parses Playwright JSON report. Supports two test case linking methods: (1) test annotations with `type: "test case"` and URL description, (2) marker in test name. Handles nested suites recursively. Extracts top-level `errors[]` as run level error logs. - `allureParser.ts` — Parses Allure 2 JSON results directories (`*-result.json` files only; containers/XML/images ignored). Supports test case linking via TMS links (`type: "tms"`) or marker in test name, maps Allure statuses to QA Sphere result statuses (`unknown→open`, `broken→blocked`), strips ANSI codes and HTML-escapes messages, and resolves attachments via `attachments[].source`. Uses `formatMarker()` from `MarkerParser`. -- `types.ts` — Shared `TestCaseResult` and `Attachment` interfaces used by both parsers. +- `types.ts` — Shared `TestCaseResult`, `ParseResult`, and `Attachment` interfaces used by both parsers. ### API Layer (src/api/) diff --git a/README.md b/README.md index 2e5932d..2e24eb8 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The QAS CLI is a command-line tool for submitting your test automation results to [QA Sphere](https://qasphere.com/). It provides the most efficient way to collect and report test results from your test automation workflow, CI/CD pipeline, and build servers. -The tool can upload test case results from JUnit XML files, Playwright JSON files, and Allure result directories to QA Sphere test runs by matching test case references to QA Sphere test cases. +The tool can upload test case results from JUnit XML files, Playwright JSON files, and Allure result directories to QA Sphere test runs by matching test case references to QA Sphere test cases. It also automatically detects global or suite-level failures (e.g., setup/teardown errors) and uploads them as run-level logs. ## Installation @@ -264,6 +264,13 @@ Allure results use one `*-result.json` file per test in a results directory. `al Only Allure 2 JSON (`*-result.json`) is supported. Legacy Allure 1 XML files are ignored. +## Run-Level Logs + +The CLI automatically detects global or suite-level failures and uploads them as run-level logs to QA Sphere. These failures are typically caused by setup/teardown issues that aren't tied to specific test cases. + +- **JUnit XML**: Suite-level `` elements and empty-name `` entries with `` or `` (synthetic entries from setup/teardown failures, e.g., Maven Surefire) are extracted as run-level logs. Empty-name testcases are excluded from individual test case results. +- **Playwright JSON**: Top-level `errors` array entries (global setup/teardown failures) are extracted as run-level logs. + ## Development (for those who want to contribute to the tool) 1. Install and build: `npm install && npm run build && npm link` diff --git a/src/api/run.ts b/src/api/run.ts index 013ada2..9673007 100644 --- a/src/api/run.ts +++ b/src/api/run.ts @@ -17,6 +17,10 @@ export interface CreateRunResponse { id: number } +export interface CreateRunLogRequest { + comment: string +} + export const createRunApi = (fetcher: typeof fetch) => { fetcher = withJson(fetcher) return { @@ -36,6 +40,12 @@ export const createRunApi = (fetcher: typeof fetch) => { method: 'POST', body: JSON.stringify(req), }).then((r) => jsonResponse(r)), + + createRunLog: (projectCode: ResourceId, runId: ResourceId, req: CreateRunLogRequest) => + fetcher(`/api/public/v0/project/${projectCode}/run/${runId}/log`, { + method: 'POST', + body: JSON.stringify(req), + }).then((r) => jsonResponse<{ id: string }>(r)), } } diff --git a/src/tests/allure-parsing.spec.ts b/src/tests/allure-parsing.spec.ts index 67537ed..3b11a2d 100644 --- a/src/tests/allure-parsing.spec.ts +++ b/src/tests/allure-parsing.spec.ts @@ -43,7 +43,7 @@ afterEach(async () => { describe('Allure parsing', () => { test('Should parse matching directory with marker extraction and status mapping', async () => { const dir = `${allureBasePath}/matching-tcases` - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', allowPartialParse: false, @@ -85,7 +85,7 @@ describe('Allure parsing', () => { ), }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -107,13 +107,13 @@ describe('Allure parsing', () => { ), }) - const skippedOnSuccess = await parseAllureResults(dir, dir, { + const { testCaseResults: skippedOnSuccess } = await parseAllureResults(dir, dir, { skipStdout: 'on-success', skipStderr: 'on-success', }) expect(skippedOnSuccess[0].message).toBe('') - const neverSkip = await parseAllureResults(dir, dir, { + const { testCaseResults: neverSkip } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -159,7 +159,7 @@ describe('Allure parsing', () => { '005-result.json': JSON.stringify(makeResult({ name: 'TEST-305 has no labels', labels: [] })), }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -173,7 +173,7 @@ describe('Allure parsing', () => { test('Should keep attachment errors without crashing parse', async () => { const dir = `${allureBasePath}/missing-attachments` - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -230,7 +230,7 @@ describe('Allure parsing', () => { stop: 20, }), }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', allowPartialParse: true, @@ -267,7 +267,7 @@ describe('Allure parsing', () => { ), }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -290,7 +290,7 @@ describe('Allure parsing', () => { }) ), }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -311,7 +311,7 @@ describe('Allure parsing', () => { }) ), }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -333,7 +333,7 @@ describe('Allure parsing', () => { }) ), }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -349,7 +349,7 @@ describe('Allure parsing', () => { 'report.xml': '', 'screenshot.png': 'fake-png-data', }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -359,7 +359,7 @@ describe('Allure parsing', () => { test('Should return empty array for empty directory', async () => { const dir = await createTempAllureDir({}) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -379,7 +379,7 @@ describe('Allure parsing', () => { }) ), }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'on-success', skipStderr: 'on-success', }) @@ -397,7 +397,7 @@ describe('Allure parsing', () => { }) ), }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -412,7 +412,7 @@ describe('Allure parsing', () => { }) ), }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -429,7 +429,7 @@ describe('Allure parsing', () => { }) ), }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -445,7 +445,7 @@ describe('Allure parsing', () => { }) ), }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -462,7 +462,7 @@ describe('Allure parsing', () => { }) ), }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -487,7 +487,7 @@ describe('Allure parsing', () => { ), 'step-log.txt': 'step log content', }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) @@ -530,7 +530,7 @@ describe('Allure parsing', () => { 'level1.txt': 'level1', 'level2.txt': 'level2', }) - const testcases = await parseAllureResults(dir, dir, { + const { testCaseResults: testcases } = await parseAllureResults(dir, dir, { skipStdout: 'never', skipStderr: 'never', }) diff --git a/src/tests/fixtures/junit-xml/suite-level-errors.xml b/src/tests/fixtures/junit-xml/suite-level-errors.xml new file mode 100644 index 0000000..3f7cc18 --- /dev/null +++ b/src/tests/fixtures/junit-xml/suite-level-errors.xml @@ -0,0 +1,16 @@ + + + + Failed to initialize database connection +java.sql.SQLException: Connection refused + + java.lang.RuntimeException: BeforeAll setup failed + at com.example.SetupFailureTest.setup(SetupFailureTest.java:15) + + + + + Expected true but got false + + + diff --git a/src/tests/junit-xml-parsing.spec.ts b/src/tests/junit-xml-parsing.spec.ts index aa0f90c..b340247 100644 --- a/src/tests/junit-xml-parsing.spec.ts +++ b/src/tests/junit-xml-parsing.spec.ts @@ -10,7 +10,7 @@ describe('Junit XML parsing', () => { const xmlContent = await readFile(xmlPath, 'utf8') // This should not throw any exceptions - const testcases = await parseJUnitXml(xmlContent, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xmlContent, xmlBasePath, { skipStdout: 'never', skipStderr: 'never', }) @@ -55,7 +55,7 @@ describe('Junit XML parsing', () => { const xmlPath = `${xmlBasePath}/comprehensive-test.xml` const xmlContent = await readFile(xmlPath, 'utf8') - const testcases = await parseJUnitXml(xmlContent, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xmlContent, xmlBasePath, { skipStdout: 'never', skipStderr: 'never', }) @@ -86,7 +86,7 @@ describe('Junit XML parsing', () => { const xmlPath = `${xmlBasePath}/empty-system-err.xml` const xmlContent = await readFile(xmlPath, 'utf8') - const testcases = await parseJUnitXml(xmlContent, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xmlContent, xmlBasePath, { skipStdout: 'never', skipStderr: 'never', }) @@ -103,7 +103,7 @@ describe('Junit XML parsing', () => { const xmlPath = `${xmlBasePath}/jest-failure-type-missing.xml` const xmlContent = await readFile(xmlPath, 'utf8') - const testcases = await parseJUnitXml(xmlContent, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xmlContent, xmlBasePath, { skipStdout: 'never', skipStderr: 'never', }) @@ -132,7 +132,7 @@ describe('Junit XML parsing', () => { const xmlPath = `${xmlBasePath}/pytest-style.xml` const xmlContent = await readFile(xmlPath, 'utf8') - const testcases = await parseJUnitXml(xmlContent, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xmlContent, xmlBasePath, { skipStdout: 'never', skipStderr: 'never', }) @@ -168,7 +168,7 @@ describe('Junit XML parsing', () => { ` - const testcases = await parseJUnitXml(xml, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xml, xmlBasePath, { skipStdout: 'never', skipStderr: 'never', }) @@ -181,7 +181,7 @@ describe('Junit XML parsing', () => { const xmlPath = `${xmlBasePath}/webdriverio-real.xml` const xmlContent = await readFile(xmlPath, 'utf8') - const testcases = await parseJUnitXml(xmlContent, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xmlContent, xmlBasePath, { skipStdout: 'never', skipStderr: 'never', }) @@ -203,7 +203,7 @@ describe('Junit XML parsing', () => { const xmlPath = `${xmlBasePath}/empty-system-err.xml` const xmlContent = await readFile(xmlPath, 'utf8') - const testcases = await parseJUnitXml(xmlContent, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xmlContent, xmlBasePath, { skipStdout: 'never', skipStderr: 'never', }) @@ -218,7 +218,7 @@ describe('Junit XML parsing', () => { const xmlPath = `${xmlBasePath}/empty-system-err.xml` const xmlContent = await readFile(xmlPath, 'utf8') - const testcases = await parseJUnitXml(xmlContent, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xmlContent, xmlBasePath, { skipStdout: 'on-success', skipStderr: 'never', }) @@ -241,7 +241,7 @@ describe('Junit XML parsing', () => { ` - const testcases = await parseJUnitXml(xml, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xml, xmlBasePath, { skipStdout: 'never', skipStderr: 'on-success', }) @@ -266,7 +266,7 @@ describe('Junit XML parsing', () => { ` - const testcases = await parseJUnitXml(xml, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xml, xmlBasePath, { skipStdout: 'on-success', skipStderr: 'on-success', }) @@ -291,7 +291,7 @@ describe('Junit XML parsing', () => { ` - const testcases = await parseJUnitXml(xml, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xml, xmlBasePath, { skipStdout: 'on-success', skipStderr: 'on-success', }) @@ -311,7 +311,7 @@ describe('Junit XML parsing', () => { ` - const testcases = await parseJUnitXml(xml, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xml, xmlBasePath, { skipStdout: 'never', skipStderr: 'never', }) @@ -330,7 +330,7 @@ describe('Junit XML parsing', () => { ` - const testcases = await parseJUnitXml(xml, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xml, xmlBasePath, { skipStdout: 'never', skipStderr: 'never', }) @@ -350,7 +350,7 @@ describe('Junit XML parsing', () => { ` - const testcases = await parseJUnitXml(xml, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xml, xmlBasePath, { skipStdout: 'never', skipStderr: 'never', }) @@ -372,7 +372,7 @@ describe('Junit XML parsing', () => { ` - const testcases = await parseJUnitXml(xml, xmlBasePath, { + const { testCaseResults: testcases } = await parseJUnitXml(xml, xmlBasePath, { skipStdout: 'never', skipStderr: 'never', }) @@ -381,4 +381,44 @@ describe('Junit XML parsing', () => { expect(testcases[0].name).toBe('test1') expect(testcases[0].folder).toBe('') }) + + test('Should return empty runFailureLogs when no suite-level errors', async () => { + const xml = ` + + + + + +` + + const { runFailureLogs } = await parseJUnitXml(xml, xmlBasePath, { + skipStdout: 'never', + skipStderr: 'never', + }) + + expect(runFailureLogs).toBe('') + }) + + test('Should extract suite-level system-err and empty-name testcase errors into runFailureLogs', async () => { + const xmlPath = `${xmlBasePath}/suite-level-errors.xml` + const xmlContent = await readFile(xmlPath, 'utf8') + + const { testCaseResults, runFailureLogs } = await parseJUnitXml(xmlContent, xmlBasePath, { + skipStdout: 'never', + skipStderr: 'never', + }) + + // Empty-name testcase should be excluded from testCaseResults + expect(testCaseResults).toHaveLength(2) + expect(testCaseResults.every((tc) => tc.name !== '')).toBe(true) + + // Suite name header should appear only once, followed by both system-err and empty-name testcase error + expect(runFailureLogs).toBe( + '

com.example.SetupFailureTest

' + + '
Failed to initialize database connection\n' +
+				'java.sql.SQLException: Connection refused
' + + '
java.lang.RuntimeException: BeforeAll setup failed\n' +
+				'\tat com.example.SetupFailureTest.setup(SetupFailureTest.java:15)
' + ) + }) }) diff --git a/src/tests/playwright-json-parsing.spec.ts b/src/tests/playwright-json-parsing.spec.ts index 89f2184..02f7d2d 100644 --- a/src/tests/playwright-json-parsing.spec.ts +++ b/src/tests/playwright-json-parsing.spec.ts @@ -10,7 +10,7 @@ describe('Playwright JSON parsing', () => { const jsonContent = await readFile(jsonPath, 'utf8') // This should not throw any exceptions - const testcases = await parsePlaywrightJson(jsonContent, '', { + const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', { skipStdout: 'never', skipStderr: 'never', }) @@ -53,7 +53,7 @@ describe('Playwright JSON parsing', () => { const jsonPath = `${playwrightJsonBasePath}/empty-tsuite.json` const jsonContent = await readFile(jsonPath, 'utf8') - const testcases = await parsePlaywrightJson(jsonContent, '', { + const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', { skipStdout: 'never', skipStderr: 'never', }) @@ -108,7 +108,7 @@ describe('Playwright JSON parsing', () => { ], }) - const testcases = await parsePlaywrightJson(jsonContent, '', { + const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', { skipStdout: 'never', skipStderr: 'never', }) @@ -185,7 +185,7 @@ describe('Playwright JSON parsing', () => { ], }) - const testcases = await parsePlaywrightJson(jsonContent, '', { + const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', { skipStdout: 'never', skipStderr: 'never', }) @@ -252,7 +252,7 @@ describe('Playwright JSON parsing', () => { ], }) - const testcases = await parsePlaywrightJson(jsonContent, '', { + const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', { skipStdout: 'never', skipStderr: 'never', }) @@ -363,7 +363,7 @@ describe('Playwright JSON parsing', () => { ], }) - const testcases = await parsePlaywrightJson(jsonContent, '', { + const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', { skipStdout: 'never', skipStderr: 'never', }) @@ -498,7 +498,7 @@ describe('Playwright JSON parsing', () => { ], }) - const testcases = await parsePlaywrightJson(jsonContent, '', { + const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', { skipStdout: 'never', skipStderr: 'never', }) @@ -545,7 +545,7 @@ describe('Playwright JSON parsing', () => { ], }) - const testcases = await parsePlaywrightJson(jsonContent, '', { + const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', { skipStdout: 'never', skipStderr: 'never', }) @@ -597,7 +597,7 @@ describe('Playwright JSON parsing', () => { ], }) - const testcases = await parsePlaywrightJson(jsonContent, '', { + const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', { skipStdout: 'on-success', skipStderr: 'never', }) @@ -643,7 +643,7 @@ describe('Playwright JSON parsing', () => { ], }) - const testcases = await parsePlaywrightJson(jsonContent, '', { + const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', { skipStdout: 'never', skipStderr: 'on-success', }) @@ -689,7 +689,7 @@ describe('Playwright JSON parsing', () => { ], }) - const testcases = await parsePlaywrightJson(jsonContent, '', { + const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', { skipStdout: 'on-success', skipStderr: 'on-success', }) @@ -736,7 +736,7 @@ describe('Playwright JSON parsing', () => { ], }) - const testcases = await parsePlaywrightJson(jsonContent, '', { + const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', { skipStdout: 'on-success', skipStderr: 'on-success', }) @@ -747,4 +747,82 @@ describe('Playwright JSON parsing', () => { expect(testcases[0].message).not.toContain('stderr content') expect(testcases[0].message).toBe('') }) + + test('Should return empty runFailureLogs when no top-level errors', async () => { + const jsonContent = JSON.stringify({ + suites: [ + { + title: 'test.spec.ts', + specs: [ + { + title: 'Simple test', + tags: [], + tests: [ + { + annotations: [], + expectedStatus: 'passed', + projectName: 'chromium', + results: [ + { + status: 'passed', + errors: [], + stdout: [], + stderr: [], + retry: 0, + duration: 1000, + attachments: [], + }, + ], + status: 'expected', + }, + ], + }, + ], + suites: [], + }, + ], + }) + + const { runFailureLogs } = await parsePlaywrightJson(jsonContent, '', { + skipStdout: 'never', + skipStderr: 'never', + }) + + expect(runFailureLogs).toBe('') + }) + + test('Should extract top-level errors into runFailureLogs', async () => { + const jsonContent = JSON.stringify({ + suites: [], + errors: [ + { message: 'Error in global setup: Connection refused' }, + { message: 'Failed to start server' }, + ], + }) + + const { testCaseResults, runFailureLogs } = await parsePlaywrightJson(jsonContent, '', { + skipStdout: 'never', + skipStderr: 'never', + }) + + expect(testCaseResults).toHaveLength(0) + expect(runFailureLogs).toBe( + '
Error in global setup: Connection refused
' + + '
Failed to start server
' + ) + }) + + test('Should strip ANSI codes from top-level errors in runFailureLogs', async () => { + const jsonContent = JSON.stringify({ + suites: [], + errors: [{ message: '\x1b[31mError: Global setup failed\x1b[0m' }], + }) + + const { runFailureLogs } = await parsePlaywrightJson(jsonContent, '', { + skipStdout: 'never', + skipStderr: 'never', + }) + + expect(runFailureLogs).toBe('
Error: Global setup failed
') + }) }) diff --git a/src/tests/result-upload.spec.ts b/src/tests/result-upload.spec.ts index 27801a4..1d8bdbd 100644 --- a/src/tests/result-upload.spec.ts +++ b/src/tests/result-upload.spec.ts @@ -116,6 +116,10 @@ const server = setupServer( }) } ), + http.post(`${baseURL}/api/public/v0/project/${projectCode}/run/${runId}/log`, ({ request }) => { + expect(request.headers.get('Authorization')).toEqual('ApiKey QAS_TOKEN') + return HttpResponse.json({ id: 'log-1' }) + }), http.post(`${baseURL}/api/public/v0/file/batch`, async ({ request }) => { expect(request.headers.get('Authorization')).toEqual('ApiKey QAS_TOKEN') expect(request.headers.get('Content-Type')).includes('multipart/form-data') @@ -149,6 +153,8 @@ const countResultUploadApiCalls = () => countMockedApiCalls(server, (req) => new URL(req.url).pathname.endsWith('/result/batch')) const countCreateTCasesApiCalls = () => countMockedApiCalls(server, (req) => new URL(req.url).pathname.endsWith('/tcase/bulk')) +const countRunLogApiCalls = () => + countMockedApiCalls(server, (req) => new URL(req.url).pathname.endsWith(`/run/${runId}/log`)) const getMappingFiles = () => new Set( @@ -608,3 +614,23 @@ describe('Allure invalid result file handling', () => { expect(numResultUploadCalls()).toBe(1) // 1 valid result total }) }) + +describe('Run-level log upload', () => { + const junitBasePath = './src/tests/fixtures/junit-xml' + + test('Should upload run-level log when suite-level errors exist', async () => { + const numRunLogCalls = countRunLogApiCalls() + const numResultUploadCalls = countResultUploadApiCalls() + await run(`junit-upload -r ${runURL} --force ${junitBasePath}/suite-level-errors.xml`) + expect(numRunLogCalls()).toBe(1) + expect(numResultUploadCalls()).toBe(1) + }) + + test('Should not upload run-level log when no suite-level errors exist', async () => { + const numRunLogCalls = countRunLogApiCalls() + const numResultUploadCalls = countResultUploadApiCalls() + await run(`junit-upload -r ${runURL} ${junitBasePath}/matching-tcases.xml`) + expect(numRunLogCalls()).toBe(0) + expect(numResultUploadCalls()).toBe(1) + }) +}) diff --git a/src/utils/result-upload/ResultUploadCommandHandler.ts b/src/utils/result-upload/ResultUploadCommandHandler.ts index 67744de..77e4a99 100644 --- a/src/utils/result-upload/ResultUploadCommandHandler.ts +++ b/src/utils/result-upload/ResultUploadCommandHandler.ts @@ -6,7 +6,7 @@ import { parseRunUrl, printErrorThenExit, processTemplate } from '../misc' import { MarkerParser } from './MarkerParser' import { Api, createApi } from '../../api' import { TCase } from '../../api/schemas' -import { TestCaseResult } from './types' +import { ParseResult, TestCaseResult } from './types' import { ResultUploader } from './ResultUploader' import { parseJUnitXml } from './junitXmlParser' import { parsePlaywrightJson } from './playwrightJsonParser' @@ -28,7 +28,7 @@ export type Parser = ( data: string, attachmentBaseDirectory: string, options: ParserOptions -) => Promise +) => Promise export type ResultUploadCommandArgs = { type: UploadCommandType @@ -52,6 +52,7 @@ export type ResultUploadCommandArgs = { interface FileResults { file: string results: TestCaseResult[] + runFailureLogs: string } interface TestCaseResultWithSeqAndFile { @@ -130,7 +131,8 @@ export class ResultUploadCommandHandler { } const results = fileResults.flatMap((fileResult) => fileResult.results) - await this.uploadResults(projectCode, runId, results) + const runFailureLogs = fileResults.map((fr) => fr.runFailureLogs).join('') + await this.uploadResults({ projectCode, runId, results, runFailureLogs }) } protected async parseFiles(): Promise { @@ -146,12 +148,16 @@ export class ResultUploadCommandHandler { const isDirectoryInput = directoryInputTypes.has(this.type) const fileData = isDirectoryInput ? file : readFileSync(file).toString() const attachmentBaseDir = isDirectoryInput ? file : dirname(file) - const fileResults = await commandTypeParsers[this.type]( + const parseResult = await commandTypeParsers[this.type]( fileData, attachmentBaseDir, parserOptions ) - results.push({ file, results: fileResults }) + results.push({ + file, + results: parseResult.testCaseResults, + runFailureLogs: parseResult.runFailureLogs, + }) } return results @@ -264,7 +270,7 @@ export class ResultUploadCommandHandler { } } - if (tcaseIds.length === 0) { + if (tcaseIds.length === 0 && !fileResults.some((fr) => fr.runFailureLogs)) { return printErrorThenExit('No valid test cases found in any of the files') } @@ -428,9 +434,19 @@ export class ResultUploadCommandHandler { } } - private async uploadResults(projectCode: string, runId: number, results: TestCaseResult[]) { + private async uploadResults({ + projectCode, + runId, + results, + runFailureLogs, + }: { + projectCode: string + runId: number + results: TestCaseResult[] + runFailureLogs: string + }) { const runUrl = `${this.baseUrl}/project/${projectCode}/run/${runId}` const uploader = new ResultUploader(this.markerParser, this.type, { ...this.args, runUrl }) - await uploader.handle(results) + await uploader.handle(results, runFailureLogs) } } diff --git a/src/utils/result-upload/ResultUploader.ts b/src/utils/result-upload/ResultUploader.ts index a382772..6c1c2a4 100644 --- a/src/utils/result-upload/ResultUploader.ts +++ b/src/utils/result-upload/ResultUploader.ts @@ -30,7 +30,7 @@ export class ResultUploader { this.api = createApi(url, apiToken) } - async handle(results: TestCaseResult[]) { + async handle(results: TestCaseResult[], runFailureLogs?: string) { const tcases = await this.api.runs .getRunTCases(this.project, this.run) .catch(printErrorThenExit) @@ -44,8 +44,16 @@ export class ResultUploader { .map((f) => chalk.green(f)) .join(', ')}] to run [${chalk.green(this.run)}] of project [${chalk.green(this.project)}]` ) - await this.uploadTestCases(mappedResults) - console.log(`Uploaded ${mappedResults.length} test cases`) + + if (runFailureLogs) { + await this.api.runs.createRunLog(this.project, this.run, { comment: runFailureLogs }) + console.log(`Uploaded run failure logs`) + } + + if (mappedResults.length) { + await this.uploadTestCases(mappedResults) + console.log(`Uploaded ${mappedResults.length} test cases`) + } } private validateAndPrintMissingTestCases(missing: TestCaseResult[]) { diff --git a/src/utils/result-upload/allureParser.ts b/src/utils/result-upload/allureParser.ts index d4efd26..26cda36 100644 --- a/src/utils/result-upload/allureParser.ts +++ b/src/utils/result-upload/allureParser.ts @@ -7,7 +7,7 @@ import { ResultStatus } from '../../api/schemas' import { parseTCaseUrl } from '../misc' import { formatMarker, getMarkerFromText } from './MarkerParser' import { Parser, ParserOptions } from './ResultUploadCommandHandler' -import { Attachment, TestCaseResult } from './types' +import { Attachment, ParseResult, TestCaseResult } from './types' import { getAttachments } from './utils' const allureStatusSchema = z.enum(['passed', 'failed', 'broken', 'skipped', 'unknown']) @@ -91,7 +91,7 @@ export const parseAllureResults: Parser = async ( resultsDirectory: string, attachmentBaseDirectory: string, options: ParserOptions -): Promise => { +): Promise => { let resultFiles: string[] try { resultFiles = readdirSync(resultsDirectory) @@ -162,7 +162,7 @@ export const parseAllureResults: Parser = async ( testcases[tcaseIndex].attachments = tcaseAttachment }) - return testcases + return { testCaseResults: testcases, runFailureLogs: '' } } const collectStepAttachmentPaths = (steps: AllureStep[] | null | undefined): string[] => { diff --git a/src/utils/result-upload/junitXmlParser.ts b/src/utils/result-upload/junitXmlParser.ts index af1173d..08845c2 100644 --- a/src/utils/result-upload/junitXmlParser.ts +++ b/src/utils/result-upload/junitXmlParser.ts @@ -1,15 +1,16 @@ import escapeHtml from 'escape-html' import xml from 'xml2js' import z from 'zod' -import { Attachment, TestCaseResult } from './types' +import { Attachment, ParseResult, TestCaseResult } from './types' import { Parser, ParserOptions } from './ResultUploadCommandHandler' import { ResultStatus } from '../../api/schemas' import { getAttachments } from './utils' -// Note about junit xml schema: -// there are multiple schemas on the internet, and apparently some are more strict than others -// we have to use LESS strict schema (see one from Jest, based on Jenkins JUnit schema) -// see https://github.com/jest-community/jest-junit/blob/master/__tests__/lib/junit.xsd#L42 +// There is no official JUnit XML schema — multiple popular variants exist with varying strictness: +// - Jenkins/Jest: https://github.com/jest-community/jest-junit/blob/master/__tests__/lib/junit.xsd +// - Windyroad: https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd +// - Maven Surefire: https://maven.apache.org/surefire/maven-surefire-plugin/xsd/surefire-test-report-3.0.xsd +// We use a lenient schema that accepts the union of common elements/attributes across these variants. const stringContent = z.object({ _: z.string().optional(), @@ -39,16 +40,18 @@ const skippedSchema = z.union([ }), ]) +// Some JUnit producers emit empty tags like which +// xml2js may parse as empty strings. Accept both object and string forms. +const systemErrOutSchema = z.array(z.union([stringContent, z.string()])).optional() + const testCaseSchema = z.object({ $: z.object({ name: z.string().optional(), classname: z.string().optional(), time: z.string().optional(), }), - // Some JUnit producers emit empty tags like which - // xml2js may parse as empty strings. Accept both object and string forms. - 'system-out': z.array(z.union([stringContent, z.string()])).optional(), - 'system-err': z.array(z.union([stringContent, z.string()])).optional(), + 'system-out': systemErrOutSchema, + 'system-err': systemErrOutSchema, failure: z.array(failureErrorSchema).optional(), skipped: z.array(skippedSchema).optional(), error: z.array(failureErrorSchema).optional(), @@ -73,6 +76,7 @@ const junitXmlSchema = z.object({ }) .optional(), testcase: z.array(testCaseSchema).optional(), + 'system-err': systemErrOutSchema, }) ), }), @@ -82,7 +86,7 @@ export const parseJUnitXml: Parser = async ( xmlString: string, attachmentBaseDirectory: string, options: ParserOptions -): Promise => { +): Promise => { const xmlData = await xml.parseStringPromise(xmlString, { explicitCharkey: true, includeWhiteChars: true, @@ -100,9 +104,43 @@ export const parseJUnitXml: Parser = async ( index: number promise: Promise }> = [] + const runFailureLogParts: string[] = [] for (const suite of validated.testsuites.testsuite) { + const suiteName = suite.$?.name ?? '' + let runLevelFailureLogsFound = false + + // Extract suite-level system-err into runFailureLogParts + for (const err of suite['system-err'] ?? []) { + const content = (typeof err === 'string' ? err : (err._ ?? '')).trim() + if (content) { + if (suiteName && !runLevelFailureLogsFound) + runFailureLogParts.push(`

${escapeHtml(suiteName)}

`) + runFailureLogParts.push(`
${escapeHtml(content)}
`) + runLevelFailureLogsFound = true + } + } + for (const tcase of suite.testcase ?? []) { + const tcaseName = tcase.$.name ?? '' + + // Empty-name testcases with error/failure can be synthetic entries from + // setup/teardown failures (e.g., Maven Surefire). Extract into runFailureLogParts + // and exclude from testCaseResults. + if (!tcaseName && (tcase.error || tcase.failure)) { + const elements = tcase.error ?? tcase.failure ?? [] + for (const element of elements) { + const content = (typeof element === 'string' ? element : (element._ ?? '')).trim() + if (content) { + if (suiteName && !runLevelFailureLogsFound) + runFailureLogParts.push(`

${escapeHtml(suiteName)}

`) + runFailureLogParts.push(`
${escapeHtml(content)}
`) + runLevelFailureLogsFound = true + } + } + continue + } + const result = getResult(tcase, options) const timeTakenSeconds = Number.parseFloat(tcase.$.time ?? '') // Use classname as folder when available, as it provides more meaningful @@ -114,7 +152,7 @@ export const parseJUnitXml: Parser = async ( testcases.push({ ...result, folder, - name: tcase.$.name ?? '', + name: tcaseName, timeTaken: Number.isFinite(timeTakenSeconds) && timeTakenSeconds >= 0 ? Math.round(timeTakenSeconds * 1000) @@ -168,7 +206,7 @@ export const parseJUnitXml: Parser = async ( testcases[tcaseIndex].attachments = tcaseAttachment }) - return testcases + return { testCaseResults: testcases, runFailureLogs: runFailureLogParts.join('') } } const getResult = ( @@ -204,10 +242,10 @@ const getResult = ( messageOptions.push(mainResult) } if (includeStdout) { - messageOptions.push({ result: out, type: 'code' }) + messageOptions.push({ header: 'Output (stdout):', result: out, type: 'code' }) } if (includeStderr) { - messageOptions.push({ result: err, type: 'code' }) + messageOptions.push({ header: 'Output (stderr):', result: err, type: 'code' }) } return { @@ -217,6 +255,7 @@ const getResult = ( } interface GetResultMessageOption { + header?: string result?: ( | string | Partial> @@ -226,23 +265,28 @@ interface GetResultMessageOption { } const getResultMessage = (...options: GetResultMessageOption[]): string => { - let message = '' + const parts: string[] = [] options.forEach((option) => { + const sectionParts: string[] = [] option.result?.forEach((r) => { // Handle both string and object formats from xml2js parsing const content = (typeof r === 'string' ? r : r._)?.trim() if (!content) return if (!option.type || option.type === 'paragraph') { - message += `

${escapeHtml(content)}

` - return + sectionParts.push(`

${escapeHtml(content)}

`) } else if (option.type === 'code') { - message += `
${escapeHtml(content)}
` - return + sectionParts.push(`
${escapeHtml(content)}
`) } }) + if (sectionParts.length > 0) { + if (option.header) { + parts.push(`

${option.header}

`) + } + parts.push(...sectionParts) + } }) - return message + return parts.join('') } const extractAttachmentPaths = (content: string) => { diff --git a/src/utils/result-upload/playwrightJsonParser.ts b/src/utils/result-upload/playwrightJsonParser.ts index c4c50a2..a6d0587 100644 --- a/src/utils/result-upload/playwrightJsonParser.ts +++ b/src/utils/result-upload/playwrightJsonParser.ts @@ -1,7 +1,7 @@ import z from 'zod' import escapeHtml from 'escape-html' import stripAnsi from 'strip-ansi' -import { Attachment, TestCaseResult } from './types' +import { Attachment, ParseResult, TestCaseResult } from './types' import { Parser, ParserOptions } from './ResultUploadCommandHandler' import { ResultStatus } from '../../api/schemas' import { parseTCaseUrl } from '../misc' @@ -78,13 +78,14 @@ const suiteSchema: z.ZodType = z.object({ const playwrightJsonSchema = z.object({ suites: suiteSchema.array(), + errors: reportErrorSchema.array().optional(), }) export const parsePlaywrightJson: Parser = async ( jsonString: string, attachmentBaseDirectory: string, options: ParserOptions -): Promise => { +): Promise => { const jsonData = JSON.parse(jsonString) const validated = playwrightJsonSchema.parse(jsonData) const testcases: TestCaseResult[] = [] @@ -153,7 +154,14 @@ export const parsePlaywrightJson: Parser = async ( testcases[tcaseIndex].attachments = tcaseAttachment }) - return testcases + // Build runFailureLogs from top-level errors + const runFailureLogParts: string[] = [] + for (const error of validated.errors ?? []) { + const cleanMessage = stripAnsi(error.message) + runFailureLogParts.push(`
${escapeHtml(cleanMessage)}
`) + } + + return { testCaseResults: testcases, runFailureLogs: runFailureLogParts.join('') } } const getTCaseMarkerFromAnnotations = (annotations: Annotation[]) => { @@ -183,18 +191,18 @@ const mapPlaywrightStatus = (status: Status): ResultStatus => { } const buildMessage = (result: Result, status: ResultStatus, options: ParserOptions) => { - let message = '' + const parts: string[] = [] if (result.retry) { - message += `

Test passed in ${result.retry + 1} attempts

` + parts.push(`

Test passed in ${result.retry + 1} attempts

`) } if (result.errors.length > 0) { - message += '

Errors:

' + parts.push('

Errors:

') result.errors.forEach((error) => { if (error.message) { const cleanMessage = stripAnsi(error.message) - message += `
${escapeHtml(cleanMessage)}
` + parts.push(`
${escapeHtml(cleanMessage)}
`) } }) } @@ -202,12 +210,12 @@ const buildMessage = (result: Result, status: ResultStatus, options: ParserOptio // Conditionally include stdout based on status and options const includeStdout = !(status === 'passed' && options.skipStdout === 'on-success') if (includeStdout && result.stdout.length > 0) { - message += '

Output:

' + parts.push('

Output (stdout):

') result.stdout.forEach((out) => { const content = 'text' in out ? out.text : out.buffer if (content) { const cleanContent = stripAnsi(content) - message += `
${escapeHtml(cleanContent)}
` + parts.push(`
${escapeHtml(cleanContent)}
`) } }) } @@ -215,15 +223,15 @@ const buildMessage = (result: Result, status: ResultStatus, options: ParserOptio // Conditionally include stderr based on status and options const includeStderr = !(status === 'passed' && options.skipStderr === 'on-success') if (includeStderr && result.stderr.length > 0) { - message += '

Errors (stderr):

' + parts.push('

Output (stderr):

') result.stderr.forEach((err) => { const content = 'text' in err ? err.text : err.buffer if (content) { const cleanContent = stripAnsi(content) - message += `
${escapeHtml(cleanContent)}
` + parts.push(`
${escapeHtml(cleanContent)}
`) } }) } - return message + return parts.join('') } diff --git a/src/utils/result-upload/types.ts b/src/utils/result-upload/types.ts index d46e3fc..a16f5da 100644 --- a/src/utils/result-upload/types.ts +++ b/src/utils/result-upload/types.ts @@ -17,3 +17,8 @@ export interface TestCaseResult { timeTaken: number | null // In milliseconds attachments: Attachment[] } + +export interface ParseResult { + testCaseResults: TestCaseResult[] + runFailureLogs: string // HTML string, empty if no global/suite-level issues +}