Reports that are generated from test runs via the custom test reporters stored as part of the repository will output a custom format. This format allows us to capture all of the desired information in a consistent way so that the mapping from test run data to back-end storage is easier. For details on how data is stored in AWS Timestream, please see Storage Schema.
Note
All JSON below is simply a pseudo representation of the actual schemas which
are stored in the schemas folder.
Report versions 3 and 4 accept both release and CI build numbers. The
prefix has two digits, a dot, two digits, a dot, and one or two digits.
Release builds append a dot and exactly five digits, including leading zeros
(for example, 20.26.9.00001 or 20.26.11.16663).
CI builds append lowercase -ci and an unpadded integer from 1 to 99999
(for example, 20.26.9-ci1 or 20.26.11-ci16663).
The build number is optional and is preserved unchanged, including the -ci
label, when reports are loaded. Legacy report versions 1 and 2 accept only
dotted release build numbers, including when LMS metadata is injected before
upgrading.
{
"id": "<GUID>",
"version": 3,
"summary": {
"status": "<Must be 'passed' or 'failed'>",
"github": {
"organization": "<Non-empty string matching pattern '[A-Za-z0-9_.-]+'>",
"repository": "<Non-empty string matching pattern '[A-Za-z0-9_.-]+'>",
"workflow": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$'>",
"runId": "<Positive integer, can be 0>",
"runAttempt": "<Positive integer, must be 1 or greater>"
},
"git": {
"branch": "<Git branch this was generated for>",
"sha": "<Git SHA hash representing the commit this was generated for>"
},
"lms": {
"buildNumber": "<LMS release or CI build number, optional; see LMS build numbers above>",
"instanceUrl": "<URI of the LMS instance, optional>"
},
"framework": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$>'",
"operatingSystem": "<Must be one of 'linux', 'windows' or 'mac'>",
"started": "<UTC timestamp>",
"duration": {
"total": "<Positive integer representing milliseconds, can be 0>"
},
"count": {
"passed": "<Positive integer, can be 0>",
"failed": "<Positive integer, can be 0>",
"skipped": "<Positive integer, can be 0>",
"flaky": "<Positive integer, can be 0>"
}
},
"details": [
{
"name": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$>'",
"status": "<Must be 'passed', 'skipped' or 'failed'>",
"location": {
"file": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$'>",
"line": "<Positive integer, can be 0, optional>",
"column": "<Positive integer, can be 0, optional>"
},
"taxonomy": {
"tool": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$', optional>",
"type": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$', optional>"
},
"browser": "<Can be 'chromium', 'chrome', 'firefox', 'webkit', 'safari' or 'edge', optional>",
"configuration": {
"timeout": "<Positive integer representing milliseconds, can be 0, optional>"
},
"started": "<Timestamp in UTC time>",
"duration": {
"total": "<Positive integer representing milliseconds, can be 0>",
"final": "<Positive integer representing milliseconds, can be 0>"
},
"github": {
"codeowners": [
"<Codeowner matching pattern '^@\\S+$', optional>"
]
},
"retries": "<Positive integer, can be 0>"
}
]
}Version 2
{
"id": "<GUID>",
"version": 2,
"summary": {
"status": "<Must be 'passed' or 'failed'>",
"github": {
"organization": "<Non-empty string matching pattern '[A-Za-z0-9_.-]+'>",
"repository": "<Non-empty string matching pattern '[A-Za-z0-9_.-]+'>",
"workflow": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$'>",
"runId": "<Positive integer, can be 0>",
"runAttempt": "<Positive integer, must be 1 or greater>"
},
"git": {
"branch": "<Git branch this was generated for>",
"sha": "<Git SHA hash representing the commit this was generated for>"
},
"framework": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$>'",
"operatingSystem": "<Must be one of 'linux', 'windows' or 'mac'>",
"started": "<UTC timestamp>",
"duration": {
"total": "<Positive integer representing milliseconds, can be 0>"
},
"count": {
"passed": "<Positive integer, can be 0>",
"failed": "<Positive integer, can be 0>",
"skipped": "<Positive integer, can be 0>",
"flaky": "<Positive integer, can be 0>"
}
},
"details": [
{
"name": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$>'",
"status": "<Must be 'passed', 'skipped' or 'failed'>",
"location": {
"file": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$'>",
"line": "<Positive integer, can be 0, optional>",
"column": "<Positive integer, can be 0, optional>"
},
"browser": "<Can be 'chromium', 'chrome', 'firefox', 'webkit', 'safari' or 'edge', optional>",
"timeout": "<Positive integer representing milliseconds, can be 0, optional>",
"started": "<Timestamp in UTC time>",
"duration": {
"total": "<Positive integer representing milliseconds, can be 0>",
"final": "<Positive integer representing milliseconds, can be 0>"
},
"tool": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$', optional>",
"experience": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$', optional>",
"type": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$', optional>",
"retries": "<Positive integer, can be 0>"
}
]
}Version 1
{
"id": "<GUID>",
"version": 1,
"summary": {
"githubOrganization": "<Non-empty string matching pattern '[A-Za-z0-9_.-]+'>",
"githubRepository": "<Non-empty string matching pattern '[A-Za-z0-9_.-]+'>",
"githubWorkflow": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$'>",
"githubRunId": "<Positive integer, can be 0>",
"githubRunAttempt": "<Positive integer, must be 1 or greater>",
"gitBranch": "<Git branch this was generated for>",
"gitSha": "<Git SHA hash representing the commit this was generated for>",
"countPassed": "<Positive integer, can be 0>",
"countSkipped": "<Positive integer, can be 0>",
"countFlaky": "<Positive integer, can be 0>",
"countFailed": "<Positive integer, can be 0>",
"totalDuration": "<Positive integer representing milliseconds, can be 0>",
"status": "<Must be 'passed' or 'failed'>",
"framework": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$>'",
"operatingSystem": "<Must be one of 'linux', 'windows' or 'mac'>",
"started": "<UTC timestamp>"
},
"details": [
{
"location": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$'>",
"started": "<Timestamp in UTC time>",
"totalDuration": "<Positive integer representing milliseconds, can be 0>",
"duration": "<Positive integer representing milliseconds, can be 0>",
"name": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$>'",
"status": "<Must be 'passed', 'skipped' or 'failed'>",
"retries": "<Positive integer, can be 0>",
"browser": "<Can be 'chromium', 'chrome', 'firefox', 'webkit', 'safari' or 'edge', optional>",
"type": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$', optional>",
"experience": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$', optional>",
"tool": "<Non-empty string matching pattern '^(?!\\s).+(?<!\\s)$', optional>"
}
]
}