Skip to content

Commit 34d9724

Browse files
horenmarhorenmar
committed
Add experimental CMake script for sharding tests in binaries
1 parent 5d26904 commit 34d9724

File tree

3 files changed

+164
-1
lines changed

3 files changed

+164
-1
lines changed

‎docs/cmake-integration.md‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,17 +63,19 @@ target_link_libraries(tests PRIVATE Catch2::Catch2WithMain)
6363

6464
## Automatic test registration
6565

66-
Catch2's repository also contains two CMake scripts that help users
66+
Catch2's repository also contains three CMake scripts that help users
6767
with automatically registering their `TEST_CASE`s with CTest. They
6868
can be found in the `extras` folder, and are
6969

7070
1) `Catch.cmake` (and its dependency `CatchAddTests.cmake`)
7171
2) `ParseAndAddCatchTests.cmake` (deprecated)
72+
3) `CatchShardTests.cmake` (and its dependency `CatchShardTestsImpl.cmake`)
7273

7374
If Catch2 has been installed in system, both of these can be used after
7475
doing `find_package(Catch2 REQUIRED)`. Otherwise you need to add them
7576
to your CMake module path.
7677

78+
<a id="catch_discover_tests"></a>
7779
### `Catch.cmake` and `CatchAddTests.cmake`
7880

7981
`Catch.cmake` provides function `catch_discover_tests` to get tests from
@@ -257,6 +259,49 @@ unset(OptionalCatchTestLauncher)
257259
ParseAndAddCatchTests(bar)
258260
```
259261

262+
263+
### `CatchShardTests.cmake`
264+
265+
> `CatchShardTests.cmake` was introduced in Catch2 X.Y.Z.
266+
267+
`CatchShardTests.cmake` provides a function
268+
`catch_add_sharded_tests(TEST_BINARY)` that splits tests from `TEST_BINARY`
269+
into multiple shards. The tests in each shard and their order is randomized,
270+
and the seed changes every invocation of CTest.
271+
272+
Currently there are 3 customization points for this script:
273+
274+
* SHARD_COUNT - number of shards to split target's tests into
275+
* REPORTER - reporter spec to use for tests
276+
* TEST_SPEC - test spec used for filtering tests
277+
278+
Example usage:
279+
280+
```
281+
include(CatchShardTests)
282+
283+
catch_add_sharded_tests(foo-tests
284+
SHARD_COUNT 4
285+
REPORTER "xml::out=-"
286+
TEST_SPEC "A"
287+
)
288+
289+
catch_add_sharded_tests(tests
290+
SHARD_COUNT 8
291+
REPORTER "xml::out=-"
292+
TEST_SPEC "B"
293+
)
294+
```
295+
296+
This registers total of 12 CTest tests (4 + 8 shards) to run shards
297+
from `foo-tests` test binary, filtered by a test spec.
298+
299+
_Note that this script is currently a proof-of-concept for reseeding
300+
shards per CTest run, and thus does not support (nor does it currently
301+
aim to support) all customization points from
302+
[`catch_discover_tests`](#catch_discover_tests)._
303+
304+
260305
## CMake project options
261306

262307
Catch2's CMake project also provides some options for other projects

‎extras/CatchShardTests.cmake‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
2+
# Copyright Catch2 Authors
3+
# Distributed under the Boost Software License, Version 1.0.
4+
# (See accompanying file LICENSE_1_0.txt or copy at
5+
# https://www.boost.org/LICENSE_1_0.txt)
6+
7+
# SPDX-License-Identifier: BSL-1.0
8+
9+
# Supported optional args:
10+
# * SHARD_COUNT - number of shards to split target's tests into
11+
# * REPORTER - reporter spec to use for tests
12+
# * TEST_SPEC - test spec used for filtering tests
13+
function(catch_add_sharded_tests TARGET)
14+
if (${CMAKE_VERSION} VERSION_LESS "3.10.0")
15+
message(FATAL_ERROR "add_sharded_catch_tests only supports CMake versions 3.10.0 and up")
16+
endif()
17+
18+
cmake_parse_arguments(
19+
""
20+
""
21+
"SHARD_COUNT;REPORTER;TEST_SPEC"
22+
""
23+
${ARGN}
24+
)
25+
26+
if (NOT DEFINED _SHARD_COUNT)
27+
set(_SHARD_COUNT 2)
28+
endif()
29+
30+
# Generate a unique name based on the extra arguments
31+
string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS} ${_REPORTER} ${_OUTPUT_DIR} ${_OUTPUT_PREFIX} ${_OUTPUT_SUFFIX} ${_SHARD_COUNT}")
32+
string(SUBSTRING ${args_hash} 0 7 args_hash)
33+
34+
set(ctest_include_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}-sharded-tests-include-${args_hash}.cmake")
35+
set(ctest_tests_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}-sharded-tests-impl-${args_hash}.cmake")
36+
37+
file(WRITE "${ctest_include_file}"
38+
"if(EXISTS \"${ctest_tests_file}\")\n"
39+
" include(\"${ctest_tests_file}\")\n"
40+
"else()\n"
41+
" add_test(${TARGET}_NOT_BUILT-${args_hash} ${TARGET}_NOT_BUILT-${args_hash})\n"
42+
"endif()\n"
43+
)
44+
45+
set_property(DIRECTORY
46+
APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
47+
)
48+
49+
set(shard_impl_script_file "${CMAKE_CURRENT_LIST_DIR}/CatchShardTestsImpl.cmake")
50+
51+
add_custom_command(
52+
TARGET ${TARGET} POST_BUILD
53+
BYPRODUCTS "${ctest_tests_file}"
54+
COMMAND "${CMAKE_COMMAND}"
55+
-D "TARGET_NAME=${TARGET}"
56+
-D "TEST_BINARY=$<TARGET_FILE:${TARGET}>"
57+
-D "CTEST_FILE=${ctest_tests_file}"
58+
-D "SHARD_COUNT=${_SHARD_COUNT}"
59+
-D "REPORTER_SPEC=${_REPORTER}"
60+
-D "TEST_SPEC=${_TEST_SPEC}"
61+
-P "${shard_impl_script_file}"
62+
VERBATIM
63+
)
64+
65+
66+
endfunction()

‎extras/CatchShardTestsImpl.cmake‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
2+
# Copyright Catch2 Authors
3+
# Distributed under the Boost Software License, Version 1.0.
4+
# (See accompanying file LICENSE_1_0.txt or copy at
5+
# https://www.boost.org/LICENSE_1_0.txt)
6+
7+
# SPDX-License-Identifier: BSL-1.0
8+
9+
# Indirection for CatchShardTests that allows us to delay the script
10+
# file generation until build time.
11+
12+
# Expected args:
13+
# * TEST_BINARY - full path to the test binary to run sharded
14+
# * CTEST_FILE - full path to ctest script file to write to
15+
# * TARGET_NAME - name of the target to shard (used for test names)
16+
# * SHARD_COUNT - number of shards to split the binary into
17+
# Optional args:
18+
# * REPORTER_SPEC - reporter specs to be passed down to the binary
19+
# * TEST_SPEC - test spec to pass down to the test binary
20+
21+
if(NOT EXISTS "${TEST_BINARY}")
22+
message(FATAL_ERROR
23+
"Specified test binary '${TEST_BINARY}' does not exist"
24+
)
25+
endif()
26+
27+
set(other_args "")
28+
if (TEST_SPEC)
29+
set(other_args "${other_args} ${TEST_SPEC}")
30+
endif()
31+
if (REPORTER_SPEC)
32+
set(other_args "${other_args} --reporter ${REPORTER_SPEC}")
33+
endif()
34+
35+
# foreach RANGE in cmake is inclusive of the end, so we have to adjust it
36+
math(EXPR adjusted_shard_count "${SHARD_COUNT} - 1")
37+
38+
file(WRITE "${CTEST_FILE}"
39+
"string(RANDOM LENGTH 8 ALPHABET \"0123456789abcdef\" rng_seed)\n"
40+
"\n"
41+
"foreach(shard_idx RANGE ${adjusted_shard_count})\n"
42+
" add_test(${TARGET_NAME}-shard-" [[${shard_idx}]] "/${adjusted_shard_count}\n"
43+
" ${TEST_BINARY}"
44+
" --shard-index " [[${shard_idx}]]
45+
" --shard-count ${SHARD_COUNT}"
46+
" --rng-seed " [[0x${rng_seed}]]
47+
" --order rand"
48+
"${other_args}"
49+
"\n"
50+
" )\n"
51+
"endforeach()\n"
52+
)

0 commit comments

Comments
 (0)