Skip to content

cmake: Introduce ctest labels for testing #29844

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wjwithagen wants to merge 3 commits into from
Closed

cmake: Introduce ctest labels for testing #29844

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ead4604
doc: Add manual for LABELS and COST to developer guide
wjwithagen Feb 9, 2020
cfd7e86
cmake: Adding COST, LABELS and TIMEOUT to CephTest's
wjwithagen Feb 9, 2020
9f6518e
cmake: Add LABELS and COST to some long running tests
wjwithagen Feb 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 29 additions & 10 deletions cmake/modules/AddCephTest.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,25 @@
#AddCephTest is a module for adding tests to the "make check" target which runs CTest

macro(set_test_props labels cost cost_default timeout)
if("${timeout}" GREATER 0)
set_property(TEST ${test_name} PROPERTY TIMEOUT ${timeout})
else()
set_property(TEST ${test_name} PROPERTY TIMEOUT ${CEPH_TEST_TIMEOUT})
endif()
if("${cost}" GREATER 0)
set_property(TEST ${test_name} PROPERTY COST ${cost})
else()
set_property(TEST ${test_name} PROPERTY COST ${cost_default})
endif()
if(NOT "${labels}" EQUAL "")
set_property(TEST ${test_name} APPEND PROPERTY LABELS "${labels}")
endif()
endmacro()

#adds makes target/script into a test, test to check target, sets necessary environment variables
function(add_ceph_test test_name test_path)
add_test(NAME ${test_name} COMMAND ${test_path} ${ARGN})
cmake_parse_arguments(PROPS "" "COST;TIMEOUT" "LABELS" ${ARGN})
add_test(NAME ${test_name} COMMAND ${test_path} ${PROPS_UNPARSED_ARGUMENTS})
if(TARGET ${test_name})
add_dependencies(tests ${test_name})
endif()
Expand All @@ -17,10 +34,7 @@ function(add_ceph_test test_name test_path)
PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:${CMAKE_SOURCE_DIR}/src:$ENV{PATH}
PYTHONPATH=${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cython_modules/lib.3:${CMAKE_SOURCE_DIR}/src/pybind
CEPH_BUILD_VIRTUALENV=${CEPH_BUILD_VIRTUALENV})
# none of the tests should take more than 1 hour to complete
set_property(TEST
${test_name}
PROPERTY TIMEOUT ${CEPH_TEST_TIMEOUT})
set_test_props("${PROPS_LABELS}" "${PROPS_COST}" 5.0 "${PROPS_TIMEOUT}")
endfunction()

option(WITH_GTEST_PARALLEL "Enable running gtest based tests in parallel" OFF)
Expand All @@ -43,20 +57,22 @@ if(WITH_GTEST_PARALLEL)
endif()

#sets uniform compiler flags and link libraries
function(add_ceph_unittest unittest_name)
set(UNITTEST "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${unittest_name}")
function(add_ceph_unittest test_name)
cmake_parse_arguments(PROPS "" "COST;TIMEOUT" "LABELS" ${ARGN})
set(UNITTEST "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${test_name}")
# If the second argument is "parallel", it means we want a parallel run
if(WITH_GTEST_PARALLEL AND "${ARGV1}" STREQUAL "parallel")
set(UNITTEST ${GTEST_PARALLEL_COMMAND} ${UNITTEST})
endif()
add_ceph_test(${unittest_name} "${UNITTEST}")
target_link_libraries(${unittest_name} ${UNITTEST_LIBS})
add_ceph_test(${test_name} "${UNITTEST}" "unittest" LABELS "unittest")
target_link_libraries(${test_name} ${UNITTEST_LIBS})
set_test_props("${PROPS_LABELS}" "${PROPS_COST}" 1.0 "${PROPS_TIMEOUT}")
endfunction()

function(add_tox_test name)
set(test_name run-tox-${name})
set(venv_path ${CEPH_BUILD_VIRTUALENV}/${name}-virtualenv)
cmake_parse_arguments(TOXTEST "" "TOX_PATH" "TOX_ENVS" ${ARGN})
cmake_parse_arguments(TOXTEST "" "TOX_PATH;COST;TIMEOUT" "TOX_ENVS;LABELS" ${ARGN})
if(DEFINED TOXTEST_TOX_PATH)
set(tox_path ${TOXTEST_TOX_PATH})
else()
Expand Down Expand Up @@ -95,4 +111,7 @@ function(add_tox_test name)
PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:${CMAKE_SOURCE_DIR}/src:$ENV{PATH}
PYTHONPATH=${CMAKE_SOURCE_DIR}/src/pybind)
list(APPEND tox_test run-tox-${name})
set_tests_properties(${test_name} PROPERTIES LABELS "tox")
set_test_props("${TOXTEST_LABELS}" "${TOXTEST_COST}" 25.0 "${TOXTEST_TIMEOUT}")
endfunction()

32 changes: 32 additions & 0 deletions doc/dev/developer_guide/tests-unit-tests.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,38 @@ functions, which are themselves defined in
others are binaries that are compiled during the build process. The
``add_ceph_test`` function is used to declare unit test scripts, while
``add_ceph_unittest`` is used for unit test binaries.
``add_tox_test`` declare tests that require ``tox`` for testing and sets up
the full environment for this during testing.
::

add_ceph_test(test_name test_path <args> <options>)
add_ceph_unittest(test_name <args> <options>)
add_tox_test(test_name <args> <options>)
args: any parameter, not being an options keyword
options:
[COST <float>]
Order of testing is set by COST.
Highest COST are tested first.
[TIMEOUT sec]
Restrict the max runtime of this test to <sec>
[LABELS "labels"]
Append "labels" to a test.
Labels can be selected with ctest -L or -LE

Since COST is used to determine the testing order, values can be chosen rather
arbitrarily. Currently long running programs are given a value equal to the
Copy link
Contributor

@anthonyeleven anthonyeleven Jul 10, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Zac may have a different view, but I'd make it long-running.

execution time in a OpenStack VM. The idea is to first run the long standing
tests, and then fill up free cores with programs with shorter execution time.
Turning this into a bin-packing challenge, So the exact value is not important,
it is about the relative COST value.
Copy link
Contributor

@anthonyeleven anthonyeleven Jul 10, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

an OpenStack. But conceptually I'm a bit unclear, how is OpenStack involved? Wouldn't runtime in an OpenStack instance be a function of the aggregate's overcommit policy, the instance flavor, and and the underlying CPU model? Where does the numerical COST come from? Historical data? Enquiring minds want to know.

"The idea is to first run the long-running tests, and then fill unused cores with tests with shorter execution time.
This is effectively a bin-packing challenge, so the exact values are not important, bur rather relative COST value.

Copy link
Contributor Author

@wjwithagen wjwithagen Nov 13, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

an OpenStack. But conceptually I'm a bit unclear, how is OpenStack involved? Wouldn't runtime in an OpenStack instance be a function of the aggregate's overcommit policy, the instance flavor, and and the underlying CPU model? Where does the numerical COST come from? Historical data? Enquiring minds want to know.

"The idea is to first run the long-running tests, and then fill unused cores with tests with shorter execution time. This is effectively a bin-packing challenge, so the exact values are not important, bur rather relative COST value.

The main purpose here is to prevent that run in a multi-core environment and try to schedule to very long running jobs at the end of test-run, since that could extend the test-run time with like 5 - 10 minutes.

'Openstack ' refers to the fact that I was running these builds in an OpenStack system. But like I indicated COST is only used to steer the bin-packing algorithm, no use in getting exact numbers, order of magnitude will already work.
And I used the times that I found in my runs.


Currently some LABELS are introduced::

unittest as declared above (DEF COST = 5)
tox the test used tox to do the testing (DEF COST = 25)
long when the test executes for more than 60 secs (DEF COST = 50)
vstart when the test uses vsstart.sh to setup a cluster


Unit testing of CLI tools
-------------------------
Expand Down
2 changes: 1 addition & 1 deletion qa/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,5 @@ endif()

if(WITH_TESTS)
include(AddCephTest)
add_tox_test(qa TOX_ENVS flake8 import-tasks)
add_tox_test(qa TOX_ENVS flake8 import-tasks COST 215.0)
endif()
4 changes: 2 additions & 2 deletions src/pybind/mgr/dashboard/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,6 @@ add_custom_target(mgr-dashboard-frontend-build
add_dependencies(tests mgr-dashboard-frontend-build)

if(WITH_TESTS)
include(AddCephTest)
add_tox_test(mgr-dashboard TOX_ENVS lint check)
include(AddCephTest)
add_tox_test(mgr-dashboard TOX_ENVS lint check LABELS "long" COST 300.0)
endif()
20 changes: 10 additions & 10 deletions src/test/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -515,25 +515,25 @@ endif()
if(WITH_RBD)
# Run rbd-unit-tests separate so they an run in parallel
# For values see: src/include/rbd/features.h
add_ceph_test(run-rbd-unit-tests-N.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh N)
add_ceph_test(run-rbd-unit-tests-0.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 0)
add_ceph_test(run-rbd-unit-tests-1.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 1)
add_ceph_test(run-rbd-unit-tests-61.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 61)
add_ceph_test(run-rbd-unit-tests-109.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 109)
add_ceph_test(run-rbd-unit-tests-127.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 127)
add_ceph_test(run-rbd-unit-tests-N.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh N COST 500.0)
add_ceph_test(run-rbd-unit-tests-0.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 0 COST 181.0)
add_ceph_test(run-rbd-unit-tests-1.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 1 COST 255.0)
add_ceph_test(run-rbd-unit-tests-61.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 61 COST 267.0)
add_ceph_test(run-rbd-unit-tests-109.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 109 COST 346.0)
add_ceph_test(run-rbd-unit-tests-127.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh 127 COST 350.0)
if(FREEBSD)
add_ceph_test(rbd-ggate.sh ${CMAKE_CURRENT_SOURCE_DIR}/rbd-ggate.sh)
add_ceph_test(rbd-ggate.sh ${CMAKE_CURRENT_SOURCE_DIR}/rbd-ggate.sh COST 160.0)
endif(FREEBSD)
endif(WITH_RBD)
add_ceph_test(run-cli-tests ${CMAKE_CURRENT_SOURCE_DIR}/run-cli-tests)
add_ceph_test(run-cli-tests ${CMAKE_CURRENT_SOURCE_DIR}/run-cli-tests COST 264.0)

# flaky, see https://tracker.ceph.com/issues/44243
#add_ceph_test(test_objectstore_memstore.sh ${CMAKE_CURRENT_SOURCE_DIR}/test_objectstore_memstore.sh)

# flaky
#add_ceph_test(test_pidfile.sh ${CMAKE_CURRENT_SOURCE_DIR}/test_pidfile.sh)

add_ceph_test(smoke.sh ${CMAKE_CURRENT_SOURCE_DIR}/smoke.sh)
add_ceph_test(smoke.sh ${CMAKE_CURRENT_SOURCE_DIR}/smoke.sh COST 230.0)

set_property(
TEST ${tox_tests}
Expand Down Expand Up @@ -657,7 +657,7 @@ add_executable(unittest_bufferlist
bufferlist.cc
$<TARGET_OBJECTS:unit-main>
)
add_ceph_unittest(unittest_bufferlist)
add_ceph_unittest(unittest_bufferlist LABELS "long" COST 400.0)
target_link_libraries(unittest_bufferlist global)

# compiletest_cxx11_client
Expand Down
6 changes: 4 additions & 2 deletions src/test/encoding/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# scripts
add_ceph_test(check-generated.sh ${CMAKE_CURRENT_SOURCE_DIR}/check-generated.sh)
add_ceph_test(readable.sh ${CMAKE_CURRENT_SOURCE_DIR}/readable.sh)
add_ceph_test(check-generated.sh ${CMAKE_CURRENT_SOURCE_DIR}/check-generated.sh
LABELS "long" COST 619.0)
add_ceph_test(readable.sh ${CMAKE_CURRENT_SOURCE_DIR}/readable.sh
LABELS "long" COST 813.0)
2 changes: 1 addition & 1 deletion src/test/erasure-code/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ target_link_libraries(unittest_erasure_code_shec
add_executable(unittest_erasure_code_shec_all
TestErasureCodeShec_all.cc
)
add_ceph_unittest(unittest_erasure_code_shec_all parallel)
add_ceph_unittest(unittest_erasure_code_shec_all parallel LABELS "long" COST 560.0)
target_link_libraries(unittest_erasure_code_shec_all
global
${CMAKE_DL_LIBS}
Expand Down
2 changes: 1 addition & 1 deletion src/test/mgr/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ target_link_libraries(unittest_mgr_mgrcap global)
#scripts
if(WITH_MGR_DASHBOARD_FRONTEND)
if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm|ARM")
add_ceph_test(mgr-dashboard-frontend-unittests ${CMAKE_SOURCE_DIR}/src/pybind/mgr/dashboard/run-frontend-unittests.sh)
add_ceph_test(mgr-dashboard-frontend-unittests ${CMAKE_SOURCE_DIR}/src/pybind/mgr/dashboard/run-frontend-unittests.sh COST 764.0)
endif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm|ARM")

add_ceph_test(mgr-dashboard-smoke.sh ${CMAKE_CURRENT_SOURCE_DIR}/mgr-dashboard-smoke.sh)
Expand Down
6 changes: 3 additions & 3 deletions src/test/objectstore/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ if(WITH_BLUESTORE)
fastbmap_allocator_test.cc
$<TARGET_OBJECTS:unit-main>
)
add_ceph_unittest(unittest_fastbmap_allocator)
add_ceph_unittest(unittest_fastbmap_allocator LABELS "long" COST 102.0)
target_link_libraries(unittest_fastbmap_allocator os global)

set_target_properties(unittest_fastbmap_allocator PROPERTIES COMPILE_FLAGS
Expand All @@ -130,7 +130,7 @@ if(WITH_BLUESTORE)
add_executable(unittest_bluefs
test_bluefs.cc
)
add_ceph_unittest(unittest_bluefs)
add_ceph_unittest(unittest_bluefsi COST 702.0)
target_link_libraries(unittest_bluefs os global)

# unittest_bluestore_types
Expand Down Expand Up @@ -159,7 +159,7 @@ target_link_libraries(unittest_transaction os ceph-common)
add_executable(unittest_memstore_clone
test_memstore_clone.cc
$<TARGET_OBJECTS:store_test_fixture>)
add_ceph_unittest(unittest_memstore_clone)
add_ceph_unittest(unittest_memstore_clone 195.0)
target_link_libraries(unittest_memstore_clone os global)

if(WITH_BLUESTORE)
Expand Down
7 changes: 4 additions & 3 deletions src/test/osd/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,14 @@ install(TARGETS
DESTINATION ${CMAKE_INSTALL_BINDIR})

# scripts
add_ceph_test(safe-to-destroy.sh ${CMAKE_CURRENT_SOURCE_DIR}/safe-to-destroy.sh)
add_ceph_test(safe-to-destroy.sh ${CMAKE_CURRENT_SOURCE_DIR}/safe-to-destroy.sh
LABELS "long" COST 441.0)

# unittest_osdmap
add_executable(unittest_osdmap
TestOSDMap.cc
)
add_ceph_unittest(unittest_osdmap)
add_ceph_unittest(unittest_osdmap LABELS "long" COST 91.0)
target_link_libraries(unittest_osdmap global ${BLKID_LIBRARIES})

# unittest_osd_types
Expand Down Expand Up @@ -71,7 +72,7 @@ add_executable(unittest_pglog
$<TARGET_OBJECTS:unit-main>
$<TARGET_OBJECTS:store_test_fixture>
)
add_ceph_unittest(unittest_pglog)
add_ceph_unittest(unittest_pglog COST 214.0)
target_link_libraries(unittest_pglog osd os global ${CMAKE_DL_LIBS} ${BLKID_LIBRARIES})

# unittest_hitset
Expand Down
3 changes: 2 additions & 1 deletion src/test/pybind/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
add_ceph_test(test_ceph_daemon.py ${CMAKE_CURRENT_SOURCE_DIR}/test_ceph_daemon.py)
add_ceph_test(test_ceph_argparse.py ${CMAKE_CURRENT_SOURCE_DIR}/test_ceph_argparse.py)
add_ceph_test(test_ceph_argparse.py ${CMAKE_CURRENT_SOURCE_DIR}/test_ceph_argparse.py
LABELS "long" COST 104.0)
2 changes: 1 addition & 1 deletion src/test/rbd_mirror/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ add_executable(unittest_rbd_mirror
image_sync/test_mock_SyncPointPruneRequest.cc
pool_watcher/test_mock_RefreshImagesRequest.cc
)
add_ceph_unittest(unittest_rbd_mirror)
add_ceph_unittest(unittest_rbd_mirror LABELS "long" COST 196.0)

add_dependencies(unittest_rbd_mirror
cls_journal
Expand Down
Loading