cmake: Introduce ctest TEST labels for testing and COST

Tests in CMake/Ctest can have labels on which can be filtered

  ctest  --print-labels
  Test project /home/wjw/work/ceph/build
  All Labels:
    freebsd
    long
    run-tox
    unittest
    vstart

Which allows to run certain subsets of by using RE filters
in ctest.

COST will determine the order in which the tests are executed.
Current settings will start with long running tests.

Signed-off-by: Willem Jan Withagen <[email protected]>

Author: wjwithagen

cmake/modules/AddCephTest.cmake

@@ -1,8 +1,30 @@
 #AddCephTest is a module for adding tests to the "make check" target which runs CTest
 
+macro(set_test_props labels cost cost_default timeout)
+  if(DEFINED timeout)
+    set_property(TEST ${test_name} PROPERTY TIMEOUT ${timeout})
+  else()
+    set_property(TEST ${test_name} PROPERTY TIMEOUT ${CEPH_TEST_TIMEOUT})
+  endif()
+  if(DEFINED cost)
+    set_property(TEST ${test_name} PROPERTY COST ${cost})
+  else()
+    set_property(TEST ${test_name} PROPERTY COST ${cost_default})
+  endif()
+  if(DEFINED labels)
+    set_property(TEST ${test_name} PROPERTY LABELS ${labels})
+  endif()
+endmacro()
+
 #adds makes target/script into a test, test to check target, sets necessary environment variables
 function(add_ceph_test test_name test_path)
-  add_test(NAME ${test_name} COMMAND ${test_path} ${ARGN})
+  set(options "")
+  set(oneValueArgs COST TIMEOUT)
+  set(multiValueArgs LABELS)
+  cmake_parse_arguments(PROPS "${options}" "${oneValueArgs}"
+                        "${multiValueArgs}" ${ARGN} )
+
+  add_test(NAME ${test_name} COMMAND ${test_path} ${PROPS_UNPARSED_ARGUMENTS})
   if(TARGET ${test_name})
     add_dependencies(tests ${test_name})
   endif()
@@ -17,10 +39,7 @@ function(add_ceph_test test_name test_path)
     PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:${CMAKE_SOURCE_DIR}/src:$ENV{PATH}
     PYTHONPATH=${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cython_modules/lib.${Python${PYTHON_VERSION}_VERSION_MAJOR}:${CMAKE_SOURCE_DIR}/src/pybind
     CEPH_BUILD_VIRTUALENV=${CEPH_BUILD_VIRTUALENV})
-  # none of the tests should take more than 1 hour to complete
-  set_property(TEST
-    ${test_name}
-    PROPERTY TIMEOUT ${CEPH_TEST_TIMEOUT})
+    set_test_props("${PROPS_LABELS}" "${PROPS_COST}" 5.0 "${PROPS_TIMEOUT}")
 endfunction()
 
 option(WITH_GTEST_PARALLEL "Enable running gtest based tests in parallel" OFF)
@@ -44,19 +63,21 @@ endif()
 
 #sets uniform compiler flags and link libraries
 function(add_ceph_unittest unittest_name)
+  cmake_parse_arguments(PROPS "" "COST;TIMEOUT" "LABELS" ${ARGN})
   set(UNITTEST "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${unittest_name}")
   # If the second argument is "parallel", it means we want a parallel run
   if(WITH_GTEST_PARALLEL AND "${ARGV1}" STREQUAL "parallel")
     set(UNITTEST ${GTEST_PARALLEL_COMMAND} ${UNITTEST})
   endif()
-  add_ceph_test(${unittest_name} "${UNITTEST}")
+  add_ceph_test(${unittest_name} "${UNITTEST}" "unittest" LABELS "unittest")
   target_link_libraries(${unittest_name} ${UNITTEST_LIBS})
+  set_test_props("${PROPS_LABELS}" "${PROPS_COST}" 1.0 "${PROPS_TIMEOUT}")
 endfunction()
 
 function(add_tox_test name)
   set(test_name run-tox-${name})
   set(venv_path ${CEPH_BUILD_VIRTUALENV}/${name}-virtualenv)
-  cmake_parse_arguments(TOXTEST "" "TOX_PATH" "TOX_ENVS" ${ARGN})
+  cmake_parse_arguments(TOXTEST "" "TOX_PATH;COST;TIMEOUT" "TOX_ENVS;LABELS" ${ARGN})
   if(DEFINED TOXTEST_TOX_PATH)
     set(tox_path ${TOXTEST_TOX_PATH})
   else()
@@ -99,4 +120,6 @@ function(add_tox_test name)
     PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:${CMAKE_SOURCE_DIR}/src:$ENV{PATH}
     PYTHONPATH=${CMAKE_SOURCE_DIR}/src/pybind)
   list(APPEND tox_test run-tox-${name})
+  set_tests_properties(${test_name} PROPERTIES LABELS "tox")
+  set_test_props("${TOXTEST_LABELS}" "${TOXTEST_COST}" 25.0 "${TOXTEST_TIMEOUT}")
 endfunction()

doc/dev/developer_guide/index.rst

@@ -777,7 +777,25 @@ Unit tests are declared in the ``CMakeLists.txt`` files (multiple files under
 which are themselves defined in ``./cmake/modules/AddCephTest.cmake``. Some
 unit tests are scripts, while others are binaries that are compiled during the
 build process.  The ``add_ceph_test`` function is used to declare unit test
-scripts, while ``add_ceph_unittest`` is used for unit test binaries.
+scripts, while ``add_ceph_unittest`` is used for unit test binaries. 
+``add_tox_test`` declare tests that require ``tox`` for testing and sets up 
+the full environment for this during testing.
+::
+
+        add_ceph_test(test_name test_path <args> <options>)
+        add_ceph_unittest(test_name <args> <options>)
+        add_tox_test(test_name <args> <options>)
+        args: any parameter, not being an options keyword
+        options:
+            [COST <float>]
+               Order of testing is set by COST. 
+               Highest COST are tested first.
+            [TIMEOUT sec]
+               Restrict the max runtime of this test to <sec>
+            [LABELS "labels"]
+               Append "labels" to a test.
+               Labels can be selected with ctest -L or -LE
+
 
 Unit testing of CLI tools
 ^^^^^^^^^^^^^^^^^^^^^^^^^

src/pybind/mgr/dashboard/CMakeLists.txt

@@ -98,5 +98,5 @@ endif(WITH_MGR_DASHBOARD_FRONTEND AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch6
 
 if(WITH_TESTS)
   include(AddCephTest)
-  add_tox_test(mgr-dashboard TOX_ENVS lint check)
+  add_tox_test(mgr-dashboard TOX_ENVS lint check LABELS "long" COST 50.0)
 endif()

src/test/CMakeLists.txt

@@ -527,18 +527,18 @@ if(WITH_FIO OR WITH_SYSTEM_FIO)
 endif()
 
 if(WITH_RBD)
-  add_ceph_test(run-rbd-unit-tests.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh)
+  add_ceph_test(run-rbd-unit-tests.sh ${CMAKE_CURRENT_SOURCE_DIR}/run-rbd-unit-tests.sh LABELS "long" COST 1000.0)
   if(FREEBSD)
-    add_ceph_test(rbd-ggate.sh ${CMAKE_CURRENT_SOURCE_DIR}/rbd-ggate.sh)
+    add_ceph_test(rbd-ggate.sh ${CMAKE_CURRENT_SOURCE_DIR}/rbd-ggate.sh LABELS "vstart;freebsd" COST 160.0)
   endif(FREEBSD)
 endif(WITH_RBD)
-add_ceph_test(run-cli-tests ${CMAKE_CURRENT_SOURCE_DIR}/run-cli-tests)
+add_ceph_test(run-cli-tests ${CMAKE_CURRENT_SOURCE_DIR}/run-cli-tests LABELS "long" COST 519.0)
 add_ceph_test(test_objectstore_memstore.sh ${CMAKE_CURRENT_SOURCE_DIR}/test_objectstore_memstore.sh)
 
 # flaky
 #add_ceph_test(test_pidfile.sh ${CMAKE_CURRENT_SOURCE_DIR}/test_pidfile.sh)
 
-add_ceph_test(smoke.sh ${CMAKE_CURRENT_SOURCE_DIR}/smoke.sh)
+add_ceph_test(smoke.sh ${CMAKE_CURRENT_SOURCE_DIR}/smoke.sh LABELS "long" COST 230)
 
 set_property(
   TEST ${tox_tests}
@@ -662,7 +662,7 @@ add_executable(unittest_bufferlist
   bufferlist.cc
   $<TARGET_OBJECTS:unit-main>
   )
-add_ceph_unittest(unittest_bufferlist)
+add_ceph_unittest(unittest_bufferlist LABELS "long" COST 630)
 target_link_libraries(unittest_bufferlist global)
 
 # compiletest_cxx11_client

src/test/ObjectMap/CMakeLists.txt

@@ -3,7 +3,7 @@ add_executable(ceph_test_object_map
   test_object_map.cc
   KeyValueDBMemory.cc
   )
-add_ceph_unittest(ceph_test_object_map)
+add_ceph_unittest(ceph_test_object_map LABELS "long" COST 350.0)
 target_link_libraries(ceph_test_object_map
   os
   ceph-common

src/test/encoding/CMakeLists.txt

@@ -1,3 +1,5 @@
 # scripts
-add_ceph_test(check-generated.sh ${CMAKE_CURRENT_SOURCE_DIR}/check-generated.sh)
-add_ceph_test(readable.sh ${CMAKE_CURRENT_SOURCE_DIR}/readable.sh)
+add_ceph_test(check-generated.sh ${CMAKE_CURRENT_SOURCE_DIR}/check-generated.sh
+              LABELS "long" COST 487.0)
+add_ceph_test(readable.sh ${CMAKE_CURRENT_SOURCE_DIR}/readable.sh
+              LABELS "long" COST 561.0)

src/test/erasure-code/CMakeLists.txt

@@ -201,7 +201,7 @@ target_link_libraries(unittest_erasure_code_shec
 add_executable(unittest_erasure_code_shec_all
   TestErasureCodeShec_all.cc
   )
-add_ceph_unittest(unittest_erasure_code_shec_all parallel)
+add_ceph_unittest(unittest_erasure_code_shec_all parallel LABELS "long" COST 750.0)
 target_link_libraries(unittest_erasure_code_shec_all
   global
   ${CMAKE_DL_LIBS}

src/test/objectstore/CMakeLists.txt

@@ -103,7 +103,7 @@ if(WITH_BLUESTORE)
     fastbmap_allocator_test.cc
     $<TARGET_OBJECTS:unit-main>
     )
-  add_ceph_unittest(unittest_fastbmap_allocator)
+  add_ceph_unittest(unittest_fastbmap_allocator LABELS "long" COST 102.0)
   target_link_libraries(unittest_fastbmap_allocator os global)
 
   add_executable(unittest_bluefs

src/test/osd/CMakeLists.txt

@@ -34,13 +34,14 @@ install(TARGETS
   DESTINATION ${CMAKE_INSTALL_BINDIR})
 
 # scripts
-add_ceph_test(safe-to-destroy.sh ${CMAKE_CURRENT_SOURCE_DIR}/safe-to-destroy.sh)
+add_ceph_test(safe-to-destroy.sh ${CMAKE_CURRENT_SOURCE_DIR}/safe-to-destroy.sh
+              LABELS "long" COST 367.0)
 
 # unittest_osdmap
 add_executable(unittest_osdmap
   TestOSDMap.cc
   )
-add_ceph_unittest(unittest_osdmap)
+add_ceph_unittest(unittest_osdmap LABELS "long" COST 91.0)
 target_link_libraries(unittest_osdmap global ${BLKID_LIBRARIES})
 
 # unittest_osd_types

src/test/pybind/CMakeLists.txt

@@ -1,2 +1,3 @@
 add_ceph_test(test_ceph_daemon.py ${CMAKE_CURRENT_SOURCE_DIR}/test_ceph_daemon.py)
-add_ceph_test(test_ceph_argparse.py ${CMAKE_CURRENT_SOURCE_DIR}/test_ceph_argparse.py)
+add_ceph_test(test_ceph_argparse.py ${CMAKE_CURRENT_SOURCE_DIR}/test_ceph_argparse.py
+              LABELS "long" COST 104.0)