Update cmake script

src/CMakeLists.txt

@@ -9,55 +9,139 @@
 # Enbable debug mode by passing -DCMAKE_BUILD_TYPE=Debug to CMake, default is
 # Release
 
-cmake_minimum_required (VERSION 3.11)
+cmake_minimum_required (VERSION 3.24)
 
-option(USE_GPU "user-set flag to compile in GPU code" FALSE)
+project (BRANSON
+  VERSION 0.8
+  DESCRIPTION "Branson can be used to study different algorithms for parallel Monte Carlo transport. Currently it contains particle passing and mesh passing methods for domain decomposition."
+)
 
-if(DEFINED ENV{CUDADIR}
-   OR DEFINED ENV{CUDACXX}
-   OR DEFINED ENV{CUDA_HOME})
-  if(USE_GPU)
-    set(GPU_DBS_STRING "CUDA" CACHE STRING "If CUDA is available, this variable is 'CUDA'")
-    enable_language(CUDA)
-  endif()
-elseif(
-  "$ENV{LOADEDMODULES}" MATCHES "rocmcc"
-  OR DEFINED ENV{HIPCXX}
-  OR DEFINED CMAKE_HIP_COMPILER
-  OR DEFINED ENV{ROCM_PATH})
-  if(USE_GPU)
-    set(GPU_DBS_STRING "HIP" CACHE STRING "If HIP is available, this variable is 'HIP'")
-    #if(CMAKE_HIP_COMPILER MATCHES "CC")
-    #  set(CMAKE_HIP_FLAGS "-x hip")
-    #endif()
-    enable_language(HIP)
+## Fail if someone tries to config an in-source build.
+if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}")
+   message(FATAL_ERROR "In-source builds are not supported. Please remove "
+                       "CMakeCache.txt from the 'src' dir and configure an "
+                       "out-of-source build in another directory.")
+endif()
+
+# Set the build type to Release by default if not set
+if(NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE "Release")
+endif()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+string(REPLACE " " ";" CMAKE_BRANSON_CXXFLAGS "${CMAKE_CXX_FLAGS}")
+
+option(ENABLE_CUDA "Use CUDA" FALSE)
+option(ENABLE_HIP "Use HIP" FALSE)
+option(ENABLE_CALIPER "Enable Caliper" FALSE)
+option(ENABLE_OPENMP "Enable OpenMP" FALSE)
+option(ENABLE_METIS "Enable METIS" FALSE)
+option(ENABLE_VIZ "Enable VIZ" FALSE)
+
+set(CMAKE_VERBOSE_MAKEFILE ON)
+
+find_package(MPI REQUIRED)
+list(APPEND branson_deps
+  MPI::MPI_CXX)
+
+##############################################################################
+# OpenMP
+##############################################################################
+if(ENABLE_OPENMP)
+  message(STATUS "Looking for OpenMP...")
+  find_package(OpenMP REQUIRED)
+  if(OpenMP_FOUND)
+    message(STATUS "Looking for openmp......found ${OpenMP_CXX_FLAGS}")
+    set(USE_OPENMP ON)
+    list(APPEND branson_deps
+      OpenMP::OpenMP_CXX
+    )
   endif()
+else()
+  message(STATUS "Disabling OpenMP...")
+  set(USE_OPENMP OFF)
 endif()
 
-message("GPU_DBS_STRING: ${GPU_DBS_STRING}")
-project (BRANSON
-  VERSION 0.8
-  DESCRIPTION "Branson can be used to study different algorithms for parallel Monte Carlo transport. Currently it contains particle passing and mesh passing methods for domain decomposition."
-#  HOMEPAGE URL "https://github.com/lanl/branson" # needs cmake 3.12+
-  LANGUAGES CXX C ${GPU_DBS_STRING})
-
-get_property(_LANGUAGES_ GLOBAL PROPERTY ENABLED_LANGUAGES)
-
-message("Languages: ${_LANGUAGES_}")
-if((_LANGUAGES_ MATCHES CUDA OR _LANGUAGES_ MATCHES HIP) AND USE_GPU)
-  message("CUDA/HIP module found (CUDA/HIP environment variables set) and USE_GPU is on, making GPU BRANSON")
-  add_compile_definitions(HAS_GPU)
-elseif(USE_GPU)
-  message(FATAL_ERROR "CUDA/HIP module NOT found (CUDA/HIP environment variables set) but USE_GPU is on, reconfigure with USE_GPU off or fix modules")
-elseif(NOT USE_GPU)
-  message("GPU mode not requested, making CPU only BRANSON")
+if(ENABLE_CUDA)
+  enable_language(CXX)
+
+  find_package(CUDAToolkit REQUIRED)
+  list(APPEND branson_deps
+    CUDA::cudart)
+
+  include(CheckLanguage)
+  check_language(CUDA)
+
+  set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
+
+  if(NOT CMAKE_CUDA_COMPILER)
+    message(FATAL_ERROR "Unable to find the nvcc compiler. Please use"
+                        "CMAKE_CUDA_COMPILER to provide the nvcc compiler.")
+  endif()
+
+  enable_language(CUDA)
+  set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD})
+  set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+  set(CMAKE_CUDA_ARCHITECTURES "${CUDA_ARCH}")
+  set_source_files_properties("main.cc" PROPERTIES LANGUAGE CUDA)
+  set(CMAKE_CUDA_FLAGS "-g --expt-relaxed-constexpr ${CMAKE_CUDA_FLAGS}")
+  set(CMAKE_CUDA_FLAGS "-expt-extended-lambda ${CMAKE_CUDA_FLAGS}")
+  list(APPEND CMAKE_BRANSON_CXXFLAGS -DUSE_GPU)
+  list(APPEND CMAKE_BRANSON_CXXFLAGS -DHAS_GPU)
+
+  set(GPU_DBS_STRING "CUDA" CACHE STRING "If CUDA is available, this variable is 'CUDA'")
+
+  message("Making GPU(CUDA) BRANSON")
+
+elseif(ENABLE_HIP)
+  enable_language(CXX)
+
+  if(NOT ROCM_PATH)
+    if(DEFINED ENV{ROCM_PATH})
+      set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCM has been installed")
+    else()
+      message(FATAL_ERROR "Unable to find rocm. Please use "
+                          "ROCM_PATH to provide the rocm installation.")
+    endif()
+  endif()
+
+  set(CMAKE_HIP_COMPILER "${ROCM_PATH}/llvm/bin/clang++")
+  list(APPEND CMAKE_PREFIX_PATH "${ROCM_PATH}/lib/cmake")
+
+  find_package(hip REQUIRED)
+  list(APPEND branson_deps
+    hip::host
+    hip::device)
+
+  include(CheckLanguage)
+  check_language(HIP)
+
+  set(CMAKE_HIP_HOST_COMPILER ${CMAKE_CXX_COMPILER})
+
+  enable_language(HIP)
+  set(CMAKE_HIP_STANDARD ${CMAKE_CXX_STANDARD})
+  set(CMAKE_HIP_STANDARD_REQUIRED ON)
+  set(CMAKE_HIP_ARCHITECTURES "${HIP_ARCH}")
+  set_source_files_properties("main.cc" PROPERTIES LANGUAGE HIP)
+  set(CMAKE_HIP_FLAGS "-g ${CMAKE_HIP_FLAGS}")
+  list(APPEND CMAKE_BRANSON_CXXFLAGS -DUSE_GPU)
+  list(APPEND CMAKE_BRANSON_CXXFLAGS -DHAS_GPU)
+
+  set(GPU_DBS_STRING "HIP" CACHE STRING "If HIP is available, this variable is 'HIP'")
+
+  message("Making GPU(HIP) BRANSON")
+
 else()
-  message("CUDA/HIP module not found and GPU mode not requested, making CPU only BRANSON")
+  message("Making CPU BRANSON")
+
 endif()
 
+message("GPU_DBS_STRING: ${GPU_DBS_STRING}")
 
 # Build system support files are located here.
-set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/config )
+set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/config ${CMAKE_MODULE_PATH})
 
 message( "
 This is ${CMAKE_PROJECT_NAME} v. ${PROJECT_VERSION}.
@@ -88,24 +172,6 @@ string( REGEX REPLACE "([A-z0-9]+).*" "\\1" SITENAME ${SITENAME} )
 include (lanl-setup)
 set( SITENAME ${SITENAME} CACHE "STRING" "Name of the current machine" FORCE)
 
-#------------------------------------------------------------------------------#
-# Setup compiler options
-set( CXX_STANDARD_REQUIRED ON )
-set( CMAKE_CXX_STANDARD 17 )
-
-# Do not enable extensions (e.g.: --std=gnu++11)
-set( CMAKE_CXX_EXTENSIONS OFF )
-set( CMAKE_C_EXTENSIONS   OFF )
-
-#------------------------------------------------------------------------------#
-# Build type and custom compiler flags
-
-if ( "${CMAKE_BUILD_TYPE}notset" STREQUAL "notset" AND
-    NOT DEFINED CMAKE_CONFIGURATION_TYPES )
-  set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Release|Debug|RelWithDebInfo"
-    FORCE)
-endif ()
-
 # GCC options for address or undefined sanitizing
 #set(GCC_SANITIZE_COMPILE_FLAGS "-fsanitize=address")
 #set(GCC_SANITIZE_LINK_FLAGS "-fsanitize=address")
@@ -136,14 +202,17 @@ elseif( CMAKE_CXX_COMPILER_ID STREQUAL "Cray" )
   set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -DR123_USE_GNU_UINT128=0")
 endif()
 
+if(ENABLE_CUDA)
+  string(JOIN " " CMAKE_BRANSON_CUDAFLAGS ${CMAKE_BRANSON_CXXFLAGS})
+  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler ${CMAKE_BRANSON_CUDAFLAGS}")
+else()
+  add_compile_options(${CMAKE_BRANSON_CXXFLAGS})
+endif()
 
 #------------------------------------------------------------------------------#
 # Look for Third Party Libraries (metis, etc.)
 add_subdirectory(pugixml)
 
-include(find_tpls)
-setupTPLs()
-
 #------------------------------------------------------------------------------#
 # Report build configuration
 
@@ -166,83 +235,82 @@ else()
 endif()
 message( STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
 message( STATUS "Compiler        : ${CMAKE_CXX_COMPILER}")
+string( REPLACE ";" " " CMAKE_BRANSON_CXXFLAGS_STR "${CMAKE_BRANSON_CXXFLAGS}")
 if( CMAKE_CONFIGURATION_TYPES )
-  message( STATUS "Compiler Flags (All)    : ${CMAKE_CXX_FLAGS}")
+  message( STATUS "Compiler Flags (All)    : ${CMAKE_CXX_FLAGS} ${CMAKE_BRANSON_CXXFLAGS_STR}")
   message( STATUS "Compiler Flags (Debug)  : ${CMAKE_CXX_FLAGS_DEBUG}")
   message( STATUS "Compiler Flags (Release): ${CMAKE_CXX_FLAGS_RELEASE}")
 else()
   string( TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER )
-  message( STATUS "Compiler Flags  : ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}")
+  message( STATUS "Compiler Flags  : ${CMAKE_CXX_FLAGS} ${CMAKE_BRANSON_CXXFLAGS_STR} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UPPER}}")
 endif()
 message("\n")
 
-#------------------------------------------------------------------------------#
-# Set up libraries for Cray RMA routines
-
-# set(DMAPP_DYNAMIC -Wl,--whole-archive,-ldmapp,--no-whole-archive)
-
-
-#------------------------------------------------------------------------------#
-# Targets
-file(GLOB headers *.h)
-add_executable(BRANSON main.cc ${headers})
-target_include_directories( BRANSON PRIVATE
-  $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}> ${PROJECT_SOURCE_DIR}/pugixml/src/ ${PROJECT_SOURCE_DIR}/random123/ )
-set( branson_deps
-  MPI::MPI_CXX
-  MPI::MPI_C
-  pugixml)
-
-if(OpenMP_FOUND)
-  set(branson_deps "OpenMP::OpenMP_CXX;${branson_deps}")
-endif()
-
-if(METIS_FOUND)
-  set( branson_deps "METIS::metis;${branson_deps}")
-endif()
-
-if( VIZ_LIBRARIES_FOUND )
-  set( branson_deps "Silo::silo;${HDF5_LIBRARIES};${branson_deps}" )
-endif()
+list(APPEND branson_deps pugixml)
+
+##############################################################################
+# caliper/adiak
+##############################################################################
+if(ENABLE_CALIPER)
+  message(STATUS "Looking for caliper...")
+  find_package(caliper REQUIRED)
+  if(caliper_FOUND)
+    message(STATUS "Looking for caliper......found ${caliper_DIR}")
+    list(APPEND branson_deps
+      caliper)
+  endif()
 
-if(caliper_FOUND)
-  set( branson_deps "caliper;${branson_deps}")
+  message(STATUS "Looking for adiak...")
+  find_package(adiak REQUIRED)
+  if(adiak_FOUND)
+    message(STATUS "Looking for adiak......found ${adiak_DIR}")
+    list(APPEND branson_deps
+      adiak::adiak)
+  endif()
+else()
+  message(STATUS "Disabling caliper...")
 endif()
 
-#------------------------------------------------------------------------------#
-# Generate config.h
-
-configure_file(config.h.in ${PROJECT_BINARY_DIR}/config.h)
-#------------------------------------------------------------------------------#
-
-if("${GPU_DBS_STRING}" STREQUAL "CUDA" )
-  message("Setting CUDA compiler options")
-  #set_target_properties(BRANSON PROPERTIES CUDA_ARCHITECTURES "70") # V100
-  set_target_properties(BRANSON PROPERTIES CUDA_ARCHITECTURES "80") # A100
-  set_target_properties(BRANSON PROPERTIES CUDA_STANDARD 17)
-  string(APPEND CMAKE_CUDA_FLAGS " -g --expt-relaxed-constexpr")
-  string(APPEND CMAKE_CUDA_FLAGS " --expt-extended-lambda"  )
-  set_source_files_properties("main.cc" PROPERTIES LANGUAGE CUDA)
-elseif("${GPU_DBS_STRING}" STREQUAL "HIP" )
-  message("Setting HIP compiler options")
-  if(CMAKE_HIP_COMPILER MATCHES "CC")
-    set(CMAKE_HIP_FLAGS "-x hip")
+##############################################################################
+# metis
+##############################################################################
+if(ENABLE_METIS)
+  message(STATUS "Looking for METIS...")
+  find_package(METIS REQUIRED)
+  if(METIS_FOUND)
+    message(STATUS "Looking for METIS......found ${METIS_LIBRARIES}")
+    list(APPEND branson_deps
+      METIS::metis)
   endif()
-  set_target_properties(BRANSON PROPERTIES HIP_ARCHITECTURES "gfx942") # MI300
-  set_target_properties(BRANSON PROPERTIES HIP_STANDARD 17)
-  string(APPEND CMAKE_HIP_FLAGS " -g -mllvm=--disable-peephole")
-  #string(APPEND CMAKE_HIP_FLAGS " --expt-extended-lambda"  )
-  set_source_files_properties("main.cc" PROPERTIES LANGUAGE HIP)
 else()
-  message("GPU Options: Not a GPU build or GPU_DBS_STRING not recognized")
+  message(STATUS "Disabling METIS...")
 endif()
 
-target_link_libraries( BRANSON PRIVATE ${branson_deps} )
-
-#------------------------------------------------------------------------------#
-# Testing
+##############################################################################
+# Silo and HDF5 libraries
+##############################################################################
+if(ENABLE_VIZ)
+  message(STATUS "Looking for HDF5...")
+  find_package(HDF5 REQUIRED)
+  if(HDF5_FOUND)
+    list(GET HDF5_LIBRARIES 0 hdf5lib)
+    message(STATUS "Looking for HDF5..found ${hdf5lib}")
+    unset(hdf5lib)
+    message(STATUS "Looking for Silo...")
+    find_package(Silo REQUIRED)
+    if(Silo_FOUND)
+      message(STATUS "Looking for Silo..found ${Silo_LIBRARY}")
+    endif()
+    list(APPEND branson_deps
+      Silo::silo
+      ${HDF5_LIBRARIES}
+    )
+  endif()
+else ()
+  message(STATUS "Disabling visualization libraries...")
+endif()
 
-option( BUILD_TESTING "Should we compile the tests?" ON )
+option(BUILD_TESTING "Should we compile the tests?" ON)
 if(BUILD_TESTING)
   enable_testing()
   add_subdirectory(test)
@@ -253,11 +321,24 @@ else()
     "Building tests disabled, set BUILD_TESTING=TRUE or don't set BUILD_TESTING to enable test builds")
 endif()
 
+#------------------------------------------------------------------------------#
+# Generate config.h
+
+configure_file(config.h.in ${PROJECT_BINARY_DIR}/config.h)
+#------------------------------------------------------------------------------#
+
+#------------------------------------------------------------------------------#
+# Targets
+file(GLOB headers *.h)
+add_executable(BRANSON main.cc ${headers})
+target_link_libraries(BRANSON PUBLIC ${branson_deps})
+target_include_directories(BRANSON PRIVATE
+  ${CMAKE_BINARY_DIR} ${PROJECT_SOURCE_DIR}/pugixml/src/ ${HIP_INCLUDE_DIRS})
 
 #------------------------------------------------------------------------------#
 # Targets for installation
 
-install(TARGETS BRANSON DESTINATION bin)
+install(TARGETS BRANSON DESTINATION ${CMAKE_INSTALL_BINDIR})
 
 #------------------------------------------------------------------------------#
 # End src/CMakeLists.txt