Use enable_language(HIP) (#702)

Co-authored-by: Chris White <[email protected]>

Author: davidbeckingsale

.gitlab/build_ruby.yml

@@ -2,7 +2,7 @@
 # This is the share configuration of jobs for ruby
 .on_ruby:
   variables:
-    SCHEDULER_PARAMETERS: "--res=ci --exclusive=user --deadline=now+1hour -N1 -t ${ALLOC_TIME}"
+    SCHEDULER_PARAMETERS: "--reservation=ci --exclusive=user --deadline=now+1hour -N1 -t ${ALLOC_TIME}"
   tags:
     - batch
     - ruby

.gitlab/build_tioga.yml

@@ -27,16 +27,22 @@
 
 ####
 # Build jobs
-tioga-clang_16_0_0_hip:
+tioga-clang_18_0_0_hip:
   variables:
-    HOST_CONFIG: "clang@16.0.0_hip.cmake"
+    HOST_CONFIG: "clang@18.0.0_hip.cmake"
   extends: .src_build_on_tioga
 
 
 ####
 # HIP project tests
-tioga-clang_16.0.0_hip_config_test:
+tioga-clang_18.0.0_hip_config_test:
   variables:
     ENABLED_BLT_TESTS: "hip-config-test"
-    HOST_CONFIG: "[email protected]_hip.cmake"
+    HOST_CONFIG: "[email protected]_hip.cmake"
+  extends: [.run_project_integration_test_on_tioga]
+
+tioga-gcc_10.3.1_clang_17.0.0_hip_config_test:
+  variables:
+    ENABLED_BLT_TESTS: "hip-config-test"
+    HOST_CONFIG: "[email protected][email protected]_hip.cmake"
   extends: [.run_project_integration_test_on_tioga]

RELEASE-NOTES.md

@@ -11,9 +11,12 @@ The project release numbers follow [Semantic Versioning](http://semver.org/spec/
 
 ### Added
 - Added ``WORKING_DIRECTORY`` option to ``blt_add_test`` and ``blt_add_benchmark``.
+- Added `BLT_CXX_FILE_EXTS` variable (split out of `BLT_C_FILE_EXTS`) for use in modified `blt_split_source_list_by_language` macro.
 
 ### Changed
 - Modified `blt_convert_to_system_includes` to handle multiple targets and recursively update includes for dependencies.
+- Modified `blt_split_source_list_by_language` to accept a `CXX_LIST` argument for splitting out C and CXX sources. If not specified, the `C_LIST` will contain both C and C++ sources.
+- HIP support now uses enable_language(HIP), and specifying a HIP compiler must use the variable `CMAKE_HIP_COMPILER`.
 - Updated GoogleBenchmark to 1.9.1
 
 ### Fixed

SetupBLT.cmake

@@ -193,8 +193,10 @@ if (NOT BLT_LOADED)
     # File extension lists used to filter sources based on languages for code checks
     # and filtering Fortran sources out of cuda/hip source lists
     # Note: this filtering is case-insensitive
-    set(BLT_C_FILE_EXTS ".cpp" ".hpp" ".cxx" ".hxx" ".c" ".h" ".cc" ".hh" ".inl" ".cu" ".cuh"
-               CACHE STRING "List of known file extensions used for C/CXX sources")
+    set(BLT_C_FILE_EXTS ".c" ".h"
+               CACHE STRING "List of known file extensions used for C sources")
+    set(BLT_CXX_FILE_EXTS ".cpp" ".hpp" ".cxx" ".hxx" ".cc" ".hh" ".inl" ".cu" ".cuh"
+               CACHE STRING "List of known file extensions used for CXX sources")
     set(BLT_Fortran_FILE_EXTS ".f" ".f90"
                CACHE STRING "List of known file extensions used for Fortran sources")
     set(BLT_Python_FILE_EXTS ".py"

azure-pipelines.yml

@@ -43,7 +43,7 @@ strategy:
       MPI_DIR: '/usr'
       CMAKE_FLAGS: '$(C_COMPILERS) $(MPI_FLAGS) -DENABLE_GTEST_DEATH_TESTS=OFF -DENABLE_OPENMP=ON'
     osx_gcc:
-      VM_ImageName: 'macos-12'
+      VM_ImageName: 'macos-14'
       CMAKE_FLAGS: ''
     windows:
       VM_ImageName: 'windows-2022'

cmake/BLTMacros.cmake

@@ -205,6 +205,13 @@ macro(blt_add_library)
                 DEPENDS_ON   ${arg_DEPENDS_ON}
                 LIBRARY_TYPE ${_lib_type})
         endif()
+
+        if(BLT_ENABLE_HIP)
+            blt_setup_hip_target(
+                NAME         ${arg_NAME}
+                SOURCES      ${arg_SOURCES}
+                DEPENDS_ON   ${arg_DEPENDS_ON})
+        endif()
     else()
         #
         #  Header-only library support
@@ -320,6 +327,13 @@ macro(blt_add_executable)
             DEPENDS_ON   ${arg_DEPENDS_ON})
     endif()
 
+    if(BLT_ENABLE_HIP)
+        blt_setup_hip_target(
+            NAME         ${arg_NAME}
+            SOURCES      ${arg_SOURCES}
+            DEPENDS_ON   ${arg_DEPENDS_ON})
+    endif()
+    
     # CMake wants to load with C++ if any of the libraries are C++.
     # Force to load with Fortran if the first file is Fortran.
     list(GET arg_SOURCES 0 _first)

cmake/BLTPrivateMacros.cmake

@@ -353,6 +353,51 @@ macro(blt_setup_cuda_target)
     endif()
 endmacro(blt_setup_cuda_target)
 
+##------------------------------------------------------------------------------
+## blt_setup_hip_target(NAME <name of target>
+##                       SOURCES <list of sources>
+##                       DEPENDS_ON <list of dependencies>
+##------------------------------------------------------------------------------
+macro(blt_setup_hip_target)
+
+    set(options)
+    set(singleValueArgs NAME)
+    set(multiValueArgs SOURCES DEPENDS_ON)
+
+    # Parse the arguments
+    cmake_parse_arguments(arg "${options}" "${singleValueArgs}"
+                            "${multiValueArgs}" ${ARGN} )
+
+    # Check arguments
+    if ( NOT DEFINED arg_NAME )
+        message( FATAL_ERROR "Must provide a NAME argument to the 'blt_setup_hip_target' macro")
+    endif()
+
+    if ( NOT DEFINED arg_SOURCES )
+        message( FATAL_ERROR "Must provide SOURCES to the 'blt_setup_hip_target' macro")
+    endif()
+
+    # Determine if hip is in DEPENDS_ON
+    list(FIND arg_DEPENDS_ON "blt_hip" _hip_index)
+    list(FIND arg_DEPENDS_ON "blt::hip" _hip_index2)
+    set(_depends_on_hip FALSE)
+    if(${_hip_index} GREATER -1 OR ${_hip_index2} GREATER -1)
+        set(_depends_on_hip TRUE)
+    endif()
+
+    if (${_depends_on_hip})
+        set(_hip_sources)
+        set(_non_hip_sources)
+        blt_split_source_list_by_language(SOURCES      ${arg_SOURCES}
+                                          C_LIST       _non_hip_sources
+                                          CXX_LIST     _hip_sources
+                                          Fortran_LIST _non_hip_sources)
+
+        set_source_files_properties( ${_hip_sources} PROPERTIES LANGUAGE HIP)
+
+    endif()
+endmacro(blt_setup_hip_target)
+
 
 ##-----------------------------------------------------------------------------
 ## blt_make_file_ext_regex( EXTENSIONS   [ext1 [ext2 ...]]
@@ -399,6 +444,7 @@ endmacro(blt_make_file_ext_regex)
 ##------------------------------------------------------------------------------
 ## blt_split_source_list_by_language( SOURCES <sources>
 ##                                    C_LIST <list name>
+##                                    CXX_LIST <list name>
 ##                                    Fortran_LIST <list name>
 ##                                    Python_LIST <list name>
 ##                                    CMAKE_LIST <list name>)
@@ -414,7 +460,7 @@ endmacro(blt_make_file_ext_regex)
 macro(blt_split_source_list_by_language)
 
     set(options)
-    set(singleValueArgs C_LIST Fortran_LIST Python_LIST CMAKE_LIST)
+    set(singleValueArgs C_LIST CXX_LIST Fortran_LIST Python_LIST CMAKE_LIST)
     set(multiValueArgs SOURCES)
 
     # Parse the arguments
@@ -430,6 +476,9 @@ macro(blt_split_source_list_by_language)
     set(BLT_C_FILE_REGEX)
     blt_make_file_ext_regex(EXTENSIONS   ${BLT_C_FILE_EXTS}
                             OUTPUT_REGEX BLT_C_FILE_REGEX)
+    set(BLT_CXX_FILE_REGEX)
+    blt_make_file_ext_regex(EXTENSIONS   ${BLT_CXX_FILE_EXTS}
+                            OUTPUT_REGEX BLT_CXX_FILE_REGEX)
     set(BLT_Fortran_FILE_REGEX)
     blt_make_file_ext_regex(EXTENSIONS   ${BLT_Fortran_FILE_EXTS}
                             OUTPUT_REGEX BLT_Fortran_FILE_REGEX)
@@ -463,6 +512,13 @@ macro(blt_split_source_list_by_language)
             if (DEFINED arg_C_LIST)
                 list(APPEND ${arg_C_LIST} "${_file}")
             endif()
+        elseif("${_lower_file}" MATCHES "${BLT_CXX_FILE_REGEX}")
+            if (DEFINED arg_CXX_LIST)
+                list(APPEND ${arg_CXX_LIST} "${_file}")
+            # maintain backwards compatibilty by adding to C list if CXX list not set
+            else()
+                list(APPEND ${arg_C_LIST} "${_file}")
+            endif()
         elseif("${_lower_file}" MATCHES "${BLT_Fortran_FILE_REGEX}")
             if (DEFINED arg_Fortran_LIST)
                 list(APPEND ${arg_Fortran_LIST} "${_file}")

cmake/thirdparty/BLTSetupHIP.cmake

@@ -6,6 +6,12 @@
 ################################
 # HIP
 ################################
+ 
+if( ${CMAKE_VERSION} VERSION_LESS "3.21.0" )
+  message(FATAL_ERROR "HIP support requires CMake >= 3.21.0")
+endif ()
+
+enable_language(HIP)
 
 if(NOT ROCM_PATH)
     # First try finding paths given by the user
@@ -41,32 +47,7 @@ message(STATUS "ROCM path:        ${ROCM_PATH}")
 message(STATUS "HIP version:      ${hip_VERSION}")
 
 # AMDGPU_TARGETS should be defined in the hip-config.cmake that gets "included" via find_package(hip)
-# This file is also what hardcodes the --offload-arch flags we're removing here
 if(DEFINED AMDGPU_TARGETS)
-    # If we haven't selected a particular architecture via CMAKE_HIP_ARCHITECTURES,
-    # we want to remove the unconditionally added compile/link flags from the hip::device target.
-    # FIXME: This may cause problems for targets whose HIP_ARCHITECTURES property differs
-    # from CMAKE_HIP_ARCHITECTURES - this only happens when a user manually modifies
-    # the property after it is initialized
-    get_target_property(_hip_compile_options hip::device INTERFACE_COMPILE_OPTIONS)
-    get_target_property(_hip_link_libs hip::device INTERFACE_LINK_LIBRARIES)
-
-    foreach(_target ${AMDGPU_TARGETS})
-        if(NOT "${CMAKE_HIP_ARCHITECTURES}" MATCHES "${_target}")
-            set(_flag "--offload-arch=${_target}")
-            set(_generator_compile_flag "$<$<COMPILE_LANGUAGE:CXX>:SHELL:${_flag}>")
-            set(_generator_link_flag "$<$<LINK_LANGUAGE:CXX>:${_flag}>")
-
-            list(REMOVE_ITEM _hip_compile_options ${_generator_compile_flag})
-            list(REMOVE_ITEM _hip_compile_options ${_flag})
-            list(REMOVE_ITEM _hip_link_libs ${_generator_link_flag})
-            list(REMOVE_ITEM _hip_link_libs ${_flag})
-        endif()
-    endforeach()
-    
-    set_property(TARGET hip::device PROPERTY INTERFACE_COMPILE_OPTIONS ${_hip_compile_options})
-    set_property(TARGET hip::device PROPERTY INTERFACE_LINK_LIBRARIES ${_hip_link_libs})
-
     if(DEFINED CMAKE_HIP_ARCHITECTURES)
         set(AMDGPU_TARGETS "${CMAKE_HIP_ARCHITECTURES}" CACHE STRING "" FORCE)
     endif()
@@ -87,14 +68,10 @@ else()
 endif()
 
 blt_import_library(NAME          blt_hip
-                   COMPILE_FLAGS ${_blt_hip_compile_flags}
                    EXPORTABLE    ${BLT_EXPORT_THIRDPARTY}
+                   DEPENDS_ON    blt_hip_runtime
                    GLOBAL        ${_blt_hip_is_global})
 
-# Hard-copy inheritable properties instead of depending on hip::device so that we can export
-# all required information in our target blt_hip
-blt_inherit_target_info(TO blt_hip FROM hip::device OBJECT FALSE)
-
 add_library(blt::hip ALIAS blt_hip)
 
 

docs/tutorial/common_hpc_dependencies.rst

@@ -167,8 +167,9 @@ compilers be set as the main compilers. This will change soon.
 **Important Setup Variables**
 
 * ``ENABLE_HIP`` : Enables HIP support in BLT
-* ``HIP_ROOT_DIR`` : Root directory for HIP installation
+* ``ROCM_ROOT_DIR`` : Root directory for ROCM installation
 * ``CMAKE_HIP_ARCHITECTURES`` : GPU architecture to use when generating HIP/ROCm code
+* ``CMAKE_HIP_COMPILER``: Compiler to use for HIP compilation, required when the C++ compiler does not support HIP.
 
 **BLT Targets**
 

host-configs/llnl/toss_4_x86_64_ib_cray/[email protected]_hip.cmake renamed to host-configs/llnl/toss_4_x86_64_ib_cray/[email protected]_hip.cmake

@@ -7,11 +7,11 @@
 #------------------------------------------------------------------------------
 # Compilers
 #------------------------------------------------------------------------------
-# Compiler Spec: clang@16.0.0
+# Compiler Spec: clang@18.0.0
 #------------------------------------------------------------------------------
 
 #_blt_tutorial_hip_compiler_start
-set(_compiler_root "/opt/rocm-5.6.0/llvm")
+set(_compiler_root "/opt/rocm-6.2.1/llvm")
 
 set(CMAKE_C_COMPILER "${_compiler_root}/bin/amdclang" CACHE PATH "")
 
@@ -28,7 +28,7 @@ set(ENABLE_FORTRAN ON CACHE BOOL "")
 # MPI
 #------------------------------------------------------------------------------
 
-set(_mpi_root "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.28-rocmcc-5.6.0")
+set(_mpi_root "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.31-rocmcc-6.2.1")
 
 set(MPI_C_COMPILER "${_mpi_root}/bin/mpicc" CACHE PATH "")
 
@@ -53,15 +53,14 @@ set(ENABLE_MPI ON CACHE BOOL "")
 #------------------------------------------------------------------------------
 
 #_blt_tutorial_useful_hip_variables_start
-set(_rocm_root "/opt/rocm-5.6.0")
+set(ROCM_ROOT_DIR "/opt/rocm-6.2.1" CACHE PATH "")
 
 set(ENABLE_HIP ON CACHE BOOL "")
 
-set(HIP_ROOT_DIR "${_rocm_root}/hip" CACHE STRING "")
-
 set(CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "")
 
-set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags -L${_rocm_root}/hip/../llvm/lib -L${_rocm_root}/hip/lib -Wl,-rpath,${_rocm_root}/hip/../llvm/lib:${_rocm_root}/hip/lib -lpgmath -lflang -lflangrti -lompstub -lamdhip64  -L${_rocm_root}/hip/../lib64 -Wl,-rpath,${_rocm_root}/hip/../lib64  -L${_rocm_root}/hip/../lib -Wl,-rpath,${_rocm_root}/hip/../lib -lamd_comgr -lhsa-runtime64 " CACHE STRING "")
+set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags -L${ROCM_ROOT_DIR}/llvm/lib -L${ROCM_ROOT_DIR}/lib -Wl,-rpath,${ROCM_ROOT_DIR}/llvm/lib:${ROCM_ROOT_DIR}/lib -lpgmath -lflang -lflangrti -lompstub -lamdhip64  -L${ROCM_ROOT_DIR}/lib64 -Wl,-rpath,${ROCM_ROOT_DIR}/lib64  -L${ROCM_ROOT_DIR}/lib -Wl,-rpath,${ROCM_ROOT_DIR}/lib -lamd_comgr -lhsa-runtime64 " CACHE STRING "")
+
 #_blt_tutorial_useful_hip_variables_end
 
 #------------------------------------------------