Refactor initialization logic to allow for automatic detection in Fedora (#788)

Author: gbaraldi

docs/src/index.md

@@ -48,6 +48,18 @@ Pkg.add("AMDGPU")
 !!! note "On Windows AMD Software"
     Adrenalin Edition contains HIP library itself, while ROCm provides support for other functionality.
 
+!!! note "On Fedora ROCm packages"
+    Although not included in the AMD's list of supported Linux distributions,
+    Fedora provides its own ROCM packages.
+    ```
+    sudo dnf install rocminfo rccl-devel rocblas-devel rocfft-devel rocsparse-devel rocsolver-devel rocrand-devel roctracer-devel miopen-devel rocm-hip-devel
+    ```
+    However, the libraries are not installed in the usual location (under 
+    `/opt/rocm`) so for `AMDGPU` to find them you must set an environment variable.
+    ```
+    export ROCM_PATH=/usr/lib64
+    ```
+
 ## Test
 
 To ensure that everything works, you can run the test suite:

docs/src/install_tips.md

@@ -35,6 +35,10 @@ Depending on your GPU model and the functionality you want to use, you may have
 to force the GPU architecture by setting the `HSA_OVERRIDE_GFX_VERSION`
 variable to a compatible version.
 
+You may also have more than one type of GPU, for example a dedicated AMD GPU and an integrated one.
+In that case use `rocminfo` to find which device they are and setting `HIP_VISIBLE_DEVICES` to the specific device you want to use.
+Otherwise the runtime may crash if it sees two different architectures.
+
 ## Extra Setup Details
 
 List of additional steps that may be needed to take to ensure everything is working:

src/discovery/discovery.jl

@@ -44,18 +44,20 @@ function _hip_runtime_version()
     VersionNumber(major, minor, patch)
 end
 
+global rel_libdir::String = ""
+global libhsaruntime::String = ""
+global lld_path::String = ""
+global lld_artifact::Bool = false
+global libhip::String = ""
+global libdevice_libs::String = ""
+global librocblas::String = ""
+global librocsparse::String = ""
+global librocsolver::String = ""
+global librocrand::String = ""
+global librocfft::String = ""
+global libMIOpen_path::String = ""
+
 function __init__()
-    global libhsaruntime = ""
-    global lld_path = ""
-    global lld_artifact = ""
-    global libhip = ""
-    global libdevice_libs = ""
-    global librocblas = ""
-    global librocsparse = ""
-    global librocsolver = ""
-    global librocrand = ""
-    global librocfft = ""
-    global libMIOpen_path = ""
 
     if Sys.islinux() && isdir("/sys/class/kfd/kfd/topology/nodes/")
         for node_id in readdir("/sys/class/kfd/kfd/topology/nodes/")

src/discovery/utils.jl

@@ -1,12 +1,36 @@
-"""
-Find root ROCm directory.
-"""
+# use amdhip as query for a valid rocm_path
+function check_rocm_path(path::String)
+    libname = (Sys.islinux() ? "libamdhip64" : "amdhip64") * "." * dlext
+    path2 = path
+    isfile(joinpath(path2, libname)) && @goto success
+    path2 = joinpath(path, "lib")
+    isfile(joinpath(path2, libname)) && @goto success
+    path2 = joinpath(path, "bin")
+    isfile(joinpath(path2, libname)) && @goto success
+    path2 = joinpath(path, "lib64")
+    isfile(joinpath(path2, libname)) && @goto success
+    return ""
+    @label success
+    @assert isdir(path2)
+    return path2
+end
+
+# Find root ROCm directory.
 function find_roc_path()::String
     env_dir = get(ENV, "ROCM_PATH", "")
-    isdir(env_dir) && return env_dir
+    isdir(env_dir) && check_rocm_path(env_dir) != "" && return env_dir
 
     if Sys.islinux()
-        isdir("/opt/rocm") && return "/opt/rocm" # shim for Ubuntu rocm packages.
+        hipconfig = Sys.which("hipconfig")
+        if !isnothing(hipconfig)
+            rocm_path = read(`$hipconfig --rocmpath`, String)
+            rocm_path = check_rocm_path(rocm_path)
+            isdir(rocm_path) && return rocm_path
+        end
+        rocm_path = check_rocm_path("/opt/rocm")
+        isdir(rocm_path) && return rocm_path
+        rocm_path = check_rocm_path("/usr")
+        isdir(rocm_path) && return rocm_path
     elseif Sys.iswindows()
         disk_dir = dirname(dirname(homedir())) # Disk C root directory.
         rocm_dir = joinpath(disk_dir, "Program Files", "AMD", "ROCm")
@@ -24,22 +48,52 @@ function find_roc_path()::String
     return ""
 end
 
+# use hip.bc as query for a valid device libs dir
+function check_device_libs(path::String)
+    if isdir(path)
+        file_path = joinpath(path, "hip" * ".bc")
+        if !ispath(file_path)
+            file_path = joinpath(path, "hip"* ".amdgcn.bc")
+            if !ispath(file_path)
+                # failed to find matching bitcode file
+                return ""
+            end
+        end
+        return path
+    else
+        return ""
+    end
+end
+
 function find_device_libs(rocm_path::String)::String
     env_dir = get(ENV, "ROCM_PATH", "")
     if isdir(env_dir)
         path = joinpath(env_dir, "amdgcn", "bitcode")
+        path = check_device_libs(path)
         isdir(path) && return path
     end
-
     # Might be set by tools like Spack or the user
     hip_devlibs_path = get(ENV, "HIP_DEVICE_LIB_PATH", "")
     hip_devlibs_path !== "" && return hip_devlibs_path
     devlibs_path = get(ENV, "DEVICE_LIB_PATH", "")
     devlibs_path !== "" && return devlibs_path
-
+    # Try using hipconfig to find the device libraries.
     # Try the canonical location.
     canonical_dir = joinpath(rocm_path, "amdgcn", "bitcode")
+    canonical_dir = check_device_libs(canonical_dir)
     isdir(canonical_dir) && return canonical_dir
+    # Fedora might put it in a weird place
+    hipconfig = Sys.which("hipconfig")
+    if !isnothing(hipconfig)
+        clang_path = read(`$hipconfig --hipclangpath`, String)
+        lib_path = joinpath(clang_path ,".." , "lib","clang")
+        if isdir(lib_path)
+            lib_path = joinpath(lib_path, only(readdir(lib_path)))
+            lib_path = joinpath(lib_path, "amdgcn", "bitcode")
+            lib_path = check_device_libs(lib_path)
+            isdir(lib_path) && return lib_path
+        end
+    end
     return ""
 end
 
@@ -52,21 +106,27 @@ function find_rocm_library(libs::Vector; rocm_path::String, ext::String = dlext)
 end
 
 function find_rocm_library(lib::String; rocm_path::String, ext::String = dlext)::String
-    libdir = joinpath(rocm_path, Sys.islinux() ? "lib" : "bin")
-    isdir(libdir) || return ""
-
+    libdir = joinpath(rocm_path, rel_libdir)
+    @assert isdir(libdir)
     for file in readdir(libdir; join=true)
         fname = basename(file)
-        matched = startswith(fname, lib) && endswith(fname, ext)
+        matched = startswith(fname, lib) && contains(fname, ext)
         matched && return file
     end
     return ""
 end
 
 function find_ld_lld(rocm_path::String)::String
     lld_name = "ld.lld" * (Sys.iswindows() ? ".exe" : "")
-    for subdir in (joinpath("llvm", "bin"), "bin")
-        exp_ld_path = joinpath(rocm_path, subdir, lld_name)
+
+    dirs = (joinpath(rocm_path,"llvm", "bin"), joinpath(rocm_path,"bin"))
+    hipconfig = Sys.which("hipconfig")
+     if !isnothing(hipconfig)
+        clang_path = read(`$hipconfig --hipclangpath`, String)
+        dirs = (dirs ..., clang_path)
+     end
+    for dir in dirs
+        exp_ld_path = joinpath(dir, lld_name)
         ispath(exp_ld_path) || continue
         try
             tmpfile = tempname(;cleanup=false)