diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index b8979a49aa..3d9b395c59 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -8,6 +8,7 @@ on: permissions: contents: read # to fetch code (actions/checkout) env: + EESSI_VERSION: '2023.06' EESSI_ACCELERATOR_TARGETS: | x86_64/amd/zen2: - nvidia/cc80 @@ -15,23 +16,33 @@ env: - nvidia/cc80 jobs: check_missing: - runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: - EESSI_VERSION: - - 2023.06 - EESSI_SOFTWARE_SUBDIR_OVERRIDE: - - aarch64/generic - - aarch64/neoverse_n1 - - aarch64/neoverse_v1 - - x86_64/amd/zen2 - - x86_64/amd/zen3 - - x86_64/amd/zen4 - - x86_64/intel/haswell - - x86_64/intel/sapphirerapids - - x86_64/intel/skylake_avx512 - - x86_64/generic + include: + # Arm CPU targets + - runs_on: ubuntu-24.04-arm + EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/generic + - runs_on: ubuntu-24.04-arm + EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/neoverse_n1 + - runs_on: ubuntu-24.04-arm + EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/neoverse_v1 + # x86_64 CPU targets + - runs_on: ubuntu-24.04 + EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen2 + - runs_on: ubuntu-24.04 + EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen3 + - runs_on: ubuntu-24.04 + EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen4 + - runs_on: ubuntu-24.04 + EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/haswell + - runs_on: ubuntu-24.04 + EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/sapphirerapids + - runs_on: ubuntu-24.04 + EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/skylake_avx512 + - runs_on: ubuntu-24.04 + EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/generic + runs-on: ${{ matrix.runs_on }} steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -46,20 +57,20 @@ jobs: - name: Test check_missing_installations.sh script run: | export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} - source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash), # to prevent issues with checks in the Easybuild configuration that use this variable export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*} module load EasyBuild which eb eb --version - export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${EESSI_VERSION} export EESSI_OS_TYPE=linux env | grep ^EESSI | sort # first check the CPU-only builds for this CPU target - echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)" - for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do + echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${EESSI_VERSION}/eessi-${EESSI_VERSION}-*.yml with latest EasyBuild release)" + for easystack_file in $(EESSI_VERSION=${EESSI_VERSION} .github/workflows/scripts/only_latest_easystacks.sh); do echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? @@ -74,7 +85,7 @@ jobs: for accel in ${accelerators}; do module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" - for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do + for easystack_file in $(EESSI_VERSION=${EESSI_VERSION} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? @@ -87,14 +98,14 @@ jobs: - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} - source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash), # to prevent issues with checks in the Easybuild configuration that use this variable export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*} module load EasyBuild which eb eb --version - export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${EESSI_VERSION} export EESSI_OS_TYPE=linux env | grep ^EESSI | sort # create dummy easystack file with a single entry (something that is not installed in EESSI) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 713fcba883..40b6e5c50f 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -354,7 +354,7 @@ else echo -e "Processing easystack file ${easystack_file}...\n\n" # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file - eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') + eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g') # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 770b7e4117..47edaa8bfc 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -108,7 +108,7 @@ if [ $EUID -eq 0 ]; then else for easystack_file in ${changed_easystacks_rebuilds}; do # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file - eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') + eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g') # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 6acef9bffa..f3d11aebe0 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -231,73 +231,6 @@ end end -local function using_eessi_accel_stack () - local modulepath = os.getenv("MODULEPATH") or "" - local accel_stack_in_modulepath = false - - -- Check if we are using an EESSI version 2023 accelerator stack by checking if the $MODULEPATH contains - -- a path that starts with /cvmfs/software.eessi.io and contains accel/nvidia/ccNN - for path in string.gmatch(modulepath, '(.-):') do - if string.sub(path, 1, 41) == "/cvmfs/software.eessi.io/versions/2023.06" then - if string.find(path, "accel/nvidia/cc%d%d") then - accel_stack_in_modulepath = true - break - end - end - end - return accel_stack_in_modulepath -end - -local function eessi_removed_module_warning_startup_hook(usrCmd) - if usrCmd == 'load' and not os.getenv("EESSI_SKIP_REMOVED_MODULES_CHECK") then - local CUDA_RELOCATION_MSG = [[All CUDA installations and modules depending on CUDA have been relocated to GPU-specific stacks. - Please see https://www.eessi.io/docs/site_specific_config/gpu/ for more information.]] - - local RELOCATED_CUDA_MODULES = { - ['NCCL'] = CUDA_RELOCATION_MSG, - ['NCCL/2.18.3-GCCcore-12.3.0-CUDA-12.1.1'] = CUDA_RELOCATION_MSG, - ['UCX-CUDA'] = CUDA_RELOCATION_MSG, - ['UCX-CUDA/1.14.1-GCCcore-12.3.0-CUDA-12.1.1'] = CUDA_RELOCATION_MSG, - -- we also have non-CUDA versions of OSU Micro Benchmarks, so only match the CUDA version - ['OSU-Micro-Benchmarks/7.2-gompi-2023a-CUDA-12.1.1'] = CUDA_RELOCATION_MSG, - ['UCC-CUDA'] = CUDA_RELOCATION_MSG, - ['UCC-CUDA/1.2.0-GCCcore-12.3.0-CUDA-12.1.1'] = CUDA_RELOCATION_MSG, - ['CUDA'] = CUDA_RELOCATION_MSG, - ['CUDA/12.1.1'] = CUDA_RELOCATION_MSG, - ['CUDA-Samples'] = CUDA_RELOCATION_MSG, - ['CUDA-Samples/12.1-GCC-12.3.0-CUDA-12.1.1'] = CUDA_RELOCATION_MSG, - } - - local REMOVED_MODULES = { - ['ipympl/0.9.3-foss-2023a'] = 'This module has been replaced by ipympl/0.9.3-gfbf-2023a', - } - - local masterTbl = masterTbl() - local error_msg = "" - -- The CUDA messages should only be shown if the accelerator stack is NOT being used - if not using_eessi_accel_stack() then - for _, module in pairs(masterTbl.pargs) do - if RELOCATED_CUDA_MODULES[module] ~= nil then - error_msg = error_msg .. module .. ': ' .. RELOCATED_CUDA_MODULES[module] .. '\\n\\n' - end - end - end - for _, module in pairs(masterTbl.pargs) do - if REMOVED_MODULES[module] ~= nil then - error_msg = error_msg .. module .. ': ' .. REMOVED_MODULES[module] .. '\\n\\n' - end - end - if error_msg ~= "" then - LmodError('\\n' .. error_msg .. 'If you know what you are doing and you want to ignore this check for removed/relocated modules, set $EESSI_SKIP_REMOVED_MODULES_CHECK to any value.') - end - end -end - -function eessi_startup_hook(usrCmd) - eessi_removed_module_warning_startup_hook(usrCmd) -end - -hook.register("startup", eessi_startup_hook) hook.register("load", eessi_load_hook) """ @@ -320,7 +253,7 @@ modT.fullName:match("GCC%-([0-9]*.[0-9]*.[0-9]*)") or modT.fullName:match("GCCcore%-([0-9]*.[0-9]*.[0-9]*)") - -- if nothing matches, return + -- if nothing matches, return if tcver == nil then return end -- if we have matches, check if the toolchain version is either 2022b or 12.2.0 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 537c7705d7..4e7ef512bf 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -59,3 +59,4 @@ easyconfigs: # without any from-commit this will use a version from April 6, 2024 # https://github.com/easybuilders/easybuild-easyconfigs/blob/88f6f9c7439c535e62461e6e71b1961c7be118b8/easybuild/easyconfigs/l/lit/lit-18.1.2-GCCcore-12.3.0.eb # EB 5.0.0 has a few changes to that + - cutadapt-4.9-GCCcore-12.3.0.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.0.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.0.0-2023a.yml index e08f6a8773..50a9a74733 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.0.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.0.0-2023a.yml @@ -4,3 +4,13 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22292 # and https://github.com/easybuilders/easybuild-easyconfigs/pull/22646 from-commit: b212c00fdc3983678037429719f1b210cb978b42 + - MUSCLE-5.1.0-GCCcore-12.3.0.eb: + options: + # See https://github.com/easybuilders/easybuild-easyconfigs/pull/22780 + from-commit: ff159780fbaa881fb442b0ee1b37cd9e3d981ca9 + - bcgTree-1.2.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22707 + # and https://github.com/easybuilders/easybuild-easyconfigs/pull/22540 + # and https://github.com/easybuilders/easybuild-easyconfigs/pull/22781 + from-commit: 720c5b5b8280bf342d827617ee14738f3d231097 diff --git a/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.0-2023a.yml new file mode 100644 index 0000000000..fa1b9b388e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.0-2023a.yml @@ -0,0 +1,4 @@ +easyconfigs: +#This is an outdated version of the Z3-4.12.2-GCCcore-12.3.0 easyconfig, adding it to keep sync with the other stacks +#see https://github.com/easybuilders/easybuild-easyconfigs/pull/20050 + - Z3-4.12.2-GCCcore-12.3.0-Python-3.11.3.eb diff --git a/easystacks/software.eessi.io/2023.06/icelake_cclake/001-eb-4.9.4.yml b/easystacks/software.eessi.io/2023.06/icelake_cclake/001-eb-4.9.4.yml new file mode 100644 index 0000000000..afe5d33ce3 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/icelake_cclake/001-eb-4.9.4.yml @@ -0,0 +1,16 @@ +easyconfigs: + - /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/4.8.2/easybuild/reprod/EasyBuild-4.8.2.eb: + options: + include-easyblocks: /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/4.8.2/easybuild/reprod/easyblocks/*.py + - /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/4.9.0/easybuild/reprod/EasyBuild-4.9.0.eb: + options: + include-easyblocks: /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/4.9.0/easybuild/reprod/easyblocks/*.py + - /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/4.9.1/easybuild/reprod/EasyBuild-4.9.1.eb: + options: + include-easyblocks: /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/4.9.1/easybuild/reprod/easyblocks/*.py + - /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/5.0.0/easybuild/reprod/EasyBuild-5.0.0.eb: + options: + include-easyblocks: /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/5.0.0/easybuild/reprod/easyblocks/*.py + - /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/4.9.4/easybuild/reprod/EasyBuild-4.9.4.eb: + options: + include-easyblocks: /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/4.9.4/easybuild/reprod/easyblocks/*.py diff --git a/easystacks/software.eessi.io/2023.06/icelake_cclake/002-eb-4.9.1.yml b/easystacks/software.eessi.io/2023.06/icelake_cclake/002-eb-4.9.1.yml new file mode 100644 index 0000000000..ce43ff7da5 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/icelake_cclake/002-eb-4.9.1.yml @@ -0,0 +1,4 @@ +easyconfigs: + - /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/4.9.2/easybuild/reprod/EasyBuild-4.9.2.eb: + options: + include-easyblocks: /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/4.9.2/easybuild/reprod/easyblocks/*.py diff --git a/easystacks/software.eessi.io/2023.06/icelake_cclake/003-eb-4.9.2.yml b/easystacks/software.eessi.io/2023.06/icelake_cclake/003-eb-4.9.2.yml new file mode 100644 index 0000000000..083bc53fef --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/icelake_cclake/003-eb-4.9.2.yml @@ -0,0 +1,4 @@ +easyconfigs: + - /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/4.9.3/easybuild/reprod/EasyBuild-4.9.3.eb: + options: + include-easyblocks: /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/intel/haswell/software/EasyBuild/4.9.3/easybuild/reprod/easyblocks/*.py diff --git a/eb_hooks.py b/eb_hooks.py index 5f5405c173..947ac90e47 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -373,40 +373,6 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs): print_msg("Changed toolchainopts for %s: %s", ec.name, ec['toolchainopts']) -def parse_hook_lammps_remove_deps_for_aarch64(ec, *args, **kwargs): - """ - Remove x86_64 specific dependencies for the CI and missing installations to pass on aarch64 - """ - if ec.name == 'LAMMPS': - if ec.version in ('2Aug2023_update2', '29Aug2024'): - if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': - # ScaFaCoS and tbb are not compatible with aarch64/* CPU targets, - # so remove them as dependencies for LAMMPS (they're optional); - # see also https://github.com/easybuilders/easybuild-easyconfigs/pull/19164 + - # https://github.com/easybuilders/easybuild-easyconfigs/pull/19000; - # we need this hook because we check for missing installations for all CPU targets - # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true) - ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('ScaFaCoS', 'tbb',)] - else: - raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") - - -def parse_hook_CP2K_remove_deps_for_aarch64(ec, *args, **kwargs): - """ - Remove x86_64 specific dependencies for the CI and missing installations to pass on aarch64 - """ - if ec.name == 'CP2K' and ec.version in ('2023.1',): - if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': - # LIBXSMM is not supported on ARM with GCC 12.2.0 and 12.3.0 - # See https://www.cp2k.org/dev:compiler_support - # See https://github.com/easybuilders/easybuild-easyconfigs/pull/20951 - # we need this hook because we check for missing installations for all CPU targets - # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true) - ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('libxsmm',)] - else: - raise EasyBuildError("CP2K-specific hook triggered for non-CP2K easyconfig?!") - - def parse_hook_zen4_module_only(ec, eprefix): """ Use --force --module-only if building a foss-2022b-based EasyConfig for Zen4. @@ -1192,8 +1158,6 @@ def post_module_hook(self, *args, **kwargs): 'fontconfig': parse_hook_fontconfig_add_fonts, 'FreeImage': parse_hook_freeimage_aarch64, 'grpcio': parse_hook_grpcio_zlib, - 'LAMMPS': parse_hook_lammps_remove_deps_for_aarch64, - 'CP2K': parse_hook_CP2K_remove_deps_for_aarch64, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, diff --git a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh index 741ead0559..8b87f630e0 100755 --- a/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh +++ b/scripts/gpu_support/nvidia/install_cuda_and_libraries.sh @@ -96,7 +96,7 @@ for EASYSTACK_FILE in ${TOPDIR}/easystacks/eessi-*CUDA*.yml; do echo -e "Processing easystack file ${easystack_file}...\n\n" # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file - eb_version=$(echo ${EASYSTACK_FILE} | sed 's/.*eb-\([0-9.]*\).*/\1/g') + eb_version=$(echo ${EASYSTACK_FILE} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g') # Load EasyBuild version for this easystack file _before_ loading EESSI-extend module_avail_out=${tmpdir}/ml.out