Update testing infrastructure for OpenHPC 3.4 compatibility

These changes are necessary for proper testing of OpenHPC 3.4:

- Replace power reset with power off/on cycle for better hardware
  reliability in Huawei and Lenovo launch scripts
- Add distribution-specific network interface handling for leap vs
  other distributions (eth0 vs enp189s0f0)
- Update repository paths and mount configurations for consistency
- Add openEuler 22.03 kernel firmware fixes and dracut rebuild for
  proper hardware support
- Update ISO paths to use latest versions for Rocky and AlmaLinux
- Improve network routing configuration with additional routes for
  test infrastructure
- Standardize autoyast template naming by removing cluster-specific
  variants
- Add hypre test disabling for openEuler 22.03 with OpenPBS due to
  compatibility issues in 3.4
- Various CI and testing infrastructure improvements for 3.4 release

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>
Signed-off-by: Adrian Reber <[email protected]>

Author: adrianreber

ansible/roles/obs/ohpc-lenovo-c3.yml

@@ -26,7 +26,7 @@
     - name: Import ohpc-lenovo-common.yml
       ansible.builtin.import_tasks: ../test/ohpc-lenovo-common.yml
 
-    - name: Create OBS directories
+    - name: Create OBS directories before mounting
       ansible.builtin.file:
         dest: "{{ item }}"
         mode: "0755"
@@ -52,7 +52,7 @@
         - /etc/systemd/system/obsworker-podman.service.d
         - /etc/systemd/system/[email protected]
 
-    - name: Install packages
+    - name: Install packages (first epel-release)
       ansible.builtin.package:
         state: present
         name:

ansible/roles/obs/ohpc-lenovo-repo.yml

@@ -34,7 +34,7 @@
         - /etc/systemd/system/obsworker-podman.service.d
         - /etc/systemd/system/[email protected]
 
-    - name: Install packages
+    - name: Install packages (epel-release first)
       ansible.builtin.package:
         state: present
         name:

ansible/roles/repos/repos-aarch64.yml

@@ -42,7 +42,7 @@
       ansible.builtin.user:
         name: rhel
 
-    - name: Create directories
+    - name: Create directories before mounting
       ansible.builtin.file:
         dest: "{{ item }}"
         mode: "0755"
@@ -95,7 +95,7 @@
         state: mounted
         fstype: none
 
-    - name: Create directories
+    - name: Create directories after mounting
       ansible.builtin.file:
         dest: "/data{{ item }}"
         mode: "0755"
@@ -184,19 +184,19 @@
         setype: httpd_sys_content_t
         state: present
 
-    - name: Allow apache to modify files in /stats
+    - name: Allow apache to modify files in /results
       community.general.sefcontext:
         target: '/results(/.*)?'
         setype: httpd_sys_content_t
         state: present
 
-    - name: Allow apache to modify files in /stats
+    - name: Allow apache to modify files in /repos
       community.general.sefcontext:
         target: '/repos(/.*)?'
         setype: public_content_t
         state: present
 
-    - name: Allow apache to modify files in /stats
+    - name: Allow apache to modify files in /repos/repositories
       community.general.sefcontext:
         target: '/rhel/repositories(/.*)?'
         setype: httpd_sys_content_t

ansible/roles/test/files/huawei_launch_sms.sh

@@ -57,8 +57,14 @@ ssh "${BOOT_SERVER}" systemctl start kea-dhcp4
 echo -n "----> Switching boot device to PXE: "
 export IPMI_PASSWORD=${SMS_IPMI_PASSWORD}
 /usr/bin/ipmitool -C 3 -E -I lanplus -H "${BMC}" -U root chassis bootdev pxe
-echo -n "----> Rebooting ${TARGET}: "
-/usr/bin/ipmitool -C 3 -E -I lanplus -H "${BMC}" -U root chassis power reset
+echo -n "----> Power Off ${TARGET}: "
+/usr/bin/ipmitool -C 3 -E -I lanplus -H "${BMC}" -U root chassis power off
+for i in $(seq 5 -1 1); do
+	echo "-----> ${i}"
+	sleep 1
+done
+echo -n "----> Power On ${TARGET}: "
+/usr/bin/ipmitool -C 3 -E -I lanplus -H "${BMC}" -U root chassis power on
 echo "----> done"
 
 echo "--> Waiting for ${TARGET} to finish installation"

ansible/roles/test/files/lenovo_launch_sms.sh

@@ -55,8 +55,14 @@ ssh "${BOOT_SERVER}" systemctl start kea-dhcp4
 echo -n "----> Switching boot device to PXE: "
 export IPMI_PASSWORD=${SMS_IPMI_PASSWORD}
 /usr/bin/ipmitool -E -I lanplus -H "${BMC}" -U "${SMS_IPMI_USER}" chassis bootdev pxe options=efiboot
-echo -n "----> Rebooting ${TARGET}: "
-/usr/bin/ipmitool -E -I lanplus -H "${BMC}" -U "${SMS_IPMI_USER}" chassis power reset
+echo -n "----> Power Off ${TARGET}: "
+/usr/bin/ipmitool -C 3 -E -I lanplus -H "${BMC}" -U root chassis power off
+for i in $(seq 5 -1 1); do
+	echo "-----> ${i}"
+	sleep 1
+done
+echo -n "----> Power On ${TARGET}: "
+/usr/bin/ipmitool -C 3 -E -I lanplus -H "${BMC}" -U root chassis power on
 echo "----> done"
 
 echo "--> Waiting for ${TARGET} to finish installation"

ansible/roles/test/files/run-ci.sh

@@ -209,7 +209,11 @@ if [[ "${SMS}" == "ohpc-huawei-sms" ]]; then
 	((TIMEOUT += 100))
 	GATEWAY="175.200.16.14"
 	SMS_INTERNAL="${SMS}"
-	SMS_ETH_INTERNAL="enp189s0f0"
+	if [[ "${DISTRIBUTION}" == "leap"* ]]; then
+		SMS_ETH_INTERNAL="eth0"
+	else
+		SMS_ETH_INTERNAL="enp189s0f0"
+	fi
 else
 	TEST_ARCH=$(uname -m)
 	CI_CLUSTER=lenovo
@@ -349,6 +353,10 @@ if [[ "${CI_CLUSTER}" == "huawei" ]]; then
 	USER_TEST_OPTIONS="${USER_TEST_OPTIONS} --disable-spack --disable-easybuild"
 fi
 
+if [[ "${CI_CLUSTER}" == "huawei" ]] && [[ "${DISTRIBUTION}" == "openEuler_22.03" ]] && [[ "${RMS}" == "openpbs" ]]; then
+	USER_TEST_OPTIONS="${USER_TEST_OPTIONS} --disable-hypre"
+fi
+
 print_overview
 
 # Save old history
@@ -407,10 +415,10 @@ if [[ "${PROVISIONER}" == "confluent" ]]; then
 	} >>"${VARS}"
 
 	if [[ "${DISTRIBUTION}" == "rocky"* ]]; then
-		echo "export iso_path=/root/Rocky-9.4-x86_64-dvd.iso" >>"${VARS}"
+		echo "export iso_path=/root/Rocky-9-latest-x86_64-dvd.iso" >>"${VARS}"
 	fi
 	if [[ "${DISTRIBUTION}" == "almalinux"* ]]; then
-		echo "export iso_path=/root/AlmaLinux-9.5-x86_64-dvd.iso" >>"${VARS}"
+		echo "export iso_path=/root/AlmaLinux-9-latest-x86_64-dvd.iso" >>"${VARS}"
 	fi
 	if [[ "${DISTRIBUTION}" == "almalinux10" ]]; then
 		echo "export iso_path=/root/AlmaLinux-10.0-x86_64-dvd.iso" >>"${VARS}"

ansible/roles/test/files/support_functions.sh

@@ -151,7 +151,7 @@ run_root_level_tests() {
 		fi
 	fi
 
-	if [ "x${USER_TEST_OPTIONS}" != "x" ]; then
+	if [ "${USER_TEST_OPTIONS}" != "" ]; then
 		echo " "
 		echo "adding ${USER_TEST_OPTIONS} to root tests"
 		echo " "
@@ -193,7 +193,7 @@ run_user_level_tests() {
 		config_opts="$config_opts --with-compiler-families='gnu12 arm1'"
 	fi
 
-	if [ "x${USER_TEST_OPTIONS}" != "x" ]; then
+	if [ "${USER_TEST_OPTIONS}" != "" ]; then
 		echo " "
 		echo "adding ${USER_TEST_OPTIONS} to user tests"
 		echo " "
@@ -294,7 +294,12 @@ install_openHPC_cluster() {
 		if [ "${Provisioner}" == "warewulf" ]; then
 			echo "CI Customization: console=tty0 breaks the compute nodes"
 			sed '/dnf -y install ohpc-warewulf/a sed -e "s,# \\(database chunk size\\),\\1,g" -i /etc/warewulf/database.conf' -i "${recipeFile}"
-			sed '/dnf -y install ohpc-warewulf/a sed -e "s,console=tty0,,g" -i /usr/share/perl5/vendor_perl/Warewulf/Provision/Pxe.pm' -i "${recipeFile}"
+			if [[ "${DISTRIBUTION}" == "leap"* ]]; then
+				sed '/dnf -y install ohpc-warewulf/a sed -e "s,console=tty0,,g" -i /usr/lib/perl5/vendor_perl/*/Warewulf/Provision/Pxe.pm' -i "${recipeFile}"
+			else
+				sed '/dnf -y install ohpc-warewulf/a sed -e "s,console=tty0,,g" -i /usr/share/perl5/vendor_perl/Warewulf/Provision/Pxe.pm' -i "${recipeFile}"
+			fi
+
 		fi
 		if [ "${Provisioner}" == "warewulf4" ]; then
 			echo "CI Customization: Switch to http in repository definition"
@@ -666,6 +671,7 @@ wait_for_computes() {
 					ipmitool -E -I lanplus -H "${c_bmc[$j]}" -U "${bmc_username}" -P "${bmc_password}" chassis bootdev pxe options=efiboot
 				fi
 				ipmitool -E -I lanplus -H "${c_bmc[$j]}" -U "${bmc_username}" -P "${bmc_password}" power off
+				local_sleep 5
 				ipmitool -E -I lanplus -H "${c_bmc[$j]}" -U "${bmc_username}" -P "${bmc_password}" power on
 			done
 		fi
@@ -730,11 +736,7 @@ enable_repo() {
 	if [[ "${VERSION_MAJOR}" == "2" ]]; then
 		OBS_KEY="https://obs.openhpc.community/projects/OpenHPC/public_key"
 	else
-		if [[ "${VERSION_MINOR}" != "0" ]]; then
-			OBS_KEY="https://obs.openhpc.community/projects/OpenHPC${VERSION_MAJOR}/public_key"
-		else
-			OBS_KEY="http://obs.openhpc.community:82/OpenHPC${VERSION_MAJOR}:/${VERSION_MAJOR}.0:/Factory/${os_repo}/repodata/repomd.xml.key"
-		fi
+		OBS_KEY="http://obs.openhpc.community:82/OpenHPC${VERSION_MAJOR}:/${VERSION_MAJOR}.0:/Factory/${os_repo}/repodata/repomd.xml.key"
 	fi
 	STAGING_REPO_KEY="${STAGING_REPO}/${os_repo}/repodata/repomd.xml.key"
 	RELEASE_REPO_KEY="${RELEASE_REPO}/${os_repo}/repodata/repomd.xml.key"

ansible/roles/test/ohpc-common-repo.yml

@@ -5,8 +5,8 @@
     dest: "{{ item }}"
     state: directory
   with_items:
-    - "/var/www/html/openSUSE-Leap-15.3-DVD-{{ arch }}-Media"
-    - "/var/www/html/openSUSE-Leap-15.5-DVD-{{ arch }}-Media"
+    - "/var/www/html/openSUSE-Leap-15.3-DVD-{{ arch }}-Current"
+    - "/var/www/html/openSUSE-Leap-15.5-DVD-{{ arch }}-Current"
     - /var/www/html/openEuler-22.03-LTS-SP3-everything
     - /var/www/html/openEuler-24.03-LTS-SP2-everything
     - /var/www/html/AlmaLinux-10-latest
@@ -101,7 +101,6 @@
     opts: defaults,ro
     state: mounted
     fstype: iso9660
-  when: inventory_hostname_short.startswith('ohpc-lenovo-repo')
 
 - name: Setup tftp booting
   ansible.builtin.copy:
@@ -163,8 +162,8 @@
     dest: /var/lib/tftpboot/
     mode: "0644"
   with_items:
-    - "/var/www/html/openSUSE-Leap-15.5-DVD-{{ arch }}-Current/boot/{{ arch }}/loader/initrd"
-    - "/var/www/html/openSUSE-Leap-15.5-DVD-{{ arch }}-Current/boot/{{ arch }}/loader/linux"
+    - "/var/www/html/openSUSE-Leap-15.5-DVD-{{ arch }}-Current/boot/{{ arch }}/{%- if arch == 'x86_64' -%}loader/{%- endif -%}initrd"
+    - "/var/www/html/openSUSE-Leap-15.5-DVD-{{ arch }}-Current/boot/{{ arch }}/{%- if arch == 'x86_64' -%}loader/{%- endif -%}linux"
   when: distro == "leap15.5"
 
 - name: Setup tftp booting for leap15.3
@@ -187,7 +186,7 @@
 
 - name: Copy autoyast for "{{ distro }}"
   ansible.builtin.template:
-    src: "{{ distro }}-autoyast.{{ cluster }}"
+    src: "{{ distro }}-autoyast"
     dest: "/var/www/html/{{ distro }}-autoyast"
     mode: "0644"
   when: distro.startswith("leap15")

ansible/roles/test/templates/el-kickstart

@@ -129,7 +129,12 @@ services --enabled="chronyd"
 
 rootpw --iscrypted {{ root_password_crypted }}
 
+{% if type == 'openeuler' and inventory_hostname_short.startswith('ohpc-lenovo-repo') %}
+# the network card firmware requires a real reboot
+reboot
+{% else %}
 reboot --kexec
+{% endif %}
 
 %post
 set -x
@@ -153,6 +158,10 @@ export http_proxy=http://{{ proxy }}
 	{% endif %}
 	/usr/bin/crb enable
 {% elif type == 'openeuler' %}
+	# the openeuler 22.03 kernel cannot handle zstd compressed firmware, but openeuler ships it that way
+	zstd -d /usr/lib/firmware/qed/qed_init_values_zipped-8.42.2.0.bin.zst || true
+	dracut --force --kver $(rpm -q --queryformat '%{version}-%{release}.%{arch}' kernel)
+
 	echo "gpgcheck=0" >> /etc/yum.repos.d/DVD.repo
 	echo "gpgcheck=0" >> /etc/yum.repos.d/everythingMirror.repo
 	echo "gpgcheck=0" >> /etc/yum.repos.d/updateMirror.repo

ansible/roles/test/templates/huawei.mapping

@@ -28,9 +28,9 @@ internal_network=175.200.0.0
 {% if distro.startswith('rocky') or distro.startswith('almalinux') %}
 eth_provision=enp189s0f0
 {% elif distro.startswith('openEuler') %}
-eth_provision=eth2
+eth_provision=eth0
 {% elif distro.startswith('leap') %}
-eth_provision=eth2
+eth_provision=eth0
 {% endif %}