test: Extend bare metal deployment (#7661)

This PR adds many improvements to the reload_host.sh script and adds a
library for redeploying HostOS and GuestOS. A CLI tool is coming in a
follow-up PR.

Notable changes:
- Create deploy_to_bare_metal function for deploying HostOS and/or
GuestOS from a URL or a file
- Make all input images optional in reload_hostos.sh
- Support loading images from a local file in reload_hostos.sh

reload_hostos will be renamed in the future, for now it's still called
reload_hostos to preserve git history.

---------

Co-authored-by: IDX GitHub Automation <[email protected]>

Author: frankdavid

Cargo.lock

@@ -1,6 +1,6 @@
 load("//ic-os/components:setupos.bzl", "component_files")
 
-package(default_visibility = ["//rs/tests:__subpackages__"])
+package(default_visibility = ["//rs:ic-os-pkg"])
 
 # Components required by the reload_hostos script to reload HostOS
 all_component_files = component_files | {

ic-os/testing/BUILD.bazel

@@ -11,11 +11,13 @@ TARGET_BOOT_PARTITION_MOUNT=/tmp/target_boot_partition
 GUESTOS_EXTRACT_DIR=/tmp/guestos_img
 
 # Temporary download paths
-HOSTOS_UPGRADE_TAR=/tmp/hostos-upgrade-img.tar.zst
-GUESTOS_TAR=/tmp/guestos.tar.zst
+SETUPOS_CONFIG_IMG_PATH=/tmp/setupos-config.img
+HOSTOS_UPGRADE_IMG_TAR_PATH=/tmp/hostos-upgrade-img.tar.zst
+GUESTOS_IMG_TAR_PATH=/tmp/guestos.tar.zst
 
 usage() {
-    echo "Usage: $0 --setupos-config-img=<path-to-setupos-config-image> --hostos-upgrade-img=<path-to-hostos-upgrade-image> --guestos-img=<path-to-guestos-image>" >&2
+    echo "Usage: $0 [--setupos-config-img=<path-to-setupos-config-image>] [--hostos-upgrade-img=<path-to-hostos-upgrade-image>] [--guestos-img=<path-to-guestos-image>]" >&2
+    echo "At least one of --hostos-upgrade-img or --guestos-img must be provided." >&2
 }
 
 cleanup() {
@@ -27,18 +29,42 @@ cleanup() {
     umount /media 2>/dev/null || true
     umount "$SETUPOS_CONFIG_MOUNT" 2>/dev/null || true
     umount "$TARGET_BOOT_PARTITION_MOUNT" 2>/dev/null || true
-    rm -rf "$TARGET_BOOT_PARTITION_MOUNT" 2>/dev/null || true
-    rm -rf "$GUESTOS_EXTRACT_DIR" 2>/dev/null || true
-    rm -f "$HOSTOS_UPGRADE_TAR" "$GUESTOS_TAR" 2>/dev/null || true
+    rm -rf "$TARGET_BOOT_PARTITION_MOUNT" "$GUESTOS_EXTRACT_DIR" 2>/dev/null || true
+    rm -f "$SETUPOS_CONFIG_IMG_PATH" "$HOSTOS_UPGRADE_IMG_TAR_PATH" "$GUESTOS_IMG_TAR_PATH" 2>/dev/null || true
 }
 
-download_images() {
-    echo "Downloading HostOS upgrade image..."
-    curl "$HOSTOS_UPGRADE_IMG" -o "$HOSTOS_UPGRADE_TAR" --fail --silent --show-error --clobber
+fetch_file() {
+    # Fetch a file from a URL or local filesystem and store it at the specified path.
+    local source="$1"
+    local dest="$2"
+    local name="$3"
+
+    if echo "$source" | grep -q "://"; then
+        echo "Downloading $name from $source..."
+        curl "$source" -o "$dest" --fail --silent --show-error --clobber
+    else
+        if [ ! -f "$source" ]; then
+            echo "Local file $source does not exist!" >&2
+            exit 1
+        fi
+        echo "Using local $name from $source..."
+        if [[ "$source" != "$dest" ]]; then
+            mv "$source" "$dest"
+        fi
+    fi
+}
+
+fetch_images() {
+    if [ -n "$SETUPOS_CONFIG_IMG_SRC" ]; then
+        fetch_file "$SETUPOS_CONFIG_IMG_SRC" "$SETUPOS_CONFIG_IMG_PATH" "SetupOS config image"
+    fi
+
+    if [ -n "$HOSTOS_UPGRADE_IMG_TAR_SRC" ]; then
+        fetch_file "$HOSTOS_UPGRADE_IMG_TAR_SRC" "$HOSTOS_UPGRADE_IMG_TAR_PATH" "HostOS upgrade image"
+    fi
 
-    if [ -n "$GUESTOS_IMG" ]; then
-        echo "Downloading GuestOS image..."
-        curl "$GUESTOS_IMG" -o "$GUESTOS_TAR" --fail --silent --show-error --clobber
+    if [ -n "$GUESTOS_IMG_TAR_SRC" ]; then
+        fetch_file "$GUESTOS_IMG_TAR_SRC" "$GUESTOS_IMG_TAR_PATH" "GuestOS image"
     fi
 }
 
@@ -50,19 +76,24 @@ setup_temp_mounts() {
 }
 
 install_new_hostos() {
+    if [ -z "$HOSTOS_UPGRADE_IMG_TAR_SRC" ]; then
+        echo "No HostOS upgrade image specified, skipping HostOS upgrade."
+        return
+    fi
+
     echo "Invoking HostOS upgrade..."
-    /opt/ic/bin/manageboot.sh hostos upgrade-install "$HOSTOS_UPGRADE_TAR"
+    /opt/ic/bin/manageboot.sh hostos upgrade-install "$HOSTOS_UPGRADE_IMG_TAR_PATH"
 }
 
 install_new_guestos() {
-    if [ -z "$GUESTOS_IMG" ]; then
+    if [ -z "$GUESTOS_IMG_TAR_SRC" ]; then
         echo "No GuestOS image specified, skipping GuestOS upgrade."
         return
     fi
 
     echo "Installing GuestOS image..."
     mkdir -p "$GUESTOS_EXTRACT_DIR"
-    tar -xavf "$GUESTOS_TAR" -C "$GUESTOS_EXTRACT_DIR"
+    tar -xavf "$GUESTOS_IMG_TAR_PATH" -C "$GUESTOS_EXTRACT_DIR"
 
     echo "Stopping GuestOS service..."
     systemctl stop guestos.service || true
@@ -83,6 +114,11 @@ mount_target_boot_partition() {
 }
 
 setup_config() {
+    if [ -z "$SETUPOS_CONFIG_IMG_SRC" ]; then
+        echo "No SetupOS config image provided, skipping configuration setup..."
+        return
+    fi
+
     echo "Setting up configuration environment..."
 
     # Mount over existing script dir and use the shipped scripts instead of the ones already deployed on the node
@@ -94,7 +130,7 @@ setup_config() {
 
     # Mount SetupOS config image
     mkdir -p "$SETUPOS_CONFIG_MOUNT"
-    mount "$SETUPOS_CONFIG_IMG" "$SETUPOS_CONFIG_MOUNT"
+    mount "$SETUPOS_CONFIG_IMG_PATH" "$SETUPOS_CONFIG_MOUNT"
 
     # Preload and create config
     /opt/ic/bin/preload-config.sh
@@ -119,25 +155,25 @@ commit_and_reboot() {
         --command-line="$boot_args"
 
     echo "Scheduling reboot via kexec..."
-    nohup bash -c 'sleep 5; systemctl start kexec.target' >/dev/null 2>&1 &
+    nohup bash -c 'sleep 2; systemctl start kexec.target' >/dev/null 2>&1 &
 }
 
-SETUPOS_CONFIG_IMG=""
-HOSTOS_UPGRADE_IMG=""
-GUESTOS_IMG=""
+SETUPOS_CONFIG_IMG_SRC=""
+HOSTOS_UPGRADE_IMG_TAR_SRC=""
+GUESTOS_IMG_TAR_SRC=""
 
 while [ "$#" -gt 0 ]; do
     case "$1" in
         --setupos-config-img=*)
-            SETUPOS_CONFIG_IMG="${1#*=}"
+            SETUPOS_CONFIG_IMG_SRC="${1#*=}"
             shift
             ;;
         --hostos-upgrade-img=*)
-            HOSTOS_UPGRADE_IMG="${1#*=}"
+            HOSTOS_UPGRADE_IMG_TAR_SRC="${1#*=}"
             shift
             ;;
         --guestos-img=*)
-            GUESTOS_IMG="${1#*=}"
+            GUESTOS_IMG_TAR_SRC="${1#*=}"
             shift
             ;;
         --)
@@ -147,40 +183,46 @@ while [ "$#" -gt 0 ]; do
         *)
             echo "Unknown option: $1" >&2
             usage
-            exit 2
+            exit 1
             ;;
     esac
 done
 
-if [ -z "$SETUPOS_CONFIG_IMG" ]; then
-    echo "Missing required --setupos-config-img argument" >&2
+if [ -z "$HOSTOS_UPGRADE_IMG_TAR_SRC" ] && [ -z "$GUESTOS_IMG_TAR_SRC" ] && [ -z "$SETUPOS_CONFIG_IMG_SRC" ]; then
+    echo "At least one of --hostos-upgrade-img or --guestos-img or --setupos-config-img must be provided" >&2
     usage
-    exit 2
-fi
-if [ -z "$HOSTOS_UPGRADE_IMG" ]; then
-    echo "Missing required --hostos-upgrade-img argument" >&2
-    usage
-    exit 2
+    exit 1
 fi
 
 trap cleanup EXIT
 
 mount -o remount,rw /
 cleanup
 
-download_images
+fetch_images
 setup_temp_mounts
 install_new_hostos
 install_new_guestos
-
-# Determine target alternative and mount its boot partition
-target_alternative="$(/opt/ic/bin/manageboot.sh hostos target)"
-echo "Will update HostOS into: $target_alternative"
-mount_target_boot_partition "$target_alternative"
-
-# Read boot arguments from new HostOS
-eval "$(cat "$TARGET_BOOT_PARTITION_MOUNT/boot_args")"
-boot_args_var=BOOT_ARGS_${target_alternative}
-
 setup_config
-commit_and_reboot "${!boot_args_var}"
+
+if [ -n "$HOSTOS_UPGRADE_IMG_TAR_SRC" ]; then
+    # Determine target alternative and mount its boot partition
+    target_alternative="$(/opt/ic/bin/manageboot.sh hostos target)"
+    echo "Will update HostOS into: $target_alternative"
+    mount_target_boot_partition "$target_alternative"
+
+    # Read boot arguments from new HostOS
+    eval "$(cat "$TARGET_BOOT_PARTITION_MOUNT/boot_args")"
+    boot_args_var=BOOT_ARGS_${target_alternative}
+
+    commit_and_reboot "${!boot_args_var}"
+elif [ -n "$SETUPOS_CONFIG_IMG_SRC" ]; then
+    # Config-only update: reboot with current kernel
+    echo "Preparing kexec reboot with current kernel..."
+    kexec -l /boot/vmlinuz --initrd=/boot/initrd.img --reuse-cmdline
+    nohup bash -c 'sleep 2; systemctl start kexec.target' >/dev/null 2>&1 &
+elif [ -n "$GUESTOS_IMG_TAR_SRC" ]; then
+    echo "Only GuestOS was updated."
+    echo "Starting GuestOS..."
+    systemctl start guestos.service
+fi

ic-os/testing/reload_hostos.sh

@@ -1,16 +1,35 @@
+load("@rules_rust//cargo:defs.bzl", "cargo_build_script")
 load("@rules_rust//rust:defs.bzl", "rust_library")
 
 package(default_visibility = ["//rs:ic-os-pkg"])
 
 rust_library(
     name = "bare_metal_deployment",
-    srcs = ["src/lib.rs"],
+    srcs = [
+        "src/deploy.rs",
+        "src/lib.rs",
+    ],
+    compile_data = ["//ic-os/testing:reload_hostos_cmd"],
+    rustc_env = {
+        "RELOAD_HOSTOS_CMD": "$(execpath //ic-os/testing:reload_hostos_cmd)",
+    },
+    target_compatible_with = ["@platforms//os:linux"],  # requires Linux-specific dependencies
     visibility = [
         "//rs:ic-os-pkg",
         "//rs:system-tests-pkg",
     ],
     deps = [
+        ":build_script",
         "@crate_index//:anyhow",
+        "@crate_index//:http",
+        "@crate_index//:rand",
         "@crate_index//:rexpect",
+        "@crate_index//:slog",
+        "@crate_index//:ssh2",
     ],
 )
+
+cargo_build_script(
+    name = "build_script",
+    srcs = ["build.rs"],
+)

rs/ic_os/dev_test_tools/bare_metal_deployment/BUILD.bazel

@@ -3,5 +3,8 @@ name = "bare_metal_deployment"
 edition = "2021"
 
 [dependencies]
-anyhow = "1.0"
+anyhow = { workspace = true }
+http = { workspace = true }
+rand = { workspace = true }
 rexpect = { workspace = true }
+ssh2 = { workspace = true }

rs/ic_os/dev_test_tools/bare_metal_deployment/Cargo.toml

@@ -0,0 +1,20 @@
+use std::env;
+use std::fs;
+use std::path::PathBuf;
+
+fn main() {
+    let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
+    let dest_path = out_dir.join("reload_hostos_cmd.rs");
+
+    if let Ok(path) = env::var("RELOAD_HOSTOS_CMD") {
+        // Bazel build: include the actual file
+        let code = format!(
+            r#"const RELOAD_HOSTOS_CMD: &[u8] = include_bytes!("{}");"#,
+            path
+        );
+        fs::write(&dest_path, code).unwrap();
+    } else {
+        // Cargo build: use empty bytes
+        fs::write(&dest_path, r#"const RELOAD_HOSTOS_CMD: &[u8] = b"";"#).unwrap();
+    }
+}