diff --git a/src/eessi/__init__.py b/src/eessi/__init__.py index 485f44a..d3ec452 100644 --- a/src/eessi/__init__.py +++ b/src/eessi/__init__.py @@ -1 +1 @@ -__version__ = "0.1.1" +__version__ = "0.2.0" diff --git a/src/eessi/cli/install.py b/src/eessi/cli/install.py new file mode 100644 index 0000000..e4aa41b --- /dev/null +++ b/src/eessi/cli/install.py @@ -0,0 +1,393 @@ +# license (SPDX): GPL-2.0-only +# +# authors: +# - Alex Domingo (Vrije Universiteit Brussel) + +import os +import re +import tempfile +import typing as t +import urllib.error as urlerr +import urllib.request as urlreq +from enum import Enum + +import typer +from rich import print as rich_print +from rich.padding import Padding +from rich.progress import Progress, SpinnerColumn, TextColumn +from rich.prompt import Confirm +from rich.tree import Tree + +from eessi.cli.help import help_callback +from eessi.print import report_error +from eessi.run import CmdRunner + +app = typer.Typer() +runner = CmdRunner() + +# Default values for CernVM-FS configuration +class CVMFSClientProfiles(str, Enum): + single = "single" + cluster = "cluster" + +DEFAULT_CVMFS_CLIENT_FILE = "/etc/cvmfs/default.local" +DEFAULT_CVMFS_CLIENT_PROFILE = CVMFSClientProfiles.single +DEFAULT_CVMFS_QUOTA_LIMIT = 10000 +DEFAULT_CVMFS_CACHE_DIR = "/var/lib/cvmfs" + +URL_CVMFS_RELEASE_RPM = "https://cvmrepo.s3.cern.ch/cvmrepo/yum/cvmfs-release-latest.noarch.rpm" +URL_CVMFS_RELEASE_DEB = "https://cvmrepo.s3.cern.ch/cvmrepo/apt/cvmfs-release-latest_all.deb" +URL_CVMFS_EESSI_RPM = "https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm" +URL_CVMFS_EESSI_DEB = "https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb" + + +def download_remote_file(url: str, filename: t.Optional[str] = None) -> str: + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + transient=True, + ) as progress: + progress.add_task(description=f"Downloading: {url}", total=None) + try: + local_path, _ = urlreq.urlretrieve(url, filename=filename) + except urlerr.HTTPError as e: + report_error(f"Download failed with HTTP error: {e.code}") + except urlerr.URLError as e: + report_error(f"Failed to reach a server: {e.reason}") + except urlerr.ContentTooShortError as e: + report_error(f"Download interrupted: {e.content}") + else: + rich_print(f":white_check_mark: Downloaded {url}") + return local_path + + +def download_and_install_remote_deb(url: str) -> None: + """ + Download DEB file from given URL and install it with dpkg + Use temporary directory + """ + filename = os.path.basename(url) + if os.path.splitext(filename)[1] != ".deb": + report_error(f"Cannot install file '{filename}' in Debian-based system") + + original_workdir = os.getcwd() + with tempfile.TemporaryDirectory() as tmp_workdir: + os.chdir(tmp_workdir) + local_pkg = download_remote_file(url, filename=filename) + runner.run_cmd( + f"dpkg -i {local_pkg}", + use_sudo=True, + ) + os.chdir(original_workdir) + + +def get_package_manager() -> t.Optional[str]: + """ + Determine which package manager is available on the system + """ + supported_package_managers = ["yum", "dnf", "apt"] + + for pkgmgr in supported_package_managers: + try: + _, _, exit_code = runner.run_cmd( + f"command -v {pkgmgr}", + check=False, + show_cmd=False, + ) + if exit_code == 0: + return pkgmgr + except Exception: + pass + + return None + + +def is_cvmfs_installed() -> bool: + """Check if CernVM-FS is installed""" + _, _, exit_code = runner.run_cmd( + "cvmfs_config showconfig", + check=False, + show_cmd=False, + ) + return exit_code == 0 + + +def is_eessi_config_installed() -> bool: + """Check if EESSI configuration is installed""" + return os.path.exists("/etc/cvmfs/domain.d/eessi.io.conf") + + +def is_client_config_installed(config_file: str = DEFAULT_CVMFS_CLIENT_FILE) -> bool: + """ + Check if client configuration file exists and contains required settings + """ + client_profile = None + quota_limit = None + + if os.path.exists(config_file): + try: + with open(config_file, 'r') as fp: + content = fp.read() + client_profile = re.search("CVMFS_CLIENT_PROFILE", content) + quota_limit = re.search("CVMFS_QUOTA_LIMIT", content) + except PermissionError: + report_error(f"Failed to read existing client configuration file for CernVM-FS: {config_file}") + + return client_profile and quota_limit + + +def install_cvmfs(answer_yes: bool = None) -> None: + """Install CernVM-FS based on the Linux distribution""" + + if is_cvmfs_installed(): + rich_print(":white_check_mark: CernVM-FS is already installed") + return + + install_confirmation = answer_yes + if install_confirmation is None: + install_confirmation = Confirm.ask( + "CernVM-FS not found. Do you want to install CernVM-FS in this system?" + ) + if not install_confirmation: + raise typer.Abort() + + rich_print(":package: Installing CernVM-FS packages...") + + package_manager = get_package_manager() + + if package_manager in ["yum", "dnf"]: + rich_print(":white_check_mark: Detected RHEL-based distribution") + runner.run_cmd( + f"{package_manager} install -y {URL_CVMFS_RELEASE_RPM}", + use_sudo=True, + ) + runner.run_cmd( + f"{package_manager} install -y cvmfs", + use_sudo=True, + ) + + elif package_manager == "apt": + rich_print(":white_check_mark: Detected Debian-based distribution") + download_and_install_remote_deb(URL_CVMFS_RELEASE_DEB) + runner.run_cmd( + "apt update", + use_sudo=True, + ) + runner.run_cmd( + "apt install -y cvmfs", + use_sudo=True, + ) + else: + report_error( + "No supported package manager found. Only yum/dnf (RHEL-based) and apt (Debian-based) " + "distributions are supported." + ) + + if is_cvmfs_installed(): + rich_print(":tada: CernVM-FS installed successfully") + else: + report_error("verification of CernVM-FS installation failed") + + +def install_eessi_config(answer_yes: bool = None) -> None: + """Install EESSI configuration for CernVM-FS""" + + if is_eessi_config_installed(): + rich_print(":white_check_mark: EESSI configuration is already installed") + return + + install_confirmation = answer_yes + if install_confirmation is None: + install_confirmation = Confirm.ask( + "EESSI configuration not found. Do you want to install it in this system?" + ) + if not install_confirmation: + raise typer.Abort() + + rich_print(":package: Installing EESSI configuration for CernVM-FS...") + + package_manager = get_package_manager() + + if package_manager in ["yum", "dnf"]: + rich_print(":white_check_mark: Detected RHEL-based distribution") + runner.run_cmd( + f"{package_manager} install -y {URL_CVMFS_EESSI_RPM}", + use_sudo=True, + ) + elif package_manager == "apt": + rich_print(":white_check_mark: Detected Debian-based distribution") + download_and_install_remote_deb(URL_CVMFS_EESSI_DEB) + else: + report_error( + "No supported package manager found. Only yum/dnf (RHEL-based) and apt (Debian-based) " + "distributions are supported." + ) + + if is_eessi_config_installed(): + rich_print(":tada: EESSI configuration for CernVM-FS installed successfully") + else: + report_error("verification of EESSI configuration for CernVM-FS failed") + + +def create_client_config( + answer_yes: bool = None, + cache_dir: str = DEFAULT_CVMFS_CACHE_DIR, + config_file: str = DEFAULT_CVMFS_CLIENT_FILE, + quota_limit: int = DEFAULT_CVMFS_QUOTA_LIMIT, + client_profile: str = DEFAULT_CVMFS_CLIENT_PROFILE, +) -> None: + """Create client configuration file for CernVM-FS""" + + if is_client_config_installed(config_file=config_file): + rich_print(":white_check_mark: CernVM-FS client configuration is already installed") + return + + install_confirmation = answer_yes + if install_confirmation is None: + install_confirmation = Confirm.ask( + "CernVM-FS client configuration not found. Do you want to install it in this system?" + ) + if not install_confirmation: + raise typer.Abort() + + rich_print(":package: Creating client configuration file...") + + config_content = [ + f"CVMFS_CLIENT_PROFILE='{client_profile.value}'", + f"CVMFS_QUOTA_LIMIT={quota_limit}", + ] + # Add cache directory if it's not the default + if cache_dir != DEFAULT_CVMFS_CACHE_DIR: + config_content.append(f"CVMFS_CACHE_BASE={cache_dir}") + + with tempfile.NamedTemporaryFile(mode="w", delete_on_close=False) as fp: + fp.writelines(f"{line}\n" for line in config_content) + fp.close() + rich_print(":white_check_mark: Configuration parameters added to temporary file") + runner.run_cmd( + f"cp {fp.name} {config_file}", + use_sudo=True, + ) + runner.run_cmd( + f"chmod 644 {config_file}", + show_cmd=False, + use_sudo=True, + ) + + if is_client_config_installed(config_file=config_file): + rich_print(f":tada: Client configuration file created at {config_file}") + else: + report_error("verification of client configuration for CernVM-FS failed") + + +def setup_eessi() -> None: + """Run cvmfs_config setup to make EESSI CernVM-FS repository accessible""" + + rich_print(":package: Running cvmfs_config setup to configure EESSI repositories...") + runner.run_cmd( + "cvmfs_config setup", + use_sudo=True, + ) + + rich_print(":white_check_mark: EESSI repositories configured successfully") + + +def native_install( + answer_yes: bool = None, + cache_dir: str = DEFAULT_CVMFS_CACHE_DIR, + config_file: str = DEFAULT_CVMFS_CLIENT_FILE, + quota_limit: int = DEFAULT_CVMFS_QUOTA_LIMIT, + client_profile: str = DEFAULT_CVMFS_CLIENT_PROFILE, +) -> None: + """ + Install EESSI natively on the local host by: + 1. Installing CernVM-FS if not already installed + 2. Installing EESSI configuration for CernVM-FS if not already installed + 3. Creating client configuration file for CernVM-FS if not already installed + """ + rich_print(Padding(":rocket: Launching native EESSI installation...", (1, 0))) + + task_list = Tree("EESSI requires the following components:") + task_list.add("[underline blue][link=https://cernvm.cern.ch/fs]CernVM File System[/link][/]") + task_list.add("configuration files of EESSI for CernVM-FS") + task_list.add("client configuration files for CernVM-FS") + task_list.add("EESSI repositories") + rich_print(task_list) + + # Check and install CernVM-FS if needed + rich_print(Padding(":jigsaw: [green]Step 1 of 4: [bold]CernVM-FS[/]", (1, 0))) + install_cvmfs(answer_yes) + + # Check and install EESSI configuration if needed + rich_print(Padding(":jigsaw: [green]Step 2 of 4: [bold]EESSI configuration for CernVM-FS[/]", (1, 0))) + install_eessi_config(answer_yes) + + # Check and create client configuration if needed + rich_print(Padding(":jigsaw: [green]Step 3 of 4: [bold]client configuration for CernVM-FS[/]", (1, 0))) + create_client_config(answer_yes, cache_dir, config_file, quota_limit, client_profile) + + # Setup EESSI repositories + rich_print(Padding(":jigsaw: [green]Step 4 of 4: [bold]EESSI repositories[/]", (1, 0))) + setup_eessi() + + rich_print(Padding(":checkered_flag: Native EESSI installation completed successfully!", (1, 0))) + + +@app.command() +def install( + help: bool = typer.Option( + None, # default value + "-h", + "--help", + help="Show this message and exit.", + callback=help_callback, + is_eager=True, + ), + answer_yes: bool = typer.Option( + None, + "-y", + "--yes", + help="Automatically answer all questions with yes", + ), + cache_dir: str = typer.Option( + DEFAULT_CVMFS_CACHE_DIR, + help="Directory for CernVM-FS client cache", + ), + config_file: str = typer.Option( + DEFAULT_CVMFS_CLIENT_FILE, + help="Path to client configuration file for CernVM-FS", + ), + quota_limit: int = typer.Option( + DEFAULT_CVMFS_QUOTA_LIMIT, + help="CernVM-FS client cache quota limit in MB", + ), + client_profile: CVMFSClientProfiles = typer.Option( + DEFAULT_CVMFS_CLIENT_PROFILE, + help="CernVM-FS client operation profile", + ), +): + """ + Install EESSI natively on the local host + + This command will carry out the following tasks: + 1. Install CernVM-FS system-wide + 2. Install EESSI configuration for CernVM-FS + 3. Configure CernVM-FS client + + [i]Note: This command requires [red]root privileges[/]. + """ + native_install( + answer_yes=answer_yes, + cache_dir=cache_dir, + config_file=config_file, + quota_limit=quota_limit, + client_profile=client_profile, + ) + rich_print("You can now use EESSI by running 'eessi init' or 'eessi shell'") + + # stop timer in CmdRunner + runner.sudo_reset_timer.cancel() + + +if __name__ == "__main__": + app() diff --git a/src/eessi/cli/main.py b/src/eessi/cli/main.py index 801ba01..02e0b13 100644 --- a/src/eessi/cli/main.py +++ b/src/eessi/cli/main.py @@ -9,6 +9,7 @@ from eessi.cli.check import app as check_app from eessi.cli.help import help_callback, version_callback from eessi.cli.init import app as init_app +from eessi.cli.install import app as install_app from eessi.cli.shell import app as shell_app app = typer.Typer( @@ -21,6 +22,7 @@ app.add_typer(check_app) app.add_typer(init_app) +app.add_typer(install_app) app.add_typer(shell_app) diff --git a/src/eessi/print.py b/src/eessi/print.py new file mode 100644 index 0000000..df45d67 --- /dev/null +++ b/src/eessi/print.py @@ -0,0 +1,18 @@ +# license (SPDX): GPL-2.0-only +# +# authors: +# - Alex Domingo (Vrije Universiteit Brussel) + +import sys +import typing as t + +import typer +from rich import print as rich_print + + +def report_error(msg: str) -> t.NoReturn: + """ + Report error and exit with specified non-zero exit code + """ + rich_print(f":collision: [bold red]{msg}[/]", file=sys.stderr) + raise typer.Abort() diff --git a/src/eessi/run.py b/src/eessi/run.py new file mode 100644 index 0000000..04ee419 --- /dev/null +++ b/src/eessi/run.py @@ -0,0 +1,154 @@ +# license (SPDX): GPL-2.0-only +# +# authors: +# - Alex Domingo (Vrije Universiteit Brussel) + +import subprocess +import threading + +from rich import print as rich_print +from rich.progress import Progress, SpinnerColumn, TextColumn +from rich.prompt import Prompt + +from eessi.print import report_error + +SUDO_PWD_TIMEOUT_SEC = 30.0 + + +class CmdRunner: + """ + Execute shell commands: + - as active user or as root (with sudo) + - hidden from output or shown with a spinner + + Password for sudo is stored in a single class attribute and it is + automatically cleared after a timeout + """ + def __init__(self): + self.sudo_password = None + self.sudo_reset_timer = None + + def clear_sudo_password(self) -> None: + self.sudo_password = None + + def ask_sudo_password(self, cmd: str) -> None: + """ + Prompt the user to input the password for sudo + and trigger the reset of the password after a timeout + """ + rich_print( + ":rotating_light: The following command requires [bold red]root permissions[/]: " + f"[dim cyan]{cmd}[/]" + ) + self.sudo_password = Prompt.ask( + ":key: Enter your [bold yellow]user password[/] in this system:", + password=True, + ) + # trigger reset of sudo password + self.sudo_reset_timer = threading.Timer(SUDO_PWD_TIMEOUT_SEC, self.clear_sudo_password) + self.sudo_reset_timer.start() + + def run_cmd_user(self, cmd: str) -> tuple[str, str, int]: + """ + Execute shell command as user + + Returns stdout, stderr, and exit code + """ + res = subprocess.run(cmd, shell=True, capture_output=True, text=True) + return res.stdout, res.stderr, res.returncode + + def run_cmd_root(self, cmd: str) -> tuple[str, str, int]: + """ + Execute shell command as root + Supports sudo with password input + + Returns stdout, stderr, and exit code + """ + if self.sudo_password is None: + self.ask_sudo_password(cmd) + + proc = subprocess.Popen( + ["sudo", "-S"] + cmd.split(), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + stdout, stderr = proc.communicate(input=f"{self.sudo_password}\n") + return stdout, stderr, proc.returncode + + def run_cmd_spinner(self, cmd: str, use_sudo: bool = False) -> tuple[str, str, int]: + """ + Execute shell command + Print command with a spinner in output + + - cmd: command to execute + - use_sudo: whether to use sudo for this command + + Returns stdout, stderr, and exit code + """ + cmd_runner = self.run_cmd_user + description = f"Executing command: [dim cyan]{cmd}[/]" + if use_sudo: + cmd_runner = self.run_cmd_root + description = f"Executing command as [bold red]root[/]: [dim cyan]{cmd}[/]" + # password prompt must happen before spinner, otherwise it gets drawn over + if self.sudo_password is None: + self.ask_sudo_password(cmd) + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + transient=True, + ) as progress: + progress.add_task(description=description, total=None) + res = cmd_runner(cmd) + cmd_status_mark = "white_check_mark" if res[2] == 0 else "collision" + rich_print(f":{cmd_status_mark}: Command: [dim cyan]{cmd}[/]") + + return res + + def run_cmd_hidden(self, cmd: str, use_sudo: bool = False) -> tuple[str, str, int]: + """ + Execute shell command + Keep execution hidden, do not print feedback on output + + - cmd: command to execute + - use_sudo: whether to use sudo for this command + + Returns stdout, stderr, and exit code + """ + cmd_runner = self.run_cmd_user + if use_sudo: + cmd_runner = self.run_cmd_root + + return cmd_runner(cmd) + + def run_cmd( + self, + cmd: str, + check: bool = True, + show_cmd: bool = True, + use_sudo: bool = False, + ) -> tuple[str, str, int]: + """ + Generic execution of shell command + Switches to specific runners depending on given options + + - cmd: Command to execute + - check: Whether to check for errors and report them + - show_cmd: Whether to show the executing command with a spinner + - use_sudo: Whether to use sudo for this command + + Returns stdout, stderr, and exit code + """ + cmd_runner = self.run_cmd_hidden + if show_cmd: + cmd_runner = self.run_cmd_spinner + + res = cmd_runner(cmd, use_sudo=use_sudo) + + if check and res[2] != 0: + report_error(f"Command failed: {cmd}; Output: {res[0]}; Error: {res[1]}") + + return res