From 2744498c418cc59e6e043f8f83f2b59e2154707b Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Thu, 12 Feb 2026 04:55:06 +0000 Subject: [PATCH 1/9] Progress on tier1 container site. --- configs/sites/tier1/container/Dockerfile | 185 ++++++++++++++++++ configs/sites/tier1/container/README.md | 115 +++++++++++ configs/sites/tier1/container/config.yaml | 9 + configs/sites/tier1/container/modules.yaml | 7 + configs/sites/tier1/container/packages.yaml | 63 ++++++ .../sites/tier1/container/packages_gcc.yaml | 18 ++ .../sites/tier1/container/packages_intel.yaml | 76 +++++++ 7 files changed, 473 insertions(+) create mode 100644 configs/sites/tier1/container/Dockerfile create mode 100644 configs/sites/tier1/container/README.md create mode 100644 configs/sites/tier1/container/config.yaml create mode 100644 configs/sites/tier1/container/modules.yaml create mode 100644 configs/sites/tier1/container/packages.yaml create mode 100644 configs/sites/tier1/container/packages_gcc.yaml create mode 100644 configs/sites/tier1/container/packages_intel.yaml diff --git a/configs/sites/tier1/container/Dockerfile b/configs/sites/tier1/container/Dockerfile new file mode 100644 index 000000000..f7f0c20b6 --- /dev/null +++ b/configs/sites/tier1/container/Dockerfile @@ -0,0 +1,185 @@ +# Spack-Stack Container Build for tier1/container site +# +# This Dockerfile builds a spack-stack environment using the tier1/container +# See building instructions in README.md +# +# Docker build args: +# BUILD_JOBS: Build parallelism (default 4) +# SPACK_STACK_TEMPLATE: Environment template from spack-stack (default unified-dev). +# COMPILER: Compiler to install (default gcc). +# - gcc: GCC 13.4.0 +# - intel: Intel OneAPI 2025.3.0 + +FROM ubuntu:24.04 AS ubuntu_base + +ARG BUILD_JOBS=4 +ARG SPACK_STACK_TEMPLATE=unified-dev +ARG COMPILER=gcc + +SHELL ["/bin/bash", "-c"] + + +ENV DEBIAN_FRONTEND=noninteractive \ + TZ=Etc/UTC \ + LANGUAGE=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 + +# Install core packages. +RUN set -euo pipefail; \ + apt-get -yqq update && \ + apt-get -yqq upgrade && \ + apt-get -yqq install --no-install-recommends \ + # Core build tools. + build-essential \ + gcc-13 \ + gcc++-13 \ + gfortran-13 \ + cpp-13 \ + make \ + llvm-14 \ + autoconf \ + # External dependencies. + libcurl4-openssl-dev \ + libmysqlclient-dev \ + libqt5svg5-dev \ + qt5-qmake \ + qt5dxcb-plugin \ + qtbase5-dev \ + zstd \ + # Shell tools, source retrieval and networking. + sed \ + file \ + less \ + bzip2 \ + unzip \ + ca-certificates \ + curl \ + git \ + git-lfs \ + gpg \ + environment-modules \ + tcl \ + tcl-dev \ + vim \ + wget \ + nano \ + locales && \ + locale-gen en_US.UTF-8 && \ + # Update alternatives for compilers and tools + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \ + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ + # Install the intel + if [ "${COMPILER:-}" = "intel" ]; then \ + apt update && \ + apt install -y apt-utils gpg && \ + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ + | gpg --dearmor \ + | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ + echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \ + | tee /etc/apt/sources.list.d/oneAPI.list && \ + apt update && \ + apt install -y \ + intel-oneapi-compiler-dpcpp-cpp-2025.3 \ + intel-oneapi-compiler-fortran-2025.3 \ + intel-oneapi-mpi-devel-2021.17 \ + intel-oneapi-tbb-devel-2022.3 \ + intel-oneapi-mkl-devel-2025.3 && \ + echo "source /etc/profile.d/modules.sh && module use /opt/intel/oneapi/modulefiles" > /etc/profile.d/z01_oneapi_modules.sh; \ + fi && \ + rm -rf /var/lib/apt/lists/* + +#RUN source${SPACK_STACK_DIR}/setup.sh \ +# && cp ${SPACK_ROOT}/share/spack/docker/modules.yaml \ +# /root/.spack/modules.yaml \ + + +# TODO(evan): do I need this? +# https://superuser.com/questions/1241548/ +#RUN [ -f ~/.profile ] \ +# && sed -i 's/mesg n/( tty -s \&\& mesg n || true )/g' ~/.profile \ +# || true + + +FROM ubuntu_base AS builder + +ENV SPACK_STACK_DIR=/opt/spack-stack \ + SPACK_ROOT=/opt/spack-stack/spack + + +# copy spack-stack from build context +# The build context must be the spack-stack repo root (use +# "$(git rev-parse --show-toplevel)" as the context path). +# This copies the full checkout including the spack submodule, +# configs, spack-ext, templates, etc. +COPY . ${SPACK_STACK_DIR} + +# Create spack-stack environment +WORKDIR /opt/spack-stack +RUN mkdir -p /tmp/spack-stack && \ + source setup.sh && \ + spack stack create env \ + --site container \ + --template ${SPACK_STACK_TEMPLATE} \ + --name container \ + --prefix /opt/spack-software/ \ + --compiler $COMPILER && \ + cd ${SPACK_STACK_DIR}/envs/container && \ + spack env activate . && \ + spack concretize 2>&1 | tee log.concretize && \ + spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install && \ + spack module tcl refresh -y && \ + spack stack setup-meta-modules && \ + # Save output of spack find. + spack find 2>&1 | tee /opt/spack-software/spack_find.out + +## ---------- record installed packages ---------- +# FROM spack_base AS runtime +# COPY --from=builder /opt/view /opt/view +#RUN . ${SPACK_STACK_DIR}/setup.sh \ + +# +## ---------- clean build caches ---------- +#RUN . ${SPACK_STACK_DIR}/setup.sh \ +# && spack clean --all +# +## ---------- create nonroot user for MPI ---------- +#RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 \ +# && echo "ulimit -s unlimited" >> /home/nonroot/.bashrc \ +# && echo "ulimit -v unlimited" >> /home/nonroot/.bashrc \ +# && echo "export CC=gcc" >> /home/nonroot/.bashrc \ +# && echo "export CXX=g++" >> /home/nonroot/.bashrc \ +# && echo "export FC=gfortran" >> /home/nonroot/.bashrc \ +# && printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig \ +# && mkdir /home/nonroot/.pmix \ +# && echo "rmaps_default_mapping_policy=:oversubscribe" >> /home/nonroot/.pmix/mca-params.conf \ +# && chown -R nonroot:nonroot /home/nonroot/.gitconfig /home/nonroot/.pmix +# +## ---------- container-wide rc script ---------- +## Sourced by all shells: compiler defaults, MPI policy, module paths. +#RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \ +# && echo "ulimit -v unlimited" >> /etc/spack_container_rc.sh \ +# && echo "export CC=gcc" >> /etc/spack_container_rc.sh \ +# && echo "export CXX=g++" >> /etc/spack_container_rc.sh \ +# && echo "export FC=gfortran" >> /etc/spack_container_rc.sh \ +# && echo "# OpenMPI settings for running as root and oversubscription." >> /etc/spack_container_rc.sh \ +# && echo "export OMPI_ALLOW_RUN_AS_ROOT=1" >> /etc/spack_container_rc.sh \ +# && echo "export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1" >> /etc/spack_container_rc.sh \ +# && echo "export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe" >> /etc/spack_container_rc.sh \ +# && echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \ +# && echo "export MODULEPATH=/opt/spack-stack/envs/unified-gcc/install/modules/Core" >> /etc/spack_container_rc.sh \ +# && echo "source /etc/spack_container_rc.sh" >> /etc/bash.bashrc \ +# && printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig \ +# && mkdir /root/.pmix \ +# && echo "rmaps_default_mapping_policy=:oversubscribe" >> /root/.pmix/mca-params.conf +# +## Ensure the container rc is run by non-login shells too. +#ENV BASH_ENV=/etc/spack_container_rc.sh +# +#LABEL "app"="spack-stack" \ +# "mpi"="openmpi" +# +#CMD ["/bin/bash"] +# \ No newline at end of file diff --git a/configs/sites/tier1/container/README.md b/configs/sites/tier1/container/README.md new file mode 100644 index 000000000..3013576e5 --- /dev/null +++ b/configs/sites/tier1/container/README.md @@ -0,0 +1,115 @@ +# Container Site + +This site config is used to build the official Spack-stack containers. This new tier-1 container site superceedes the legacy spack-based container builds and orients the container builder around common configs and a common site definition. The motivation for this site is to have our shared containers more closely match sites with loadable modules and developer tools-pre installed. + +# Outline of plan for implementation. +1) Review current docker-ubuntu-gcc-openmpi.yaml +2) Review content of //configs/sites/tier2/aws-ubuntu2404 +3) Create a new site based on the assumption that the site will be similar to the ubuntu2404 site and built with a dockerfile. Initially target only the gcc13 build system. Modules should be TCL + +## Notes + - compilers.yaml is no longer needed and is part of spack < 1.0. New compilers ARE packages and are segregated into the "packages_buildstack.yaml" config. + + +# Agent questions or notes +- The `--site container` argument to `spack stack create env` resolves by searching `configs/sites/tier1/` then `configs/sites/tier2/` for a directory named `container`. Since this site lives at `tier1/container/`, it will be found as a tier1 site. Confirmed via reading `stack_env.py:site_configs_dir()`. +- The `--compiler gcc` argument causes `packages_gcc.yaml` to be merged with `packages.yaml` for both the common and site configs. This is handled automatically by `_copy_site_includes()` and `_copy_common_includes()` in `stack_env.py`. +- The external package versions in `packages.yaml` are taken from Ubuntu 24.04 (Noble Numbat) and must match what is installed via `apt` in the Dockerfile. If the base image changes, these versions need to be updated. +- The `mysql@8.0.40` version was carried over from the old `docker-ubuntu-gcc-openmpi.yaml`. Verify this is the correct version for the Ubuntu 24.04 apt package `mysql-server`. +- The `qt@5.15.3` version was also carried over from the old container config. The actual version in Ubuntu 24.04 may differ slightly. Verify with `dpkg -l qtbase5-dev` in the container. +- The `git-lfs@3.4.1` version should be verified against the Ubuntu 24.04 apt package. +- `BUILD_JOBS` defaults to 4 in the Dockerfile. This is conservative for CI runners. Adjust via `--build-arg BUILD_JOBS=N` for faster builds on larger machines. +- The old container config had `checksum: false` in the spack config. This was NOT carried over to the new site as it is a security concern. If build mirrors are trusted, this can be re-added to `config.yaml`. +- The old container config used `concretizer: unify: true`. The unified-dev template uses `unify: when_possible`. The template setting takes precedence and should be correct for the full unified environment. +- The Dockerfile uses a single-stage build. All build tooling, spack assets, and runtime libraries remain in the final image. This keeps the image larger but ensures the spack installation, modules, and developer tools are all usable together. A future optimization could strip `.git` directories and source caches to reduce size. +- The `met` package variants `+python +grib2 +graphics +lidar2nc +modis` are set in the site `packages.yaml` to match the old container config and tier2 ubuntu2404 site. +- The `MODULEPATH` in the container rc script points to `$SPACK_STACK_DIR/envs/unified-gcc/install/modules/Core`. This assumes the environment name is `unified-gcc` as specified in the `spack stack create env --name unified-gcc` command. +- I did not add `gcc-runtime` as an external. The old `Dockerfile.edits` had it but the tier2 aws-ubuntu2404 site does not. If concretization complains about gcc-runtime, add it to `packages_gcc.yaml`. + + +# Building + +This container is not a typical site config and is installed slightly differently to accomodate the use of a dockerfile. Please follow the instructions here to build it. + +## Prerequisites + +- Docker (or Podman) installed +- At least 50GB of disk space for the build +- Sufficient RAM (8GB+ recommended) + +## Quick Build + +The build context is the spack-stack repository root. The Dockerfile +copies the entire local checkout (including the submodules) into +the image, so any local changes to configs, templates, or spack-ext +are automatically reflected in the container. + +```bash +docker build \ + -t spack-stack-gcc:test \ + -f "$(git rev-parse --show-toplevel)/configs/sites/tier1/container/Dockerfile" \ + --build-arg SPACK_STACK_TEMPLATE=unified-dev \ + --build-arg BUILD_JOBS=10 \ + --build-arg COMPILER=gcc \ + "$(git rev-parse --show-toplevel)" + + +docker build \ + -t spack-stack-intel:test \ + -f "$(git rev-parse --show-toplevel)/configs/sites/tier1/container/Dockerfile" \ + --build-arg SPACK_STACK_TEMPLATE=unified-dev \ + --build-arg BUILD_JOBS=10 \ + --build-arg COMPILER=intel \ + "$(git rev-parse --show-toplevel)" +``` + +### Build Arguments + +| Argument | Default | Description | +|---|---|---| +| `BUILD_JOBS` | `4` | Number of parallel build jobs | + +## Running the Container + +```bash +# Run as root (for quick testing) +docker run -it spack-stack:unified-gcc + +# Run as nonroot user (recommended for MPI jobs) +docker run -it --user nonroot spack-stack:unified-gcc +``` + +Once inside the container, load modules: + +```bash +# Modules are automatically available via MODULEPATH set in /etc/spack_container_rc.sh +module avail + +# Example: load the stack compiler and MPI, then a JEDI environment +module load stack-gcc/13.3.0 +module load stack-openmpi/5.0.8 +module load jedi-fv3-env +``` + +## File Inventory + +| File | Purpose | +|---|---| +| `Dockerfile` | Single-stage Docker build for the container | +| `packages.yaml` | External system packages + container preferences (target arch, MPI provider) | +| `packages_gcc.yaml` | GCC 13.3.0 compiler definition (merged when `--compiler gcc` is used) | +| `modules.yaml` | Enables TCL module system | +| `config.yaml` | Container-specific spack config (build jobs, cache paths) | +| `README.md` | This file | + +## Relationship to Other Configs + +This site config is designed to work with the `spack stack create env` workflow. When the environment is created, spack-stack automatically: + +1. Copies and merges `configs/common/packages.yaml` + `configs/common/packages_gcc.yaml` into `common/packages.yaml` +2. Copies and merges `configs/common/modules.yaml` + `configs/common/modules_tcl.yaml` into `common/modules.yaml` +3. Copies and merges this site's `packages.yaml` + `packages_gcc.yaml` into `site/packages.yaml` +4. Copies this site's `modules.yaml` (with TCL enable) into `site/modules.yaml` +5. Uses the `unified-dev` template's `spack.yaml` as the base spec list + +The site configs take precedence over common configs. The template `spack.yaml` has the highest precedence. diff --git a/configs/sites/tier1/container/config.yaml b/configs/sites/tier1/container/config.yaml new file mode 100644 index 000000000..52ecf712a --- /dev/null +++ b/configs/sites/tier1/container/config.yaml @@ -0,0 +1,9 @@ +config: + build_jobs: 10 + + # Container-specific paths for build and staging areas. + # Use /tmp for transient build data (not preserved in final image). + build_stage: /tmp/spack-stack/cache/build_stage + test_stage: /tmp/spack-stack/cache/test_stage + source_cache: /tmp/spack-stack/cache/source_cache + misc_cache: /tmp/spack-stack/cache/misc_cache diff --git a/configs/sites/tier1/container/modules.yaml b/configs/sites/tier1/container/modules.yaml new file mode 100644 index 000000000..b134d3669 --- /dev/null +++ b/configs/sites/tier1/container/modules.yaml @@ -0,0 +1,7 @@ +modules: + default: + enable:: + - tcl + tcl: + include: + - python diff --git a/configs/sites/tier1/container/packages.yaml b/configs/sites/tier1/container/packages.yaml new file mode 100644 index 000000000..2aba0519a --- /dev/null +++ b/configs/sites/tier1/container/packages.yaml @@ -0,0 +1,63 @@ +packages: + all: + target: [core2] + met: + variants: +python +grib2 +graphics +lidar2nc +modis + # System packages installed during the container build. + gmake: + buildable: false + externals: + - spec: gmake@4.3 + prefix: /usr + grep: + externals: + - spec: grep@3.11 + prefix: /usr + findutils: + externals: + - spec: findutils@4.9.0 + prefix: /usr + coreutils: + externals: + - spec: coreutils@9.4 + prefix: /usr + autoconf: + externals: + - spec: autoconf@2.71 + prefix: /usr + diffutils: + buildable: false + externals: + - spec: diffutils@3.10 + prefix: /usr + git: + buildable: false + externals: + - spec: git@2.43.0~tcltk + prefix: /usr + git-lfs: + buildable: false + externals: + - spec: git-lfs@3.4.1 + prefix: /usr + llvm: + buildable: false + externals: + - spec: llvm@14.0.6 + prefix: /usr + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr + version: [5.15.3] + wget: + buildable: false + externals: + - spec: wget@1.21.4 + prefix: /usr + perl: + buildable: false + externals: + - spec: perl@5.38.2 + prefix: /usr \ No newline at end of file diff --git a/configs/sites/tier1/container/packages_gcc.yaml b/configs/sites/tier1/container/packages_gcc.yaml new file mode 100644 index 000000000..d17749d31 --- /dev/null +++ b/configs/sites/tier1/container/packages_gcc.yaml @@ -0,0 +1,18 @@ +packages: + all: + prefer: + - '%gcc' + providers: + mpi: [openmpi@5.0.8] + mpi: + require: [openmpi@5.0.8] + gcc: + externals: + - spec: gcc@13.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + buildable: false diff --git a/configs/sites/tier1/container/packages_intel.yaml b/configs/sites/tier1/container/packages_intel.yaml new file mode 100644 index 000000000..dfd726e28 --- /dev/null +++ b/configs/sites/tier1/container/packages_intel.yaml @@ -0,0 +1,76 @@ +packages: + all: + prefer: + - '%oneapi' + providers: + mpi: [intel-oneapi-mpi@2021.13] + mpi: + require: [intel-oneapi-mpi@2021.17] + # Compilers and meta packages. + intel-oneapi-compilers: + buildable: false + externals: + - spec: intel-oneapi-compilers@2025.3.0 + prefix: /opt/intel/oneapi + modules: + - umf/1.0.2 + - tbb/2022.3 + - compiler-rt/2025.3.0 + - compiler/2025.3.0 + extra_attributes: + compilers: + fortran: /opt/intel/oneapi/compiler/2025.3/bin/ifx + c: /opt/intel/oneapi/compiler/2025.3/bin/icx + cxx: /opt/intel/oneapi/compiler/2025.3/bin/icpx + gcc: + externals: + - spec: gcc@13.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc + cxx: /usr/bin/g++ + fortran: /usr/bin/gfortran + buildable: false + intel-oneapi-mkl: + buildable: false + externals: + - spec: intel-oneapi-mkl@2025.3 + prefix: /opt/intel/oneapi + modules: + - mkl/2025.3 + intel-oneapi-mpi: + buildable: false + externals: + - spec: intel-oneapi-mpi@2021.17 + modules: + - mpi/2021.17 + prefix: /opt/intel/oneapi + intel-oneapi-tbb: + buildable: false + externals: + - spec: intel-oneapi-tbb@2022.3 + prefix: /opt/intel/oneapi + modules: + - tbb/2022.3 + mpich: + buildable: false + mkl: + require: [intel-oneapi-mkl@2025.3] + openmpi: + buildable: false + icu4c: + require: + - '%gcc' + libmd: + require: + - '%gcc' + libbsd: + require: + - '%gcc' + bison: + require: + - '%gcc' + py-numpy: + require: + - '^intel-oneapi-mkl' \ No newline at end of file From ebcc7c40991ab929cc3b98404b8c24304262f1e9 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Fri, 13 Feb 2026 20:37:31 +0000 Subject: [PATCH 2/9] Final container environment --- configs/sites/tier1/container/Dockerfile | 185 ------------------ configs/sites/tier1/container/Dockerfile.gcc | 142 ++++++++++++++ .../sites/tier1/container/Dockerfile.oneapi | 158 +++++++++++++++ configs/sites/tier1/container/README.md | 94 +-------- configs/sites/tier1/container/packages.yaml | 5 - ...ckages_intel.yaml => packages_oneapi.yaml} | 17 +- 6 files changed, 311 insertions(+), 290 deletions(-) delete mode 100644 configs/sites/tier1/container/Dockerfile create mode 100644 configs/sites/tier1/container/Dockerfile.gcc create mode 100644 configs/sites/tier1/container/Dockerfile.oneapi rename configs/sites/tier1/container/{packages_intel.yaml => packages_oneapi.yaml} (88%) diff --git a/configs/sites/tier1/container/Dockerfile b/configs/sites/tier1/container/Dockerfile deleted file mode 100644 index f7f0c20b6..000000000 --- a/configs/sites/tier1/container/Dockerfile +++ /dev/null @@ -1,185 +0,0 @@ -# Spack-Stack Container Build for tier1/container site -# -# This Dockerfile builds a spack-stack environment using the tier1/container -# See building instructions in README.md -# -# Docker build args: -# BUILD_JOBS: Build parallelism (default 4) -# SPACK_STACK_TEMPLATE: Environment template from spack-stack (default unified-dev). -# COMPILER: Compiler to install (default gcc). -# - gcc: GCC 13.4.0 -# - intel: Intel OneAPI 2025.3.0 - -FROM ubuntu:24.04 AS ubuntu_base - -ARG BUILD_JOBS=4 -ARG SPACK_STACK_TEMPLATE=unified-dev -ARG COMPILER=gcc - -SHELL ["/bin/bash", "-c"] - - -ENV DEBIAN_FRONTEND=noninteractive \ - TZ=Etc/UTC \ - LANGUAGE=en_US.UTF-8 \ - LANG=en_US.UTF-8 \ - LC_ALL=en_US.UTF-8 - -# Install core packages. -RUN set -euo pipefail; \ - apt-get -yqq update && \ - apt-get -yqq upgrade && \ - apt-get -yqq install --no-install-recommends \ - # Core build tools. - build-essential \ - gcc-13 \ - gcc++-13 \ - gfortran-13 \ - cpp-13 \ - make \ - llvm-14 \ - autoconf \ - # External dependencies. - libcurl4-openssl-dev \ - libmysqlclient-dev \ - libqt5svg5-dev \ - qt5-qmake \ - qt5dxcb-plugin \ - qtbase5-dev \ - zstd \ - # Shell tools, source retrieval and networking. - sed \ - file \ - less \ - bzip2 \ - unzip \ - ca-certificates \ - curl \ - git \ - git-lfs \ - gpg \ - environment-modules \ - tcl \ - tcl-dev \ - vim \ - wget \ - nano \ - locales && \ - locale-gen en_US.UTF-8 && \ - # Update alternatives for compilers and tools - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \ - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ - update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \ - update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ - # Install the intel - if [ "${COMPILER:-}" = "intel" ]; then \ - apt update && \ - apt install -y apt-utils gpg && \ - wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ - | gpg --dearmor \ - | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ - echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \ - | tee /etc/apt/sources.list.d/oneAPI.list && \ - apt update && \ - apt install -y \ - intel-oneapi-compiler-dpcpp-cpp-2025.3 \ - intel-oneapi-compiler-fortran-2025.3 \ - intel-oneapi-mpi-devel-2021.17 \ - intel-oneapi-tbb-devel-2022.3 \ - intel-oneapi-mkl-devel-2025.3 && \ - echo "source /etc/profile.d/modules.sh && module use /opt/intel/oneapi/modulefiles" > /etc/profile.d/z01_oneapi_modules.sh; \ - fi && \ - rm -rf /var/lib/apt/lists/* - -#RUN source${SPACK_STACK_DIR}/setup.sh \ -# && cp ${SPACK_ROOT}/share/spack/docker/modules.yaml \ -# /root/.spack/modules.yaml \ - - -# TODO(evan): do I need this? -# https://superuser.com/questions/1241548/ -#RUN [ -f ~/.profile ] \ -# && sed -i 's/mesg n/( tty -s \&\& mesg n || true )/g' ~/.profile \ -# || true - - -FROM ubuntu_base AS builder - -ENV SPACK_STACK_DIR=/opt/spack-stack \ - SPACK_ROOT=/opt/spack-stack/spack - - -# copy spack-stack from build context -# The build context must be the spack-stack repo root (use -# "$(git rev-parse --show-toplevel)" as the context path). -# This copies the full checkout including the spack submodule, -# configs, spack-ext, templates, etc. -COPY . ${SPACK_STACK_DIR} - -# Create spack-stack environment -WORKDIR /opt/spack-stack -RUN mkdir -p /tmp/spack-stack && \ - source setup.sh && \ - spack stack create env \ - --site container \ - --template ${SPACK_STACK_TEMPLATE} \ - --name container \ - --prefix /opt/spack-software/ \ - --compiler $COMPILER && \ - cd ${SPACK_STACK_DIR}/envs/container && \ - spack env activate . && \ - spack concretize 2>&1 | tee log.concretize && \ - spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install && \ - spack module tcl refresh -y && \ - spack stack setup-meta-modules && \ - # Save output of spack find. - spack find 2>&1 | tee /opt/spack-software/spack_find.out - -## ---------- record installed packages ---------- -# FROM spack_base AS runtime -# COPY --from=builder /opt/view /opt/view -#RUN . ${SPACK_STACK_DIR}/setup.sh \ - -# -## ---------- clean build caches ---------- -#RUN . ${SPACK_STACK_DIR}/setup.sh \ -# && spack clean --all -# -## ---------- create nonroot user for MPI ---------- -#RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 \ -# && echo "ulimit -s unlimited" >> /home/nonroot/.bashrc \ -# && echo "ulimit -v unlimited" >> /home/nonroot/.bashrc \ -# && echo "export CC=gcc" >> /home/nonroot/.bashrc \ -# && echo "export CXX=g++" >> /home/nonroot/.bashrc \ -# && echo "export FC=gfortran" >> /home/nonroot/.bashrc \ -# && printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig \ -# && mkdir /home/nonroot/.pmix \ -# && echo "rmaps_default_mapping_policy=:oversubscribe" >> /home/nonroot/.pmix/mca-params.conf \ -# && chown -R nonroot:nonroot /home/nonroot/.gitconfig /home/nonroot/.pmix -# -## ---------- container-wide rc script ---------- -## Sourced by all shells: compiler defaults, MPI policy, module paths. -#RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \ -# && echo "ulimit -v unlimited" >> /etc/spack_container_rc.sh \ -# && echo "export CC=gcc" >> /etc/spack_container_rc.sh \ -# && echo "export CXX=g++" >> /etc/spack_container_rc.sh \ -# && echo "export FC=gfortran" >> /etc/spack_container_rc.sh \ -# && echo "# OpenMPI settings for running as root and oversubscription." >> /etc/spack_container_rc.sh \ -# && echo "export OMPI_ALLOW_RUN_AS_ROOT=1" >> /etc/spack_container_rc.sh \ -# && echo "export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1" >> /etc/spack_container_rc.sh \ -# && echo "export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe" >> /etc/spack_container_rc.sh \ -# && echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \ -# && echo "export MODULEPATH=/opt/spack-stack/envs/unified-gcc/install/modules/Core" >> /etc/spack_container_rc.sh \ -# && echo "source /etc/spack_container_rc.sh" >> /etc/bash.bashrc \ -# && printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig \ -# && mkdir /root/.pmix \ -# && echo "rmaps_default_mapping_policy=:oversubscribe" >> /root/.pmix/mca-params.conf -# -## Ensure the container rc is run by non-login shells too. -#ENV BASH_ENV=/etc/spack_container_rc.sh -# -#LABEL "app"="spack-stack" \ -# "mpi"="openmpi" -# -#CMD ["/bin/bash"] -# \ No newline at end of file diff --git a/configs/sites/tier1/container/Dockerfile.gcc b/configs/sites/tier1/container/Dockerfile.gcc new file mode 100644 index 000000000..51e992471 --- /dev/null +++ b/configs/sites/tier1/container/Dockerfile.gcc @@ -0,0 +1,142 @@ +# Spack-Stack Container Build for tier1/container Intel oneapi site +# See the README.md for building instructions. + +## ---------- build base container used by builder and runtime ---------- +FROM ubuntu:24.04 AS ubuntu_base + +ARG BUILD_JOBS=4 +ARG SPACK_STACK_TEMPLATE=unified-dev + +SHELL ["/bin/bash", "-c"] + + +ENV DEBIAN_FRONTEND=noninteractive \ + TZ=Etc/UTC \ + LANGUAGE=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + COMPILER=gcc + +# Install core packages. +RUN set -euo pipefail; \ + apt-get -yqq update && \ + apt-get -yqq upgrade && \ + apt-get -yqq install --no-install-recommends \ + # Core build tools. + build-essential \ + gcc-13 \ + gcc++-13 \ + gfortran-13 \ + cpp-13 \ + make \ + llvm-14 \ + autoconf \ + # External dependencies. + libcurl4-openssl-dev \ + libmysqlclient-dev \ + libqt5svg5-dev \ + qt5-qmake \ + qt5dxcb-plugin \ + qtbase5-dev \ + zstd \ + # Shell tools, source retrieval and networking. + sed \ + file \ + less \ + bzip2 \ + unzip \ + ca-certificates \ + curl \ + git \ + git-lfs \ + gpg \ + environment-modules \ + tcl \ + tcl-dev \ + vim \ + wget \ + nano \ + locales && \ + locale-gen en_US.UTF-8 && \ + # Update alternatives for compilers and tools + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \ + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ + rm -rf /var/lib/apt/lists/* + + +## ---------- build spack-stack environment ---------- +FROM ubuntu_base AS builder + +ENV SPACK_STACK_DIR=/opt/spack-stack \ + SPACK_ROOT=/opt/spack-stack/spack + + +# Copy spack-stack from build context. Note the build context must be the +# root of the spack-stack repository (see the README.md for details). +COPY . ${SPACK_STACK_DIR} + +# Create spack-stack environment +WORKDIR /opt/spack-stack +RUN mkdir -p /tmp/spack-stack && \ + source setup.sh && \ + spack stack create env \ + --site container \ + --template ${SPACK_STACK_TEMPLATE} \ + --name container \ + --prefix /opt/spack-software/ \ + --compiler $COMPILER && \ + cd ${SPACK_STACK_DIR}/envs/container && \ + spack env activate . && \ + spack concretize 2>&1 | tee log.concretize && \ + spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install && \ + && cd ${SPACK_STACK_DIR}/envs/container \ + && spack env activate . \ + spack module tcl refresh -y && \ + spack stack setup-meta-modules && \ + # Save output of spack find. + spack find 2>&1 | tee /opt/spack-software/spack_find.out && \ + spack clean --all + +## ---------- Create runtime container ---------- +FROM spack_base AS runtime + +COPY --from=builder /opt/spack-software/ /opt/spack-software/ + +ENV CC=gcc \ + CXX=g++ \ + FC=gfortran + +# Container-wide rc script with compiler defaults, MPI policy, module paths. +RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \ + && echo "ulimit -v unlimited" >> /etc/spack_container_rc.sh \ + && echo "export CC=gcc" >> /etc/spack_container_rc.sh \ + && echo "export CXX=g++" >> /etc/spack_container_rc.sh \ + && echo "export FC=gfortran" >> /etc/spack_container_rc.sh \ + && echo "# OpenMPI settings for running as root and oversubscription." >> /etc/spack_container_rc.sh \ + && echo "export OMPI_ALLOW_RUN_AS_ROOT=1" >> /etc/spack_container_rc.sh \ + && echo "export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1" >> /etc/spack_container_rc.sh \ + && echo "export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe" >> /etc/spack_container_rc.sh \ + && echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \ + && echo "export MODULEPATH=/opt/spack-software/modules/Core" >> /etc/spack_container_rc.sh \ + && echo "source /etc/spack_container_rc.sh" >> /etc/bash.bashrc \ + && printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig \ + && mkdir /root/.pmix \ + && echo "rmaps_default_mapping_policy=:oversubscribe" >> /root/.pmix/mca-params.conf + +# Nonroot user for MPI +RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 \ + && echo "ulimit -s unlimited" >> /home/nonroot/.bashrc \ + && echo "ulimit -v unlimited" >> /home/nonroot/.bashrc \ + && printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig \ + && mkdir /home/nonroot/.pmix \ + && chown -R nonroot:nonroot /home/nonroot/.gitconfig /home/nonroot/.pmix + +# Ensure the container rc is run by non-login shells too. +ENV BASH_ENV=/etc/spack_container_rc.sh + +LABEL "app"="spack-stack" \ + "mpi"="openmpi" + +CMD ["/bin/bash"] diff --git a/configs/sites/tier1/container/Dockerfile.oneapi b/configs/sites/tier1/container/Dockerfile.oneapi new file mode 100644 index 000000000..78334d8f8 --- /dev/null +++ b/configs/sites/tier1/container/Dockerfile.oneapi @@ -0,0 +1,158 @@ +# Spack-Stack Container Build for tier1/container Intel oneapi site +# See the README.md for building instructions. + +## ---------- build base container used by builder and runtime ---------- +FROM ubuntu:24.04 AS ubuntu_base + +ARG BUILD_JOBS=4 +ARG SPACK_STACK_TEMPLATE=unified-dev + +SHELL ["/bin/bash", "-c"] + + +ENV DEBIAN_FRONTEND=noninteractive \ + TZ=Etc/UTC \ + LANGUAGE=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + COMPILER=intel + +# Install core packages. +RUN set -euo pipefail; \ + apt-get -yqq update && \ + apt-get -yqq upgrade && \ + apt-get -yqq install --no-install-recommends \ + # Core build tools. + build-essential \ + gcc-13 \ + gcc++-13 \ + gfortran-13 \ + cpp-13 \ + make \ + llvm-14 \ + autoconf \ + # External dependencies. + libcurl4-openssl-dev \ + libmysqlclient-dev \ + libqt5svg5-dev \ + qt5-qmake \ + qt5dxcb-plugin \ + qtbase5-dev \ + zstd \ + # Shell tools, source retrieval and networking. + sed \ + file \ + less \ + bzip2 \ + unzip \ + ca-certificates \ + curl \ + git \ + git-lfs \ + gpg \ + environment-modules \ + tcl \ + tcl-dev \ + vim \ + wget \ + nano \ + locales && \ + locale-gen en_US.UTF-8 && \ + # Update alternatives for compilers and tools + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ + update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \ + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ + # Install the intel + if [ "${COMPILER:-}" = "intel" ]; then \ + apt update && \ + apt install -y apt-utils gpg && \ + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ + | gpg --dearmor \ + | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ + echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \ + | tee /etc/apt/sources.list.d/oneAPI.list && \ + apt update && \ + apt install -y \ + intel-oneapi-compiler-dpcpp-cpp-2025.3 \ + intel-oneapi-compiler-fortran-2025.3 \ + intel-oneapi-mpi-devel-2021.17 \ + intel-oneapi-tbb-devel-2022.3 \ + intel-oneapi-mkl-devel-2025.3 && \ + echo "source /etc/profile.d/modules.sh && module use /opt/intel/oneapi/modulefiles" > /etc/profile.d/z01_oneapi_modules.sh; \ + fi && \ + rm -rf /var/lib/apt/lists/* + + +## ---------- build spack-stack environment ---------- +FROM ubuntu_base AS builder + +ENV SPACK_STACK_DIR=/opt/spack-stack \ + SPACK_ROOT=/opt/spack-stack/spack + + +# Copy spack-stack from build context. Note the build context must be the +# root of the spack-stack repository (see the README.md for details). +COPY . ${SPACK_STACK_DIR} + +# Create spack-stack environment +WORKDIR /opt/spack-stack +RUN mkdir -p /tmp/spack-stack && \ + source setup.sh && \ + spack stack create env \ + --site container \ + --template ${SPACK_STACK_TEMPLATE} \ + --name container \ + --prefix /opt/spack-software/ \ + --compiler oneapi && \ + cd ${SPACK_STACK_DIR}/envs/container && \ + spack env activate . && \ + spack concretize 2>&1 | tee log.concretize && \ + spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install && \ + && cd ${SPACK_STACK_DIR}/envs/container \ + && spack env activate . \ + spack module tcl refresh -y && \ + spack stack setup-meta-modules && \ + # Save output of spack find. + spack find 2>&1 | tee /opt/spack-software/spack_find.out && \ + spack clean --all + +## ---------- Create runtime container ---------- +FROM spack_base AS runtime + +COPY --from=builder /opt/spack-software/ /opt/spack-software/ + +ENV CC=icx \ + CXX=icpx \ + FC=ifx + +# Container-wide rc script with compiler defaults, MPI policy, module paths. +RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \ + && echo "ulimit -v unlimited" >> /etc/spack_container_rc.sh \ + && echo "export CC=icx" >> /etc/spack_container_rc.sh \ + && echo "export CXX=icpx" >> /etc/spack_container_rc.sh \ + && echo "export FC=ifx" >> /etc/spack_container_rc.sh \ + && echo "# OpenMPI settings for running as root and oversubscription." >> /etc/spack_container_rc.sh \ + && echo "export OMPI_ALLOW_RUN_AS_ROOT=1" >> /etc/spack_container_rc.sh \ + && echo "export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1" >> /etc/spack_container_rc.sh \ + && echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \ + && echo "export MODULEPATH=/opt/spack-software/modules/Core" >> /etc/spack_container_rc.sh \ + && echo "source /etc/spack_container_rc.sh" >> /etc/bash.bashrc \ + && printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig \ + && mkdir /root/.pmix \ + +# Nonroot user for MPI +RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 \ + && echo "ulimit -s unlimited" >> /home/nonroot/.bashrc \ + && echo "ulimit -v unlimited" >> /home/nonroot/.bashrc \ + && printf "[credential]\n helper = cache --timeout=7200\n" >> /home/nonroot/.gitconfig \ + && mkdir /home/nonroot/.pmix \ + && chown -R nonroot:nonroot /home/nonroot/.gitconfig /home/nonroot/.pmix + +# Ensure the container rc is run by non-login shells too. +ENV BASH_ENV=/etc/spack_container_rc.sh + +LABEL "app"="spack-stack" \ + "mpi"="openmpi" + +CMD ["/bin/bash"] diff --git a/configs/sites/tier1/container/README.md b/configs/sites/tier1/container/README.md index 3013576e5..c388bd8fa 100644 --- a/configs/sites/tier1/container/README.md +++ b/configs/sites/tier1/container/README.md @@ -2,39 +2,14 @@ This site config is used to build the official Spack-stack containers. This new tier-1 container site superceedes the legacy spack-based container builds and orients the container builder around common configs and a common site definition. The motivation for this site is to have our shared containers more closely match sites with loadable modules and developer tools-pre installed. -# Outline of plan for implementation. -1) Review current docker-ubuntu-gcc-openmpi.yaml -2) Review content of //configs/sites/tier2/aws-ubuntu2404 -3) Create a new site based on the assumption that the site will be similar to the ubuntu2404 site and built with a dockerfile. Initially target only the gcc13 build system. Modules should be TCL - -## Notes - - compilers.yaml is no longer needed and is part of spack < 1.0. New compilers ARE packages and are segregated into the "packages_buildstack.yaml" config. - - -# Agent questions or notes -- The `--site container` argument to `spack stack create env` resolves by searching `configs/sites/tier1/` then `configs/sites/tier2/` for a directory named `container`. Since this site lives at `tier1/container/`, it will be found as a tier1 site. Confirmed via reading `stack_env.py:site_configs_dir()`. -- The `--compiler gcc` argument causes `packages_gcc.yaml` to be merged with `packages.yaml` for both the common and site configs. This is handled automatically by `_copy_site_includes()` and `_copy_common_includes()` in `stack_env.py`. -- The external package versions in `packages.yaml` are taken from Ubuntu 24.04 (Noble Numbat) and must match what is installed via `apt` in the Dockerfile. If the base image changes, these versions need to be updated. -- The `mysql@8.0.40` version was carried over from the old `docker-ubuntu-gcc-openmpi.yaml`. Verify this is the correct version for the Ubuntu 24.04 apt package `mysql-server`. -- The `qt@5.15.3` version was also carried over from the old container config. The actual version in Ubuntu 24.04 may differ slightly. Verify with `dpkg -l qtbase5-dev` in the container. -- The `git-lfs@3.4.1` version should be verified against the Ubuntu 24.04 apt package. -- `BUILD_JOBS` defaults to 4 in the Dockerfile. This is conservative for CI runners. Adjust via `--build-arg BUILD_JOBS=N` for faster builds on larger machines. -- The old container config had `checksum: false` in the spack config. This was NOT carried over to the new site as it is a security concern. If build mirrors are trusted, this can be re-added to `config.yaml`. -- The old container config used `concretizer: unify: true`. The unified-dev template uses `unify: when_possible`. The template setting takes precedence and should be correct for the full unified environment. -- The Dockerfile uses a single-stage build. All build tooling, spack assets, and runtime libraries remain in the final image. This keeps the image larger but ensures the spack installation, modules, and developer tools are all usable together. A future optimization could strip `.git` directories and source caches to reduce size. -- The `met` package variants `+python +grib2 +graphics +lidar2nc +modis` are set in the site `packages.yaml` to match the old container config and tier2 ubuntu2404 site. -- The `MODULEPATH` in the container rc script points to `$SPACK_STACK_DIR/envs/unified-gcc/install/modules/Core`. This assumes the environment name is `unified-gcc` as specified in the `spack stack create env --name unified-gcc` command. -- I did not add `gcc-runtime` as an external. The old `Dockerfile.edits` had it but the tier2 aws-ubuntu2404 site does not. If concretization complains about gcc-runtime, add it to `packages_gcc.yaml`. - # Building This container is not a typical site config and is installed slightly differently to accomodate the use of a dockerfile. Please follow the instructions here to build it. ## Prerequisites - -- Docker (or Podman) installed -- At least 50GB of disk space for the build +- Docker +- At least 100 GB of disk space for the build - Sufficient RAM (8GB+ recommended) ## Quick Build @@ -46,70 +21,17 @@ are automatically reflected in the container. ```bash docker build \ - -t spack-stack-gcc:test \ - -f "$(git rev-parse --show-toplevel)/configs/sites/tier1/container/Dockerfile" \ + -t spack-stack-gcc:local \ + -f "$(git rev-parse --show-toplevel)/configs/sites/tier1/container/Dockerfile.gcc" \ --build-arg SPACK_STACK_TEMPLATE=unified-dev \ --build-arg BUILD_JOBS=10 \ - --build-arg COMPILER=gcc \ - "$(git rev-parse --show-toplevel)" + "$(git rev-parse --show-toplevel)" 2>&1 | tee ${HOME}/log.docker_gcc docker build \ - -t spack-stack-intel:test \ - -f "$(git rev-parse --show-toplevel)/configs/sites/tier1/container/Dockerfile" \ + -t spack-stack-oneapi:local \ + -f "$(git rev-parse --show-toplevel)/configs/sites/tier1/container/Dockerfile.oneapi" \ --build-arg SPACK_STACK_TEMPLATE=unified-dev \ --build-arg BUILD_JOBS=10 \ - --build-arg COMPILER=intel \ - "$(git rev-parse --show-toplevel)" -``` - -### Build Arguments - -| Argument | Default | Description | -|---|---|---| -| `BUILD_JOBS` | `4` | Number of parallel build jobs | - -## Running the Container - -```bash -# Run as root (for quick testing) -docker run -it spack-stack:unified-gcc - -# Run as nonroot user (recommended for MPI jobs) -docker run -it --user nonroot spack-stack:unified-gcc + "$(git rev-parse --show-toplevel)" 2>&1 | tee ${HOME}/log.docker_oneapi ``` - -Once inside the container, load modules: - -```bash -# Modules are automatically available via MODULEPATH set in /etc/spack_container_rc.sh -module avail - -# Example: load the stack compiler and MPI, then a JEDI environment -module load stack-gcc/13.3.0 -module load stack-openmpi/5.0.8 -module load jedi-fv3-env -``` - -## File Inventory - -| File | Purpose | -|---|---| -| `Dockerfile` | Single-stage Docker build for the container | -| `packages.yaml` | External system packages + container preferences (target arch, MPI provider) | -| `packages_gcc.yaml` | GCC 13.3.0 compiler definition (merged when `--compiler gcc` is used) | -| `modules.yaml` | Enables TCL module system | -| `config.yaml` | Container-specific spack config (build jobs, cache paths) | -| `README.md` | This file | - -## Relationship to Other Configs - -This site config is designed to work with the `spack stack create env` workflow. When the environment is created, spack-stack automatically: - -1. Copies and merges `configs/common/packages.yaml` + `configs/common/packages_gcc.yaml` into `common/packages.yaml` -2. Copies and merges `configs/common/modules.yaml` + `configs/common/modules_tcl.yaml` into `common/modules.yaml` -3. Copies and merges this site's `packages.yaml` + `packages_gcc.yaml` into `site/packages.yaml` -4. Copies this site's `modules.yaml` (with TCL enable) into `site/modules.yaml` -5. Uses the `unified-dev` template's `spack.yaml` as the base spec list - -The site configs take precedence over common configs. The template `spack.yaml` has the highest precedence. diff --git a/configs/sites/tier1/container/packages.yaml b/configs/sites/tier1/container/packages.yaml index 2aba0519a..af5a307f5 100644 --- a/configs/sites/tier1/container/packages.yaml +++ b/configs/sites/tier1/container/packages.yaml @@ -4,11 +4,6 @@ packages: met: variants: +python +grib2 +graphics +lidar2nc +modis # System packages installed during the container build. - gmake: - buildable: false - externals: - - spec: gmake@4.3 - prefix: /usr grep: externals: - spec: grep@3.11 diff --git a/configs/sites/tier1/container/packages_intel.yaml b/configs/sites/tier1/container/packages_oneapi.yaml similarity index 88% rename from configs/sites/tier1/container/packages_intel.yaml rename to configs/sites/tier1/container/packages_oneapi.yaml index dfd726e28..c163bc96a 100644 --- a/configs/sites/tier1/container/packages_intel.yaml +++ b/configs/sites/tier1/container/packages_oneapi.yaml @@ -55,22 +55,11 @@ packages: - tbb/2022.3 mpich: buildable: false - mkl: - require: [intel-oneapi-mkl@2025.3] openmpi: buildable: false - icu4c: - require: - - '%gcc' - libmd: - require: - - '%gcc' - libbsd: - require: - - '%gcc' - bison: + mkl: require: - - '%gcc' + - intel-oneapi-mkl py-numpy: require: - - '^intel-oneapi-mkl' \ No newline at end of file + - '^intel-oneapi-mkl' From d37d342326fca5c00bdedb276debd36734340f6c Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Fri, 13 Feb 2026 21:56:28 +0000 Subject: [PATCH 3/9] Fixes --- configs/sites/tier1/container/Dockerfile.gcc | 5 +---- configs/sites/tier1/container/Dockerfile.oneapi | 6 ------ configs/sites/tier1/container/packages_oneapi.yaml | 4 ++-- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/configs/sites/tier1/container/Dockerfile.gcc b/configs/sites/tier1/container/Dockerfile.gcc index 51e992471..8bdd5761b 100644 --- a/configs/sites/tier1/container/Dockerfile.gcc +++ b/configs/sites/tier1/container/Dockerfile.gcc @@ -1,4 +1,4 @@ -# Spack-Stack Container Build for tier1/container Intel oneapi site +# Spack-Stack Container Build for tier1/container GCC site # See the README.md for building instructions. ## ---------- build base container used by builder and runtime ---------- @@ -136,7 +136,4 @@ RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 # Ensure the container rc is run by non-login shells too. ENV BASH_ENV=/etc/spack_container_rc.sh -LABEL "app"="spack-stack" \ - "mpi"="openmpi" - CMD ["/bin/bash"] diff --git a/configs/sites/tier1/container/Dockerfile.oneapi b/configs/sites/tier1/container/Dockerfile.oneapi index 78334d8f8..6dce0928c 100644 --- a/configs/sites/tier1/container/Dockerfile.oneapi +++ b/configs/sites/tier1/container/Dockerfile.oneapi @@ -132,9 +132,6 @@ RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \ && echo "export CC=icx" >> /etc/spack_container_rc.sh \ && echo "export CXX=icpx" >> /etc/spack_container_rc.sh \ && echo "export FC=ifx" >> /etc/spack_container_rc.sh \ - && echo "# OpenMPI settings for running as root and oversubscription." >> /etc/spack_container_rc.sh \ - && echo "export OMPI_ALLOW_RUN_AS_ROOT=1" >> /etc/spack_container_rc.sh \ - && echo "export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1" >> /etc/spack_container_rc.sh \ && echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \ && echo "export MODULEPATH=/opt/spack-software/modules/Core" >> /etc/spack_container_rc.sh \ && echo "source /etc/spack_container_rc.sh" >> /etc/bash.bashrc \ @@ -152,7 +149,4 @@ RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 # Ensure the container rc is run by non-login shells too. ENV BASH_ENV=/etc/spack_container_rc.sh -LABEL "app"="spack-stack" \ - "mpi"="openmpi" - CMD ["/bin/bash"] diff --git a/configs/sites/tier1/container/packages_oneapi.yaml b/configs/sites/tier1/container/packages_oneapi.yaml index c163bc96a..44c1482c9 100644 --- a/configs/sites/tier1/container/packages_oneapi.yaml +++ b/configs/sites/tier1/container/packages_oneapi.yaml @@ -3,7 +3,7 @@ packages: prefer: - '%oneapi' providers: - mpi: [intel-oneapi-mpi@2021.13] + mpi: [intel-oneapi-mpi@2021.17] mpi: require: [intel-oneapi-mpi@2021.17] # Compilers and meta packages. @@ -19,9 +19,9 @@ packages: - compiler/2025.3.0 extra_attributes: compilers: - fortran: /opt/intel/oneapi/compiler/2025.3/bin/ifx c: /opt/intel/oneapi/compiler/2025.3/bin/icx cxx: /opt/intel/oneapi/compiler/2025.3/bin/icpx + fortran: /opt/intel/oneapi/compiler/2025.3/bin/ifx gcc: externals: - spec: gcc@13.3.0 languages:='c,c++,fortran' From 82531c2f65a399adae09f09d4a0385109dd5f457 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Fri, 13 Feb 2026 23:04:48 +0000 Subject: [PATCH 4/9] Fix oneapi module files --- .../sites/tier1/container/Dockerfile.oneapi | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/configs/sites/tier1/container/Dockerfile.oneapi b/configs/sites/tier1/container/Dockerfile.oneapi index 6dce0928c..365276823 100644 --- a/configs/sites/tier1/container/Dockerfile.oneapi +++ b/configs/sites/tier1/container/Dockerfile.oneapi @@ -64,23 +64,21 @@ RUN set -euo pipefail; \ update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-13 100 && \ update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 100 && \ # Install the intel - if [ "${COMPILER:-}" = "intel" ]; then \ - apt update && \ - apt install -y apt-utils gpg && \ - wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ - | gpg --dearmor \ - | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ - echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \ - | tee /etc/apt/sources.list.d/oneAPI.list && \ - apt update && \ - apt install -y \ - intel-oneapi-compiler-dpcpp-cpp-2025.3 \ - intel-oneapi-compiler-fortran-2025.3 \ - intel-oneapi-mpi-devel-2021.17 \ - intel-oneapi-tbb-devel-2022.3 \ - intel-oneapi-mkl-devel-2025.3 && \ - echo "source /etc/profile.d/modules.sh && module use /opt/intel/oneapi/modulefiles" > /etc/profile.d/z01_oneapi_modules.sh; \ - fi && \ + apt install -y apt-utils gpg && \ + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ + | gpg --dearmor \ + | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \ + echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \ + | tee /etc/apt/sources.list.d/oneAPI.list && \ + apt update && \ + apt install -y \ + intel-oneapi-compiler-dpcpp-cpp-2025.3 \ + intel-oneapi-compiler-fortran-2025.3 \ + intel-oneapi-mpi-devel-2021.17 \ + intel-oneapi-tbb-devel-2022.3 \ + intel-oneapi-mkl-devel-2025.3 && \ + /opt/intel/oneapi/modulefiles-setup.sh --output-dir=/opt/intel/oneapi/modulefiles && \ + echo "source /etc/profile.d/modules.sh && module use /opt/intel/oneapi/modulefiles" > /etc/profile.d/z01_oneapi_modules.sh && \ rm -rf /var/lib/apt/lists/* @@ -134,6 +132,9 @@ RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \ && echo "export FC=ifx" >> /etc/spack_container_rc.sh \ && echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \ && echo "export MODULEPATH=/opt/spack-software/modules/Core" >> /etc/spack_container_rc.sh \ + && echo "source /etc/profile.d/modules.sh" \ + && echo "module use /opt/intel/oneapi/modulefiles" \ + && echo "module use /opt/spack-software/modules/Core" \ && echo "source /etc/spack_container_rc.sh" >> /etc/bash.bashrc \ && printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig \ && mkdir /root/.pmix \ From 2181c58f0f9bd390ec8efbca16e39e08bb65fac7 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Fri, 13 Feb 2026 23:06:30 +0000 Subject: [PATCH 5/9] also gcc container --- configs/sites/tier1/container/Dockerfile.gcc | 3 ++- configs/sites/tier1/container/Dockerfile.oneapi | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/configs/sites/tier1/container/Dockerfile.gcc b/configs/sites/tier1/container/Dockerfile.gcc index 8bdd5761b..6b2f45426 100644 --- a/configs/sites/tier1/container/Dockerfile.gcc +++ b/configs/sites/tier1/container/Dockerfile.gcc @@ -119,7 +119,8 @@ RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \ && echo "export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1" >> /etc/spack_container_rc.sh \ && echo "export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe" >> /etc/spack_container_rc.sh \ && echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \ - && echo "export MODULEPATH=/opt/spack-software/modules/Core" >> /etc/spack_container_rc.sh \ + && echo "source /etc/profile.d/modules.sh" >> /etc/spack_container_rc.sh \ + && echo "module use /opt/spack-software/modules/Core" >> /etc/spack_container_rc.sh \ && echo "source /etc/spack_container_rc.sh" >> /etc/bash.bashrc \ && printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig \ && mkdir /root/.pmix \ diff --git a/configs/sites/tier1/container/Dockerfile.oneapi b/configs/sites/tier1/container/Dockerfile.oneapi index 365276823..53e1f8987 100644 --- a/configs/sites/tier1/container/Dockerfile.oneapi +++ b/configs/sites/tier1/container/Dockerfile.oneapi @@ -132,9 +132,9 @@ RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \ && echo "export FC=ifx" >> /etc/spack_container_rc.sh \ && echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \ && echo "export MODULEPATH=/opt/spack-software/modules/Core" >> /etc/spack_container_rc.sh \ - && echo "source /etc/profile.d/modules.sh" \ - && echo "module use /opt/intel/oneapi/modulefiles" \ - && echo "module use /opt/spack-software/modules/Core" \ + && echo "source /etc/profile.d/modules.sh" >> /etc/spack_container_rc.sh \ + && echo "module use /opt/intel/oneapi/modulefiles" >> /etc/spack_container_rc.sh \ + && echo "module use /opt/spack-software/modules/Core" >> /etc/spack_container_rc.sh \ && echo "source /etc/spack_container_rc.sh" >> /etc/bash.bashrc \ && printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig \ && mkdir /root/.pmix \ From 6c78e682150844294f8c8e346ca4e606e469c3dd Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Fri, 13 Feb 2026 23:19:17 +0000 Subject: [PATCH 6/9] Fix to docker base ref --- configs/sites/tier1/container/Dockerfile.gcc | 2 +- configs/sites/tier1/container/Dockerfile.oneapi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/sites/tier1/container/Dockerfile.gcc b/configs/sites/tier1/container/Dockerfile.gcc index 6b2f45426..c11186eda 100644 --- a/configs/sites/tier1/container/Dockerfile.gcc +++ b/configs/sites/tier1/container/Dockerfile.gcc @@ -100,7 +100,7 @@ RUN mkdir -p /tmp/spack-stack && \ spack clean --all ## ---------- Create runtime container ---------- -FROM spack_base AS runtime +FROM ubuntu_base AS runtime COPY --from=builder /opt/spack-software/ /opt/spack-software/ diff --git a/configs/sites/tier1/container/Dockerfile.oneapi b/configs/sites/tier1/container/Dockerfile.oneapi index 53e1f8987..26673d9cd 100644 --- a/configs/sites/tier1/container/Dockerfile.oneapi +++ b/configs/sites/tier1/container/Dockerfile.oneapi @@ -116,7 +116,7 @@ RUN mkdir -p /tmp/spack-stack && \ spack clean --all ## ---------- Create runtime container ---------- -FROM spack_base AS runtime +FROM ubuntu_base AS runtime COPY --from=builder /opt/spack-software/ /opt/spack-software/ From 4738a2518feed7f6a8f71fd1cbe2cf663f9db5a9 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Fri, 13 Feb 2026 23:35:08 +0000 Subject: [PATCH 7/9] Fix repeated env activate --- configs/sites/tier1/container/Dockerfile.gcc | 2 -- configs/sites/tier1/container/Dockerfile.oneapi | 2 -- 2 files changed, 4 deletions(-) diff --git a/configs/sites/tier1/container/Dockerfile.gcc b/configs/sites/tier1/container/Dockerfile.gcc index c11186eda..917527d7c 100644 --- a/configs/sites/tier1/container/Dockerfile.gcc +++ b/configs/sites/tier1/container/Dockerfile.gcc @@ -91,8 +91,6 @@ RUN mkdir -p /tmp/spack-stack && \ spack env activate . && \ spack concretize 2>&1 | tee log.concretize && \ spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install && \ - && cd ${SPACK_STACK_DIR}/envs/container \ - && spack env activate . \ spack module tcl refresh -y && \ spack stack setup-meta-modules && \ # Save output of spack find. diff --git a/configs/sites/tier1/container/Dockerfile.oneapi b/configs/sites/tier1/container/Dockerfile.oneapi index 26673d9cd..f0fc757b2 100644 --- a/configs/sites/tier1/container/Dockerfile.oneapi +++ b/configs/sites/tier1/container/Dockerfile.oneapi @@ -107,8 +107,6 @@ RUN mkdir -p /tmp/spack-stack && \ spack env activate . && \ spack concretize 2>&1 | tee log.concretize && \ spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install && \ - && cd ${SPACK_STACK_DIR}/envs/container \ - && spack env activate . \ spack module tcl refresh -y && \ spack stack setup-meta-modules && \ # Save output of spack find. From 427144751fc4e6a6a2493ef126173053a3802a47 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Fri, 13 Feb 2026 23:36:06 +0000 Subject: [PATCH 8/9] [skip ci] From b72a0f8d67ecf3e480fc3a242a5449d1c605f9b2 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Sat, 14 Feb 2026 05:41:16 +0000 Subject: [PATCH 9/9] [skip ci] superfluous trailing backslash --- configs/sites/tier1/container/Dockerfile.oneapi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/sites/tier1/container/Dockerfile.oneapi b/configs/sites/tier1/container/Dockerfile.oneapi index f0fc757b2..c22058808 100644 --- a/configs/sites/tier1/container/Dockerfile.oneapi +++ b/configs/sites/tier1/container/Dockerfile.oneapi @@ -135,7 +135,7 @@ RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \ && echo "module use /opt/spack-software/modules/Core" >> /etc/spack_container_rc.sh \ && echo "source /etc/spack_container_rc.sh" >> /etc/bash.bashrc \ && printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig \ - && mkdir /root/.pmix \ + && mkdir /root/.pmix # Nonroot user for MPI RUN useradd -U -k /etc/skel -s /bin/bash -d /home/nonroot -m nonroot --uid 43891 \