Merge remote-tracking branch 'origin/develop' into features/branson

Author: Riyaz Haque

.gitlab-ci.yml

@@ -47,6 +47,8 @@ variables:
   FLUX_ARGS: -N 1 --exclusive
   # Dane
   DANE_SHARED_ALLOC: -N 1 -t 04:00:00 --exclusive --reservation=ci
+  # Matrix (NODES ARE SHARED WITHOUT --exclusive)
+  MATRIX_SHARED_ALLOC: -N 1 -t 04:00:00 --exclusive
   # Tuolumne (max 4hr policy)
   TUOLUMNE_SHARED_ALLOC: -N 1 -t 2h --queue=pbatch --exclusive --setattr=gpumode=${GPUMODE}
     --conf=resource.rediscover=true

.gitlab/tests/nightly.yml

@@ -5,6 +5,7 @@ include:
   variables:
     DASHBOARD_NAME: Nightly [benchpark-develop]
     DANE_PARAMS: -N 1 -t 01:00:00 --exclusive --reservation=ci
+    MATRIX_PARAMS: -N 1 -t 01:00:00 --exclusive
     ELCAP_PARAMS: -N 1 -t 60m --queue=pci --exclusive
   parallel:
     matrix:
@@ -60,6 +61,22 @@ include:
           - raja-perf
           - salmon-tddft
         VARIANT: [+openmp]
+      # +cuda tests
+      - HOST: matrix
+        ARCHCONFIG: llnl-matrix
+        SCHEDULER_PARAMETERS: $MATRIX_PARAMS
+        BENCHMARK:
+          - amg2023
+          - babelstream
+          - gromacs
+          - kripke
+          - laghos
+          - lammps
+          - osu-micro-benchmarks
+          - raja-perf
+          - remhos
+          - saxpy
+        VARIANT: [+cuda]
       #############
       # Other tests
       #############

.gitlab/tests/shared_slurm_clusters.yml

@@ -9,35 +9,83 @@ workflow:
 #############################
 # Resource Allocation Section
 #############################
-allocate_resources_dane:
-  tags: [shell, dane]
-  needs: [dane-up-check]
+.allocate_resources_slurm:
   extends: .on_host_template
   stage: allocate-resources
   script:
-    - sbatch ${DANE_SHARED_ALLOC} --wait-all-nodes=1 --job-name=${ALLOC_NAME} --wrap="srun
-      -N 1 -n 1 flux start sleep inf"
+    - set -e
+    # submit and grab JobID
+    - JOBID=$(sbatch --parsable ${SHARED_ALLOC} --wait-all-nodes=1 --job-name="${ALLOC_NAME}"
+      --wrap='srun -N 1 -n 1 flux start sleep inf')
+    - echo "Slurm job = ${JOBID}"
+    # wait until SLURM job running
+    # 1..2160 is up to 6 hours
+    - for i in {1..2160}; do state=$(sacct -j ${JOBID} -n -o State 2>/dev/null | head
+      -1 | awk '{print $1}'); [[ "$state" == "RUNNING" ]] && break; echo "waiting...
+      total duration has been $(( (i-1)*10 ))s" && sleep 10; done
+    #- for i in {1..1080}; do flux proxy slurm:${JOBID} flux ping -c1 >/dev/null 2>&1 && break; echo "waiting..." sleep 10; done
+    #- flux proxy slurm:${JOBID} flux ping -c1  # final check (fails the job if still not ready)
   after_script:
-    - |-
+    - |
       if [[ "$CI_JOB_STATUS" == "canceled" ]]; then
         . .gitlab/utils/cancel-slurm.sh
       fi
+allocate_resources_dane:
+  extends: .allocate_resources_slurm
+  variables:
+    HOST: dane
+    SHARED_ALLOC: $DANE_SHARED_ALLOC
+  tags: [shell, dane]
+  needs: [dane-up-check]
+allocate_resources_matrix:
+  extends: .allocate_resources_slurm
+  variables:
+    HOST: matrix
+    SHARED_ALLOC: $MATRIX_SHARED_ALLOC
+  tags: [shell, matrix]
+  needs: [matrix-up-check]
 ##########################
 # Resource Release Section
 ##########################
-release_resources_dane:
-  tags: [shell, dane]
-  needs: [allocate_resources_dane, run_tests_slurm_dane]
+.release_resources_slurm:
   extends: .on_host_template
   stage: release-resources
   # Do not cleanup dir yet
-  script:
-    - . .gitlab/utils/cancel-slurm.sh --no-clean
+  script: [. .gitlab/utils/cancel-slurm.sh --no-clean]
+release_resources_dane:
+  extends: .release_resources_slurm
+  tags: [shell, dane]
+  needs: [allocate_resources_dane, run_tests_slurm_dane]
+release_resources_matrix:
+  extends: .release_resources_slurm
+  tags: [shell, matrix]
+  needs: [allocate_resources_matrix, run_tests_slurm_matrix]
 ##################
 # Test Run Section
 ##################
-.run_tests_slurm: &run_tests_slurm
+.run_tests_slurm:
   extends: .on_host_template
+  stage: test
+  before_script:
+    - !reference [.report_status, before_script]
+  script:
+    - |
+      echo -e "### Allocation name is ${ALLOC_NAME}"
+      export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
+      echo -e "### Job ID is ${JOBID}"
+    - |
+      PROXY="$( [[ -n "${JOBID}" ]] && echo "flux proxy slurm:${JOBID}" || echo "" )"
+      ${PROXY} flux watch $( ${PROXY} flux batch -o output.stdout.type=kvs ${FLUX_ARGS} .gitlab/utils/run-experiment.sh )
+  after_script:
+    - !reference [.report_status, after_script]
+    - |
+      if [[ "$CI_JOB_STATUS" == "canceled" ]]; then
+        . .gitlab/utils/cancel-slurm.sh
+      fi
+run_tests_slurm_dane:
+  extends: .run_tests_slurm
+  tags: [shell, dane]
+  needs: [allocate_resources_dane]
   parallel:
     matrix:
       - HOST: dane
@@ -58,24 +106,19 @@ release_resources_dane:
         ARCHCONFIG: llnl-cluster
         BENCHMARK: [kripke]
         VARIANT: ['caliper=mpi,time hwloc=on affinity=on']
-run_tests_slurm_dane:
-  tags: [shell, dane]
-  needs: [allocate_resources_dane]
-  stage: test
-  <<: *run_tests_slurm
-  before_script:
-    - !reference [.report_status, before_script]
-  script:
-    - |
-      echo -e "### Allocation name is ${ALLOC_NAME}"
-      export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A)
-      echo -e "### Job ID is ${JOBID}"
-    - |
-      PROXY="$( [[ -n "${JOBID}" ]] && echo "flux proxy slurm:${JOBID}" || echo "" )"
-      ${PROXY} flux watch $( ${PROXY} flux batch -o output.stdout.type=kvs ${FLUX_ARGS} .gitlab/utils/run-experiment.sh )
-  after_script:
-    - !reference [.report_status, after_script]
-    - |-
-      if [[ "$CI_JOB_STATUS" == "canceled" ]]; then
-        . .gitlab/utils/cancel-slurm.sh
-      fi
+run_tests_slurm_matrix:
+  extends: .run_tests_slurm
+  tags: [shell, matrix]
+  needs: [allocate_resources_matrix]
+  parallel:
+    matrix:
+      - HOST: matrix
+        ARCHCONFIG: llnl-matrix
+        BENCHMARK:
+          - amg2023
+          - babelstream
+          - kripke
+          - laghos
+          - raja-perf
+          - saxpy
+        VARIANT: [+cuda]

.gitlab/utils/machine_checks.yml

@@ -28,3 +28,8 @@ dane-up-check:
   variables:
     CI_MACHINE: dane
   extends: [.machine-check]
+matrix-up-check:
+  stage: machine-checks
+  variables:
+    CI_MACHINE: matrix
+  extends: [.machine-check]

.gitlab/utils/run-experiment.sh

@@ -4,7 +4,7 @@ set -e
 # Activate Virtual Environment
 . /usr/workspace/benchpark-dev/benchpark-venv/$SYS_TYPE/bin/activate
 
-echo "./bin/benchpark system init --dest=${HOST} ${ARCHCONFIG} cluster=$HOST $SYSTEM_ARGS"
+echo "./bin/benchpark system init --dest=${HOST} ${ARCHCONFIG} $([ "$HOST" != "matrix" ] && echo "cluster=$HOST") $SYSTEM_ARGS"
 echo "./bin/benchpark experiment init --dest=${BENCHMARK} --system=${HOST} ${BENCHMARK} ${VARIANT}"
 echo "./bin/benchpark setup ${BENCHMARK} wkp/"
 echo ". wkp/setup.sh"
@@ -14,7 +14,7 @@ echo "ramble --disable-logger --workspace-dir . on --executor '{execute_experime
 echo "ramble --disable-logger --workspace-dir . workspace analyze --format json yaml text"
 
 # Initialize System
-./bin/benchpark system init --dest=${HOST} ${ARCHCONFIG} cluster=$HOST $SYSTEM_ARGS
+./bin/benchpark system init --dest=${HOST} ${ARCHCONFIG} $([ "$HOST" != "matrix" ] && echo "cluster=$HOST") $SYSTEM_ARGS
 
 # Initialize Experiment
 BV=""

docs/add-a-system-config.rst

@@ -237,7 +237,7 @@ this function, followed by our AWS example:
    ``compiler_section_for()``.
 3. Merge the compiler definitions with merge_dicts (this part is unnecessary if you have
    only one type of compiler).
-4. Generally, you will want to compose a minimal list of compilers: e.g. if you want to
+4. Generally, you will want to compose a minimal list of compilers: e.g., if you want to
    compile your benchmark with the oneAPI compiler, and have multiple versions to choose
    from, you would add a variant to the system, and the config would expose only one of
    them.
@@ -247,9 +247,9 @@ For our AWS system, the compiler we define is ``[email protected]``. For the
 
 1. ``spec`` - Similar to package specs, ``name@version``. GCC in particular also needs
    the ``languages`` variant, where the list of languages depends on the available
-   ``exes`` (e.g. do not include "fortran" if ``gfortran`` is not available). If you are
-   **not** using GCC or Spack as your package manager, ``languages`` is unnecessary.
-2. ``prefix`` - Prefix to the compiler binary directory, e.g. ``/usr/`` for
+   ``exes`` (e.g., do not include "fortran" if ``gfortran`` is not available). If you
+   are **not** using GCC or Spack as your package manager, ``languages`` is unnecessary.
+2. ``prefix`` - Prefix to the compiler binary directory, e.g., ``/usr/`` for
    ``/usr/bin/gcc``
 3. ``exes`` - Dictionary to map ``c``, ``cxx``, and ``fortran`` to the appropriate file
    found in the prefix.
@@ -320,23 +320,27 @@ instead. For each package that you include, you need to define its spec ``name@v
 and the system path ``prefix`` to the package. Additionally for Spack, you need to set
 ``buildable: False`` to tell Spack not to build that package.
 
-At minimum, we recommend you define externals for ``cmake`` and ``mpi`` (users also
-typically define externals for other libraries, e.g. math libraries like ``blas`` and
-``lapack``). This is because certain packages (e.g. ``cmake``) can take a long time to
+At minimum, we recommend to define externals for ``cmake`` and ``mpi`` (users also
+typically define externals for other libraries, e.g., math libraries like ``blas`` and
+``lapack``). This is because certain packages (e.g., ``cmake``) can take a long time to
 build, and packages such as ``mpi``, ``blas``, and ``lapack`` can influence runtime
-performance significantly. Additionally, for systems with accelerators, define externals
-for CUDA and ROCm runtime libraries (see externals examples for a `CUDA system
+performance significantly so it is prudent to use the versions optimized for our system.
+Additionally, for systems with accelerators, define externals for CUDA and ROCm runtime
+libraries (see externals examples for a `CUDA system
 <https://github.com/LLNL/benchpark/blob/e82e3a26aef54855cf281c088b8f149ab7d87d9d/systems/llnl-matrix/system.py#L274>`_,
 or a `ROCm system
 <https://github.com/LLNL/benchpark/blob/e82e3a26aef54855cf281c088b8f149ab7d87d9d/systems/llnl-elcapitan/system.py#L483>`_).
-Also, see :ref:`adding-sys-packages`, for help on how to search for the packages
-available on your system.
+See :ref:`adding-sys-packages`, for help on how to search for the packages available on
+your system.
 
 .. note::
 
-    For ``mpi``, you need to define ``"mpi": {"buildable": False},`` as a virtual
-    package, and then define your MPI package as we have for the ``openmpi`` package.
-    This is to ensure Spack uses our MPI, and does not try to build another MPI package.
+    For packages that declare virtual dependencies, e.g., ``depends_on("mpi")``, you
+    need to define a virtual package ``"mpi": {"buildable": False},``, followed by a
+    definition of at least one provider of this package (see the provider definition for
+    ``openmpi`` in our example). This is to ensure Spack uses the provider we specified,
+    and does not try to build another MPI package. See a similar example for ``blas``,
+    ``lapack``, and their provider ``atlas``.
 
 ::
 
@@ -354,6 +358,11 @@ available on your system.
         def compute_packages_section(self):
             return {
                 "packages": {
+                    "blas": {"buildable": False},
+                    "lapack": {"buildable": False},
+                    "atlas": {
+                        "externals": [{"spec": "[email protected]", "prefix": "/usr"}],
+                    },
                     "mpi": {"buildable": False},
                     "openmpi": {
                         "externals": [
@@ -364,7 +373,7 @@ available on your system.
                         ]
                     },
                     "cmake": {
-                        "externals": [{"spec": "cmake@4.0.2", "prefix": "/usr"}],
+                        "externals": [{"spec": "cmake@4.1.1", "prefix": "/usr"}],
                         "buildable": False,
                     },
                     ...
@@ -462,7 +471,7 @@ cluster=<cluster>``:
     'tar': {'buildable': False,
             'externals': [{'prefix': '/usr', 'spec': '[email protected]'}]}}
 
-where the command should be ran on a cluster that is defined for the given system, e.g.
+where the command should be ran on a cluster that is defined for the given system, e.g.,
 ruby for llnl-cluster. Use this output to update your package definitions in your
 ``system.py``'s ``compute_package_section()``.
 

systems/aws-tutorial/system.py

@@ -75,9 +75,10 @@ def compute_packages_section(self):
                     "buildable": False,
                 },
                 "gmake": {"externals": [{"spec": "[email protected]", "prefix": "/usr"}]},
-                "lapack": {
-                    "externals": [{"spec": "[email protected]", "prefix": "/usr"}],
-                    "buildable": False,
+                "blas": {"buildable": False},
+                "lapack": {"buildable": False},
+                "atlas": {
+                    "externals": [{"spec": "[email protected]", "prefix": "/usr"}],
                 },
                 "mpi": {"buildable": False},
                 "openmpi": {
@@ -89,7 +90,7 @@ def compute_packages_section(self):
                     ]
                 },
                 "cmake": {
-                    "externals": [{"spec": "cmake@4.0.2", "prefix": "/usr"}],
+                    "externals": [{"spec": "cmake@4.1.1", "prefix": "/usr"}],
                     "buildable": False,
                 },
                 "git": {