Spackify JEDI components for NEPTUNE-JEDI

.github/workflows/ubuntu-ci-x86_64-gnu.yaml

@@ -70,9 +70,10 @@ jobs:
             spack stack create env --site linux.default --template ${TEMPLATE} --name ${ENVNAME} --compiler gcc
             spack env activate ${ENVDIR}
 
-            # Turn off variant "adp" for neptune-dev (NRL internal only)
+            # Turn off variants "adp" and "jedi" for neptune-dev (NRL internal only)
             if [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then
               sed -i 's/+adp/~adp/g' ${ENVDIR}/spack.yaml
+              sed -i 's/+jedi/~jedi/g' ${ENVDIR}/spack.yaml
             fi
 
             unset SPACK_DISABLE_LOCAL_CONFIG

.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml

@@ -70,9 +70,10 @@ jobs:
             spack stack create env --site linux.default --template ${TEMPLATE} --name ${ENVNAME} --compiler oneapi
             spack env activate ${ENVDIR}
 
-            # Turn off variant "adp" for neptune-dev (NRL internal only)
+            # Turn off variants "adp" and "jedi" for neptune-dev (NRL internal only)
             if [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then
               sed -i 's/+adp/~adp/g' ${ENVDIR}/spack.yaml
+              sed -i 's/+jedi/~jedi/g' ${ENVDIR}/spack.yaml
             fi
 
             unset SPACK_DISABLE_LOCAL_CONFIG

.github/workflows/ubuntu-ci-x86_64-oneapi.yaml

@@ -70,9 +70,10 @@ jobs:
             spack stack create env --site linux.default --template ${TEMPLATE} --name ${ENVNAME} --compiler oneapi
             spack env activate ${ENVDIR}
 
-            # Turn off variant "adp" for neptune-dev (NRL internal only)
+            # Turn off variants "adp" and "jedi" for neptune-dev (NRL internal only)
             if [[ "${TEMPLATE}" == *"neptune-dev"* ]]; then
               sed -i 's/+adp/~adp/g' ${ENVDIR}/spack.yaml
+              sed -i 's/+jedi/~jedi/g' ${ENVDIR}/spack.yaml
             fi
 
             unset SPACK_DISABLE_LOCAL_CONFIG

.gitmodules

@@ -4,5 +4,7 @@
   branch = spack-stack-dev
 [submodule "repos/builtin"]
   path = repos/builtin
-  url = https://github.com/jcsda/spack-packages
-  branch = spack-stack-dev
+  #url = https://github.com/jcsda/spack-packages
+  #branch = spack-stack-dev
+  url = https://github.com/climbfuji/spack-packages
+  branch = feature/crtm_checks

configs/common/packages_gcc.yaml

@@ -23,3 +23,6 @@ packages:
     ectrans:
       require:
       - +fftw
+    oops:
+      require:
+      - ~mkl

configs/common/packages_oneapi.yaml

@@ -51,6 +51,9 @@ packages:
     nco:
       require:
       - '%c,cxx=gcc'
+    oops:
+      require:
+      - +mkl
     parallel-netcdf:
       require:
       - +shared-intel

configs/templates/neptune-dev-llvm/spack.yaml

@@ -19,10 +19,7 @@ spack:
   #- neptune-python-env    ^neptune-env ~debug +openmp +espc +ncview ^esmf@=8.9.1
   #- jedi-neptune-env +adp ^neptune-env ~debug +openmp +espc +ncview ^esmf@=8.9.1
   - py-numpy
-  # Let's assume/hope that by the time FALCON starts
-  # testing with LLVM, we have moved from crtm v2 to v3
-  - crtm@3.1.2
-  #- crtm@v2.4.1-jedi.2
+  - crtm@3.1.3
 
   packages:
     # Turn on python variant for esmf

configs/templates/neptune-dev/spack.yaml

@@ -13,9 +13,8 @@ spack:
   - neptune-env ~debug ~openmp +espc +ncview ^esmf@=8.9.1
   - neptune-env +debug ~openmp +espc +ncview ^esmf@=8.9.1
   - neptune-python-env +gittools ^neptune-env ~debug +openmp +espc +ncview ^esmf@=8.9.1
-  - jedi-neptune-env +adp        ^neptune-env ~debug +openmp +espc +ncview ^esmf@=8.9.1
-  - crtm@3.1.2
-  - crtm@v2.4.1-jedi.2
+  - jedi-neptune-env +adp +jedi  ^neptune-env ~debug +openmp +espc +ncview ^esmf@=8.9.1
+  - crtm@3.1.3
 
   packages:
     # Turn on python variant for esmf

configs/templates/unified-dev/spack.yaml

@@ -11,9 +11,10 @@ spack:
   - ewok-env
   #- ai-env
   - geos-gcm-env          ^esmf@=8.9.1
-  - global-workflow-env   ^esmf@=8.6.1 ^crtm@=3.1.2
+  - global-workflow-env   ^esmf@=8.6.1 ^crtm@=3.1.3
+  # Is GMAO ready to move to crtm@3.1.3?
   - gmao-swell-env                     ^crtm@=v2.4.1-jedi.2
-  - gsi-env                            ^crtm@=3.1.2
+  - gsi-env                            ^crtm@=3.1.3
   - jedi-fv3-env
   - jedi-geos-env         ^esmf@=8.9.1
   - jedi-mpas-env
@@ -24,12 +25,12 @@ spack:
   - neptune-env           ^esmf@=8.9.1
   - neptune-python-env    ^esmf@=8.9.1
   - soca-env
-  - ufs-srw-app-env       ^esmf@=8.8.0 ^crtm@=3.1.2
-  - ufs-weather-model-env ^esmf@=8.8.0 ^crtm@=3.1.2
+  - ufs-srw-app-env       ^esmf@=8.8.0 ^crtm@=3.1.3
+  - ufs-weather-model-env ^esmf@=8.8.0 ^crtm@=3.1.3
 
   # Various crtm tags (list all to avoid duplicate packages)
   - crtm@v2.4.1-jedi.2
-  - crtm@3.1.2
+  - crtm@3.1.3
 
   # Mapl with various esmf tags
   - mapl@2.53.0 ^esmf@8.6.1

repos/spack_stack/spack_repo/spack_stack/packages/ioda/ioda_cmake_import.patch

@@ -0,0 +1,18 @@
+--- a/cmake/ioda-import.cmake.in
++++ b/cmake/ioda-import.cmake.in
+@@ -65,6 +65,15 @@
+ if(@odc_FOUND@)
+     find_dependency( odc 1.0.2 REQUIRED )
+ endif()
++
++if(@bufr_FOUND@)
++    find_dependency( bufr 12.0.1 REQUIRED )
++endif()
++
++if(@bufr_query_FOUND@)
++    find_dependency( bufr_query 0.0.4 REQUIRED )
++endif()
++
+ # Header-only. Not exposed.
+ #find_dependency( Boost 1.64.0 )
+

repos/spack_stack/spack_repo/spack_stack/packages/ioda/package.py

@@ -0,0 +1,111 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack_repo.builtin.build_systems.cmake import CMakePackage, generator
+
+from spack.package import *
+
+
+class Ioda(CMakePackage):
+    """Interface for Observation Data Access"""
+
+    homepage = "https://github.com/JCSDA/ioda"
+    git = "https://github.com/JCSDA/ioda.git"
+
+    maintainers("climbfuji")
+
+    version("develop", branch="develop", no_cache=True)
+    version("2.9.0.20250826", commit="6e76616001067384f7d0ca4341ad78e81527af8b")
+
+    patch("ioda_cmake_import.patch", when="@2.9.0.20250826")
+
+    variant("doc", default=False, description="Build IODA documentation")
+    # Let's always assume IODA_BUILD_LANGUAGE_FORTRAN=on.
+    # variant('fortran', default=True, description='Build the ioda Fortran interface')
+    variant("odc", default=True, description="Build ODC bindings")
+    # ioda has no explicit OpenMP calls, but header files from Eigen and oops do use openmp.
+    variant("openmp", default=True, description="Build with OpenMP support")
+    # Let's always BUILD_PYTHON_BINDINGS.
+    # variant('python', default=True, description='Build the ioda Python interface')
+
+    generator("make")
+
+    # Project doesn't list "c" as a dependency in CMakeLists.txt, but cmake step fails w/o it
+    depends_on("c", type=("build"))
+    depends_on("cxx", type=("build"))
+    depends_on("fortran", type=("build"))
+
+    depends_on("boost@1.64.0:")
+    depends_on("bufr")
+    depends_on("bufr@12.0.1:", when="@2.9:")
+    depends_on("bufr-query@0.0.4:", when="@2.9:")
+    depends_on("cmake", type=("build"))
+    depends_on("cmake@3.14:", type=("build"), when="@2.9:")
+    depends_on("ecbuild", type=("build"))
+    depends_on("ecbuild@3.3.2:", type=("build"), when="@2.9:")
+    depends_on("eckit")
+    depends_on("eckit@1.23.0:", when="@2.9:")
+    depends_on("eigen")
+    depends_on("fckit")
+    depends_on("fckit@0.10.1:", when="@2.9:")
+    depends_on("gsl-lite")
+    depends_on("hdf5@1.12.0: +mpi")
+    depends_on("hdf5@1.14.0: +mpi", when="@2.9:")
+    depends_on("ioda-data", type=("build", "test"))
+    depends_on("ioda-data@2.9.0.20250805", type=("build", "test"), when="@2.9:")
+    depends_on("jedi-cmake", type=("build"))
+    depends_on("llvm-openmp", when="+openmp %apple-clang", type=("build", "link", "run"))
+    depends_on("mpi")
+    depends_on("nccmp", type=("build", "test"))
+    depends_on("netcdf-cxx", when="@2.9:")
+    depends_on("odc", when="+odc")
+    depends_on("odc@1.4.6:", when="@2.9: +odc")
+    depends_on("oops+openmp", when="+openmp")
+    depends_on("oops~openmp", when="~openmp")
+    depends_on("oops@1.10", when="@2.9:")
+    depends_on("python")
+    depends_on("python@3.9:3.11", when="@2.9:")
+    depends_on("py-pybind11")
+    depends_on("py-pycodestyle", type=("build", "test"))
+    depends_on("py-netcdf4", type=("build", "test"))
+    depends_on("udunits")
+    depends_on("udunits@2.2.0:", when="@2.9:")
+
+    def cmake_args(self):
+        res = [
+            self.define("BUILD_TESTING", self.run_tests),
+            self.define_from_variant("ENABLE_IODA_DOC", "doc"),
+        ]
+        return res
+
+    def setup_build_environment(self, env: EnvironmentModifications) -> None:
+        """This needs to be set at build time, not at test time,
+        to prevent IODA from downloading test data from S4"""
+        env.set("IODA_TESTFILES", self.spec["ioda-data"].prefix)
+
+    def check(self):
+        skipped_tests = None
+        with when("@2.9.0.20250826"):
+            # No time to deal with the bufr Python dependency
+            skipped_tests = [
+                "test_ioda_bufr_python_encoder",
+                "test_ioda_bufr_python_parallel",
+            ]
+
+        ctest = Executable(self.spec["cmake"].prefix.bin.ctest)
+        with working_dir(self.build_directory):
+            if skipped_tests:
+                ctest("--timeout", "120", "-E", "|".join(skipped_tests))
+            else:
+                ctest("--timeout", "120")
+
+    @run_after("install")
+    def fix_ioda_yaml_root_path(self):
+        with when("@2.9.0.20250826"):
+            filter_file(
+                join_path(self.build_directory, "share/test/testinput/"),
+                join_path(self.prefix, "share/ioda/yaml"),
+                join_path(self.prefix, "lib64/cmake/ioda/ioda-import.cmake"),
+            )

repos/spack_stack/spack_repo/spack_stack/packages/ioda_converters/package.py

@@ -0,0 +1,82 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack_repo.builtin.build_systems.cmake import CMakePackage, generator
+
+from spack.package import *
+
+
+class IodaConverters(CMakePackage):
+    """Interface for Observation Data Access"""
+
+    homepage = "https://github.com/JCSDA/ioda-converters"
+    git = "https://github.com/JCSDA/ioda-converters.git"
+
+    maintainers("climbfuji")
+
+    version("develop", branch="develop", no_cache=True)
+    version("0.0.1.20250830", commit="a91f432d9d50940910605e689cd1cf93a1ce3798")
+
+    patch(
+        "https://github.com/JCSDA/ioda-converters/commit/2c09857aac09b7dd9029fdd23e33f712933c40c4.patch?full_index=1",
+        sha256="c31342a5bcffdcb77a99ee5f16a5ba2d74e9d822f1fe2d44b3ee772765800d83",
+        when="@0.0.1.20250830",
+    )
+
+    generator("make")
+
+    # Project doesn't list "c" as a dependency in CMakeLists.txt, but cmake step fails w/o it
+    depends_on("c", type=("build"))
+    depends_on("cxx", type=("build"))
+    depends_on("fortran", type=("build"))
+
+    extends("python")
+
+    depends_on("ecbuild", type=("build"))
+    depends_on("ecbuild@3.3.2:", type=("build"), when="@0.0.1:")
+
+    depends_on("bufr@12:")
+    depends_on("eccodes")
+    depends_on("eckit")
+    depends_on("eigen@3")
+    depends_on("gsl-lite")
+    depends_on("ioda")
+    depends_on("jedi-cmake", type=("build"))
+    depends_on("mpi")
+    depends_on("netcdf-cxx")
+    depends_on("netcdf-fortran")
+    depends_on("oops")
+    depends_on("py-cartopy")
+    depends_on("py-pybind11")
+
+    # For running checks
+    depends_on("nccmp", type=("build", "test"))
+    depends_on("py-pycodestyle", type=("build", "test"))
+    depends_on("py-eccodes", type=("build", "test"))
+    depends_on("py-h5py", type=("build", "test"))
+    depends_on("py-netcdf4", type=("build", "test"))
+    depends_on("py-pandas", type=("build", "test"))
+    depends_on("py-pyhdf", type=("build", "test"))
+    depends_on("py-pyyaml", type=("build", "test"))
+    depends_on("py-xarray", type=("build", "test"))
+
+    def cmake_args(self):
+        res = [
+            self.define("BUILD_TESTING", self.run_tests),
+        ]
+        return res
+
+    def check(self):
+        skipped_tests = None
+        #with when("@0.0.1.20250830"):
+        #    skipped_tests = [
+        #    ]
+        #
+        ctest = Executable(self.spec["cmake"].prefix.bin.ctest)
+        with working_dir(self.build_directory):
+            if skipped_tests:
+                ctest("--timeout", "120", "-E", "|".join(skipped_tests))
+            else:
+                ctest("--timeout", "120")

repos/spack_stack/spack_repo/spack_stack/packages/ioda_data/package.py

@@ -0,0 +1,51 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack_repo.builtin.build_systems.cmake import CMakePackage, generator
+
+from spack.package import *
+
+
+class IodaData(CMakePackage):
+    """Test data for IODA (Interface for Observation Data Access)"""
+
+    homepage = "https://github.com/JCSDA-internal/ioda-data"
+    git = "https://github.com/JCSDA-internal/ioda-data.git"
+
+    maintainers("climbfuji")
+
+    version("develop", branch="develop", no_cache=True)
+    version("2.9.0.20250805", commit="c6f8842648ea473eebc9f66d7c27e2204e5220d6")
+
+#    patch("ioda_cmake_import.patch", when="@2.9.0.20250826")
+#
+#    variant("doc", default=False, description="Build IODA documentation")
+#    # Let's always assume IODA_BUILD_LANGUAGE_FORTRAN=on.
+#    # variant('fortran', default=True, description='Build the ioda Fortran interface')
+#    variant("odc", default=True, description="Build ODC bindings")
+#    # ioda has no explicit OpenMP calls, but header files from Eigen and oops do use openmp.
+#    variant("openmp", default=True, description="Build with OpenMP support")
+#    # Let's always BUILD_PYTHON_BINDINGS.
+#    # variant('python', default=True, description='Build the ioda Python interface')
+
+    generator("make")
+
+    depends_on("c", type=("build"))
+    depends_on("cxx", type=("build"))
+
+    depends_on("cmake", type=("build"))
+    depends_on("cmake@3.12:", type=("build"), when="@2.9:")
+    depends_on("ecbuild", type=("build"))
+    depends_on("ecbuild@3.3.2:", type=("build"), when="@2.9:")
+
+    def install(self, spec, prefix):
+        install_tree(self.stage.source_path, prefix)
+        # JEDI dependencies expect ioda test data in subdirectory
+        # ioda-data of the IODA_TESTFILES environment variable
+        mkdirp(join_path(prefix, "ioda-data"))
+        symlink(
+            join_path(prefix, "testinput_tier_1"), 
+            join_path(prefix, "ioda-data", "testinput_tier_1"),
+        )