Update spack v1.2.2 + Docker fixes (#359)

* Update spack to v1.2.2

* Add unified build.sh and HIP support to the TPL superbuild

- build.sh: single driver for z5tux (cpu/cuda/hip) and LC (matrix, tioga,
  tuo-cpu, tuo-gpu), cmake+ninja by default, --spack for uberenv builds

- CMakeLists: HIP plumbing for raja, chai, caliper and hypre

- Make python explicit in the py-sphinx externals (spack v1.2 deprecation)

* Remove blueos spack config

* Remove Lassen and ESSL support

* Use the new spack installer on the LC machines

* Disable Trilinos by default

* Install xz-devel in the Rocky TPL image

* Fix Ubuntu CUDA TPL image builds

- Inject the streak2 CA bundle into Ubuntu Dockerfiles, not only Rocky
- Patch fmt 11.2.0 so nvcc + clang does not fail on _BitInt
- Keep extra .dockerignore rules and add CPU/memory limits to reproduce_ci.sh

* Add a logged --all sweep to reproduce_ci.sh

* Limit Spack build parallelism

* Use Ubuntu CA bundle

---------

Co-authored-by: Randolph Settgast <settgast1@llnl.gov>

Author: victorapm

.dockerignore

@@ -0,0 +1,13 @@
+.git
+build
+build-*
+install
+install-*
+*_tpls
+.rocm-ci-local.*
+*.cmake
+*.log
+__pycache__
+vtk-9.6.2-local/
+tplMirror/*.tar.gz
+tplMirror/chai-develop-060425/

.github/workflows/docker_build_tpls.yml

@@ -167,8 +167,15 @@ jobs:
         lfs: true
 
     - name: Set up Docker Buildx
+      if: matrix.RUNS_ON != 'streak2'
       uses: docker/setup-buildx-action@v4
 
+    - name: Set up Docker Buildx
+      if: matrix.RUNS_ON == 'streak2'
+      uses: docker/setup-buildx-action@v4
+      with:
+        driver-opts: network=host
+
     - name: Print environment
       run: printenv
 
@@ -180,8 +187,38 @@ jobs:
     - name: Inject CA certificate into Docker build
       if: matrix.RUNS_ON == 'streak2'
       run: |
+        set -euo pipefail
         cp /etc/pki/tls/certs/ca-bundle.crt ./ca-bundle.crt
-        sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' ${{ matrix.TPL_DOCKERFILE }}
+        dockerfile="${{ matrix.TPL_DOCKERFILE }}"
+        if grep -q 'RUN dnf clean all' "${dockerfile}"; then
+          sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' "${dockerfile}"
+        else
+          # Ubuntu CUDA jobs also run on streak2. Only COPY the host bundle;
+          # tpl-ubuntu.Dockerfile configures APT before its first update and
+          # installs the bundle after ca-certificates so the package postinst
+          # cannot wipe it. ENV covers pip/curl/git.
+          awk '
+            BEGIN { inserted=0 }
+            {
+              print
+              if (!inserted && $0 ~ /^FROM /) {
+                print "COPY ca-bundle.crt /etc/ssl/certs/llnl-ca-bundle.crt"
+                print "ENV SSL_CERT_FILE=/etc/ssl/certs/llnl-ca-bundle.crt \\"
+                print "    REQUESTS_CA_BUNDLE=/etc/ssl/certs/llnl-ca-bundle.crt \\"
+                print "    CURL_CA_BUNDLE=/etc/ssl/certs/llnl-ca-bundle.crt \\"
+                print "    GIT_SSL_CAINFO=/etc/ssl/certs/llnl-ca-bundle.crt \\"
+                print "    PIP_CERT=/etc/ssl/certs/llnl-ca-bundle.crt"
+                inserted=1
+              }
+            }
+            END {
+              if (!inserted) {
+                print "ERROR: no FROM line in Dockerfile" > "/dev/stderr"
+                exit 1
+              }
+            }
+          ' "${dockerfile}" > "${dockerfile}.ca" && mv "${dockerfile}.ca" "${dockerfile}"
+        fi
 
     - name: Run the docker build docker script
       env:
@@ -196,6 +233,7 @@ jobs:
         COMMIT: ${{ github.sha }}
         BUILD_DIR: ${{ github.workspace }}
         DOCKER_TAG: ${{ github.event.number }}-${{ github.run_number }}
+        DOCKER_NETWORK: ${{ matrix.RUNS_ON == 'streak2' && 'host' || '' }}
       run: bash -x ./scripts/docker-build.sh
 
     - name: Login to DockerHub

.gitignore

@@ -4,6 +4,7 @@
 .project
 .pydevproject
 .vscode/
+spack/
 spack-*.txt
 __pycache__
 uberenv_*

.uberenv_config.json

@@ -6,8 +6,8 @@
 "spack_configs_path": "scripts/spack_configs",
 "spack_packages_path": "scripts/spack_packages/packages",
 "spack_url": "https://github.com/spack/spack",
-"spack_commit": "2e2169d5282d166f63e3ee4db8d4446c43cefa8a",
-"spack_commit_note": "v1.1.1 (Jan 14th 2026)",
-"spack_packages_commit": "3dd98680871078353a28ee508fa76c7554f918fa",
-"spack_packages_note": "Feb 25th 2026"
+"spack_commit": "3e19345b6e12f5ff1b874f4059622fc6a1fd804a",
+"spack_commit_note": "v1.2.2 (Jul 20th 2026)",
+"spack_packages_commit": "bd9b4838fbf9e4ca01ad7b6c6df633ddcc750d71",
+"spack_packages_note": "Jul 20th 2026"
 }

CMakeLists.txt

@@ -47,10 +47,12 @@ option( BUILD_SHARED_LIBS "" OFF )
 
 option( ENABLE_CALIPER "" ON )
 option( ENABLE_HYPRE "" ON )
-option( ENABLE_TRILINOS "" ON )
+option( ENABLE_TRILINOS "" OFF )
 option( ENABLE_PETSC "" OFF )
 option( ENABLE_SCOTCH "" ON )
 option( ENABLE_SUITESPARSE "" ON )
+option( ENABLE_SUPERLU_DIST "" ON )
+option( ENABLE_HYPRE_GPU_PROFILING "" OFF )
 option( ENABLE_VTK "" ON )
 option( ENABLE_VTK_WRAP_PYTHON "" OFF )
 
@@ -125,6 +127,50 @@ else( TRUE )
 endif()
 
 
+################################
+# HIP / ROCm
+################################
+if( ENABLE_HIP )
+    if( NOT DEFINED ROCM_PATH )
+        if( DEFINED HIP_ROOT )
+            set( ROCM_PATH ${HIP_ROOT} )
+        elseif( DEFINED ENV{ROCM_PATH} )
+            set( ROCM_PATH $ENV{ROCM_PATH} )
+        else()
+            message( FATAL_ERROR "ENABLE_HIP is ON but neither ROCM_PATH nor HIP_ROOT is defined." )
+        endif()
+    endif()
+
+    if( NOT DEFINED HIP_ROOT )
+        set( HIP_ROOT ${ROCM_PATH} )
+    endif()
+
+    if( NOT DEFINED CMAKE_HIP_ARCHITECTURES )
+        message( FATAL_ERROR "ENABLE_HIP is ON but CMAKE_HIP_ARCHITECTURES is not defined (e.g. gfx90a, gfx942, gfx1100)." )
+    endif()
+
+    # hypre is configured outside of cmake, so it needs the MPI include directory explicitly
+    # when its sources are compiled for the device.
+    if( ENABLE_MPI AND NOT DEFINED MPI_INCLUDE_DIR )
+        if( DEFINED ENV{MPICH_DIR} )               # set by the cray-mpich module on LC
+            set( MPI_INCLUDE_DIR "$ENV{MPICH_DIR}/include" )
+        elseif( DEFINED ENV{MPI_ROOT} )
+            set( MPI_INCLUDE_DIR "$ENV{MPI_ROOT}/include" )
+        elseif( MPI_C_INCLUDE_DIRS )               # set by find_package( MPI ) through BLT
+            list( GET MPI_C_INCLUDE_DIRS 0 MPI_INCLUDE_DIR )
+        elseif( MPI_C_COMPILER )                   # <prefix>/bin/mpicc -> <prefix>/include
+            get_filename_component( MPI_BIN_DIR ${MPI_C_COMPILER} DIRECTORY )
+            get_filename_component( MPI_PREFIX_DIR ${MPI_BIN_DIR} DIRECTORY )
+            set( MPI_INCLUDE_DIR "${MPI_PREFIX_DIR}/include" )
+        else()
+            message( FATAL_ERROR "ENABLE_HIP is ON but MPI_INCLUDE_DIR is not defined and cannot be deduced." )
+        endif()
+        message( STATUS "MPI_INCLUDE_DIR = ${MPI_INCLUDE_DIR}" )
+    endif()
+
+    message( STATUS "ROCM_PATH = ${ROCM_PATH}" )
+    message( STATUS "CMAKE_HIP_ARCHITECTURES = ${CMAKE_HIP_ARCHITECTURES}" )
+endif()
 
 
 ################################
@@ -340,6 +386,13 @@ set(RAJA_URL "${TPL_MIRROR_DIR}/RAJA-v2025.12.0.tar.gz")
 
 message(STATUS "Building RAJA found at ${RAJA_URL}")
 
+# RAJA's vectorization layer does not compile with the ROCm toolchain.
+if( ENABLE_HIP )
+    set( RAJA_ENABLE_VECTORIZATION OFF )
+else()
+    set( RAJA_ENABLE_VECTORIZATION ON )
+endif()
+
 ExternalProject_Add( raja
                      URL ${RAJA_URL}
                      PREFIX ${PROJECT_BINARY_DIR}/raja
@@ -360,10 +413,18 @@ ExternalProject_Add( raja
                                 -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}
                                 -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES}
                                 -D CMAKE_CUDA_FLAGS:STRING=${RAJA_CUDA_FLAGS}
+                                -D ENABLE_HIP:BOOL=${ENABLE_HIP}
+                                -D RAJA_ENABLE_HIP:BOOL=${ENABLE_HIP}
+                                -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER}
+                                -D RAJA_ENABLE_VECTORIZATION:BOOL=${RAJA_ENABLE_VECTORIZATION}
+                                -D hip_DIR=${HIP_ROOT}/lib/cmake/hip
+                                -D ROCPRIM_DIR=${HIP_ROOT}
+                                -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES}
                                 -D ENABLE_TESTS:BOOL=OFF
                                 -D ENABLE_EXAMPLES:BOOL=OFF
                                 -D ENABLE_OPENMP:BOOL=${ENABLE_OPENMP}
                                 -D CMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+                                -D CMAKE_INSTALL_LIBDIR:PATH=lib
                                 -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}
                                 -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE}
                     )
@@ -398,8 +459,14 @@ ExternalProject_Add( chai
                                 -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}
                                 -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES}
                                 -D CMAKE_CUDA_FLAGS:STRING=${CHAI_CUDA_FLAGS}
+                                -D ENABLE_HIP:BOOL=${ENABLE_HIP}
+                                -D CHAI_ENABLE_HIP:BOOL=${ENABLE_HIP}
+                                -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER}
+                                -D hip_DIR=${HIP_ROOT}/lib/cmake/hip
+                                -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES}
                                 -D ENABLE_OPENMP=${ENABLE_OPENMP}
                                 -D CMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+                                -D CMAKE_INSTALL_LIBDIR:PATH=lib
                                 -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}
                                 -D ENABLE_TESTS=OFF
                                 -D ENABLE_BENCHMARKS=OFF
@@ -444,6 +511,7 @@ if (ENABLE_CALIPER)
                                     -D MPI_CXX_COMPILER:STRING=${MPI_CXX_COMPILER}
                                     -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE}
                                     -D CMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+                                    -D CMAKE_INSTALL_LIBDIR:PATH=lib
                                     -D BLT_SOURCE_DIR=${BLT_SOURCE_DIR}
                        )
 
@@ -480,14 +548,23 @@ if (ENABLE_CALIPER)
                                     -D CUDA_TOOLKIT_ROOT_DIR:PATH=${CUDA_TOOLKIT_ROOT_DIR}
                                     -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE}
                                     -D CMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+                                    -D CMAKE_INSTALL_LIBDIR:PATH=lib
                                     -D WITH_PAPI:BOOL=${ENABLE_PAPI}
                                     -D PAPI_PREFIX:STRING=${PAPI_PREFIX}
                                     -D WITH_ADIAK:BOOL=ON
                                     -D adiak_DIR:STRING=${ADIAK_DIR}/lib/cmake/adiak/
                                     -D WITH_CUPTI=${CALIPER_WITH_CUPTI}
                                     -D WITH_NVPROF=${ENABLE_CUDA}
+                                    -D WITH_ROCM=${ENABLE_ROCTX}
+                                    -D ROCM_PREFIX:PATH=${ROCM_PATH}
                        )
 
+    if( BUILD_SHARED_LIBS )
+        set( CALIPER_LIB "libcaliper.so" )
+    else()
+        set( CALIPER_LIB "libcaliper.a" )
+    endif()
+
     list(APPEND build_list caliper adiak)
 else()
     set( ENABLE_CALIPER_HYPRE OFF CACHE BOOL "" FORCE )
@@ -589,11 +666,6 @@ if (ENABLE_TRILINOS)
                                  -D TPL_MKL_LIBRARIES:STRING=${MKL_LIBRARIES})
     endif()
 
-    if ( ENABLE_ESSL )
-        set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS}
-                                 -D HAVE_dggsvd3:BOOL=ON)
-    endif()
-
     if ( DEFINED OpenMP_Fortran_FLAGS )
         set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS}
                                  -D OpenMP_Fortran_FLAGS:STRING=${OpenMP_Fortran_FLAGS})
@@ -737,7 +809,7 @@ list(APPEND build_list metis )
 ################################
 # SUPERLU --
 ################################
-if( ENABLE_MPI )
+if( ENABLE_MPI AND ENABLE_SUPERLU_DIST )
   set(SUPERLU_DIR "${CMAKE_INSTALL_PREFIX}/superlu_dist")
   set(SUPERLU_URL "${TPL_MIRROR_DIR}/superlu_dist-0f6efc3.tar.gz")
 
@@ -784,6 +856,7 @@ if( ENABLE_MPI )
                                   -D CMAKE_Fortran_FLAGS_RELEASE=${CMAKE_Fortran_FLAGS_RELEASE}
                                   -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE}
                                   -D CMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+                                  -D CMAKE_INSTALL_LIBDIR:PATH=lib
                                   -D enable_openmp=${ENABLE_OPENMP}
           )
 
@@ -814,6 +887,9 @@ if( ENABLE_HYPRE )
     set( HYPRE_C_COMPILER ${MPI_C_COMPILER} )
     set( HYPRE_CXX_COMPILER ${MPI_CXX_COMPILER} )
     set( HYPRE_Fortran_COMPILER ${MPI_Fortran_COMPILER} )
+  endif()
+
+  if( ENABLE_MPI AND ENABLE_SUPERLU_DIST )
     set( HYPRE_SUPERLU_LIBS "${SUPERLU_DIR}/lib/libsuperlu_dist.a ${PARMETIS_DIR}/lib/libparmetis.a ${METIS_DIR}/lib/libmetis.a -lstdc++" )
     if( ENABLE_OPENMP )
       set( HYPRE_SUPERLU_LIBS "${HYPRE_SUPERLU_LIBS} ${OpenMP_C_FLAGS}" ) # Sergey: why is this needed?
@@ -844,9 +920,41 @@ if( ENABLE_HYPRE )
 --with-umpire-include=${CHAI_DIR}/include \
 --with-umpire-lib-dirs=${CHAI_DIR}/lib \
 --with-umpire-libs=umpire " )
+
+      if( ENABLE_HYPRE_GPU_AWARE_MPI )
+        set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-aware-mpi" )
+      endif()
+
+      if( ENABLE_HYPRE_GPU_PROFILING )
+        set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-profiling" )
+      endif()
     endif()
   elseif( ENABLE_HYPRE_DEVICE STREQUAL "HIP" )
-    message( FATAL_ERROR "hip tpls build only supported through spack" )
+    if( NOT ENABLE_HIP )
+      message( FATAL_ERROR "Non-compatible options: HYPRE device cannot be set to HIP when ENABLE_HIP is OFF" )
+    else()
+      list( APPEND HYPRE_DEPENDS chai )
+      set( HYPRE_HIP_FLAGS "\
+ROCM_PATH=${ROCM_PATH} \
+--with-hip \
+--enable-rocblas \
+--enable-rocsparse \
+--enable-rocsolver \
+--with-gpu-arch=${CMAKE_HIP_ARCHITECTURES} \
+--with-umpire-device \
+--with-umpire-include=${CHAI_DIR}/include \
+--with-umpire-lib-dirs=${CHAI_DIR}/lib \
+--with-umpire-libs=\"umpire camp\" \
+--with-MPI-include=${MPI_INCLUDE_DIR} " )
+
+      if( ENABLE_HYPRE_GPU_AWARE_MPI )
+        set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-aware-mpi" )
+      endif()
+
+      if( ENABLE_HYPRE_GPU_PROFILING )
+        set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-profiling" )
+      endif()
+    endif()
   endif()
 
   if( CMAKE_HOST_APPLE )
@@ -856,7 +964,7 @@ if( ENABLE_HYPRE )
   endif()
 
   if( ENABLE_CALIPER_HYPRE )
-    set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/libs/caliper.a\"" )
+    set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/lib/${CALIPER_LIB}\"" )
     list( APPEND HYPRE_DEPENDS caliper )
   endif()
 
@@ -881,6 +989,7 @@ if( ENABLE_HYPRE )
   ${HYPRE_SUPERLU_FLAGS} \
   ${HYPRE_OPENMP_FLAG} \
   ${HYPRE_CUDA_FLAGS} \
+  ${HYPRE_HIP_FLAGS} \
   ${HYPRE_CALIPER_FLAGS}\n")
 
   ExternalProject_Add( hypre

cmake/blt

@@ -12,15 +12,6 @@ if( ENABLE_MKL )
     set( BLAS_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE )
     set( LAPACK_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE) 
 
-elseif( ENABLE_ESSL )
-    if ( NOT DEFINED ESSL_INCLUDE_DIRS )
-        message( FATAL_ERROR "ESSL is enabled but ESSL_INCLUDE_DIRS is not defined." )
-    elseif ( NOT DEFINED ESSL_LIBRARIES )
-        message( FATAL_ERROR "ESSL is enabled but ESSL_LIBRARIES is not defined." )
-    endif()
-
-    set( BLAS_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE )
-    set( LAPACK_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE )
 endif()
 
 if( NOT DEFINED BLAS_LIBRARIES )

cmake/thirdparty/FindMathLibraries.cmake

@@ -0,0 +1,14 @@
+openssl_conf = openssl_init
+
+[openssl_init]
+providers = provider_sect
+alg_section = algorithm_sect
+
+[provider_sect]
+default = default_sect
+
+[default_sect]
+activate = 1
+
+[algorithm_sect]
+default_properties = fips=no

docker/openssl-non-fips.cnf

@@ -111,6 +111,8 @@ spack:
       externals:
       - spec: py-sphinx@4.3.2
         prefix: /usr
+        dependencies:
+        - spec: python@3.10.12
     python:
       buildable: false
       externals: