Merge branch 'develop' into refactor/dkachuma/remove-constitutive-pas…

…s-through

.devcontainer/devcontainer.json

@@ -2,7 +2,7 @@
     "build": {
         "dockerfile": "Dockerfile",
         "args": {
-            "GEOS_TPL_TAG": "262-349"
+            "GEOS_TPL_TAG": "270-418"
         }
     },
     "runArgs": [

.github/workflows/ci_tests.yml

@@ -284,6 +284,17 @@ jobs:
             DOCKER_RUN_ARGS: "--cpus=8 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro"
             DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors"
             DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust"
+
+          # compiler error in ElasticFirstOrderWaveEquationSEMKernel::StressComputation::launch in call to FE_TYPE::computeFirstOrderStiffnessTermX
+          # - name: Rockylinux (8, clang 17.0.6, cuda 12.5)
+          #   BUILD_AND_TEST_CLI_ARGS: "--no-run-unit-tests --no-install-schema"
+          #   CMAKE_BUILD_TYPE: Release
+          #   DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.5
+          #   RUNS_ON: streak2
+          #   NPROC: 2
+          #   DOCKER_RUN_ARGS: "--cpus=1 --memory=128g --runtime=nvidia -v /etc/pki/ca-trust/source/anchors/:/etc/pki/ca-trust/source/anchors/llnl:ro"
+          #   DOCKER_CERTS_DIR: "/etc/pki/ca-trust/source/anchors"
+          #   DOCKER_CERTS_UPDATE_COMMAND: "update-ca-trust"  
 
           # Below this line, jobs that deploy to Google Cloud.
           - name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119)

host-configs/LLNL/dane-gcc-12.cmake

@@ -0,0 +1,9 @@
+include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/dane-gcc-12.cmake)
+
+# MPI
+set(MPI_HOME /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1-magic CACHE PATH "")
+
+# ATS
+set(ATS_ARGUMENTS "--machine slurm112"  CACHE STRING "")
+
+include(${CMAKE_CURRENT_LIST_DIR}/llnl-cpu-base.cmake)

host-configs/LLNL/lassen-base.cmake

@@ -31,15 +31,16 @@ set(ENABLE_CUDA_NVTOOLSEXT OFF CACHE BOOL "")
 
 # ESSL
 set(ENABLE_ESSL ON CACHE BOOL "" FORCE )
-set(ESSL_INCLUDE_DIRS /usr/tcetmp/packages/essl/essl-6.3.0.2/include CACHE STRING "" FORCE )
-set(ESSL_LIBRARIES /usr/tcetmp/packages/essl/essl-6.3.0.2/lib64/libesslsmpcuda.so
+set(ESSL_DIR /usr/tcetmp/packages/essl/essl-6.3.0.2 CACHE STRING "" FORCE )
+set(ESSL_INCLUDE_DIRS ${ESSL_DIR}/include CACHE STRING "" FORCE )
+set(ESSL_LIBRARIES ${ESSL_DIR}/lib64/libesslsmpcuda.so
                    ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcublas.so
                    ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcublasLt.so
                    ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcudart.so
-                   /usr/tcetmp/packages/essl/essl-6.3.0.2/lib64/liblapackforessl.so
-                   /usr/tcetmp/packages/essl/essl-6.3.0.2/lib64/liblapackforessl_.so
+                   ${ESSL_DIR}/lib64/liblapackforessl.so
+                   ${ESSL_DIR}/lib64/liblapackforessl_.so
                    CACHE PATH "" FORCE )
-                   
+
 # TPL
 set(ENABLE_PAPI OFF CACHE BOOL "")
 set(SILO_BUILD_TYPE powerpc64-unknown-linux-gnu CACHE STRING "")

host-configs/LLNL/lassen-clang-13-cuda-12.cmake

@@ -0,0 +1,25 @@
+include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/lassen-clang-13-cuda-12.cmake)
+
+# Fortran
+set(CMAKE_Fortran_COMPILER /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran  CACHE PATH "")
+set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG -mtune=power9" CACHE STRING "")
+set(FORTRAN_MANGLE_NO_UNDERSCORE ON CACHE BOOL "")
+set(OpenMP_Fortran_FLAGS "-fopenmp" CACHE STRING "")
+set(OpenMP_Fortran_LIB_NAMES "" CACHE STRING "")
+
+# MPI
+set(MPI_HOME /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-13.0.1-gcc-8.3.1/ CACHE PATH "")
+set(MPI_Fortran_COMPILER /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-8.3.1/bin/mpifort CACHE PATH "")
+
+include(${CMAKE_CURRENT_LIST_DIR}/lassen-base.cmake)
+
+# Overwrite options set on lassen-base.cmake
+set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE)
+set(ENABLE_CUDA_NVTOOLSEXT ON CACHE BOOL "")
+
+# Overwrite ESSL defaults from lassen-base.cmake
+# Reason: libesslsmpcuda.so depends on cuda-11
+set(ESSL_LIBRARIES ${ESSL_DIR}/lib64/libessl.so
+                   ${ESSL_DIR}/lib64/liblapackforessl.so
+                   ${ESSL_DIR}/lib64/liblapackforessl_.so
+                   CACHE PATH "" FORCE )

host-configs/LLNL/quartz-base.cmake → host-configs/LLNL/llnl-cpu-base.cmake

@@ -1,6 +1,6 @@
 ###############################################################################
 #
-# Base configuration for LC Quartz builds
+# Base configuration for LC cpu builds
 # Calling configuration file must define the following CMAKE variables:
 #
 # MPI_HOME
@@ -10,6 +10,24 @@
 # Fortran
 set(ENABLE_FORTRAN OFF CACHE BOOL "")
 
+# Fortran
+set(CMAKE_Fortran_COMPILER /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran CACHE PATH "")
+set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG -march=native -mtune=native" CACHE STRING "")
+
+# PYGEOSX
+set(ENABLE_PYGEOSX ON CACHE BOOL "")
+set(Python3_ROOT_DIR /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python CACHE PATH "")
+set(Python3_EXECUTABLE ${Python3_ROOT_DIR}/bin/python3 CACHE PATH "")
+
+# YAPF python formatting
+set(YAPF_EXECUTABLE /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python/bin/yapf CACHE PATH "" FORCE)
+
+# Sphinx
+set(SPHINX_EXECUTABLE /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python/bin/sphinx-build CACHE PATH "" FORCE)
+
+# ATS
+set(ATS_ARGUMENTS "--machine slurm56"  CACHE STRING "")
+
 # MPI
 set(ENABLE_MPI ON CACHE BOOL "")
 set(MPI_C_COMPILER ${MPI_HOME}/bin/mpicc CACHE PATH "")
@@ -20,7 +38,7 @@ set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "")
 
 # PAPI (For TPL caliper builds)
 set(ENABLE_PAPI ON CACHE BOOL "")
-set(PAPI_PREFIX /usr/tce/packages/papi/papi-5.4.3 CACHE PATH "")
+set(PAPI_PREFIX /usr/tce/packages/papi/papi-6.0.0.1/ CACHE PATH "")
 
 # OpenMP
 set(ENABLE_OPENMP ON CACHE BOOL "")
@@ -31,16 +49,9 @@ set(ENABLE_PETSC OFF CACHE BOOL "Enables PETSc." FORCE)
 
 # PYGEOSX
 set(ENABLE_PYGEOSX ON CACHE BOOL "")
-set(Python3_ROOT_DIR /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python CACHE PATH "")
+set(Python3_ROOT_DIR /usr/apps/python-3.11.5 CACHE PATH "")
 set(Python3_EXECUTABLE ${Python3_ROOT_DIR}/bin/python3 CACHE PATH "")
 
-# YAPF python formatting
-set(YAPF_EXECUTABLE /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python/bin/yapf CACHE PATH "" FORCE)
-
-# Sphinx
-set(SPHINX_EXECUTABLE /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python/bin/sphinx-build CACHE PATH "" FORCE)
-
-
 # caliper
 set(ENABLE_CALIPER ON CACHE BOOL "" FORCE)
 set(ENABLE_CALIPER_HYPRE ON CACHE BOOL "" FORCE)
@@ -55,13 +66,7 @@ set(MKL_LIBRARIES ${MKL_ROOT}/lib/intel64/libmkl_intel_lp64.so
                   CACHE STRING "")
 
 # ATS
-set(ATS_ARGUMENTS "--machine slurm36"  CACHE STRING "")
 set(USER $ENV{USER} CACHE STRING "")
 set(ATS_WORKING_DIR "/p/lustre2/${USER}/integratedTestsGEOS/${CONFIG_NAME}"  CACHE PATH "")
 set(ATS_BASELINE_DIR "/p/lustre2/${USER}/integratedTestsGEOS/baselines"  CACHE PATH "")
-
-# Temporary argument for python module change testing
-# set(GEOS_PYTHON_PACKAGES_BRANCH "feature/sherman/outOfPlaceATS" CACHE STRING "" FORCE)
-
-
 include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake)

host-configs/LLNL/ruby-clang-14.cmake

@@ -0,0 +1,6 @@
+include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/ruby-clang-14.cmake)
+
+# MPI
+set(MPI_HOME /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-14.0.6-magic CACHE PATH "")
+
+include(${CMAKE_CURRENT_LIST_DIR}/llnl-cpu-base.cmake)

host-configs/LLNL/ruby-gcc-12.cmake

@@ -0,0 +1,9 @@
+include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/ruby-gcc-12.cmake)
+
+# MPI
+set(MPI_HOME /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1-magic CACHE PATH "")
+
+# ATS
+set(ATS_ARGUMENTS "--machine slurm56"  CACHE STRING "")
+
+include(${CMAKE_CURRENT_LIST_DIR}/llnl-cpu-base.cmake)

host-configs/environment.cmake

@@ -70,5 +70,13 @@ if(ENABLE_CUDA)
 
 endif()
 
+if(DEFINED ENV{BLAS_LIBRARIES})
+  set(BLAS_LIBRARIES "$ENV{BLAS_LIBRARIES}" CACHE PATH "" FORCE)
+endif()
+
+if(DEFINED ENV{LAPACK_LIBRARIES})
+  set(LAPACK_LIBRARIES "$ENV{LAPACK_LIBRARIES}" CACHE PATH "" FORCE)
+endif()
+
 set(GEOSX_TPL_DIR "$ENV{GEOSX_TPL_DIR}" CACHE PATH "" FORCE)
 include(${CMAKE_CURRENT_LIST_DIR}/tpls.cmake)

src/cmake/thirdparty/SetupGeosxThirdParty.cmake

@@ -285,6 +285,14 @@ else()
     mandatory_tpl_doesnt_exist("pugixml" PUGIXML_DIR)
 endif()
 
+################################
+# CUDA
+################################
+if ( ENABLE_CUDA)
+  find_package(CUDAToolkit REQUIRED)
+  message( " ----> $CUDAToolkit_VERSION = ${CUDAToolkit_VERSION}")
+endif()
+
 ################################
 # CAMP ( required before raja on crusher / using spack installed tpls )
 ################################
@@ -631,41 +639,45 @@ endif()
 if(DEFINED HYPRE_DIR AND ENABLE_HYPRE)
     message(STATUS "HYPRE_DIR = ${HYPRE_DIR}")
 
-    set( HYPRE_DEPENDS blas lapack umpire)
+    set( HYPRE_DEPENDS blas lapack umpire )
     if( ENABLE_SUPERLU_DIST )
-        set( HYPRE_DEPENDS ${HYPRE_DEPENDS} superlu_dist )
+        list( APPEND HYPRE_DEPENDS superlu_dist )
     endif()
     if( ${ENABLE_HYPRE_DEVICE} STREQUAL "CUDA" )
-        set( EXTRA_LIBS ${CUDA_cusparse_LIBRARY} ${CUDA_cublas_LIBRARY} ${CUDA_curand_LIBRARY} ${CUDA_cusolver_LIBRARY} )
+        list( APPEND HYPRE_DEPENDS CUDA::cusparse CUDA::cublas CUDA::curand CUDA::cusolver )
+
+        # Add libnvJitLink when using CUDA >= 12.2.2. Note: requires cmake >= 3.26
+        if( CUDAToolkit_VERSION VERSION_GREATER_EQUAL "12.2.2" )
+           list( APPEND HYPRE_DEPENDS CUDA::nvJitLink )
+        endif()
     elseif( ${ENABLE_HYPRE_DEVICE} STREQUAL "HIP" )
         find_package( rocblas REQUIRED )
         find_package( rocsolver REQUIRED )
         find_package( rocsparse REQUIRED )
         find_package( rocrand REQUIRED )
-        set( HYPRE_DEPENDS ${HYPRE_DEPENDS} roc::rocblas roc::rocsparse roc::rocsolver roc::rocrand )
+        append( APPEND HYPRE_DEPENDS roc::rocblas roc::rocsparse roc::rocsolver roc::rocrand )
     endif( )
 
-    find_and_import(NAME hypre
-                      INCLUDE_DIRECTORIES ${HYPRE_DIR}/include
-                      LIBRARY_DIRECTORIES ${HYPRE_DIR}/lib
-                      HEADER HYPRE.h
-                      LIBRARIES HYPRE
-                      EXTRA_LIBRARIES ${EXTRA_LIBS}
-                      DEPENDS ${HYPRE_DEPENDS})
+    find_and_import( NAME hypre
+                     INCLUDE_DIRECTORIES ${HYPRE_DIR}/include
+                     LIBRARY_DIRECTORIES ${HYPRE_DIR}/lib
+                     HEADER HYPRE.h
+                     LIBRARIES HYPRE
+                     DEPENDS ${HYPRE_DEPENDS} )
 
     extract_version_from_header( NAME hypre
                                  HEADER "${HYPRE_DIR}/include/HYPRE_config.h"
                                  VERSION_STRING "HYPRE_RELEASE_VERSION" )
 
     # Extract some additional information about development version of hypre
-    file(READ ${HYPRE_DIR}/include/HYPRE_config.h header_file)
-    if("${header_file}" MATCHES "HYPRE_DEVELOP_STRING *\"([^\"]*)\"")
-        set(hypre_dev_string "${CMAKE_MATCH_1}")
-        if("${header_file}" MATCHES "HYPRE_BRANCH_NAME *\"([^\"]*)\"")
-            set(hypre_dev_branch "${CMAKE_MATCH_1}")
+    file( READ ${HYPRE_DIR}/include/HYPRE_config.h header_file )
+    if( "${header_file}" MATCHES "HYPRE_DEVELOP_STRING *\"([^\"]*)\"" )
+        set( hypre_dev_string "${CMAKE_MATCH_1}" )
+        if( "${header_file}" MATCHES "HYPRE_BRANCH_NAME *\"([^\"]*)\"" )
+            set( hypre_dev_branch "${CMAKE_MATCH_1}" )
         endif()
-        set(hypre_VERSION "${hypre_dev_string} (${hypre_dev_branch})" CACHE STRING "" FORCE)
-        message(" ----> hypre_VERSION = ${hypre_VERSION}")
+        set( hypre_VERSION "${hypre_dev_string} (${hypre_dev_branch})" CACHE STRING "" FORCE )
+        message( " ----> hypre_VERSION = ${hypre_VERSION}" )
     endif()
 
     # Prepend Hypre to link flags, fix for Umpire appearing before Hypre on the link line
@@ -682,8 +694,8 @@ if(DEFINED HYPRE_DIR AND ENABLE_HYPRE)
     #   set(ENABLE_HYPRE ON CACHE BOOL "")
     # endif()
 
-    set(ENABLE_HYPRE ON CACHE BOOL "")
-    set(thirdPartyLibs ${thirdPartyLibs} hypre ${HYPRE_DEPENDS} )
+    set( ENABLE_HYPRE ON CACHE BOOL "" )
+    set( thirdPartyLibs ${thirdPartyLibs} hypre ${HYPRE_DEPENDS} )
 else()
     if(ENABLE_HYPRE)
         message(WARNING "ENABLE_HYPRE is ON but HYPRE_DIR isn't defined.")
@@ -907,12 +919,7 @@ message(STATUS "thirdPartyLibs = ${thirdPartyLibs}")
 # NvToolExt
 ###############################
 if ( ENABLE_CUDA AND ENABLE_CUDA_NVTOOLSEXT )
-  find_package(CUDAToolkit REQUIRED)
-
-  message( " ----> $CUDAToolkit_VERSION = ${CUDAToolkit_VERSION}")
-
   set(thirdPartyLibs ${thirdPartyLibs} CUDA::nvToolsExt)
 endif()
 
 message(STATUS "thirdPartyLibs = ${thirdPartyLibs}")
-

src/coreComponents/common/CMakeLists.txt

@@ -10,6 +10,7 @@ set( common_headers
      Format.hpp
      GEOS_RAJA_Interface.hpp
      GeosxMacros.hpp
+     MemoryInfos.hpp
      Logger.hpp
      MpiWrapper.hpp
      Path.hpp
@@ -40,6 +41,7 @@ endif( )
 set( common_sources
      BufferAllocator.cpp
      DataTypes.cpp
+     MemoryInfos.cpp
      Logger.cpp
      MpiWrapper.cpp
      Path.cpp
@@ -93,6 +95,8 @@ if( ENABLE_CALIPER )
   endif()
 endif()
 
+set( dependencyList ${dependencyList} fileIO codingUtilities )
+
 blt_add_library( NAME       common
                  SOURCES    ${common_sources}
                  HEADERS    ${common_headers}