Merge branch 'master' into feature/deleteFesapi

.github/workflows/docker_build_tpls.yml

@@ -40,15 +40,15 @@ jobs:
           - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11)
             DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11
             DOCKERFILE: docker/Stanford/Dockerfile
-            DOCKER_ROOT_IMAGE: jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1
+            DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2
             HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake
-            INSTALL_DIR_ROOT: /oak/stanford/schools/ees/COLLABORATIONS/geosx/CPU
-          - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10, zlib 1.2.11)
-            DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11
+            INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU
+          - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.7.1, openblas 0.3.10, zlib 1.2.11)
+            DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11
             DOCKERFILE: docker/Stanford/Dockerfile
-            DOCKER_ROOT_IMAGE: jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1
-            HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake
-            INSTALL_DIR_ROOT: /oak/stanford/schools/ees/COLLABORATIONS/geosx/GPU
+            DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2
+            HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake
+            INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU
           - name: Ubuntu (20.04, gcc 9.3.0, open-mpi 4.0.3)
             DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9
             DOCKERFILE: docker/gcc-ubuntu/Dockerfile

CMakeLists.txt

@@ -771,7 +771,7 @@ endif()
 
 if( ENABLE_HYPRE )
   set( HYPRE_DIR "${CMAKE_INSTALL_PREFIX}/hypre" )
-  set( HYPRE_URL "${TPL_MIRROR_DIR}/hypre-v2.30.0-8-g61fa167d7.tar.gz" )
+  set( HYPRE_URL "${TPL_MIRROR_DIR}/hypre-v2.30.0-44-geab5f9f7f.tar.gz" )
   set( HYPRE_DEPENDS "" )
 
 
@@ -843,6 +843,7 @@ if( ENABLE_HYPRE )
   FC=${HYPRE_Fortran_COMPILER} \
   FCFLAGS=\"${HYPRE_Fortran_FLAGS}\" \
   --prefix=${HYPRE_DIR} \
+  --with-print-errors \
   ${HYPRE_DEBUG_FLAG} \
   ${HYPRE_INT_FLAG} \
   --with-blas-lib=\"${BLAS_LINK_LINE}\" \

docker/Stanford/Sherlock.Dockerfile

@@ -11,8 +11,8 @@ ARG GCC_VERSION=10.1.0
 ARG OPENMPI_VERSION=4.1.2
 ARG OPENBLAS_VERSION=0.3.10
 ARG ZLIB_VERSION=1.2.11
-ARG CUDA_VERSION=11.5.0
-ARG CUDA_SUBVERSION=495.29.05
+ARG CUDA_VERSION=11.7.1
+ARG CUDA_SUBVERSION=515.65.01
 
 # Main software root installation directory in SHERLOCK
 ARG SHERLOCK_ROOT_INSTALL_DIR=/share/software/user/open
@@ -182,6 +182,6 @@ ENV CC=${SHERLOCK_GCC_INSTALL_DIR}/bin/gcc \
     MPIFC=${SHERLOCK_OPENMPI_INSTALL_DIR}/bin/mpifort \
     MPIEXEC=${SHERLOCK_OPENMPI_INSTALL_DIR}/bin/mpiexec \
 # An additional `LD_LIBRARY_PATH` action is needed for the tools to work.
-    LD_LIBRARY_PATH=${SHERLOCK_OPENMPI_INSTALL_DIR}/lib:${SHERLOCK_GCC_INSTALL_DIR}/lib64:${SHERLOCK_OPENBLAS_INSTALL_DIR}/lib:${SHERLOCK_ZLIB_INSTALL_DIR}/lib:${LD_LIBRARY_PATH}
+    LD_LIBRARY_PATH=${SHERLOCK_OPENMPI_INSTALL_DIR}/lib:${SHERLOCK_GCC_INSTALL_DIR}/lib64:${SHERLOCK_OPENBLAS_INSTALL_DIR}/lib:${SHERLOCK_CUDA_INSTALL_DIR}/lib64:${SHERLOCK_ZLIB_INSTALL_DIR}/lib:${LD_LIBRARY_PATH}
 
 

docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake → docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake

@@ -1,21 +1,20 @@
 include(${CMAKE_CURRENT_LIST_DIR}/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake)
 
 # OpenMP options
-#set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE)
+set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE)
 
 # CUDA options
 set(ENABLE_CUDA ON CACHE BOOL "" FORCE)
-set(CUDA_VERSION "11.5.0" CACHE PATH "")
+set(CUDA_VERSION "11.7.1" CACHE PATH "")
 set(CUDA_HOME "${SOFTWARE_ROOT}/cuda/${CUDA_VERSION}" CACHE PATH "")
-set(CMAKE_CUDA_ARCHITECTURES "80" CACHE STRING "")
+set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "")
 set(CUDA_ARCH "sm_${CMAKE_CUDA_ARCHITECTURES}" CACHE STRING "")
 set(CUDA_TOOLKIT_ROOT_DIR "${CUDA_HOME}" CACHE STRING "")
 
 set(CONFIG_NAME "sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda${CUDA_VERSION}-${CUDA_ARCH}" CACHE PATH "" FORCE)
 
 set(CMAKE_CUDA_HOST_COMPILER ${MPI_CXX_COMPILER} CACHE STRING "")
 set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "")
-set(CMAKE_CUDA_STANDARD 14 CACHE STRING "")
 set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-extended-lambda --expt-relaxed-constexpr -Werror cross-execution-space-call,reorder,deprecated-declarations " CACHE STRING "")
 set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3" CACHE STRING "")
 set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "")
@@ -24,6 +23,6 @@ set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "")
 # LAI options
 set(GEOSX_LA_INTERFACE "Hypre" CACHE STRING "" FORCE)
 set(ENABLE_HYPRE ON CACHE BOOL "" FORCE)
-set(ENABLE_HYPRE_CUDA ON CACHE BOOL "" FORCE)
+set(ENABLE_HYPRE_DEVICE "CUDA" CACHE STRING "" FORCE)
 set(ENABLE_PETSC OFF CACHE BOOL "" FORCE)
 set(ENABLE_TRILINOS OFF CACHE BOOL "" FORCE)

scripts/updateHypre.sh

@@ -13,6 +13,10 @@ HYPRE_SRC_DIR=${HYPRE_DIR}/src
 cd ${GEOSTPL_DIR}
 rm -rf ${HYPRE_DIR}
 git clone https://github.com/hypre-space/hypre.git ${HYPRE_DIR}
+if [[ $? != "0" ]]; then
+    echo -e "git clone failed! Exiting..."
+    exit 1
+fi
 
 # Assign git variables
 cd ${HYPRE_DIR}
@@ -42,6 +46,9 @@ rm -rf ${HYPRE_DIR}/src/test/TEST_*
 # Remove git folder
 rm -rf ${HYPRE_DIR}/.git
 
+# Remove old hypre dir
+git rm ${TPL_MIRROR_DIR}/hypre*.tar.gz
+
 # Create tarball and move it to tplMirror
 HYPRE_DIR=${GEOSTPL_DIR}/hypre-${HYPRE_DEVELOP_STRING}
 cd ${GEOSTPL_DIR}
@@ -55,3 +62,7 @@ rm -rf ${HYPRE_DIR}
 # Update CMakeLists
 echo -e "Updating CMakeLists..."
 sed -i "s|set( HYPRE_URL \"\${TPL_MIRROR_DIR}/hypre-.*\.tar\.gz\" )|set( HYPRE_URL \"\${TPL_MIRROR_DIR}/hypre-${HYPRE_DEVELOP_STRING}.tar.gz\" )|" CMakeLists.txt
+
+# Stage changes
+git add ${GEOSTPL_DIR}/CMakeLists.txt
+git add ${TPL_MIRROR_DIR}/hypre*.tar.gz