Update for Sherlock (#256)

Main changes:
- Change the INSTALL_DIR_ROOT
- Change the DOCKER_ROOT_IMAGE (update to CUDA 11.7.1)

.github/workflows/docker_build_tpls.yml

@@ -40,15 +40,15 @@ jobs:
           - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11)
             DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11
             DOCKERFILE: docker/Stanford/Dockerfile
-            DOCKER_ROOT_IMAGE: jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1
+            DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2
             HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake
-            INSTALL_DIR_ROOT: /oak/stanford/schools/ees/COLLABORATIONS/geosx/CPU
-          - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10, zlib 1.2.11)
-            DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11
+            INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU
+          - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.7.1, openblas 0.3.10, zlib 1.2.11)
+            DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11
             DOCKERFILE: docker/Stanford/Dockerfile
-            DOCKER_ROOT_IMAGE: jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1
-            HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake
-            INSTALL_DIR_ROOT: /oak/stanford/schools/ees/COLLABORATIONS/geosx/GPU
+            DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2
+            HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake
+            INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU
           - name: Ubuntu (20.04, gcc 9.3.0, open-mpi 4.0.3)
             DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9
             DOCKERFILE: docker/gcc-ubuntu/Dockerfile

docker/Stanford/Sherlock.Dockerfile

@@ -11,8 +11,8 @@ ARG GCC_VERSION=10.1.0
 ARG OPENMPI_VERSION=4.1.2
 ARG OPENBLAS_VERSION=0.3.10
 ARG ZLIB_VERSION=1.2.11
-ARG CUDA_VERSION=11.5.0
-ARG CUDA_SUBVERSION=495.29.05
+ARG CUDA_VERSION=11.7.1
+ARG CUDA_SUBVERSION=515.65.01
 
 # Main software root installation directory in SHERLOCK
 ARG SHERLOCK_ROOT_INSTALL_DIR=/share/software/user/open
@@ -182,6 +182,6 @@ ENV CC=${SHERLOCK_GCC_INSTALL_DIR}/bin/gcc \
     MPIFC=${SHERLOCK_OPENMPI_INSTALL_DIR}/bin/mpifort \
     MPIEXEC=${SHERLOCK_OPENMPI_INSTALL_DIR}/bin/mpiexec \
 # An additional `LD_LIBRARY_PATH` action is needed for the tools to work.
-    LD_LIBRARY_PATH=${SHERLOCK_OPENMPI_INSTALL_DIR}/lib:${SHERLOCK_GCC_INSTALL_DIR}/lib64:${SHERLOCK_OPENBLAS_INSTALL_DIR}/lib:${SHERLOCK_ZLIB_INSTALL_DIR}/lib:${LD_LIBRARY_PATH}
+    LD_LIBRARY_PATH=${SHERLOCK_OPENMPI_INSTALL_DIR}/lib:${SHERLOCK_GCC_INSTALL_DIR}/lib64:${SHERLOCK_OPENBLAS_INSTALL_DIR}/lib:${SHERLOCK_CUDA_INSTALL_DIR}/lib64:${SHERLOCK_ZLIB_INSTALL_DIR}/lib:${LD_LIBRARY_PATH}
 
 

docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake → docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake

@@ -1,21 +1,20 @@
 include(${CMAKE_CURRENT_LIST_DIR}/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake)
 
 # OpenMP options
-#set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE)
+set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE)
 
 # CUDA options
 set(ENABLE_CUDA ON CACHE BOOL "" FORCE)
-set(CUDA_VERSION "11.5.0" CACHE PATH "")
+set(CUDA_VERSION "11.7.1" CACHE PATH "")
 set(CUDA_HOME "${SOFTWARE_ROOT}/cuda/${CUDA_VERSION}" CACHE PATH "")
-set(CMAKE_CUDA_ARCHITECTURES "80" CACHE STRING "")
+set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "")
 set(CUDA_ARCH "sm_${CMAKE_CUDA_ARCHITECTURES}" CACHE STRING "")
 set(CUDA_TOOLKIT_ROOT_DIR "${CUDA_HOME}" CACHE STRING "")
 
 set(CONFIG_NAME "sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda${CUDA_VERSION}-${CUDA_ARCH}" CACHE PATH "" FORCE)
 
 set(CMAKE_CUDA_HOST_COMPILER ${MPI_CXX_COMPILER} CACHE STRING "")
 set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "")
-set(CMAKE_CUDA_STANDARD 14 CACHE STRING "")
 set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-extended-lambda --expt-relaxed-constexpr -Werror cross-execution-space-call,reorder,deprecated-declarations " CACHE STRING "")
 set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3" CACHE STRING "")
 set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "")
@@ -24,6 +23,6 @@ set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "")
 # LAI options
 set(GEOSX_LA_INTERFACE "Hypre" CACHE STRING "" FORCE)
 set(ENABLE_HYPRE ON CACHE BOOL "" FORCE)
-set(ENABLE_HYPRE_CUDA ON CACHE BOOL "" FORCE)
+set(ENABLE_HYPRE_DEVICE "CUDA" CACHE STRING "" FORCE)
 set(ENABLE_PETSC OFF CACHE BOOL "" FORCE)
 set(ENABLE_TRILINOS OFF CACHE BOOL "" FORCE)