diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 1ab667cf..e4e73328 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -40,15 +40,15 @@ jobs: - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1 + DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2 HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake - INSTALL_DIR_ROOT: /oak/stanford/schools/ees/COLLABORATIONS/geosx/CPU - - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10, zlib 1.2.11) - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11 + INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU + - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.7.1, openblas 0.3.10, zlib 1.2.11) + DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11 DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: jafranc/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.5.0-openblas0.3.10-zlib1.2.11-no-geosx:0.0.1 - HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake - INSTALL_DIR_ROOT: /oak/stanford/schools/ees/COLLABORATIONS/geosx/GPU + DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2 + HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake + INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU - name: Ubuntu (20.04, gcc 9.3.0, open-mpi 4.0.3) DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 DOCKERFILE: docker/gcc-ubuntu/Dockerfile diff --git a/CMakeLists.txt b/CMakeLists.txt index 257f6fe1..04c119f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -771,7 +771,7 @@ endif() if( ENABLE_HYPRE ) set( HYPRE_DIR "${CMAKE_INSTALL_PREFIX}/hypre" ) - set( HYPRE_URL "${TPL_MIRROR_DIR}/hypre-v2.30.0-8-g61fa167d7.tar.gz" ) + set( HYPRE_URL "${TPL_MIRROR_DIR}/hypre-v2.30.0-44-geab5f9f7f.tar.gz" ) set( HYPRE_DEPENDS "" ) @@ -843,6 +843,7 @@ if( ENABLE_HYPRE ) FC=${HYPRE_Fortran_COMPILER} \ FCFLAGS=\"${HYPRE_Fortran_FLAGS}\" \ --prefix=${HYPRE_DIR} \ + --with-print-errors \ ${HYPRE_DEBUG_FLAG} \ ${HYPRE_INT_FLAG} \ --with-blas-lib=\"${BLAS_LINK_LINE}\" \ diff --git a/docker/Stanford/Sherlock.Dockerfile b/docker/Stanford/Sherlock.Dockerfile index aa71b3d1..ab3992f6 100644 --- a/docker/Stanford/Sherlock.Dockerfile +++ b/docker/Stanford/Sherlock.Dockerfile @@ -11,8 +11,8 @@ ARG GCC_VERSION=10.1.0 ARG OPENMPI_VERSION=4.1.2 ARG OPENBLAS_VERSION=0.3.10 ARG ZLIB_VERSION=1.2.11 -ARG CUDA_VERSION=11.5.0 -ARG CUDA_SUBVERSION=495.29.05 +ARG CUDA_VERSION=11.7.1 +ARG CUDA_SUBVERSION=515.65.01 # Main software root installation directory in SHERLOCK ARG SHERLOCK_ROOT_INSTALL_DIR=/share/software/user/open @@ -182,6 +182,6 @@ ENV CC=${SHERLOCK_GCC_INSTALL_DIR}/bin/gcc \ MPIFC=${SHERLOCK_OPENMPI_INSTALL_DIR}/bin/mpifort \ MPIEXEC=${SHERLOCK_OPENMPI_INSTALL_DIR}/bin/mpiexec \ # An additional `LD_LIBRARY_PATH` action is needed for the tools to work. - LD_LIBRARY_PATH=${SHERLOCK_OPENMPI_INSTALL_DIR}/lib:${SHERLOCK_GCC_INSTALL_DIR}/lib64:${SHERLOCK_OPENBLAS_INSTALL_DIR}/lib:${SHERLOCK_ZLIB_INSTALL_DIR}/lib:${LD_LIBRARY_PATH} + LD_LIBRARY_PATH=${SHERLOCK_OPENMPI_INSTALL_DIR}/lib:${SHERLOCK_GCC_INSTALL_DIR}/lib64:${SHERLOCK_OPENBLAS_INSTALL_DIR}/lib:${SHERLOCK_CUDA_INSTALL_DIR}/lib64:${SHERLOCK_ZLIB_INSTALL_DIR}/lib:${LD_LIBRARY_PATH} diff --git a/docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake b/docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake similarity index 84% rename from docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake rename to docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake index 71cc0965..51031871 100644 --- a/docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.5.0-sm80.cmake +++ b/docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake @@ -1,13 +1,13 @@ include(${CMAKE_CURRENT_LIST_DIR}/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake) # OpenMP options -#set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE) +set(ENABLE_OPENMP OFF CACHE BOOL "" FORCE) # CUDA options set(ENABLE_CUDA ON CACHE BOOL "" FORCE) -set(CUDA_VERSION "11.5.0" CACHE PATH "") +set(CUDA_VERSION "11.7.1" CACHE PATH "") set(CUDA_HOME "${SOFTWARE_ROOT}/cuda/${CUDA_VERSION}" CACHE PATH "") -set(CMAKE_CUDA_ARCHITECTURES "80" CACHE STRING "") +set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "") set(CUDA_ARCH "sm_${CMAKE_CUDA_ARCHITECTURES}" CACHE STRING "") set(CUDA_TOOLKIT_ROOT_DIR "${CUDA_HOME}" CACHE STRING "") @@ -15,7 +15,6 @@ set(CONFIG_NAME "sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda${CUDA_VERSION}-${C set(CMAKE_CUDA_HOST_COMPILER ${MPI_CXX_COMPILER} CACHE STRING "") set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "") -set(CMAKE_CUDA_STANDARD 14 CACHE STRING "") set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-extended-lambda --expt-relaxed-constexpr -Werror cross-execution-space-call,reorder,deprecated-declarations " CACHE STRING "") set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3" CACHE STRING "") set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "") @@ -24,6 +23,6 @@ set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "") # LAI options set(GEOSX_LA_INTERFACE "Hypre" CACHE STRING "" FORCE) set(ENABLE_HYPRE ON CACHE BOOL "" FORCE) -set(ENABLE_HYPRE_CUDA ON CACHE BOOL "" FORCE) +set(ENABLE_HYPRE_DEVICE "CUDA" CACHE STRING "" FORCE) set(ENABLE_PETSC OFF CACHE BOOL "" FORCE) set(ENABLE_TRILINOS OFF CACHE BOOL "" FORCE) diff --git a/scripts/updateHypre.sh b/scripts/updateHypre.sh index b6935890..1980745e 100755 --- a/scripts/updateHypre.sh +++ b/scripts/updateHypre.sh @@ -13,6 +13,10 @@ HYPRE_SRC_DIR=${HYPRE_DIR}/src cd ${GEOSTPL_DIR} rm -rf ${HYPRE_DIR} git clone https://github.com/hypre-space/hypre.git ${HYPRE_DIR} +if [[ $? != "0" ]]; then + echo -e "git clone failed! Exiting..." + exit 1 +fi # Assign git variables cd ${HYPRE_DIR} @@ -42,6 +46,9 @@ rm -rf ${HYPRE_DIR}/src/test/TEST_* # Remove git folder rm -rf ${HYPRE_DIR}/.git +# Remove old hypre dir +git rm ${TPL_MIRROR_DIR}/hypre*.tar.gz + # Create tarball and move it to tplMirror HYPRE_DIR=${GEOSTPL_DIR}/hypre-${HYPRE_DEVELOP_STRING} cd ${GEOSTPL_DIR} @@ -55,3 +62,7 @@ rm -rf ${HYPRE_DIR} # Update CMakeLists echo -e "Updating CMakeLists..." sed -i "s|set( HYPRE_URL \"\${TPL_MIRROR_DIR}/hypre-.*\.tar\.gz\" )|set( HYPRE_URL \"\${TPL_MIRROR_DIR}/hypre-${HYPRE_DEVELOP_STRING}.tar.gz\" )|" CMakeLists.txt + +# Stage changes +git add ${GEOSTPL_DIR}/CMakeLists.txt +git add ${TPL_MIRROR_DIR}/hypre*.tar.gz diff --git a/tplMirror/hypre-v2.30.0-44-geab5f9f7f.tar.gz b/tplMirror/hypre-v2.30.0-44-geab5f9f7f.tar.gz new file mode 100644 index 00000000..0a282714 --- /dev/null +++ b/tplMirror/hypre-v2.30.0-44-geab5f9f7f.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12c8c538e5e4bd1698b5016f1104c676516c308312ffa522af1b25ef8f65e56d +size 4385674 diff --git a/tplMirror/hypre-v2.30.0-8-g61fa167d7.tar.gz b/tplMirror/hypre-v2.30.0-8-g61fa167d7.tar.gz deleted file mode 100644 index 2c9bada8..00000000 --- a/tplMirror/hypre-v2.30.0-8-g61fa167d7.tar.gz +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4b24b9e68e43670e92b3e461d7787899d57335fe905a9f40e14b4b3b5f65241e -size 4360421