Trilinos build: enable CUDA and Tpetra-based solver packages (Belos, …

…Amesos2, Ifpack2, MueLu)
GEOS-DEV · Sep 21, 2020 · 6df6c5d · 6df6c5d
1 parent 8cc0fc3
commit 6df6c5d
Show file tree

Hide file tree

Showing 4 changed files with 54 additions and 19 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -619,8 +619,10 @@ if (ENABLE_TRILINOS)
 
     message(STATUS "Building TRILINOS found at ${TRILINOS_URL}")
 
-    if (NOT DEFINED TRILINOS_BUILD_COMMAND)
+    if (NOT DEFINED TRILINOS_NUM_PROC)
         set(TRILINOS_BUILD_COMMAND ${TPL_BUILD_COMMAND})
+    else()
+        set(TRILINOS_BUILD_COMMAND make -j ${TRILINOS_NUM_PROC})
     endif()
 
     set(TRILINOS_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS}")
@@ -650,7 +652,7 @@ if (ENABLE_TRILINOS)
 
     message(STATUS "TRILINOS_EXTRA_ARGS = ${TRILINOS_EXTRA_ARGS}")
 
-    if( ${ENABLE_MPI} )
+    if( ENABLE_MPI )
       set( TRILINOS_C_COMPILER ${MPI_C_COMPILER} )
       set( TRILINOS_CXX_COMPILER ${MPI_CXX_COMPILER} )
       set( TRILINOS_Fortran_COMPILER ${MPI_Fortran_COMPILER} )
@@ -660,6 +662,16 @@ if (ENABLE_TRILINOS)
       set( TRILINOS_Fortran_COMPILER ${CMAKE_Fortran_COMPILER} )
     endif()
 
+    if( ENABLE_CUDA )
+      set( TRILINOS_CXX_FLAGS "${TRILINOS_CXX_FLAGS} -ccbin ${TRILINOS_CXX_COMPILER} -arch=${CUDA_ARCH} --expt-extended-lambda --expt-relaxed-constexpr" )
+      set( TRILINOS_CXX_COMPILER ${CMAKE_CURRENT_BINARY_DIR}/trilinos/src/trilinos/packages/kokkos/bin/nvcc_wrapper )
+    endif()
+
+    set( TRILINOS_ENABLE_SERIAL ON )
+    if( ENABLE_OPENMP OR ENABLE_CUDA )
+      set( TRILINOS_ENABLE_SERIAL OFF )
+    endif()
+
     ExternalProject_Add( trilinos
                          PREFIX ${PROJECT_BINARY_DIR}/trilinos
                          URL ${TRILINOS_URL}
@@ -679,13 +691,15 @@ if (ENABLE_TRILINOS)
                                     -D CMAKE_Fortran_FLAGS_RELEASE:STRING=${CMAKE_Fortran_FLAGS_RELEASE}
                                     -D CMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
                                     -D CMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
-                                    -D TPL_ENABLE_MPI:BOOL=${ENABLE_MPI}
-                                    -D BUILD_SHARED_LIBS:BOOL=ON
+                                    -D CMAKE_VERBOSE_MAKEFILE:BOOL=${CMAKE_VERBOSE_MAKEFILE}
                                     -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE}
+                                    -D BUILD_SHARED_LIBS:BOOL=ON
+                                    -D CUDA_ROOT:PATH=${CUDA_TOOLKIT_ROOT_DIR}
+                                    -D TPL_ENABLE_MPI:BOOL=${ENABLE_MPI}
+                                    -D TPL_ENABLE_CUDA:BOOL=${ENABLE_CUDA}
                                     -D Trilinos_ENABLE_OpenMP:BOOL=${ENABLE_OPENMP}
                                     -D Trilinos_ENABLE_Fortran:BOOL=ON
                                     -D Trilinos_WARNINGS_AS_ERRORS_FLAGS:STRING=""
-                                    -D CMAKE_VERBOSE_MAKEFILE:BOOL=${CMAKE_VERBOSE_MAKEFILE}
                                     -D Trilinos_ENABLE_TESTS:BOOL=OFF
                                     -D Trilinos_ENABLE_Gtest:BOOL=OFF
                                     -D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF
@@ -696,14 +710,33 @@ if (ENABLE_TRILINOS)
                                     -D Trilinos_ENABLE_Kokkos:BOOL=ON
                                     -D Trilinos_ENABLE_Stratimikos:BOOL=ON
                                     -D Trilinos_ENABLE_Amesos:BOOL=ON
+                                    -D Trilinos_ENABLE_Amesos2:BOOL=ON
                                     -D Trilinos_ENABLE_AztecOO:BOOL=ON
+                                    -D Trilinos_ENABLE_Belos:BOOL=ON
                                     -D Trilinos_ENABLE_Ifpack:BOOL=ON
+                                    -D Trilinos_ENABLE_Ifpack2:BOOL=ON
                                     -D Trilinos_ENABLE_Teuchos:BOOL=ON
                                     -D Trilinos_ENABLE_ML:BOOL=ON
+                                    -D Trilinos_ENABLE_MueLu:BOOL=ON
                                     -D Trilinos_ENABLE_Thyra:BOOL=ON
                                     -D Trilinos_ENABLE_STK:BOOL=OFF
-                                    -D Trilinos_ENABLE_TrilinosCouplings:BOOL=ON
+                                    -D Trilinos_ENABLE_TrilinosCouplings:BOOL=OFF
                                     -D Trilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON
+                                    -D Trilinos_ENABLE_COMPLEX_DOUBLE:BOOL=OFF
+                                    -D KOKKOS_ENABLE_CUDA:BOOL=${ENABLE_CUDA}
+                                    -D KOKKOS_ENABLE_CUDA_LAMBDA:BOOL=${ENABLE_CUDA}
+                                    -D KOKKOS_ENABLE_CUDA_UVM:BOOL=${ENABLE_CUDA}
+                                    -D Tpetra_ENABLE_CUDA:BOOL=${ENABLE_CUDA}
+                                    -D Tpetra_INST_SERIAL:BOOL=${TRILINOS_ENABLE_SERIAL}
+                                    -D Tpetra_INST_INT_LONG_LONG:BOOL=ON
+                                    -D Tpetra_INST_INT_INT:BOOL=OFF
+                                    -D Tpetra_INST_DOUBLE:BOOL=ON
+                                    -D Tpetra_INST_COMPLEX_DOUBLE:BOOL=OFF
+                                    -D Belos_ENABLE_Tpetra:BOOL=ON
+                                    -D Belos_ENABLE_Epetra:BOOL=OFF
+                                    -D Belos_ENABLE_Xpetra:BOOL=OFF
+                                    -D Ifpack2_ENABLE_ThyraTpetraAdapters:BOOL=OFF
+                                    -D Amesos2_ENABLE_TIMERS:BOOL=ON
                                     ${TRILINOS_EXTRA_ARGS}
                         )
 
@@ -989,9 +1022,10 @@ if( ENABLE_PETSC )
 
     message(STATUS "Building PETSC found at ${PETSC_URL}")
 
-    set(PETSC_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS} ${CMAKE_C_FLAGS_RELEASE}")
-    set(PETSC_CXX_FLAGS "-fPIC ${CXX_FLAGS_NO_WARNINGS} ${CMAKE_CXX_FLAGS_RELEASE}")
-    set(PETSC_Fortran_FLAGS "${CMAKE_Fortran_FLAGS_RELEASE}")
+    string(TOUPPER "${CMAKE_BUILD_TYPE}" bt_uc)
+    set(PETSC_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS} ${CMAKE_C_FLAGS_${bt_uc}}")
+    set(PETSC_CXX_FLAGS "-fPIC ${CXX_FLAGS_NO_WARNINGS} ${CMAKE_CXX_FLAGS_${bt_uc}}")
+    set(PETSC_Fortran_FLAGS "${CMAKE_Fortran_FLAGS_${bt_uc}}")
 
     if (ENABLE_OPENMP)
         set(PETSC_OPENMP_FLAG --with-openmp=1)

diff --git a/docker/clang-cuda/Dockerfile b/docker/clang-cuda/Dockerfile
@@ -53,9 +53,9 @@ ARG TMP_DIR
 ARG TPL_SRC_DIR=${TMP_DIR}/thirdPartyLibs
 ARG TPL_BUILD_DIR=${TMP_DIR}/build
 
-ARG CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
-ARG CUDA_ARCH=sm_70
-ARG CMAKE_CUDA_COMPILER=${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc
+ENV CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
+ENV CUDA_ARCH=sm_70
+ENV CMAKE_CUDA_COMPILER=${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc
 
 COPY . ${TPL_SRC_DIR}
 RUN ${TPL_SRC_DIR}/docker/configure_tpl_build.sh \

diff --git a/docker/configure_tpl_build.sh b/docker/configure_tpl_build.sh
@@ -23,7 +23,7 @@ python ${TPL_SRC_DIR}/scripts/config-build.py \
        --buildtype Release \
        --buildpath ${TPL_BUILD_DIR} \
        --installpath ${GEOSX_TPL_DIR} \
-       -DTRILINOS_BUILD_COMMAND="make -j1" \
+       -DTRILINOS_NUM_PROC=$(nproc) \
        -DNUM_PROC=$(nproc) \
        $*
 # Note that since docker is not used for mac,\

diff --git a/docker/gcc-cuda/Dockerfile b/docker/gcc-cuda/Dockerfile
@@ -36,7 +36,7 @@ ENV NVIDIA_REQUIRE_CUDA "cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tes
 
 FROM nvidia-base AS nvidia-runtime 
 
-RUN yum install -y \
+RUN yum install -y --setopt=obsoletes=0 \
     cuda-libraries-$CUDA_PKG_VERSION \
     cuda-nvtx-$CUDA_PKG_VERSION \
     libcublas10-10.2.1.243-1 \
@@ -46,7 +46,7 @@ RUN yum install -y \
 
 FROM nvidia-runtime AS nvidia-develop 
 
-RUN yum install -y \
+RUN yum install -y --setopt=obsoletes=0 \
     cuda-nvml-dev-$CUDA_PKG_VERSION \
     cuda-command-line-tools-$CUDA_PKG_VERSION \
     cuda-libraries-dev-$CUDA_PKG_VERSION \
@@ -108,15 +108,16 @@ RUN yum install -y \
     file \
     bison \
     flex \
-    patch
+    patch \
+    which
 
 ARG TMP_DIR
 ARG TPL_SRC_DIR=${TMP_DIR}/thirdPartyLibs
 ARG TPL_BUILD_DIR=${TMP_DIR}/build
 
-ARG CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
-ARG CUDA_ARCH=sm_70
-ARG CMAKE_CUDA_COMPILER=${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc
+ENV CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
+ENV CUDA_ARCH=sm_70
+ENV CMAKE_CUDA_COMPILER=${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc
 
 COPY . ${TPL_SRC_DIR}
 RUN ${TPL_SRC_DIR}/docker/configure_tpl_build.sh \