diff --git a/CMakeLists.txt b/CMakeLists.txt index e0cb29c2..195a3b0c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -619,8 +619,10 @@ if (ENABLE_TRILINOS) message(STATUS "Building TRILINOS found at ${TRILINOS_URL}") - if (NOT DEFINED TRILINOS_BUILD_COMMAND) + if (NOT DEFINED TRILINOS_NUM_PROC) set(TRILINOS_BUILD_COMMAND ${TPL_BUILD_COMMAND}) + else() + set(TRILINOS_BUILD_COMMAND make -j ${TRILINOS_NUM_PROC}) endif() set(TRILINOS_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS}") @@ -650,7 +652,7 @@ if (ENABLE_TRILINOS) message(STATUS "TRILINOS_EXTRA_ARGS = ${TRILINOS_EXTRA_ARGS}") - if( ${ENABLE_MPI} ) + if( ENABLE_MPI ) set( TRILINOS_C_COMPILER ${MPI_C_COMPILER} ) set( TRILINOS_CXX_COMPILER ${MPI_CXX_COMPILER} ) set( TRILINOS_Fortran_COMPILER ${MPI_Fortran_COMPILER} ) @@ -660,6 +662,16 @@ if (ENABLE_TRILINOS) set( TRILINOS_Fortran_COMPILER ${CMAKE_Fortran_COMPILER} ) endif() + if( ENABLE_CUDA ) + set( TRILINOS_CXX_FLAGS "${TRILINOS_CXX_FLAGS} -ccbin ${TRILINOS_CXX_COMPILER} -arch=${CUDA_ARCH} --expt-extended-lambda --expt-relaxed-constexpr" ) + set( TRILINOS_CXX_COMPILER ${CMAKE_CURRENT_BINARY_DIR}/trilinos/src/trilinos/packages/kokkos/bin/nvcc_wrapper ) + endif() + + set( TRILINOS_ENABLE_SERIAL ON ) + if( ENABLE_OPENMP OR ENABLE_CUDA ) + set( TRILINOS_ENABLE_SERIAL OFF ) + endif() + ExternalProject_Add( trilinos PREFIX ${PROJECT_BINARY_DIR}/trilinos URL ${TRILINOS_URL} @@ -679,13 +691,15 @@ if (ENABLE_TRILINOS) -D CMAKE_Fortran_FLAGS_RELEASE:STRING=${CMAKE_Fortran_FLAGS_RELEASE} -D CMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -D CMAKE_INSTALL_PREFIX:PATH= - -D TPL_ENABLE_MPI:BOOL=${ENABLE_MPI} - -D BUILD_SHARED_LIBS:BOOL=ON + -D CMAKE_VERBOSE_MAKEFILE:BOOL=${CMAKE_VERBOSE_MAKEFILE} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} + -D BUILD_SHARED_LIBS:BOOL=ON + -D CUDA_ROOT:PATH=${CUDA_TOOLKIT_ROOT_DIR} + -D TPL_ENABLE_MPI:BOOL=${ENABLE_MPI} + -D TPL_ENABLE_CUDA:BOOL=${ENABLE_CUDA} -D Trilinos_ENABLE_OpenMP:BOOL=${ENABLE_OPENMP} -D Trilinos_ENABLE_Fortran:BOOL=ON -D Trilinos_WARNINGS_AS_ERRORS_FLAGS:STRING="" - -D CMAKE_VERBOSE_MAKEFILE:BOOL=${CMAKE_VERBOSE_MAKEFILE} -D Trilinos_ENABLE_TESTS:BOOL=OFF -D Trilinos_ENABLE_Gtest:BOOL=OFF -D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF @@ -696,14 +710,33 @@ if (ENABLE_TRILINOS) -D Trilinos_ENABLE_Kokkos:BOOL=ON -D Trilinos_ENABLE_Stratimikos:BOOL=ON -D Trilinos_ENABLE_Amesos:BOOL=ON + -D Trilinos_ENABLE_Amesos2:BOOL=ON -D Trilinos_ENABLE_AztecOO:BOOL=ON + -D Trilinos_ENABLE_Belos:BOOL=ON -D Trilinos_ENABLE_Ifpack:BOOL=ON + -D Trilinos_ENABLE_Ifpack2:BOOL=ON -D Trilinos_ENABLE_Teuchos:BOOL=ON -D Trilinos_ENABLE_ML:BOOL=ON + -D Trilinos_ENABLE_MueLu:BOOL=ON -D Trilinos_ENABLE_Thyra:BOOL=ON -D Trilinos_ENABLE_STK:BOOL=OFF - -D Trilinos_ENABLE_TrilinosCouplings:BOOL=ON + -D Trilinos_ENABLE_TrilinosCouplings:BOOL=OFF -D Trilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON + -D Trilinos_ENABLE_COMPLEX_DOUBLE:BOOL=OFF + -D KOKKOS_ENABLE_CUDA:BOOL=${ENABLE_CUDA} + -D KOKKOS_ENABLE_CUDA_LAMBDA:BOOL=${ENABLE_CUDA} + -D KOKKOS_ENABLE_CUDA_UVM:BOOL=${ENABLE_CUDA} + -D Tpetra_ENABLE_CUDA:BOOL=${ENABLE_CUDA} + -D Tpetra_INST_SERIAL:BOOL=${TRILINOS_ENABLE_SERIAL} + -D Tpetra_INST_INT_LONG_LONG:BOOL=ON + -D Tpetra_INST_INT_INT:BOOL=OFF + -D Tpetra_INST_DOUBLE:BOOL=ON + -D Tpetra_INST_COMPLEX_DOUBLE:BOOL=OFF + -D Belos_ENABLE_Tpetra:BOOL=ON + -D Belos_ENABLE_Epetra:BOOL=OFF + -D Belos_ENABLE_Xpetra:BOOL=OFF + -D Ifpack2_ENABLE_ThyraTpetraAdapters:BOOL=OFF + -D Amesos2_ENABLE_TIMERS:BOOL=ON ${TRILINOS_EXTRA_ARGS} ) @@ -989,9 +1022,10 @@ if( ENABLE_PETSC ) message(STATUS "Building PETSC found at ${PETSC_URL}") - set(PETSC_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS} ${CMAKE_C_FLAGS_RELEASE}") - set(PETSC_CXX_FLAGS "-fPIC ${CXX_FLAGS_NO_WARNINGS} ${CMAKE_CXX_FLAGS_RELEASE}") - set(PETSC_Fortran_FLAGS "${CMAKE_Fortran_FLAGS_RELEASE}") + string(TOUPPER "${CMAKE_BUILD_TYPE}" bt_uc) + set(PETSC_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS} ${CMAKE_C_FLAGS_${bt_uc}}") + set(PETSC_CXX_FLAGS "-fPIC ${CXX_FLAGS_NO_WARNINGS} ${CMAKE_CXX_FLAGS_${bt_uc}}") + set(PETSC_Fortran_FLAGS "${CMAKE_Fortran_FLAGS_${bt_uc}}") if (ENABLE_OPENMP) set(PETSC_OPENMP_FLAG --with-openmp=1) diff --git a/docker/clang-cuda/Dockerfile b/docker/clang-cuda/Dockerfile index 3b3daafe..5c48f3c3 100644 --- a/docker/clang-cuda/Dockerfile +++ b/docker/clang-cuda/Dockerfile @@ -53,9 +53,9 @@ ARG TMP_DIR ARG TPL_SRC_DIR=${TMP_DIR}/thirdPartyLibs ARG TPL_BUILD_DIR=${TMP_DIR}/build -ARG CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda -ARG CUDA_ARCH=sm_70 -ARG CMAKE_CUDA_COMPILER=${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc +ENV CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda +ENV CUDA_ARCH=sm_70 +ENV CMAKE_CUDA_COMPILER=${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc COPY . ${TPL_SRC_DIR} RUN ${TPL_SRC_DIR}/docker/configure_tpl_build.sh \ diff --git a/docker/configure_tpl_build.sh b/docker/configure_tpl_build.sh index c3014187..37a7ce4d 100755 --- a/docker/configure_tpl_build.sh +++ b/docker/configure_tpl_build.sh @@ -23,7 +23,7 @@ python ${TPL_SRC_DIR}/scripts/config-build.py \ --buildtype Release \ --buildpath ${TPL_BUILD_DIR} \ --installpath ${GEOSX_TPL_DIR} \ - -DTRILINOS_BUILD_COMMAND="make -j1" \ + -DTRILINOS_NUM_PROC=$(nproc) \ -DNUM_PROC=$(nproc) \ $* # Note that since docker is not used for mac,\ diff --git a/docker/gcc-cuda/Dockerfile b/docker/gcc-cuda/Dockerfile index 84f2d869..abdf8774 100644 --- a/docker/gcc-cuda/Dockerfile +++ b/docker/gcc-cuda/Dockerfile @@ -36,7 +36,7 @@ ENV NVIDIA_REQUIRE_CUDA "cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tes FROM nvidia-base AS nvidia-runtime -RUN yum install -y \ +RUN yum install -y --setopt=obsoletes=0 \ cuda-libraries-$CUDA_PKG_VERSION \ cuda-nvtx-$CUDA_PKG_VERSION \ libcublas10-10.2.1.243-1 \ @@ -46,7 +46,7 @@ RUN yum install -y \ FROM nvidia-runtime AS nvidia-develop -RUN yum install -y \ +RUN yum install -y --setopt=obsoletes=0 \ cuda-nvml-dev-$CUDA_PKG_VERSION \ cuda-command-line-tools-$CUDA_PKG_VERSION \ cuda-libraries-dev-$CUDA_PKG_VERSION \ @@ -108,15 +108,16 @@ RUN yum install -y \ file \ bison \ flex \ - patch + patch \ + which ARG TMP_DIR ARG TPL_SRC_DIR=${TMP_DIR}/thirdPartyLibs ARG TPL_BUILD_DIR=${TMP_DIR}/build -ARG CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda -ARG CUDA_ARCH=sm_70 -ARG CMAKE_CUDA_COMPILER=${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc +ENV CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda +ENV CUDA_ARCH=sm_70 +ENV CMAKE_CUDA_COMPILER=${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc COPY . ${TPL_SRC_DIR} RUN ${TPL_SRC_DIR}/docker/configure_tpl_build.sh \