Build sparse direct solvers without GPU support for now

awslabs · Feb 2, 2024 · 3c8c6bc · 3c8c6bc
1 parent 6677106
commit 3c8c6bc
Show file tree

Hide file tree

Showing 5 changed files with 125 additions and 208 deletions.
diff --git a/cmake/ExternalSLATE.cmake b/cmake/ExternalSLATE.cmake
diff --git a/cmake/ExternalSTRUMPACK.cmake b/cmake/ExternalSTRUMPACK.cmake
@@ -7,12 +7,12 @@
 
 # Force build order
 set(STRUMPACK_DEPENDENCIES scalapack parmetis)
-if(PALACE_WITH_CUDA OR PALACE_WITH_HIP)
-  list(APPEND STRUMPACK_DEPENDENCIES slate)
-endif()
-if(PALACE_WITH_MAGMA)
-  list(APPEND STRUMPACK_DEPENDENCIES magma)
-endif()
+# if(PALACE_WITH_CUDA OR PALACE_WITH_HIP)
+#   list(APPEND STRUMPACK_DEPENDENCIES slate)
+# endif()
+# if(PALACE_WITH_MAGMA)
+#   list(APPEND STRUMPACK_DEPENDENCIES magma)
+# endif()
 
 # Build ZFP dependency for lossy compression
 if(PALACE_WITH_STRUMPACK_ZFP)
@@ -186,60 +186,65 @@ if(NOT "${BLAS_LAPACK_LIBRARIES}" STREQUAL "")
   )
 endif()
 
-# Configure GPU support
-if(PALACE_WITH_CUDA OR PALACE_WITH_HIP)
-  list(APPEND STRUMPACK_OPTIONS
-    "-DTPL_ENABLE_SLATE=ON"
-    "-DTPL_SLATE_PREFIX=${CMAKE_INSTALL_PREFIX}"
-  )
-else()
-  list(APPEND STRUMPACK_OPTIONS
-    "-DTPL_ENABLE_SLATE=OFF"
-  )
-endif()
-if(PALACE_WITH_CUDA)
-  list(APPEND STRUMPACK_OPTIONS
-    "-DSTRUMPACK_USE_CUDA=ON"
-    "-DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}"
-    "-DCMAKE_CUDA_FLAGS=${CMAKE_CUDA_FLAGS}"
-  )
-  if(NOT "${CMAKE_CUDA_ARCHITECTURES}" STREQUAL "")
-    list(APPEND STRUMPACK_OPTIONS
-      "-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}"
-    )
-  endif()
-else()
-  list(APPEND STRUMPACK_OPTIONS
-    "-DSTRUMPACK_USE_CUDA=OFF"
-  )
-endif()
-if(PALACE_WITH_HIP)
-  list(APPEND STRUMPACK_OPTIONS
-    "-DSTRUMPACK_USE_HIP=ON"
-    "-DHIP_ROOT_DIR=${ROCM_DIR}"
-    "-DCMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER}"
-    "-DCMAKE_HIP_FLAGS=${CMAKE_HIP_FLAGS}"
-  )
-  if(NOT "${CMAKE_HIP_ARCHITECTURES}" STREQUAL "")
-    list(APPEND STRUMPACK_OPTIONS
-      "-DCMAKE_HIP_ARCHITECTURES=${CMAKE_HIP_ARCHITECTURES}"
-    )
-  endif()
-else()
-  list(APPEND STRUMPACK_OPTIONS
-    "-DSTRUMPACK_USE_HIP=OFF"
-  )
-endif()
-if(PALACE_WITH_MAGMA)
-  list(APPEND STRUMPACK_OPTIONS
-    "-DTPL_ENABLE_MAGMA=ON"
-    "-DTPL_MAGMA_PREFIX=${CMAKE_INSTALL_PREFIX}"
-  )
-else()
-  list(APPEND STRUMPACK_OPTIONS
-    "-DTPL_ENABLE_MAGMA=OFF"
-  )
-endif()
+# Configure GPU support (for now, disable since faster on CPU)
+list(APPEND STRUMPACK_OPTIONS
+  "-DTPL_ENABLE_SLATE=OFF"
+  "-DSTRUMPACK_USE_CUDA=OFF"
+  "-DSTRUMPACK_USE_HIP=OFF"
+)
+# if(PALACE_WITH_CUDA OR PALACE_WITH_HIP)
+#   list(APPEND STRUMPACK_OPTIONS
+#     "-DTPL_ENABLE_SLATE=ON"
+#     "-DTPL_SLATE_PREFIX=${CMAKE_INSTALL_PREFIX}"
+#   )
+# else()
+#   list(APPEND STRUMPACK_OPTIONS
+#     "-DTPL_ENABLE_SLATE=OFF"
+#   )
+# endif()
+# if(PALACE_WITH_CUDA)
+#   list(APPEND STRUMPACK_OPTIONS
+#     "-DSTRUMPACK_USE_CUDA=ON"
+#     "-DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}"
+#     "-DCMAKE_CUDA_FLAGS=${CMAKE_CUDA_FLAGS}"
+#   )
+#   if(NOT "${CMAKE_CUDA_ARCHITECTURES}" STREQUAL "")
+#     list(APPEND STRUMPACK_OPTIONS
+#       "-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}"
+#     )
+#   endif()
+# else()
+#   list(APPEND STRUMPACK_OPTIONS
+#     "-DSTRUMPACK_USE_CUDA=OFF"
+#   )
+# endif()
+# if(PALACE_WITH_HIP)
+#   list(APPEND STRUMPACK_OPTIONS
+#     "-DSTRUMPACK_USE_HIP=ON"
+#     "-DHIP_ROOT_DIR=${ROCM_DIR}"
+#     "-DCMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER}"
+#     "-DCMAKE_HIP_FLAGS=${CMAKE_HIP_FLAGS}"
+#   )
+#   if(NOT "${CMAKE_HIP_ARCHITECTURES}" STREQUAL "")
+#     list(APPEND STRUMPACK_OPTIONS
+#       "-DCMAKE_HIP_ARCHITECTURES=${CMAKE_HIP_ARCHITECTURES}"
+#     )
+#   endif()
+# else()
+#   list(APPEND STRUMPACK_OPTIONS
+#     "-DSTRUMPACK_USE_HIP=OFF"
+#   )
+# endif()
+# if(PALACE_WITH_MAGMA)
+#   list(APPEND STRUMPACK_OPTIONS
+#     "-DTPL_ENABLE_MAGMA=ON"
+#     "-DTPL_MAGMA_PREFIX=${CMAKE_INSTALL_PREFIX}"  ##XX TODO STRUMPACK WITH GPU...
+#   )
+# else()
+#   list(APPEND STRUMPACK_OPTIONS
+#     "-DTPL_ENABLE_MAGMA=OFF"
+#   )
+# endif()
 
 string(REPLACE ";" "; " STRUMPACK_OPTIONS_PRINT "${STRUMPACK_OPTIONS}")
 message(STATUS "STRUMPACK_OPTIONS: ${STRUMPACK_OPTIONS_PRINT}")
@@ -273,23 +278,23 @@ endif()
 if(PALACE_WITH_STRUMPACK_ZFP)
   list(APPEND _STRUMPACK_EXTRA_LIBRARIES ${_ZFP_LIBRARIES})
 endif()
-if(PALACE_WITH_CUDA OR PALACE_WITH_HIP)
-  include(GNUInstallDirs)
-  if(BUILD_SHARED_LIBS)
-    set(_SLATE_LIB_SUFFIX ${CMAKE_SHARED_LIBRARY_SUFFIX})
-  else()
-    set(_SLATE_LIB_SUFFIX ${CMAKE_STATIC_LIBRARY_SUFFIX})
-  endif()
-  set(_SLATE_LIBRARIES ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libslate${_SLATE_LIB_SUFFIX})
-  list(APPEND _SLATE_LIBRARIES ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/liblapackpp${_SLATE_LIB_SUFFIX})
-  list(APPEND _SLATE_LIBRARIES ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libblaspp${_SLATE_LIB_SUFFIX})
-  list(APPEND _STRUMPACK_EXTRA_LIBRARIES ${_SLATE_LIBRARIES})
-endif()
-if(PALACE_WITH_MAGMA)
-  set(_MAGMA_LIBRARIES ${CMAKE_INSTALL_PREFIX}/lib/libmagma_sparse${CMAKE_SHARED_LIBRARY_SUFFIX})
-  list(APPEND _MAGMA_LIBRARIES ${CMAKE_INSTALL_PREFIX}/lib/libmagma${CMAKE_SHARED_LIBRARY_SUFFIX})
-  list(APPEND _STRUMPACK_EXTRA_LIBRARIES ${_MAGMA_LIBRARIES})
-endif()
+# if(PALACE_WITH_CUDA OR PALACE_WITH_HIP)
+#   include(GNUInstallDirs)
+#   if(BUILD_SHARED_LIBS)
+#     set(_SLATE_LIB_SUFFIX ${CMAKE_SHARED_LIBRARY_SUFFIX})
+#   else()
+#     set(_SLATE_LIB_SUFFIX ${CMAKE_STATIC_LIBRARY_SUFFIX})
+#   endif()
+#   set(_SLATE_LIBRARIES ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libslate${_SLATE_LIB_SUFFIX})
+#   list(APPEND _SLATE_LIBRARIES ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/liblapackpp${_SLATE_LIB_SUFFIX})
+#   list(APPEND _SLATE_LIBRARIES ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libblaspp${_SLATE_LIB_SUFFIX})
+#   list(APPEND _STRUMPACK_EXTRA_LIBRARIES ${_SLATE_LIBRARIES})
+# endif()
+# if(PALACE_WITH_MAGMA)
+#   set(_MAGMA_LIBRARIES ${CMAKE_INSTALL_PREFIX}/lib/libmagma_sparse${CMAKE_SHARED_LIBRARY_SUFFIX})
+#   list(APPEND _MAGMA_LIBRARIES ${CMAKE_INSTALL_PREFIX}/lib/libmagma${CMAKE_SHARED_LIBRARY_SUFFIX})
+#   list(APPEND _STRUMPACK_EXTRA_LIBRARIES ${_MAGMA_LIBRARIES})
+# endif()
 if(NOT "${_STRUMPACK_EXTRA_LIBRARIES}" STREQUAL "")
   string(REPLACE ";" "$<SEMICOLON>" _STRUMPACK_EXTRA_LIBRARIES "${_STRUMPACK_EXTRA_LIBRARIES}")
   set(STRUMPACK_EXTRA_LIBRARIES ${_STRUMPACK_EXTRA_LIBRARIES} CACHE STRING

diff --git a/cmake/ExternalSuperLU_DIST.cmake b/cmake/ExternalSuperLU_DIST.cmake
@@ -51,40 +51,44 @@ if(NOT "${BLAS_LAPACK_LIBRARIES}" STREQUAL "")
   )
 endif()
 
-# Configure GPU support
-if(PALACE_WITH_CUDA)
-  list(APPEND SUPERLU_OPTIONS
-    "-DTPL_ENABLE_CUDALIB=ON"
-    "-DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}"
-    "-DCMAKE_CUDA_FLAGS=${CMAKE_CUDA_FLAGS}"
-  )
-  if(NOT "${CMAKE_CUDA_ARCHITECTURES}" STREQUAL "")
-    list(APPEND SUPERLU_OPTIONS
-      "-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}"
-    )
-  endif()
-else()
-  list(APPEND SUPERLU_OPTIONS
-    "-DTPL_ENABLE_CUDALIB=OFF"
-  )
-endif()
-if(PALACE_WITH_HIP)
-  list(APPEND SUPERLU_OPTIONS
-    "-DTPL_ENABLE_HIPLIB=ON"
-    "-DHIP_ROOT_DIR=${ROCM_DIR}"
-    "-DCMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER}"
-    "-DCMAKE_HIP_FLAGS=${CMAKE_HIP_FLAGS}"
-  )
-  if(NOT "${CMAKE_HIP_ARCHITECTURES}" STREQUAL "")
-    list(APPEND SUPERLU_OPTIONS
-      "-DCMAKE_HIP_ARCHITECTURES=${CMAKE_HIP_ARCHITECTURES}"
-    )
-  endif()
-else()
-  list(APPEND SUPERLU_OPTIONS
-    "-DTPL_ENABLE_HIPLIB=OFF"
-  )
-endif()
+# Configure GPU support (for now, disable since faster on CPU)
+list(APPEND SUPERLU_OPTIONS
+  "-DTPL_ENABLE_CUDALIB=OFF"
+  "-DTPL_ENABLE_HIPLIB=OFF"
+)
+# if(PALACE_WITH_CUDA)
+#   list(APPEND SUPERLU_OPTIONS
+#     "-DTPL_ENABLE_CUDALIB=ON"
+#     "-DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}"
+#     "-DCMAKE_CUDA_FLAGS=${CMAKE_CUDA_FLAGS}"
+#   )
+#   if(NOT "${CMAKE_CUDA_ARCHITECTURES}" STREQUAL "")
+#     list(APPEND SUPERLU_OPTIONS
+#       "-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}"
+#     )
+#   endif()
+# else()
+#   list(APPEND SUPERLU_OPTIONS
+#     "-DTPL_ENABLE_CUDALIB=OFF"
+#   )
+# endif()
+# if(PALACE_WITH_HIP)
+#   list(APPEND SUPERLU_OPTIONS
+#     "-DTPL_ENABLE_HIPLIB=ON"
+#     "-DHIP_ROOT_DIR=${ROCM_DIR}"
+#     "-DCMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER}"
+#     "-DCMAKE_HIP_FLAGS=${CMAKE_HIP_FLAGS}"
+#   )
+#   if(NOT "${CMAKE_HIP_ARCHITECTURES}" STREQUAL "")
+#     list(APPEND SUPERLU_OPTIONS
+#       "-DCMAKE_HIP_ARCHITECTURES=${CMAKE_HIP_ARCHITECTURES}"
+#     )
+#   endif()
+# else()
+#   list(APPEND SUPERLU_OPTIONS
+#     "-DTPL_ENABLE_HIPLIB=OFF"
+#   )
+# endif()
 
 string(REPLACE ";" "; " SUPERLU_OPTIONS_PRINT "${SUPERLU_OPTIONS}")
 message(STATUS "SUPERLU_OPTIONS: ${SUPERLU_OPTIONS_PRINT}")

diff --git a/extern/CMakeLists.txt b/extern/CMakeLists.txt
@@ -21,12 +21,6 @@ if(PALACE_WITH_STRUMPACK OR PALACE_WITH_MUMPS)
   include(ExternalScaLAPACK)
 endif()
 
-# Add SLATE (for STRUMPACK with GPU support)
-if(PALACE_WITH_STRUMPACK AND (PALACE_WITH_CUDA OR PALACE_WITH_HIP))
-  message(STATUS "===================== Configuring SLATE dependency =====================")
-  include(ExternalSLATE)
-endif()
-
 # Add MAGMA (for STRUMPACK with GPU support and for libCEED)
 if(PALACE_WITH_MAGMA)
   message(STATUS "===================== Configuring MAGMA dependency =====================")

diff --git a/palace/linalg/strumpack.cpp b/palace/linalg/strumpack.cpp
@@ -105,11 +105,11 @@ StrumpackSolverBase<StrumpackSolverType>::StrumpackSolverBase(
     case config::LinearSolverData::CompressionType::NONE:
       break;
   }
-  if (mfem::Device::Allows(mfem::Backend::DEVICE_MASK))
-  {
-    this->EnableGPU();
-  }
-  else
+  // if (mfem::Device::Allows(mfem::Backend::DEVICE_MASK))
+  // {
+  //   this->EnableGPU();  // XX TODO: GPU support disabled for now
+  // }
+  // else
   {
     this->DisableGPU();
   }