Feature/corbett/dense

CMakeLists.txt

@@ -35,6 +35,7 @@ if( NOT is_submodule )
 
     option( ENABLE_ADDR2LINE "Enable addr2line usage in stacktraces" ON )
     option( ENABLE_CUDA "Build with CUDA" OFF )
+    option( ENABLE_HIP "Build with HIP" OFF )
     option( ENABLE_UMPIRE "Build with UMPIRE" OFF )
     option( ENABLE_CHAI "Build with CHAI" OFF )
     option( ENABLE_CALIPER "Build with Caliper" OFF )
@@ -80,6 +81,8 @@ blt_list_append( TO lvarray_dependencies ELEMENTS chai IF ENABLE_CHAI )
 
 blt_list_append( TO lvarray_dependencies ELEMENTS cuda IF ENABLE_CUDA )
 
+blt_list_append( TO lvarray_dependencies ELEMENTS blt::hip IF ENABLE_HIP )
+
 blt_list_append( TO lvarray_dependencies ELEMENTS caliper IF ENABLE_CALIPER )
 
 

Notes.txt

@@ -0,0 +1 @@
+./scripts/uberenv/uberenv.py --prefix=../uberenv-libs/ --spack-config-dir=./scripts/uberenv/spack_configs/toss_3_x86_64_ib_python/ --spec="%[email protected] +umpire +chai +caliper +pylvarray +benchmarks +examples ^caliper ~adiak ~mpi ~libunwind ~libdw ~papi"

cmake/CMakeBasics.cmake

@@ -12,9 +12,9 @@ option( ENABLE_TOTALVIEW_OUTPUT "" OFF )
 set( LVARRAY_BUILD_OBJ_LIBS OFF CACHE BOOL "" )
 
 
-if( NOT BLT_CXX_STD STREQUAL c++14 )
-    MESSAGE(FATAL_ERROR "c++14 is NOT enabled. LvArray requires c++14")
-endif()
+# if( NOT BLT_CXX_STD STREQUAL c++14 )
+#     MESSAGE(FATAL_ERROR "c++14 is NOT enabled. LvArray requires c++14")
+# endif()
 
 
 blt_append_custom_compiler_flag( FLAGS_VAR CMAKE_CXX_FLAGS DEFAULT "${OpenMP_CXX_FLAGS}")

cmake/Config.cmake

@@ -2,8 +2,10 @@
 set( PREPROCESSOR_DEFINES UMPIRE
                           CHAI
                           CUDA
+                          HIP
                           TOTALVIEW_OUTPUT
-                          CALIPER )
+                          CALIPER
+                          MAGMA )
 
 set( USE_CONFIGFILE ON CACHE BOOL "" )
 foreach( DEP in ${PREPROCESSOR_DEFINES})

cmake/SetupTPL.cmake

@@ -1,19 +1,79 @@
+macro(find_and_register)
+    set(singleValueArgs NAME HEADER)
+    set(multiValueArgs INCLUDE_DIRECTORIES
+                       LIBRARY_DIRECTORIES
+                       LIBRARIES
+                       EXTRA_LIBRARIES
+                       DEPENDS )
+
+    ## parse the arguments
+    cmake_parse_arguments(arg
+                          "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN})
+
+    if(NOT DEFINED arg_NAME)
+        message(FATAL_ERROR "The find_and_register required parameter NAME specifies the name of the library to register.")
+    endif()
+
+    if(NOT DEFINED arg_INCLUDE_DIRECTORIES)
+        message(FATAL_ERROR "The find_and_register required parameter INCLUDE_DIRECTORIES specifies the directories to search for the given header.")
+    endif()
+
+    if(NOT DEFINED arg_LIBRARY_DIRECTORIES)
+        message(FATAL_ERROR "The find_and_register required parameter LIBRARY_DIRECTORIES specifies the directories to search for the given libraries.")
+    endif()
+
+    if(NOT DEFINED arg_HEADER)
+        message(FATAL_ERROR "The find_and_register required parameter HEADER specifies the header to search for.")
+    endif()
+
+    if(NOT DEFINED arg_LIBRARIES)
+        message(FATAL_ERROR "The find_and_register required parameter LIBRARIES specifies the libraries to search for.")
+    endif()
+
+    find_path(${arg_NAME}_INCLUDE_DIR ${arg_HEADER}
+              PATHS ${arg_INCLUDE_DIRECTORIES}
+              NO_DEFAULT_PATH
+              NO_CMAKE_ENVIRONMENT_PATH
+              NO_CMAKE_PATH
+              NO_SYSTEM_ENVIRONMENT_PATH
+              NO_CMAKE_SYSTEM_PATH)
+
+    if(${arg_NAME}_INCLUDE_DIR STREQUAL ${arg_NAME}_INCLUDE_DIR-NOTFOUND)
+        message(FATAL_ERROR "Could not find '${arg_HEADER}' in '${arg_INCLUDE_DIRECTORIES}'")
+    endif()
+
+    blt_find_libraries(FOUND_LIBS ${arg_NAME}_LIBRARIES
+                       NAMES ${arg_LIBRARIES}
+                       PATHS ${arg_LIBRARY_DIRECTORIES}
+                       REQUIRED ON)
+
+    blt_import_library(NAME ${arg_NAME}
+                         INCLUDES ${${arg_NAME}_INCLUDE_DIR}
+                         LIBRARIES ${${arg_NAME}_LIBRARIES} ${arg_EXTRA_LIBRARIES}
+                         TREAT_INCLUDES_AS_SYSTEM ON
+                         DEPENDS_ON ${arg_DEPENDS})
+
+endmacro(find_and_register)
+
 set(thirdPartyLibs "")
 
-################################
+###############################
 # CAMP
-################################
-if(NOT EXISTS ${CAMP_DIR})
-    message(FATAL_ERROR "CAMP_DIR must be defined and point to a valid directory when using CAMP.")
-endif()
+###############################
+if(CAMP_DIR STREQUAL RAJA_DIR)
+    message(STATUS "LvArray using CAMP from RAJA.")
+else()
+    if(NOT EXISTS ${CAMP_DIR})
+        message(FATAL_ERROR "CAMP_DIR must be defined and point to a valid directory when using CAMP.")
+    endif()
 
-message(STATUS "Using CAMP from ${CAMP_DIR}")
+    message(STATUS "LvArray using CAMP from ${CAMP_DIR}")
 
-find_package(camp REQUIRED PATHS ${CAMP_DIR})
+    find_package(camp REQUIRED PATHS ${CAMP_DIR})
 
-set(ENABLE_CAMP ON CACHE BOOL "")
+    set(thirdPartyLibs ${thirdPartyLibs} camp)
+endif()
 
-set(thirdPartyLibs ${thirdPartyLibs} camp)
 
 ################################
 # RAJA
@@ -22,7 +82,7 @@ if(NOT EXISTS ${RAJA_DIR})
     message(FATAL_ERROR "RAJA_DIR must be defined and point to a valid directory when using RAJA.")
 endif()
 
-message(STATUS "Using RAJA from ${RAJA_DIR}")
+message(STATUS "LvArray using RAJA from ${RAJA_DIR}")
 
 find_package(RAJA REQUIRED PATHS ${RAJA_DIR})
 
@@ -39,20 +99,26 @@ if(ENABLE_UMPIRE)
         message(FATAL_ERROR "UMPIRE_DIR must be defined and point to a valid directory when using Umpire.")
     endif()
 
-    message(STATUS "Using Umpire from ${UMPIRE_DIR}")
+    message(STATUS "LvArray using Umpire from ${UMPIRE_DIR}")
 
     find_package(umpire REQUIRED
                  PATHS ${UMPIRE_DIR})
 
     set(thirdPartyLibs ${thirdPartyLibs} umpire)
 else()
-    message(STATUS "Not using Umpire.")
+    message(STATUS "LvArray not using Umpire.")
 endif()
 
 ################################
 # CHAI
 ################################
 if(ENABLE_CHAI)
+    if(NOT EXISTS ${CHAI_DIR})
+        message(FATAL_ERROR "CHAI_DIR must be defined and point to a valid directory when using CHAI.")
+    endif()
+
+    message(STATUS "Using CHAI from ${CHAI_DIR}")
+
     if(NOT ENABLE_UMPIRE)
         message(FATAL_ERROR "Umpire must be enabled to use CHAI.")
     endif()
@@ -65,32 +131,32 @@ if(ENABLE_CHAI)
         message(FATAL_ERROR "CHAI_DIR must be defined and point to a valid directory when using CHAI.")
     endif()
 
-    message(STATUS "Using CHAI from ${CHAI_DIR}")
+    message(STATUS "LvArray using CHAI from ${CHAI_DIR}")
 
     find_package(chai REQUIRED
                  PATHS ${CHAI_DIR})
-
-    # If this isn't done chai will add -lRAJA to the link line, but we don't link to RAJA like that.
-    get_target_property(CHAI_LINK_LIBRARIES chai INTERFACE_LINK_LIBRARIES)
-    list(REMOVE_ITEM CHAI_LINK_LIBRARIES RAJA)
-    set_target_properties(chai
-                          PROPERTIES INTERFACE_LINK_LIBRARIES "${CHAI_LINK_LIBRARIES}")
+    
+    # # If this isn't done chai will add -lRAJA to the link line, but we don't link to RAJA like that.
+    # get_target_property(CHAI_LINK_LIBRARIES chai INTERFACE_LINK_LIBRARIES)
+    # list(REMOVE_ITEM CHAI_LINK_LIBRARIES RAJA)
+    # set_target_properties(chai
+    #                       PROPERTIES INTERFACE_LINK_LIBRARIES "${CHAI_LINK_LIBRARIES}")
 
     set(thirdPartyLibs ${thirdPartyLibs} chai)
 else()
-    message(STATUS "Not using CHAI.")
+    message(STATUS "LvArray not using CHAI.")
 endif()
 
 
-################################
+###############################
 # CALIPER
-################################
+###############################
 if(ENABLE_CALIPER)
     if(NOT EXISTS ${CALIPER_DIR})
         message(FATAL_ERROR "CALIPER_DIR must be defined and point to a valid directory when using caliper.")
     endif()
 
-    message(STATUS "Using caliper from ${CALIPER_DIR}")
+    message(STATUS "LvArray using caliper from ${CALIPER_DIR}")
 
     find_package(caliper REQUIRED
                  PATHS ${CALIPER_DIR})
@@ -102,22 +168,66 @@ if(ENABLE_CALIPER)
 
     set(thirdPartyLibs ${thirdPartyLibs} caliper)
 else()
-    message(STATUS "Not using caliper.")
+    message(STATUS "LvArray not using caliper.")
 endif()
 
 ################################
 # Python
 ################################
-if ( ENABLE_PYLVARRAY )
-    message( STATUS "Python3_EXECUTABLE=${Python3_EXECUTABLE}" )
-    find_package( Python3 REQUIRED
-                  COMPONENTS Development NumPy )
+if(ENABLE_PYLVARRAY)
+    message(STATUS "Python3_EXECUTABLE=${Python3_EXECUTABLE}")
+    find_package(Python3 REQUIRED
+                 COMPONENTS Development NumPy)
 
-    message( STATUS "Python3_INCLUDE_DIRS = ${Python3_INCLUDE_DIRS}" )
-    message( STATUS "Python3_LIBRARY_DIRS = ${Python3_LIBRARY_DIRS}" )
-    message( STATUS "Python3_NumPy_INCLUDE_DIRS = ${Python3_NumPy_INCLUDE_DIRS}" )
+    message(STATUS "Python3_INCLUDE_DIRS = ${Python3_INCLUDE_DIRS}")
+    message(STATUS "Python3_LIBRARY_DIRS = ${Python3_LIBRARY_DIRS}")
+    message(STATUS "Python3_NumPy_INCLUDE_DIRS = ${Python3_NumPy_INCLUDE_DIRS}")
 
-    set( thirdPartyLibs ${thirdPartyLibs} Python3::Python Python3::NumPy )
+    set(thirdPartyLibs ${thirdPartyLibs} Python3::Python Python3::NumPy)
+else()
+    message(STATUS "Not building pylvarray")
+endif()
+
+################################
+# LAPACK/BLAS
+################################
+if(ENABLE_LAPACK)
+    message(STATUS "BLAS_LIBRARIES = ${BLAS_LIBRARIES}")
+    message(STATUS "LAPACK_LIBRARIES = ${LAPACK_LIBRARIES}")
+
+    blt_import_library(NAME blas
+                       TREAT_INCLUDES_AS_SYSTEM ON
+                       LIBRARIES ${BLAS_LIBRARIES})
+
+    blt_import_library(NAME lapack
+                       DEPENDS_ON blas
+                       TREAT_INCLUDES_AS_SYSTEM ON
+                       LIBRARIES ${LAPACK_LIBRARIES})
+
+    set(thirdPartyLibs ${thirdPartyLibs} blas lapack)
+else()
+    message(STATUS "Not using LAPACK or BLAS.")
+endif()
+
+################################
+# MAGMA
+################################
+if(ENABLE_MAGMA)
+    message(STATUS "Using MAGMA from ${MAGMA_DIR}")
+
+    if(NOT ENABLE_LAPACK)
+        message(FATAL_ERROR "LAPACK must be enabled to use MAGMA.")
+    endif()
+
+    find_and_register(NAME magma
+                      INCLUDE_DIRECTORIES ${MAGMA_DIR}/include
+                      LIBRARY_DIRECTORIES ${MAGMA_DIR}/lib
+                      HEADER magma.h
+                      LIBRARIES magma)
+
+    set(thirdPartyLibs ${thirdPartyLibs} magma)
+else()
+    message(STATUS "Not using MAGMA.")
 endif()
 
 set( thirdPartyLibs ${thirdPartyLibs} CACHE STRING "" )

host-configs/LLNL/lassen-base.cmake

@@ -21,14 +21,14 @@ set(ENABLE_CUDA ON CACHE BOOL "")
 set(CUDA_TOOLKIT_ROOT_DIR /usr/tce/packages/cuda/cuda-10.1.243 CACHE STRING "")
 set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "")
 set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "")
-set(CUDA_ARCH sm_70 CACHE STRING "")
+set(CUDA_ARCHITECTURES sm_70 CACHE STRING "")
 set(CMAKE_CUDA_STANDARD 14 CACHE STRING "")
-set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" CACHE STRING "")
+set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCHITECTURES} --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" CACHE STRING "")
 set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -mcpu=powerpc64le -Xcompiler -mtune=powerpc64le" CACHE STRING "")
 set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "")
 set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "")
 
-set(CHAI_CUDA_FLAGS "-arch ${CUDA_ARCH}" CACHE STRING "" FORCE)
+set(CHAI_CUDA_FLAGS "-arch ${CUDA_ARCHITECTURES}" CACHE STRING "" FORCE)
 
 # Uncomment this line to make nvcc output register usage for each kernel.
 # set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --resource-usage" CACHE STRING "" FORCE)

host-configs/ORNL/crusher-base.cmake

@@ -0,0 +1,25 @@
+
+set(CMAKE_CXX_STANDARD 14 CACHE STRING "")
+
+set( ENABLE_MPI ON CACHE BOOL "" FORCE )
+set( ENABLE_FIND_MPI ON CACHE BOOL "" FORCE )
+
+# HIP Options
+set( ENABLE_HIP ON CACHE BOOL "" FORCE )
+
+# suppress -Werror for now
+set( ENABLE_WARNINGS_AS_ERRORS FALSE CACHE BOOL "" FORCE )
+
+# GTEST
+set(ENABLE_GTEST_DEATH_TESTS OFF CACHE BOOL "")
+set(gtest_disable_pthreads ON CACHE BOOL "")
+
+# disable most binaries and doc generation
+set(ENABLE_TESTS OFF CACHE BOOL "" FORCE)
+set(DISABLE_UNIT_TESTS ON CACHE BOOL "" FORCE)
+set(ENABLE_EXAMPLES OFF CACHE BOOL "" FORCE)
+set(ENABLE_BENCHMARKS OFF CACHE BOOL "" FORCE)
+set(ENABLE_DOCS OFF CACHE BOOL "" FORCE)
+
+# BLT trying to find MPI fails on cray with cce
+set(ENABLE_FIND_MPI FALSE CACHE BOOL "")

host-configs/ORNL/crusher-cce@13.0.1.cmake

@@ -0,0 +1,39 @@
+
+set(CONFIG_NAME "[email protected]" CACHE PATH "")
+include( ${CMAKE_CURRENT_LIST_DIR}/crusher-base.cmake )
+
+# Set up the tpls
+set(GEOSX_TPL_ROOT_DIR "/gpfs/alpine/geo127/world-shared/cray-sles15-zen2/cce-13.0.1" CACHE PATH "")
+set(GEOSX_TPL_DIR ${GEOSX_TPL_ROOT_DIR} CACHE PATH "")
+set(GEOSX_TPL_DIR2 "/gpfs/alpine/geo127/world-shared/cray-sles15-zen3/cce-13.0.1" CACHE PATH "")
+
+set(CAMP_DIR "${GEOSX_TPL_DIR2}/camp-0.2.2-oogry5gz2fts7jufeykxzmowajtmgzi3" CACHE PATH "" )
+
+set(RAJA_DIR "${GEOSX_TPL_DIR2}/raja-2022.03.0-ex5v5y6jtotfxxvwcs7bblwvy4ktjykq" CACHE PATH "" )
+
+set(ENABLE_UMPIRE TRUE CACHE BOOL "" )
+set(UMPIRE_DIR "${GEOSX_TPL_DIR2}/umpire-develop-jqqth57w2ets75sljw7lc5uxoi5wwi3c" CACHE PATH "" )
+
+set(ENABLE_CHAI TRUE CACHE BOOL "" )
+set(CHAI_DIR "${GEOSX_TPL_DIR2}/chai-2022.03.0-w7lka3bkp36mbk5kzucgtp3eowomllgl" CACHE PATH "" )
+
+set(METIS_DIR "${GEOSX_TPL_DIR}/metis-5.1.0-zcfkawg5ifqpzcihrc3i6cdrrijusc2p/" CACHE PATH "" )
+set(PARMETIS_DIR "${GEOSX_TPL_DIR}/parmetis-4.0.3-t2amifl5hh7yewre24gn2x3mlrz7qkl5/" CACHE PATH "" )
+
+# C++ options
+set(CMAKE_C_COMPILER "/opt/cray/pe/craype/2.7.13/bin/cc" CACHE PATH "")
+set(CMAKE_CXX_COMPILER "/opt/cray/pe/craype/2.7.13/bin/CC" CACHE PATH "")
+set(CMAKE_Fortran_COMPILER "/opt/cray/pe/craype/2.7.13/bin/ftn" CACHE PATH "")
+
+
+if( ENABLE_HIP )
+  set( ENABLE_CLANG_HIP ON CACHE BOOL "" FORCE ) # don't invoke hipcc, rely on cce link-time compilation
+
+  set( HIP_VERSION_STRING "4.5.2" CACHE STRING "" )
+  set( HIP_ROOT "/opt/rocm-4.5.2" CACHE PATH "" )
+
+  set( CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "" FORCE )
+  set( AMDGPU_TARGETS "${CMAKE_HIP_ARCHITECTURES}" CACHE STRING "" FORCE )
+  set( CMAKE_CXX_FLAGS "-fgpu-rdc" CACHE STRING "" FORCE )
+  set( CMAKE_CXX_LINK_FLAGS "-fgpu-rdc --hip-link" CACHE STRING "" FORCE )
+endif()