CMakeLists.txt

cmake_minimum_required(VERSION 3.0)

# Variables that can/must be passed to the cmake command:
#
# - EIGEN3_ROOT needs to be set to the path of the Eigen top-level directory.
# - -DWITH_PETSC=1 to compile with PETSc support; in which case
#     PETSC_LIB and PETSC_ARCH variables are required.
#     Note that in this case, MPI is required too.
#     It is best to pass MPI wrappers as C and CXX compilers to CMake.
# - -DNOOMP=1 to compile without OpenMP (default build is with OpenMP)
# - -DSSE=1 to compile with SSE 4.2 instructions (default)
# - -DAVX=1 to compile with AVX instructions.
# - -DAVX_2=1 to compile with AVX 2.0.
# - -DZEN=1 to compiler for AMD Zen with AVX 2.0.
# - -DSKYLAKE=1 to compile with AVX-512 instructions for Xeon Skylake CPUs.
# - -DKNL=1 to compile for Xeon Phi Knights Landing
# - -DMICKNC=1 to compile for Xeon Phi Knights Corner (deprecated).
# - -DPROFILE=1 for profiling with gprof.
# - -DSLURM=1 for running the automated tests on a system managed by Slurm
# - -DSLURMTESTTHREADS=<n> for configuring multi-threaded tests to use n threads in case of Slurm
#
# - -DBUILD_BLOCK_SIZE=<n> for some integer n, to build the block solver operations for
#     an additional block size of <n>. By default, block sizes 4 and 5 are supported.

project (blasted)

# detect C++ compiler - the C compiler must be the same type too
if("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
  set(CXX_COMPILER_GNUCLANG TRUE)
  if("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
	set(CXX_COMPILER_GNU TRUE)
	message(STATUS "GNU C++ compiler detected")
	if(NOT("${CMAKE_C_COMPILER_ID}" MATCHES "GNU"))
	  message(SEND_ERROR "C and C++ compilers should match!")
	endif()
  else()
	set(CXX_COMPILER_CLANG TRUE)
	message(STATUS "Clang C++ compiler detected")
	if(NOT("${CMAKE_C_COMPILER_ID}" MATCHES "Clang"))
	  message(SEND_ERROR "C and C++ compilers should match!")
	endif()
  endif()
elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Intel")
  set(CXX_COMPILER_INTEL TRUE)
  message(STATUS "Intel C++ compiler detected")
  if(NOT("${CMAKE_C_COMPILER_ID}" MATCHES "Intel"))
	message(SEND_ERROR "C and C++ compilers should match!")
  endif()
elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Cray")
  set(CXX_COMPILER_CRAY TRUE)
  message(STATUS "Cray C++ compiler detected")
  if(NOT("${CMAKE_C_COMPILER_ID}" MATCHES "Cray"))
	message(SEND_ERROR "C and C++ compilers should match!")
  endif()
elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "PGI")
  set(CXX_COMPILER_PGI TRUE)
  message(STATUS "PGI C++ compiler detected")
  if(NOT("${CMAKE_C_COMPILER_ID}" MATCHES "PGI"))
	message(SEND_ERROR "C and C++ compilers should match!")
  endif()
else()
  message(SEND_ERROR "Could not identify compiler!")
endif()

# Basic options
if(CXX_COMPILER_GNUCLANG OR CXX_COMPILER_INTEL)
  set (CMAKE_CXX_FLAGS "-std=c++14 -Wall -Werror")
  set (CMAKE_C_FLAGS "-std=c11 -Wall -Werror")
  set (CMAKE_CXX_FLAGS_DEBUG "-g")
  # if(CXX_COMPILER_GNUCLANG)
  # 	set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Og")
  # 	set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Og")
  # else()
  # set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O1")
  # set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O1")
  # endif()
  set (CMAKE_CXX_FLAGS_RELEASE "-O3")
elseif(CXX_COMPILER_PGI)
  set (CMAKE_CXX_FLAGS "-std=c++14")
  set (CMAKE_C_FLAGS "-std=c11")
  set (CMAKE_CXX_FLAGS_DEBUG "-g")
  set (CMAKE_CXX_FLAGS_RELEASE "-O2")
elseif(CXX_COMPILER_CRAY)
  set (CMAKE_CXX_FLAGS "-h std=c++14 -h error_on_warning")
  set (CMAKE_C_FLAGS "-h std=c11 -h error_on_warning")
  set (CMAKE_CXX_FLAGS_DEBUG "-g -O1")
  set (CMAKE_CXX_FLAGS_RELEASE "-O2")
endif()

# to compile with OpenMP
if(NOOMP)
  if(CXX_COMPILER_GNUCLANG)
	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-pragmas")
	set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -Wno-unknown-pragmas")
  endif()
  if(CXX_COMPILER_PGI)
	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nomp")
	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -nomp")
  endif()
  if(CXX_COMPILER_CRAY)
	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -hnoomp")
	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -hnoomp")
  endif()
  message(STATUS "Compiling without OpenMP")
else()
  if(CXX_COMPILER_GNUCLANG)
	set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
	set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fopenmp")
	message(STATUS "Compiling with OpenMP for GNU")
  elseif(CXX_COMPILER_INTEL)
	set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qopenmp")
	set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qopenmp")	  
	message(STATUS "Compiling with OpenMP for Intel")
  elseif(CXX_COMPILER_CRAY)
	message(STATUS "Compiling with OpenMP for Cray")
  elseif(CXX_COMPILER_PGI)
	message(STATUS "Compiling with OpenMP for PGI")
  else()
	message(SEND_ERROR "Don't know OpenMP flag for this compiler!")
  endif()
endif()

# set verbosity options
if("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopt-info-inline-vec-optimized-missed=optimizations.info")
elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Rpass=inline,vectorize")
elseif(CXX_COMPILER_INTEL)
  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qopt-report=5 -qopt-report-phase=cg,loop,vec,openmp")
elseif(CXX_COMPILER_CRAY)
  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -hlist=a -hreport=fisv")
endif()

# Architecture to target
if(MICKNC)
  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mmic")
  message(STATUS "Compiling for KNC")
  if(NOT CXX_COMPILER_INTEL)
	message(SEND_ERROR "Need to use Intel compiler for Xeon Phi!")
  endif()
  
  # Link the mic version of MKL if requested
  if(WITH_MKL)
	include_directories($ENV{MKL_ROOT}/include)
	set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_MKL -L${MKLROOT}/lib/mic -lmkl_rt -lpthread -lm -ldl")
	message(STATUS "Linking MKL for mic.")
  endif()
else()
  # Select hardware-specific optimizations
  if(AVX)
	if(CXX_COMPILER_GNUCLANG OR CXX_COMPILER_INTEL)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx")
	elseif(CXX_COMPILER_PGI)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -tp=sandybridge")
	  add_definitions(-DEIGEN_VECTORIZE_AVX)
	elseif(CXX_COMPILER_CRAY)
	  add_definitions(-DEIGEN_VECTORIZE_AVX)
	endif()
	message(STATUS "Compiling with AVX")
  elseif(AVX_2)
	if(CXX_COMPILER_INTEL)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -xcore-avx2")
	elseif(CXX_COMPILER_GNUCLANG)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
	elseif(CXX_COMPILER_PGI)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -tp=haswell")
	  add_definitions(-DEIGEN_VECTORIZE_AVX2 -DEIGEN_VECTORIZE_FMA)
	elseif(CXX_COMPILER_CRAY)
	  message(STATUS "Architecture options for Cray are to be set by the compiler wrapper.")
	  add_definitions(-DEIGEN_VECTORIZE_AVX2 -DEIGEN_VECTORIZE_FMA)
	else()
	  message(SEND_ERROR "Don't know/require option for this compiler!")
	endif()
	message(STATUS "Compiling with AVX-2")
  elseif(SKYLAKE)
	if(CXX_COMPILER_INTEL)
	  add_definitions(-DEIGEN_VECTORIZE_AVX512)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -xcore-avx512")
	elseif(CXX_COMPILER_GNUCLANG)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=skylake-avx512")
	elseif(CXX_COMPILER_PGI)
	  add_definitions(-DEIGEN_VECTORIZE_AVX512 -DEIGEN_VECTORIZE_FMA)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -tp=skylake")
	elseif(CXX_COMPILER_CRAY)
	  add_definitions(-DEIGEN_VECTORIZE_AVX512 -DEIGEN_VECTORIZE_FMA)
	  message(STATUS "Architecture options for Cray are to be set by the compiler wrapper.")
	else()
	  message(SEND_ERROR "Don't know option for this compiler!")
	endif()
	message(STATUS "Compiling with AVX-512 for Skylake")
  elseif(KNL)
	if(CXX_COMPILER_INTEL)
	  add_definitions(-DEIGEN_VECTORIZE_AVX512)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -xmic-avx512")
	elseif(CXX_COMPILER_GNUCLANG)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=knl")
	elseif(CXX_COMPILER_PGI)
	  add_definitions(-DEIGEN_VECTORIZE_AVX512 -DEIGEN_VECTORIZE_FMA)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -tp=knl")
	elseif(CXX_COMPILER_CRAY)
	  add_definitions(-DEIGEN_VECTORIZE_AVX512 -DEIGEN_VECTORIZE_FMA)
	  message(STATUS "Architecture options for Cray are to be set by the compiler wrapper.")
	else()
	  message(SEND_ERROR "Don't know/require option for this compiler!")
	endif()
	message(STATUS "Compiling with AVX-512 for KNL")
  elseif(ZEN)
	add_definitions(-DEIGEN_VECTORIZE_AVX2 -DEIGEN_VECTORIZE_FMA)
	if(CXX_COMPILER_INTEL)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -xcore-avx2")
	elseif(CXX_COMPILER_GNUCLANG)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=znver1")
	elseif(CXX_COMPILER_PGI)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -tp=zen")
	  add_definitions(-DEIGEN_VECTORIZE_AVX2 -DEIGEN_VECTORIZE_FMA)
	elseif(CXX_COMPILER_CRAY)
	  message(STATUS "Architecture options for Cray are to be set by the compiler wrapper.")
	else()
	  message(SEND_ERROR "Don't know option for this compiler!")
	endif()
	message(STATUS "Compiling with AVX 2.0 for AMD Zen")
  else()
	if(CXX_COMPILER_GNUCLANG OR CXX_COMPILER_INTEL)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
	elseif(CXX_COMPILER_PGI)
	  set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -tp=x64,nehalem")
	endif()
	message(STATUS "Compiling with SSE4")
  endif()

  # Link MKL if requested
  if(WITH_MKL)
	include_directories($ENV{MKL_ROOT}/include)
	set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_MKL -L$ENV{MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_rt -lpthread -lm -ldl")
	message(STATUS "Linking MKL.")
  endif()
endif()

# profiling
if(PROFILE)
  message(STATUS "Building with profiling options")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g")
  if(CXX_COMPILER_GNUCLANG)
	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg")
	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pg")
  elseif(CXX_COMPILER_INTEL)
	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -shared-intel -debug inline-debug-info -parallel-source-info=2")
	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -shared-intel -debug inline-debug-info -parallel-source-info=2")
  endif()
endif()

# Boost
find_package(Boost 1.57 REQUIRED COMPONENTS program_options)
include_directories(${Boost_INCLUDE_DIRS})
message(STATUS "Found Boost libraries ${Boost_LIBRARIES}")

# Eigen
find_path(EIGEN3_INCLUDE_DIR signature_of_eigen3_matrix_library
  PATHS ENV EIGEN3_ROOT DOC "Eigen3 include directory")
if(${EIGEN3_INCLUDE_DIR} STREQUAL "EIGEN3_INCLUDE_DIR-NOTFOUND")
  message(SEND_ERROR "Eigen3 not found!")
endif()
include_directories(${EIGEN3_INCLUDE_DIR})
message(STATUS "Found Eigen 3 at ${EIGEN3_INCLUDE_DIR}")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DEIGEN_DONT_PARALLELIZE")

if(SLURM)
  set(SEQEXEC "srun")
  set(SEQTASKS "-n 1")
  if(SLURMTESTTHREADS)
	  set(THREADOPTS "--cpus-per-task=${SLURMTESTTHREADS}")
  else()
	  set(THREADOPTS "--cpus-per-task=4")
  endif()
else()
  set(SEQEXEC "")
  set(SEQTASKS "")
  set(THREADOPTS "")
endif(SLURM)

# PETSc
if(WITH_PETSC)
  find_package(MPI REQUIRED)
  include_directories(${MPI_C_INCLUDE_PATH} ${MPI_CXX_INCLUDE_PATH})
  if(SLURM)
	set(MPIEXEC "srun")
	set(MPIOPTS "--cpu-bind=cores")
  else()
	set(MPIEXEC "mpirun")
	set(MPIOPTS "")
  endif(SLURM)

  # If PETSc variables were not passed to CMake, set them from environment variables
  if(NOT DEFINED PETSC_DIR)
	set(PETSC_DIR $ENV{PETSC_DIR} CACHE PATH "PETSc install directory")
  endif()
  if(NOT DEFINED PETSC_ARCH)
	set(PETSC_ARCH $ENV{PETSC_ARCH} CACHE STRING "PETSc architecture to use")
  endif()
  find_library(PETSC_LIB NAMES petsc PATHS ${PETSC_DIR}/${PETSC_ARCH}/lib
	DOC "Location of PETSc library")
  include_directories(${PETSC_DIR}/include ${PETSC_DIR}/${PETSC_ARCH}/include)
  message(STATUS "Building with PETSc found at ${PETSC_LIB}")
endif()

# Harwell's matrix ordering library - static library is assumed
if(WITH_MC64)
  if(NOT DEFINED MC64_DIR)
	set(MC64_DIR $ENV{MC64_DIR} CACHE PATH "MC64 install directory")
  endif()
  find_library(MC64_LIB libmc64.a ${MC64_DIR}/lib)
  add_definitions(-DHAVE_MC64)
  message(STATUS "Building with MC64 found at ${MC64_LIB}")
  if(CXX_COMPILER_GNUCLANG)
	set(FORTLDFLAGS "-lgfortran")
  elseif(CXX_COMPILER_INTEL)
	set(FORTLDFLAGS "-lifcore")
  else()
	message(WARNING "Don't know Fortran library for this compiler")
  endif()
endif(WITH_MC64)

# ---------------------------------------------------------------------------- #

set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG=1")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -UDEBUG -DNDEBUG")
include_directories("${PROJECT_SOURCE_DIR}/include")

if(BUILD_BLOCK_SIZE)
  add_definitions(-DBUILD_BLOCK_SIZE=${BUILD_BLOCK_SIZE})
endif()

if(CMAKE_BUILD_TYPE MATCHES "Debug")
  message(STATUS "Debug mode")
elseif(CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo")
  message(STATUS "Release with debug info mode")
else()
  message(STATUS "Release mode")
endif()

set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)

add_subdirectory(src)

# Tests
enable_testing()
add_subdirectory(tests)