Added SYCL and Aurora support (#1398)

Co-authored-by: Victor Mateevitsi <[email protected]>

scripts/build_ascent/build_ascent.sh

@@ -24,6 +24,7 @@ set -eu -o pipefail
 # shared options
 enable_cuda="${enable_cuda:=OFF}"
 enable_hip="${enable_hip:=OFF}"
+enable_sycl="${enable_sycl:=OFF}"
 enable_fortran="${enable_fortran:=OFF}"
 enable_python="${enable_python:=OFF}"
 enable_openmp="${enable_openmp:=OFF}"
@@ -81,6 +82,14 @@ if [[ "$enable_hip" == "ON" ]]; then
     ROCM_PATH="${ROCM_PATH:=/opt/rocm/}"
 
     # NOTE: this script only builds kokkos when enable_hip=ON
+    build_kokkos="${build_kokkos:=true}"
+elif [[ "$enable_sycl" == "ON" ]]; then
+    echo "*** configuring with SYCL support"
+
+    CC=`which icx`
+    CXX=`which icpx`
+    FTN=`which ifx`
+
     build_kokkos="${build_kokkos:=true}"
 else
     build_kokkos="${build_kokkos:=false}"
@@ -519,7 +528,7 @@ kokkos_build_dir=$(ospath ${build_dir}/kokkos-${kokkos_version})
 kokkos_install_dir=$(ospath ${install_dir}/kokkos-${kokkos_version}/)
 kokkos_tarball=$(ospath ${source_dir}/kokkos-${kokkos_version}.tar.gz)
 
-if [[ "$enable_hip" == "ON" ]]; then
+if [[ "$enable_hip" == "ON" ]] || [[ "$enable_sycl" == "ON" ]]; then
 # build only if install doesn't exist
 if [ ! -d ${kokkos_install_dir} ]; then
 if ${build_kokkos}; then
@@ -529,21 +538,33 @@ if [ ! -d ${kokkos_src_dir} ]; then
   tar ${tar_extra_args} -xzf ${kokkos_tarball} -C ${source_dir}
 fi
 
+kokkos_extra_cmake_args=""
+if [[ "$enable_hip" == "ON" ]]; then
+  kokkos_extra_cmake_args="-DKokkos_ENABLE_HIP=ON"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE=OFF"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ARCH_VEGA90A=ON"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_FLAGS=--amdgpu-target=${ROCM_ARCH}"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_COMPILER=${ROCM_PATH}/bin/hipcc"
+fi
+
+if [[ "$enable_sycl" == "ON" ]]; then
+  kokkos_extra_cmake_args="-DCMAKE_CXX_FLAGS=-fPIC -fp-model=precise -Wno-unused-command-line-argument -Wno-deprecated-declarations -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=128"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ENABLE_SYCL=ON"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ARCH_INTEL_PVC=ON"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_EXTENSIONS=OFF"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_STANDARD=17"
+fi
+
 # TODO: DKokkos_ARCH_VEGA90A needs to be controlled / mapped?
 
 echo "**** Configuring Kokkos ${kokkos_version}"
 cmake -S ${kokkos_src_dir} -B ${kokkos_build_dir} ${cmake_compiler_settings} \
   -DCMAKE_VERBOSE_MAKEFILE:BOOL=${enable_verbose}\
   -DCMAKE_BUILD_TYPE=${build_config} \
   -DBUILD_SHARED_LIBS=${build_shared_libs} \
-  -DKokkos_ARCH_VEGA90A=ON \
-  -DCMAKE_CXX_COMPILER=${ROCM_PATH}/bin/hipcc \
-  -DKokkos_ENABLE_HIP=ON \
   -DKokkos_ENABLE_SERIAL=ON \
-  -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE=OFF \
   -DCMAKE_INSTALL_PREFIX=${kokkos_install_dir} \
-  -DCMAKE_CXX_FLAGS="--amdgpu-target=${ROCM_ARCH}" \
-  -DBUILD_TESTING=OFF \
+  -DBUILD_TESTING=OFF "${kokkos_extra_cmake_args}" \
   -DCMAKE_INSTALL_PREFIX=${kokkos_install_dir}
 
 echo "**** Building Kokkos ${kokkos_version}"
@@ -556,7 +577,7 @@ else
   echo "**** Skipping Kokkos build, install found at: ${kokkos_install_dir}"
 fi # build_kokkos
 
-fi # if enable_hip
+fi # if enable_hip || enable_sycl
 
 ################
 # VTK-m
@@ -598,6 +619,12 @@ if [[ "$enable_hip" == "ON" ]]; then
   vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DVTKm_ENABLE_KOKKOS_THRUST=OFF"
 fi
 
+if [[ "$enable_sycl" == "ON" ]]; then
+  vtkm_extra_cmake_args="-DVTKm_ENABLE_KOKKOS=ON"
+  vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DCMAKE_PREFIX_PATH=${kokkos_install_dir}"
+  vtkm_extra_cmake_args="-DCMAKE_CXX_FLAGS=-fPIC -fp-model=precise -Wno-unused-command-line-argument -Wno-deprecated-declarations -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=128"
+fi
+
 if [[ "$enable_mpicc" == "ON" ]]; then
   vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DMPI_C_COMPILER=${mpicc_exe}"
   vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DMPI_CXX_COMPILER=${mpicxx_exe}"
@@ -617,7 +644,7 @@ cmake -S ${vtkm_src_dir} -B ${vtkm_build_dir} ${cmake_compiler_settings} \
   -DVTKm_ENABLE_RENDERING=ON \
   -DVTKm_ENABLE_TESTING=OFF\
   -DBUILD_TESTING=OFF \
-  -DVTKm_ENABLE_BENCHMARKS=OFF ${vtkm_extra_cmake_args} \
+  -DVTKm_ENABLE_BENCHMARKS=OFF "${vtkm_extra_cmake_args}" \
   -DCMAKE_INSTALL_PREFIX=${vtkm_install_dir}
 
 echo "**** Building VTK-m ${vtkm_version}"
@@ -763,6 +790,10 @@ if [[ "$enable_hip" == "ON" ]]; then
   umpire_extra_cmake_args="${umpire_extra_cmake_args} -DROCM_PATH=${ROCM_PATH}"
 fi
 
+if [[ "$enable_sycl" == "ON" ]]; then
+  umpire_extra_cmake_args="${umpire_extra_cmake_args} -DENABLE_SYCL=ON"
+fi
+
 # build only if install doesn't exist
 if [ ! -d ${umpire_install_dir} ]; then
 if ${build_umpire}; then

scripts/build_ascent/build_ascent_aurora.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -l
+
+export IGC_FunctionCloningThreshold=1
+export IGC_ControlInlineTinySize=100
+export IGC_OCLInlineThreshold=200
+export IGC_PartitionUnit=1
+export IGC_ForceOCLSIMDWidth=16
+export ZE_AFFINITY_MASK=0.0
+
+# Proxies
+export HTTP_PROXY=http://proxy.alcf.anl.gov:3128
+export HTTPS_PROXY=http://proxy.alcf.anl.gov:3128
+export http_proxy=http://proxy.alcf.anl.gov:3128
+export https_proxy=http://proxy.alcf.anl.gov:3128
+
+module reset
+module use /soft/modulefiles
+module load spack-pe-gcc cmake
+module load oneapi/eng-compiler/2023.12.15.002
+
+env enable_sycl=ON enable_mpi=ON enable_fortran=ON raja_enable_vectorization=OFF enable_tests=ON enable_verbose=ON ./build_ascent_sycl.sh

scripts/build_ascent/build_ascent_sycl.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+##############################################################################
+# Demonstrates how to manually build Ascent and its dependencies, including:
+#
+#  hdf5, conduit, vtk-m, mfem, raja, and umpire
+#
+# usage example:
+#   env enable_mpi=ON enable_openmp=ON ./build_ascent.sh
+#
+#
+# Assumes: 
+#  - cmake is in your path
+#  - selected compilers are in your path or set via env vars
+#  - [when enabled] MPI and Python (+numpy and mpi4py), are in your path
+#
+##############################################################################
+set -eu -o pipefail
+
+# 2024-02-08 SYCL support is handled by our unified script
+env enable_sycl=ON ./build_ascent.sh