aurora bv scripts (#1427)

* Added SYCL and Aurora support (#1398) Co-authored-by: Victor Mateevitsi <[email protected]> * refactor to allow more flexibility with compiler setup for sycl case * update kokkos and a few tweaks * .01 not .1 * extra echo * fix * unescape * tweaks * updates + use new rocm container for ghaction * name * name * defend against suprise new compiler * ward off change * pin ubuntu version for gcc * last one --------- Co-authored-by: Victor Mateevitsi <[email protected]> Co-authored-by: Victor Mateevitsi <[email protected]>
Alpine-DAV · Dec 13, 2024 · 01505c6 · 01505c6
1 parent 4b93212
commit 01505c6
Show file tree

Hide file tree

Showing 7 changed files with 123 additions and 26 deletions.
diff --git a/.github/workflows/build_ascent_gcc.yml b/.github/workflows/build_ascent_gcc.yml
@@ -7,7 +7,7 @@ on:
 jobs:
   build_basic:
     name: Ubuntu Build Ascent GCC
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     env:
       CC: gcc
       CXX: g++
@@ -46,6 +46,12 @@ jobs:
       uses: actions/checkout@v4
       with:
         submodules: 'recursive'
+    - name: Env Info
+      run: |
+        echo "**** Env Info"
+        cmake --version
+        gcc --version
+        which mpicc
     - name: Build TPLs
       run: |
         env enable_mpi=ON \
@@ -58,7 +64,6 @@ jobs:
             ./scripts/build_ascent/build_ascent.sh
     - name: Configure Ascent
       run: |
-        cmake --version
         echo "**** Configuring Ascent"
         cmake -S src -B build -C ascent-config.cmake -DCMAKE_INSTALL_PREFIX=install
     - name: Build Ascent
@@ -82,7 +87,7 @@ jobs:
         mpiexec -n 2 ./build/ascent_mpi_render_example
   build_basic_pyvenv:
     name: Ubuntu Build Ascent GCC with Python VENV
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     env:
       CC: gcc
       CXX: g++
@@ -121,6 +126,12 @@ jobs:
       uses: actions/checkout@v4
       with:
         submodules: 'recursive'
+    - name: Env Info
+      run: |
+        echo "**** Env Info"
+        cmake --version
+        gcc --version
+        which mpicc
     - name: Build TPLs
       run: |
         env enable_mpi=ON \
@@ -134,7 +145,6 @@ jobs:
             ./scripts/build_ascent/build_ascent.sh
     - name: Configure Ascent
       run: |
-        cmake --version
         echo "**** Configuring Ascent"
         cmake -S src -B build -C ascent-config.cmake -DCMAKE_INSTALL_PREFIX=install
     - name: Build Ascent

diff --git a/.github/workflows/build_ascent_hip.yml b/.github/workflows/build_ascent_hip.yml
@@ -7,8 +7,8 @@ on:
 jobs:
   build_cuda:
     name: Build Ascent HIP
-    runs-on: ubuntu-latest
-    container: alpinedav/ascent-ci:ubuntu-20.04-rocm-5.1.3-devel
+    runs-on: ubuntu-20.04
+    container: alpinedav/ascent-devel:ubuntu-20.04-rocm-6.3.0-x86_64
     env:
       CMAKE_VERSION: 3.23.2
     steps:
@@ -28,7 +28,7 @@ jobs:
             enable_mpi=OFF \
             enable_fortran=OFF \
             enable_tests=OFF \
-            build_ascent=OFF \
+            build_ascent=false \
             enable_verbose=OFF \
             build_jobs=2 \
             ./scripts/build_ascent/build_ascent.sh

diff --git a/.github/workflows/build_ascent_icc.yml b/.github/workflows/build_ascent_icc.yml
@@ -59,7 +59,7 @@ jobs:
         env enable_mpi=OFF \
             enable_fortran=ON \
             enable_tests=OFF \
-            build_ascent=OFF \
+            build_ascent=false \
             enable_verbose=OFF \
             build_jobs=2 \
             ./scripts/build_ascent/build_ascent.sh

diff --git a/scripts/build_ascent/build_ascent.sh b/scripts/build_ascent/build_ascent.sh
@@ -24,6 +24,7 @@ set -eu -o pipefail
 # shared options
 enable_cuda="${enable_cuda:=OFF}"
 enable_hip="${enable_hip:=OFF}"
+enable_sycl="${enable_sycl:=OFF}"
 enable_fortran="${enable_fortran:=OFF}"
 enable_python="${enable_python:=OFF}"
 enable_openmp="${enable_openmp:=OFF}"
@@ -70,17 +71,19 @@ if [[ "$enable_cuda" == "ON" ]]; then
     CUDA_ARCH_VTKM="${CUDA_ARCH_VTKM:=ampere}"
 fi
 
+# NOTE: this script only builds kokkos when enable_hip=ON or enable_cycl=ON
 if [[ "$enable_hip" == "ON" ]]; then
     echo "*** configuring with HIP support"
 
     CC="${CC:=/opt/rocm/llvm/bin/amdclang}"
     CXX="${CXX:=/opt/rocm/llvm/bin/amdclang++}"
     # FTN?
-
     ROCM_ARCH="${ROCM_ARCH:=gfx90a}"
     ROCM_PATH="${ROCM_PATH:=/opt/rocm/}"
 
-    # NOTE: this script only builds kokkos when enable_hip=ON
+    build_kokkos="${build_kokkos:=true}"
+elif [[ "$enable_sycl" == "ON" ]]; then
+    echo "*** configuring with SYCL support"
     build_kokkos="${build_kokkos:=true}"
 else
     build_kokkos="${build_kokkos:=false}"
@@ -510,16 +513,16 @@ else
   echo "**** Skipping Conduit build, install found at: ${conduit_install_dir}"
 fi # build_conduit
 
-#########################
-# Kokkos (only for hip)
-#########################
-kokkos_version=3.7.02
+###############################
+# Kokkos (only for hip or sycl)
+###############################
+kokkos_version=4.4.01
 kokkos_src_dir=$(ospath ${source_dir}/kokkos-${kokkos_version})
 kokkos_build_dir=$(ospath ${build_dir}/kokkos-${kokkos_version})
 kokkos_install_dir=$(ospath ${install_dir}/kokkos-${kokkos_version}/)
 kokkos_tarball=$(ospath ${source_dir}/kokkos-${kokkos_version}.tar.gz)
 
-if [[ "$enable_hip" == "ON" ]]; then
+if [[ "$enable_hip" == "ON" ]] || [[ "$enable_sycl" == "ON" ]]; then
 # build only if install doesn't exist
 if [ ! -d ${kokkos_install_dir} ]; then
 if ${build_kokkos}; then
@@ -529,21 +532,52 @@ if [ ! -d ${kokkos_src_dir} ]; then
   tar ${tar_extra_args} -xzf ${kokkos_tarball} -C ${source_dir}
 fi
 
-# TODO: DKokkos_ARCH_VEGA90A needs to be controlled / mapped?
+kokkos_extra_cmake_args=""
+if [[ "$enable_hip" == "ON" ]]; then
+  kokkos_extra_cmake_args="-DKokkos_ENABLE_HIP=ON"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE=OFF"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_COMPILER=${ROCM_PATH}/bin/hipcc"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_EXTENSIONS=OFF"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_STANDARD=17"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ENABLE_ROCTHRUST=OFF"
+
+  ##
+  ## build_ascent specific ROCM_ARCH Map for Kokkos options:
+  ##
+  ## TODO: Kokkos 4.5 has MI300A specific option, need to figure out how to
+  ##       map hat in when we update.
+  ##
+  ## gfx942 --> Kokkos_ARCH_AMD_GFX942 (MI300A, MI300X) 
+  ## (since Kokkos 4.2, since Kokkos 4.5 this should only be used for MI300X)
+  ##
+  ## gfx90a --> Kokkos_ARCH_AMD_GFX90A (MI200 series)
+  ## (since Kokkos 4.2)
+  ##
+  if [[ "$ROCM_ARCH" == "gfx942" ]]; then
+      kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ARCH_AMD_GFX942=ON"
+  fi
+
+  if [[ "$ROCM_ARCH" == "gfx90a" ]]; then
+      kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ARCH_AMD_GFX90A=ON"
+  fi
+fi
+
+if [[ "$enable_sycl" == "ON" ]]; then
+  kokkos_extra_cmake_args="-DCMAKE_CXX_FLAGS=-fPIC -fp-model=precise -Wno-unused-command-line-argument -Wno-deprecated-declarations -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=128"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ENABLE_SYCL=ON"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ARCH_INTEL_PVC=ON"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_EXTENSIONS=OFF"
+  kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_STANDARD=17"
+fi
 
 echo "**** Configuring Kokkos ${kokkos_version}"
 cmake -S ${kokkos_src_dir} -B ${kokkos_build_dir} ${cmake_compiler_settings} \
   -DCMAKE_VERBOSE_MAKEFILE:BOOL=${enable_verbose}\
   -DCMAKE_BUILD_TYPE=${build_config} \
   -DBUILD_SHARED_LIBS=${build_shared_libs} \
-  -DKokkos_ARCH_VEGA90A=ON \
-  -DCMAKE_CXX_COMPILER=${ROCM_PATH}/bin/hipcc \
-  -DKokkos_ENABLE_HIP=ON \
   -DKokkos_ENABLE_SERIAL=ON \
-  -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE=OFF \
   -DCMAKE_INSTALL_PREFIX=${kokkos_install_dir} \
-  -DCMAKE_CXX_FLAGS="--amdgpu-target=${ROCM_ARCH}" \
-  -DBUILD_TESTING=OFF \
+  -DBUILD_TESTING=OFF ${kokkos_extra_cmake_args} \
   -DCMAKE_INSTALL_PREFIX=${kokkos_install_dir}
 
 echo "**** Building Kokkos ${kokkos_version}"
@@ -556,7 +590,7 @@ else
   echo "**** Skipping Kokkos build, install found at: ${kokkos_install_dir}"
 fi # build_kokkos
 
-fi # if enable_hip
+fi # if enable_hip || enable_sycl
 
 ################
 # VTK-m
@@ -598,6 +632,13 @@ if [[ "$enable_hip" == "ON" ]]; then
   vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DVTKm_ENABLE_KOKKOS_THRUST=OFF"
 fi
 
+if [[ "$enable_sycl" == "ON" ]]; then
+  vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DVTKm_ENABLE_KOKKOS=ON"
+  vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DCMAKE_PREFIX_PATH=${kokkos_install_dir}"
+  vtkm_extra_cmake_args="-DCMAKE_CXX_FLAGS=-fPIC -fp-model=precise -Wno-unused-command-line-argument -Wno-deprecated-declarations -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=128"
+fi
+
+
 if [[ "$enable_mpicc" == "ON" ]]; then
   vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DMPI_C_COMPILER=${mpicc_exe}"
   vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DMPI_CXX_COMPILER=${mpicxx_exe}"
@@ -608,7 +649,6 @@ cmake -S ${vtkm_src_dir} -B ${vtkm_build_dir} ${cmake_compiler_settings} \
   -DCMAKE_VERBOSE_MAKEFILE:BOOL=${enable_verbose}\
   -DCMAKE_BUILD_TYPE=${build_config} \
   -DBUILD_SHARED_LIBS=${build_shared_libs} \
-  -DVTKm_NO_DEPRECATED_VIRTUAL=ON \
   -DVTKm_USE_64BIT_IDS=OFF \
   -DVTKm_USE_DOUBLE_PRECISION=ON \
   -DVTKm_USE_DEFAULT_TYPES_FOR_ASCENT=ON \
@@ -763,6 +803,10 @@ if [[ "$enable_hip" == "ON" ]]; then
   umpire_extra_cmake_args="${umpire_extra_cmake_args} -DROCM_PATH=${ROCM_PATH}"
 fi
 
+if [[ "$enable_sycl" == "ON" ]]; then
+  umpire_extra_cmake_args="${umpire_extra_cmake_args} -DENABLE_SYCL=ON"
+fi
+
 # build only if install doesn't exist
 if [ ! -d ${umpire_install_dir} ]; then
 if ${build_umpire}; then
@@ -797,7 +841,7 @@ fi # build_umpire
 ################
 # MFEM
 ################
-mfem_version=4.6
+mfem_version=4.7
 mfem_src_dir=$(ospath ${source_dir}/mfem-${mfem_version})
 mfem_build_dir=$(ospath ${build_dir}/mfem-${mfem_version})
 mfem_install_dir=$(ospath ${install_dir}/mfem-${mfem_version}/)
@@ -954,7 +998,6 @@ fi
 if ${build_caliper}; then
   echo 'set(CALIPER_DIR ' ${caliper_install_dir} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake
 fi
-echo 'set(BLT_CXX_STD c++14 CACHE STRING "")' >> ${root_dir}/ascent-config.cmake
 echo 'set(CONDUIT_DIR ' ${conduit_install_dir} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake
 echo 'set(VTKM_DIR ' ${vtkm_install_dir} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake
 echo 'set(CAMP_DIR ' ${camp_install_dir} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake
@@ -977,6 +1020,7 @@ fi
 
 if [[ "$enable_hip" == "ON" ]]; then
     echo 'set(ENABLE_HIP ON CACHE BOOL "")' >> ${root_dir}/ascent-config.cmake
+    echo 'set(BLT_CXX_STD c++17 CACHE STRING "")' >> ${root_dir}/ascent-config.cmake
     echo 'set(CMAKE_HIP_ARCHITECTURES ' ${ROCM_ARCH} ' CACHE STRING "")' >> ${root_dir}/ascent-config.cmake
     echo 'set(ROCM_PATH ' ${ROCM_PATH} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake
     echo 'set(KOKKOS_DIR ' ${kokkos_install_dir} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake

diff --git a/scripts/build_ascent/build_ascent_aurora.sh b/scripts/build_ascent/build_ascent_aurora.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -l
+
+export IGC_FunctionCloningThreshold=1
+export IGC_ControlInlineTinySize=100
+export IGC_OCLInlineThreshold=200
+export IGC_PartitionUnit=1
+export IGC_ForceOCLSIMDWidth=16
+export ZE_AFFINITY_MASK=0.0
+
+# Proxies
+export HTTP_PROXY=http://proxy.alcf.anl.gov:3128
+export HTTPS_PROXY=http://proxy.alcf.anl.gov:3128
+export http_proxy=http://proxy.alcf.anl.gov:3128
+export https_proxy=http://proxy.alcf.anl.gov:3128
+
+module reset
+module use /soft/modulefiles
+module load spack-pe-gcc cmake
+module load oneapi/eng-compiler/2023.12.15.002
+
+env CC=`which icx` CXX=`which icpx` FTN=`which ifx` enable_sycl=ON enable_mpi=ON enable_fortran=ON raja_enable_vectorization=OFF enable_tests=ON enable_verbose=ON ./build_ascent_sycl.sh
diff --git a/scripts/build_ascent/build_ascent_sycl.sh b/scripts/build_ascent/build_ascent_sycl.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+##############################################################################
+# Demonstrates how to manually build Ascent and its dependencies, including:
+#
+#  hdf5, conduit, vtk-m, mfem, raja, and umpire
+#
+# usage example:
+#   env enable_mpi=ON enable_openmp=ON ./build_ascent.sh
+#
+#
+# Assumes: 
+#  - cmake is in your path
+#  - selected compilers are in your path or set via env vars
+#  - [when enabled] MPI and Python (+numpy and mpi4py), are in your path
+#
+##############################################################################
+set -eu -o pipefail
+
+# 2024-02-08 SYCL support is handled by our unified script
+env enable_sycl=ON ./build_ascent.sh
diff --git a/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/Dockerfile b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/Dockerfile
@@ -65,6 +65,7 @@ RUN apt-get update && apt-get -y install \
              rocfft-dev      \
              rocprim-dev     \
              rocrand-dev     \
+             rocthrust-dev     \
              hiprand-dev     \
              openssh-server \
              language-pack-en \