From 01505c624c9332a22583b0f4dbbabd86f028f481 Mon Sep 17 00:00:00 2001 From: Cyrus Harrison Date: Thu, 12 Dec 2024 16:41:24 -0800 Subject: [PATCH] aurora bv scripts (#1427) * Added SYCL and Aurora support (#1398) Co-authored-by: Victor Mateevitsi * refactor to allow more flexibility with compiler setup for sycl case * update kokkos and a few tweaks * .01 not .1 * extra echo * fix * unescape * tweaks * updates + use new rocm container for ghaction * name * name * defend against suprise new compiler * ward off change * pin ubuntu version for gcc * last one --------- Co-authored-by: Victor Mateevitsi Co-authored-by: Victor Mateevitsi --- .github/workflows/build_ascent_gcc.yml | 18 ++++- .github/workflows/build_ascent_hip.yml | 6 +- .github/workflows/build_ascent_icc.yml | 2 +- scripts/build_ascent/build_ascent.sh | 80 ++++++++++++++----- scripts/build_ascent/build_ascent_aurora.sh | 21 +++++ scripts/build_ascent/build_ascent_sycl.sh | 21 +++++ .../Dockerfile | 1 + 7 files changed, 123 insertions(+), 26 deletions(-) create mode 100755 scripts/build_ascent/build_ascent_aurora.sh create mode 100755 scripts/build_ascent/build_ascent_sycl.sh diff --git a/.github/workflows/build_ascent_gcc.yml b/.github/workflows/build_ascent_gcc.yml index 5df955339..997ca0cec 100644 --- a/.github/workflows/build_ascent_gcc.yml +++ b/.github/workflows/build_ascent_gcc.yml @@ -7,7 +7,7 @@ on: jobs: build_basic: name: Ubuntu Build Ascent GCC - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 env: CC: gcc CXX: g++ @@ -46,6 +46,12 @@ jobs: uses: actions/checkout@v4 with: submodules: 'recursive' + - name: Env Info + run: | + echo "**** Env Info" + cmake --version + gcc --version + which mpicc - name: Build TPLs run: | env enable_mpi=ON \ @@ -58,7 +64,6 @@ jobs: ./scripts/build_ascent/build_ascent.sh - name: Configure Ascent run: | - cmake --version echo "**** Configuring Ascent" cmake -S src -B build -C ascent-config.cmake -DCMAKE_INSTALL_PREFIX=install - name: Build Ascent @@ -82,7 +87,7 @@ jobs: mpiexec -n 2 ./build/ascent_mpi_render_example build_basic_pyvenv: name: Ubuntu Build Ascent GCC with Python VENV - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 env: CC: gcc CXX: g++ @@ -121,6 +126,12 @@ jobs: uses: actions/checkout@v4 with: submodules: 'recursive' + - name: Env Info + run: | + echo "**** Env Info" + cmake --version + gcc --version + which mpicc - name: Build TPLs run: | env enable_mpi=ON \ @@ -134,7 +145,6 @@ jobs: ./scripts/build_ascent/build_ascent.sh - name: Configure Ascent run: | - cmake --version echo "**** Configuring Ascent" cmake -S src -B build -C ascent-config.cmake -DCMAKE_INSTALL_PREFIX=install - name: Build Ascent diff --git a/.github/workflows/build_ascent_hip.yml b/.github/workflows/build_ascent_hip.yml index d22507be6..9ffe0f409 100644 --- a/.github/workflows/build_ascent_hip.yml +++ b/.github/workflows/build_ascent_hip.yml @@ -7,8 +7,8 @@ on: jobs: build_cuda: name: Build Ascent HIP - runs-on: ubuntu-latest - container: alpinedav/ascent-ci:ubuntu-20.04-rocm-5.1.3-devel + runs-on: ubuntu-20.04 + container: alpinedav/ascent-devel:ubuntu-20.04-rocm-6.3.0-x86_64 env: CMAKE_VERSION: 3.23.2 steps: @@ -28,7 +28,7 @@ jobs: enable_mpi=OFF \ enable_fortran=OFF \ enable_tests=OFF \ - build_ascent=OFF \ + build_ascent=false \ enable_verbose=OFF \ build_jobs=2 \ ./scripts/build_ascent/build_ascent.sh diff --git a/.github/workflows/build_ascent_icc.yml b/.github/workflows/build_ascent_icc.yml index cf1028437..d206db783 100644 --- a/.github/workflows/build_ascent_icc.yml +++ b/.github/workflows/build_ascent_icc.yml @@ -59,7 +59,7 @@ jobs: env enable_mpi=OFF \ enable_fortran=ON \ enable_tests=OFF \ - build_ascent=OFF \ + build_ascent=false \ enable_verbose=OFF \ build_jobs=2 \ ./scripts/build_ascent/build_ascent.sh diff --git a/scripts/build_ascent/build_ascent.sh b/scripts/build_ascent/build_ascent.sh index 4eef315bf..1be8429cf 100755 --- a/scripts/build_ascent/build_ascent.sh +++ b/scripts/build_ascent/build_ascent.sh @@ -24,6 +24,7 @@ set -eu -o pipefail # shared options enable_cuda="${enable_cuda:=OFF}" enable_hip="${enable_hip:=OFF}" +enable_sycl="${enable_sycl:=OFF}" enable_fortran="${enable_fortran:=OFF}" enable_python="${enable_python:=OFF}" enable_openmp="${enable_openmp:=OFF}" @@ -70,17 +71,19 @@ if [[ "$enable_cuda" == "ON" ]]; then CUDA_ARCH_VTKM="${CUDA_ARCH_VTKM:=ampere}" fi +# NOTE: this script only builds kokkos when enable_hip=ON or enable_cycl=ON if [[ "$enable_hip" == "ON" ]]; then echo "*** configuring with HIP support" CC="${CC:=/opt/rocm/llvm/bin/amdclang}" CXX="${CXX:=/opt/rocm/llvm/bin/amdclang++}" # FTN? - ROCM_ARCH="${ROCM_ARCH:=gfx90a}" ROCM_PATH="${ROCM_PATH:=/opt/rocm/}" - # NOTE: this script only builds kokkos when enable_hip=ON + build_kokkos="${build_kokkos:=true}" +elif [[ "$enable_sycl" == "ON" ]]; then + echo "*** configuring with SYCL support" build_kokkos="${build_kokkos:=true}" else build_kokkos="${build_kokkos:=false}" @@ -510,16 +513,16 @@ else echo "**** Skipping Conduit build, install found at: ${conduit_install_dir}" fi # build_conduit -######################### -# Kokkos (only for hip) -######################### -kokkos_version=3.7.02 +############################### +# Kokkos (only for hip or sycl) +############################### +kokkos_version=4.4.01 kokkos_src_dir=$(ospath ${source_dir}/kokkos-${kokkos_version}) kokkos_build_dir=$(ospath ${build_dir}/kokkos-${kokkos_version}) kokkos_install_dir=$(ospath ${install_dir}/kokkos-${kokkos_version}/) kokkos_tarball=$(ospath ${source_dir}/kokkos-${kokkos_version}.tar.gz) -if [[ "$enable_hip" == "ON" ]]; then +if [[ "$enable_hip" == "ON" ]] || [[ "$enable_sycl" == "ON" ]]; then # build only if install doesn't exist if [ ! -d ${kokkos_install_dir} ]; then if ${build_kokkos}; then @@ -529,21 +532,52 @@ if [ ! -d ${kokkos_src_dir} ]; then tar ${tar_extra_args} -xzf ${kokkos_tarball} -C ${source_dir} fi -# TODO: DKokkos_ARCH_VEGA90A needs to be controlled / mapped? +kokkos_extra_cmake_args="" +if [[ "$enable_hip" == "ON" ]]; then + kokkos_extra_cmake_args="-DKokkos_ENABLE_HIP=ON" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE=OFF" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_COMPILER=${ROCM_PATH}/bin/hipcc" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_EXTENSIONS=OFF" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_STANDARD=17" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ENABLE_ROCTHRUST=OFF" + + ## + ## build_ascent specific ROCM_ARCH Map for Kokkos options: + ## + ## TODO: Kokkos 4.5 has MI300A specific option, need to figure out how to + ## map hat in when we update. + ## + ## gfx942 --> Kokkos_ARCH_AMD_GFX942 (MI300A, MI300X) + ## (since Kokkos 4.2, since Kokkos 4.5 this should only be used for MI300X) + ## + ## gfx90a --> Kokkos_ARCH_AMD_GFX90A (MI200 series) + ## (since Kokkos 4.2) + ## + if [[ "$ROCM_ARCH" == "gfx942" ]]; then + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ARCH_AMD_GFX942=ON" + fi + + if [[ "$ROCM_ARCH" == "gfx90a" ]]; then + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ARCH_AMD_GFX90A=ON" + fi +fi + +if [[ "$enable_sycl" == "ON" ]]; then + kokkos_extra_cmake_args="-DCMAKE_CXX_FLAGS=-fPIC -fp-model=precise -Wno-unused-command-line-argument -Wno-deprecated-declarations -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=128" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ENABLE_SYCL=ON" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DKokkos_ARCH_INTEL_PVC=ON" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_EXTENSIONS=OFF" + kokkos_extra_cmake_args="${kokkos_extra_cmake_args} -DCMAKE_CXX_STANDARD=17" +fi echo "**** Configuring Kokkos ${kokkos_version}" cmake -S ${kokkos_src_dir} -B ${kokkos_build_dir} ${cmake_compiler_settings} \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=${enable_verbose}\ -DCMAKE_BUILD_TYPE=${build_config} \ -DBUILD_SHARED_LIBS=${build_shared_libs} \ - -DKokkos_ARCH_VEGA90A=ON \ - -DCMAKE_CXX_COMPILER=${ROCM_PATH}/bin/hipcc \ - -DKokkos_ENABLE_HIP=ON \ -DKokkos_ENABLE_SERIAL=ON \ - -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE=OFF \ -DCMAKE_INSTALL_PREFIX=${kokkos_install_dir} \ - -DCMAKE_CXX_FLAGS="--amdgpu-target=${ROCM_ARCH}" \ - -DBUILD_TESTING=OFF \ + -DBUILD_TESTING=OFF ${kokkos_extra_cmake_args} \ -DCMAKE_INSTALL_PREFIX=${kokkos_install_dir} echo "**** Building Kokkos ${kokkos_version}" @@ -556,7 +590,7 @@ else echo "**** Skipping Kokkos build, install found at: ${kokkos_install_dir}" fi # build_kokkos -fi # if enable_hip +fi # if enable_hip || enable_sycl ################ # VTK-m @@ -598,6 +632,13 @@ if [[ "$enable_hip" == "ON" ]]; then vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DVTKm_ENABLE_KOKKOS_THRUST=OFF" fi +if [[ "$enable_sycl" == "ON" ]]; then + vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DVTKm_ENABLE_KOKKOS=ON" + vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DCMAKE_PREFIX_PATH=${kokkos_install_dir}" + vtkm_extra_cmake_args="-DCMAKE_CXX_FLAGS=-fPIC -fp-model=precise -Wno-unused-command-line-argument -Wno-deprecated-declarations -fsycl-device-code-split=per_kernel -fsycl-max-parallel-link-jobs=128" +fi + + if [[ "$enable_mpicc" == "ON" ]]; then vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DMPI_C_COMPILER=${mpicc_exe}" vtkm_extra_cmake_args="${vtkm_extra_cmake_args} -DMPI_CXX_COMPILER=${mpicxx_exe}" @@ -608,7 +649,6 @@ cmake -S ${vtkm_src_dir} -B ${vtkm_build_dir} ${cmake_compiler_settings} \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=${enable_verbose}\ -DCMAKE_BUILD_TYPE=${build_config} \ -DBUILD_SHARED_LIBS=${build_shared_libs} \ - -DVTKm_NO_DEPRECATED_VIRTUAL=ON \ -DVTKm_USE_64BIT_IDS=OFF \ -DVTKm_USE_DOUBLE_PRECISION=ON \ -DVTKm_USE_DEFAULT_TYPES_FOR_ASCENT=ON \ @@ -763,6 +803,10 @@ if [[ "$enable_hip" == "ON" ]]; then umpire_extra_cmake_args="${umpire_extra_cmake_args} -DROCM_PATH=${ROCM_PATH}" fi +if [[ "$enable_sycl" == "ON" ]]; then + umpire_extra_cmake_args="${umpire_extra_cmake_args} -DENABLE_SYCL=ON" +fi + # build only if install doesn't exist if [ ! -d ${umpire_install_dir} ]; then if ${build_umpire}; then @@ -797,7 +841,7 @@ fi # build_umpire ################ # MFEM ################ -mfem_version=4.6 +mfem_version=4.7 mfem_src_dir=$(ospath ${source_dir}/mfem-${mfem_version}) mfem_build_dir=$(ospath ${build_dir}/mfem-${mfem_version}) mfem_install_dir=$(ospath ${install_dir}/mfem-${mfem_version}/) @@ -954,7 +998,6 @@ fi if ${build_caliper}; then echo 'set(CALIPER_DIR ' ${caliper_install_dir} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake fi -echo 'set(BLT_CXX_STD c++14 CACHE STRING "")' >> ${root_dir}/ascent-config.cmake echo 'set(CONDUIT_DIR ' ${conduit_install_dir} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake echo 'set(VTKM_DIR ' ${vtkm_install_dir} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake echo 'set(CAMP_DIR ' ${camp_install_dir} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake @@ -977,6 +1020,7 @@ fi if [[ "$enable_hip" == "ON" ]]; then echo 'set(ENABLE_HIP ON CACHE BOOL "")' >> ${root_dir}/ascent-config.cmake + echo 'set(BLT_CXX_STD c++17 CACHE STRING "")' >> ${root_dir}/ascent-config.cmake echo 'set(CMAKE_HIP_ARCHITECTURES ' ${ROCM_ARCH} ' CACHE STRING "")' >> ${root_dir}/ascent-config.cmake echo 'set(ROCM_PATH ' ${ROCM_PATH} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake echo 'set(KOKKOS_DIR ' ${kokkos_install_dir} ' CACHE PATH "")' >> ${root_dir}/ascent-config.cmake diff --git a/scripts/build_ascent/build_ascent_aurora.sh b/scripts/build_ascent/build_ascent_aurora.sh new file mode 100755 index 000000000..7c67ea105 --- /dev/null +++ b/scripts/build_ascent/build_ascent_aurora.sh @@ -0,0 +1,21 @@ +#!/bin/bash -l + +export IGC_FunctionCloningThreshold=1 +export IGC_ControlInlineTinySize=100 +export IGC_OCLInlineThreshold=200 +export IGC_PartitionUnit=1 +export IGC_ForceOCLSIMDWidth=16 +export ZE_AFFINITY_MASK=0.0 + +# Proxies +export HTTP_PROXY=http://proxy.alcf.anl.gov:3128 +export HTTPS_PROXY=http://proxy.alcf.anl.gov:3128 +export http_proxy=http://proxy.alcf.anl.gov:3128 +export https_proxy=http://proxy.alcf.anl.gov:3128 + +module reset +module use /soft/modulefiles +module load spack-pe-gcc cmake +module load oneapi/eng-compiler/2023.12.15.002 + +env CC=`which icx` CXX=`which icpx` FTN=`which ifx` enable_sycl=ON enable_mpi=ON enable_fortran=ON raja_enable_vectorization=OFF enable_tests=ON enable_verbose=ON ./build_ascent_sycl.sh diff --git a/scripts/build_ascent/build_ascent_sycl.sh b/scripts/build_ascent/build_ascent_sycl.sh new file mode 100755 index 000000000..4f97bef9e --- /dev/null +++ b/scripts/build_ascent/build_ascent_sycl.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +############################################################################## +# Demonstrates how to manually build Ascent and its dependencies, including: +# +# hdf5, conduit, vtk-m, mfem, raja, and umpire +# +# usage example: +# env enable_mpi=ON enable_openmp=ON ./build_ascent.sh +# +# +# Assumes: +# - cmake is in your path +# - selected compilers are in your path or set via env vars +# - [when enabled] MPI and Python (+numpy and mpi4py), are in your path +# +############################################################################## +set -eu -o pipefail + +# 2024-02-08 SYCL support is handled by our unified script +env enable_sycl=ON ./build_ascent.sh diff --git a/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/Dockerfile b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/Dockerfile index 26708e861..1d4f8ada8 100644 --- a/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/Dockerfile +++ b/scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/Dockerfile @@ -65,6 +65,7 @@ RUN apt-get update && apt-get -y install \ rocfft-dev \ rocprim-dev \ rocrand-dev \ + rocthrust-dev \ hiprand-dev \ openssh-server \ language-pack-en \