aomp_common_vars

#!/bin/bash
#
#  aomp_common_vars: This file is sourced by all AOMP build scripts.
#
#     This file is stored in aomp/bin directory of the aomp repo.
#     You don't need to change this file unless you want to change
#     default values for all AOMP developers. To customize your build,
#     set environment variables defined in this file.
#
#     The bash syntax ENVNAME=${ENVNAME:-value} checks if ENVNAME is
#     already set. If not, it assigns it the value following the :-

# --- Start standard header to set AOMP environment variables ----
realpath=`realpath $0`
thisdir=`dirname $realpath`

# Individual scripts will determine what generator to use
unset CMAKE_GENERATOR

# The AOMP_COMPILER_NAME is used whenever version is queried.
# FIXME: More work is needed to propogate the use of AOMP_COMPILER_NAME.
AOMP_COMPILER_NAME=${AOMP_COMPILER_NAME:-AOMP}

# ROCM_VERSION may be set by Jenkins, if not guess a good number
ROCM_VERSION=${ROCM_VERSION:-6.0.0}

# Set the AOMP VERSION STRING
AOMP_VERSION=${AOMP_VERSION:-"21.0"}
AOMP_VERSION_MOD=${AOMP_VERSION_MOD:-"1"}
AOMP_VERSION_STRING=${AOMP_VERSION_STRING:-"$AOMP_VERSION-$AOMP_VERSION_MOD"}
ROCM_EXPECTED_MODVERSION=${ROCM_EXPECTED_MODVERSION:-6.2.4}
export AOMP_VERSION_STRING AOMP_VERSION AOMP_VERSION_MOD ROCM_VERSION ROCM_EXPECTED_MODVERSION

# Set install directory and the link directory.
# AOMP will be a symbolic link AOMP_INSTALL_DIR is the versioned dir name
AOMP=${AOMP:-$HOME/rocm/aomp}
AOMP_INSTALL_DIR=${AOMP}_${AOMP_VERSION_STRING}

GPURUN_BINDIR=${GPURUN_BINDIR:-$AOMP/bin}
if [ ! -f "$GPURUN_BINDIR/gpurun" ] || [ ! -f "$GPURUN_BINDIR/rocminfo" ] ; then
    # When using trunk, try to find gpurun and rocminfo in ROCm
    _SILENT=""
    GPURUN_BINDIR=/opt/rocm/llvm/bin
    export ROCMINFO_BINARY=/opt/rocm/bin/rocminfo
else
    export ROCMINFO_BINARY=$GPURUN_BINDIR/rocminfo
fi

# For example, if AOMP_REPOS=$HOME/git/aomp11, then the primary aomp repo
# is stored in $HOME/git/aomp11/aomp and the mono repo is stored in
# $HOME/aomp/llvm-project.
AOMP_MAJOR_VERSION=${AOMP_VERSION%.*}
export AOMP_MAJOR_VERSION
AOMP_REPOS=${AOMP_REPOS:-${HOME}/git/aomp${AOMP_VERSION}}
if [ "$AOMP_NEW" == 1 ] ; then
   AOMP_PATCH_CONTROL_FILE="$thisdir/patches/patch-control-file-new_$AOMP_VERSION.txt"
else
   AOMP_PATCH_CONTROL_FILE="$thisdir/patches/patch-control-file_$AOMP_VERSION.txt"
fi

# Because the source for all repos are tarred for release, we store
# all the test repositories in a different directory.
AOMP_REPOS_TEST=${AOMP_REPOS_TEST:-$HOME/git/aomp-test}

_ccache_bin=`which ccache`
if [ "$_ccache_bin" == "" ] ; then
AOMP_USE_CCACHE=${AOMP_USE_CCACHE:-0}
else
AOMP_USE_CCACHE=${AOMP_USE_CCACHE:-1}
fi
if [ "$AOMP_USE_CCACHE" == 0 ] ; then
  AOMP_CCACHE_OPTS=""
else
  _ccache_bin=`which ccache`
  if [ "$_ccache_bin" == "" ] ; then
    echo "ERROR: AOMP_USE_CCACHE is $AOMP_USE_CCACHE but no ccache bin found in PATH"
    echo "       Either fix path,  install ccache, or set AOMP_USE_CCACHE=0"
    echo
    exit 1
  fi
  AOMP_CCACHE_OPTS="-DCMAKE_C_COMPILER_LAUNCHER=$_ccache_bin -DCMAKE_CXX_COMPILER_LAUNCHER=$_ccache_bin"
fi

AOMP_SKIP_FLANG=${AOMP_SKIP_FLANG:-1}
AOMP_SKIP_FLANG_NEW=${AOMP_SKIP_FLANG_NEW:-0}
AOMP_FLANG_DRIVER=${AOMP_FLANG_DRIVER:-flang}

if [ "$AOMP_SKIP_FLANG_NEW" == 1 ] ; then
   AOMP_PROJECTS_LIST=${AOMP_PROJECTS_LIST:-"clang;lld;clang-tools-extra"}
else
   AOMP_PROJECTS_LIST=${AOMP_PROJECTS_LIST:-"clang;lld;clang-tools-extra;flang"}
fi
AOMP_COMPONENT_LIST=${AOMP_COMPONENT_LIST:-"prereq project"}

# AOMP_SUPP is for SUPPlemental software components needed for testing.
# such as openmpi, hdf5, and silo. Applications that need supplemental
# components can get the latest version with $AOMP_SUPP/<component name>
# For example set OPENMPI_DIR=$AOMP_SUPP/openmpi in the run script.
# Supplemental components are built with either the ROCm or AOMP
# compiler using the build_supp.sh script. The build_supp.sh script
# will use the AOMP environment variable to identify which LLVM to use
# for building the supplemental component.
# The directory structure for a supplemental component includes:
#   1. a build directory for each component,
#   2. an install directory for each version of a component,
#   3. a link from AOMP_SUPP/<component name> to latest version.
# For example: The openmpi component is built in directory
# $AOMP_SUPP_BUILD/openmpi/ and installed in directory
# $AOMP_SUPP_INSTALL/openmpi/openmpi-4.1.1/ with a symbolic link from
# $AOMP_SUPP/openmpi to $AOMP_SUPP_INSTALL/openmpi/openmpi-4.1.1/
#
# The default AOMP_SUPP directory is $HOME/local but a sysadmin with root
# root authority could set AOMP_SUPP=/usr/local
AOMP_SUPP=${AOMP_SUPP:-$HOME/local}
#
# AOMP_SUPP_BUILD could be a temp directory but defaults to $AOMP_SUPP/build
AOMP_SUPP_BUILD=${AOMP_SUPP_BUILD:-$AOMP_SUPP/build}
#
# AOMP_SUPP_INSTALL must be permanent directory.
AOMP_SUPP_INSTALL=${AOMP_SUPP_INSTALL:-$AOMP_SUPP/install}

# Set the directory location where all local AOMP git repos are stored.

# To build aomp from the release source taball, set these values to 0
AOMP_APPLY_ROCM_PATCHES=${AOMP_APPLY_ROCM_PATCHES:-1}

#  This applies ATD_ASO_full.patch to llvm-project
AOMP_APPLY_ATD_AMD_STAGING_PATCH=${AOMP_APPLY_ATD_AMD_STAGING_PATCH:-0}

CUDA=${CUDA:-/usr/local/cuda}
CUDAT=${CUDAT:-$CUDA/targets}
CUDAINCLUDE=${CUDAINCLUDE:-$CUDA/include}
CUDABIN=${CUDABIN:-$CUDA/bin}

# unless overridden with AOMP_CMAKE, get CMAKE from the
# prerequisite supplemental component cmake.
AOMP_CMAKE=${AOMP_CMAKE:-$AOMP_SUPP/cmake/bin/cmake}
if [ ! -f $AOMP_CMAKE ] ; then
  AOMP_CMAKE=`which cmake`
  if [ "$AOMP_CMAKE" == "" ] ; then
    echo "ERROR: finding cmake in aomp_common_vars."
    echo "       Missing prerequisite supplemental component cmake."
    echo "       Please run: ./build_prereq.sh cmake"
    echo "       OR set AOMP_CMAKE env variable to a valid cmake binary"
    echo "       OR build ALL prerequisite components with: ./build_prereq.sh"
    echo
    echo "NOTE:  All supplemental components are built and retrieved from "
    echo "       subdirectories of the directory defined by environment variable"
    echo "       AOMP_SUPP whose current value is: $AOMP_SUPP"
    exit 1
  else
    # FIXME: Add version check here for system cmake.
    echo "WARNING: Using system cmake $AOMP_CMAKE "
    echo "         consider building cmake with: ./build_prereq.sh cmake"
  fi
fi

AOMP_USE_NINJA=${AOMP_USE_NINJA:-0}
if [ "$AOMP_USE_NINJA" == 0 ] ; then
  AOMP_NINJA_BIN="make"
else
  AOMP_NINJA_BIN=`which ninja`
  if [ AOMP_NINJA_BIN == "" ] ; then
    echo "ERROR: AOMP_USE_NINJA is $AOMP_USE_NINJA but no ninja bin found in PATH"
    echo "       Either install ninja in PATH or set AOMP_USE_NINJA=0"
    echo
    exit 1
  fi
fi
AOMP_PROC=`uname -m`
if [ "$AOMP_PROC" == "unknown" ] ; then
   AOMP_PROC=`uname -a | awk '{print $(NF-1)}'`
fi

GCC=`which gcc`
GCCXX=`which g++`
AOMP_CC_COMPILER=${AOMP_CC_COMPILER:-$GCC}
AOMP_CXX_COMPILER=${AOMP_CXX_COMPILER:-$GCCXX}

# Non standalone builds are done with ROCm builds.
AOMP_STANDALONE_BUILD=${AOMP_STANDALONE_BUILD:-1}

if [ "$AOMP_STANDALONE_BUILD" == 1 ] ; then
  LLVM_INSTALL_LOC=$AOMP_INSTALL_DIR/lib/llvm
else
  LLVM_INSTALL_LOC=$INSTALL_PREFIX/lib/llvm
fi

# Set specified clang and clang++ compilers with env vars AOMP_CLANG*_COMPILER.
# Otherwise default to installed AOMP. If not yet installed, try rocm clang.
_clang_compiler=$LLVM_INSTALL_LOC/bin/clang
_clangxx_compiler=$LLVM_INSTALL_LOC/bin/clang++
[ ! -f $_clang_compiler ] && _clang_compiler=/opt/rocm/llvm/bin/clang
[ ! -f $_clangxx_compiler ] && _clangxx_compiler=/opt/rocm/llvm/bin/clang++
AOMP_CLANG_COMPILER=${AOMP_CLANG_COMPILER:-$_clang_compiler}
AOMP_CLANGXX_COMPILER=${AOMP_CLANGXX_COMPILER:-$_clangxx_compiler}

if [ "$AOMP_STANDALONE_BUILD" == 0 ] ; then
  # ROCm still uses standalone openmp build.
  AOMP_LEGACY_OPENMP=${AOMP_LEGACY_OPENMP:-1}
else
  # Do not build openmp as standalone unless user specifies.
  AOMP_LEGACY_OPENMP=${AOMP_LEGACY_OPENMP:-0}
fi

#  The logic for turning on AOMP_BUILD_CUDA build is here
if [ "$AOMP_PROC" == "ppc64le" ] ; then
   AOMP_BUILD_CUDA=1
   AOMP_STANDALONE_BUILD=1
fi
AOMP_BUILD_DEBUG=${AOMP_BUILD_DEBUG:-1}

if [ $AOMP_STANDALONE_BUILD == 1 ] ; then
   # Default is to build nvptx for STANDALONE build
   AOMP_BUILD_CUDA=${AOMP_BUILD_CUDA:-1}
   AOMP_BUILD_PERF=${AOMP_BUILD_PERF:-1}
   AOMP_BUILD_SANITIZER=${AOMP_BUILD_SANITIZER:-1}
else
   # Default is to NOT build nvptx for ROCm integrated build
   AOMP_BUILD_CUDA=${AOMP_BUILD_CUDA:-0}
   AOMP_BUILD_PERF=${AOMP_BUILD_PERF:-0}
   # SANITIZER is used for ROCm builds, AOMP should use AOMP_BUILD_SANITIZER=1
   if [ "$SANITIZER" == 1 ]; then
      AOMP_BUILD_SANITIZER=${AOMP_BUILD_SANITIZER:-1}
   else
      AOMP_BUILD_SANITIZER=${AOMP_BUILD_SANITIZER:-0}
   fi
fi


# AOMP uses RPATH (not) RUNPATH because LD_LIBRARY_PATH is a user feature for
# user lib development. We do not want the compiler runtime to search user paths
# first because we want assurance to recreate reported compiler runtime fails.
# While it may appear to hamper users and developers from testing experimental
# compiler runtimes, this policy enforces replacement of installed compiler
# runtime for testing instead of path manipulation. AOMP uses $ORIGIN in the
# RPATH to allow installation relocation. All scripts now use $AOMP_ORIGIN_RPATH,
# AOMP_DEBUG_ORIGIN_RPATH, AOMP_ASAN_ORIGIN_RPATH, or OPENMP_EXTRAS_LIB_RPATH.
# Spliting options with AOMP_ORIGIN_NO_DTAGS is no longer necessary.

AOMP_ORIGIN_RPATH_LIST="\$ORIGIN:\$ORIGIN/../lib:\$ORIGIN/../lib64:\$ORIGIN/../../../lib:\$ORIGIN/../llvm/lib"
AOMP_ORIGIN_RPATH="-DCMAKE_SHARED_LINKER_FLAGS='-Wl,--disable-new-dtags' -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_RPATH=$AOMP_ORIGIN_RPATH_LIST"
AOMP_DEBUG_ORIGIN_RPATH_LIST="\$ORIGIN:\$ORIGIN/../lib-debug:\$ORIGIN/../lib64:\$ORIGIN/../../../lib-debug"
AOMP_DEBUG_ORIGIN_RPATH="-DCMAKE_SHARED_LINKER_FLAGS='-Wl,--disable-new-dtags' -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_RPATH=$AOMP_DEBUG_ORIGIN_RPATH_LIST"

# Libraries and Binaries need different RUNPATH as below
if [ "$AOMP_BUILD_SANITIZER" == 1 ]; then
  ## Below 3 lines are actually only required for AOMP ASan RPATH Cmake configuration.
  AOMP_ASAN_ORIGIN_RPATH_LIST="\$ORIGIN:\$ORIGIN/../../lib:\$ORIGIN/../lib:\$ORIGIN/../lib64:\$ORIGIN/../../../../lib:\$ORIGIN/../lib/llvm"
  AOMP_ASAN_ORIGIN_RPATH="-DCMAKE_SHARED_LINKER_FLAGS='-Wl,--disable-new-dtags' -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=OFF -DCMAKE_INSTALL_RPATH=$AOMP_ASAN_ORIGIN_RPATH_LIST"
  # Please note below lines are only setup for ROCm ASan integration with AOMP.
  # Relative RUNPATH for ASAN binaries and libraries
  OPENMP_EXTRAS_LIB_RPATH='$ORIGIN:$ORIGIN/../../../lib/asan:$ORIGIN/../../../lib:$ORIGIN/../../../../lib/asan:$ORIGIN/../../../../lib'
  OPENMP_EXTRAS_EXE_RPATH='$ORIGIN/../lib/asan:$ORIGIN/../lib:$ORIGIN/../../lib/asan:$ORIGIN/../../lib:$ORIGIN/../../../lib/asan:$ORIGIN/../../../lib'
else
  # Relative RUNPATH for non-ASAN binaries and libraries
  OPENMP_EXTRAS_LIB_RPATH='$ORIGIN:$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib'
  OPENMP_EXTRAS_EXE_RPATH='$ORIGIN/../lib:$ORIGIN/../../lib:$ORIGIN/../../../lib'
fi

OPENMP_EXTRAS_SHARED_LINKER_FLAGS="-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$OPENMP_EXTRAS_LIB_RPATH"
OPENMP_EXTRAS_EXE_LINKER_FLAGS="-Wl,--enable-new-dtags,--build-id=sha1,--rpath,$OPENMP_EXTRAS_EXE_RPATH"
OPENMP_EXTRAS_ORIGIN_RPATH="-DCMAKE_SKIP_INSTALL_RPATH=TRUE -DCMAKE_SHARED_LINKER_FLAGS_INIT=${OPENMP_EXTRAS_SHARED_LINKER_FLAGS} -DCMAKE_EXE_LINKER_FLAGS_INIT=${OPENMP_EXTRAS_EXE_LINKER_FLAGS} -DCMAKE_MODULE_LINKER_FLAGS_INIT=${OPENMP_EXTRAS_SHARED_LINKER_FLAGS}"

# Regardless, turn off AOMP_BUILD_CUDA if CUDA not available
if [ ! -d "$CUDABIN" ] ; then
   AOMP_BUILD_CUDA=0
fi
# Regardless, turn off AOMP_BUILD_CUDA for arm
if [ "$AOMP_PROC" == "aarch64" ] ; then
   AOMP_BUILD_CUDA=0
fi

# Set list of default nvptx subarchitectures to build
# Only Cuda 9 and above supports sm_70
if [ "$AOMP_BUILD_CUDA" == 1 ] ; then
   NVPTXGPUS_DEFAULT="35,37,50,52,53,60,61,62,70,72,75,80,86"
   #  Developers should override NVPTXGPUS to shorten build time.
   NVPTXGPUS=${NVPTXGPUS:-"${NVPTXGPUS_DEFAULT}"}
   AOMP_NVPTX_TARGET="NVPTX;"
else
   NVPTXGPUS_DEFAULT=""
   NVPTXGPUS=""
   AOMP_NVPTX_TARGET=""
fi

# Set list of default amdgcn subarchitectures to build
# Developers may override GFXLIST to shorten build time.
GFXLIST=${GFXLIST:-"gfx700 gfx701 gfx801 gfx803 gfx900 gfx902 gfx906 gfx908 gfx90a gfx90c gfx942 gfx950 gfx1030 gfx1031 gfx1035 gfx1036 gfx1100 gfx1101 gfx1102 gfx1103 gfx1150 gfx1151 gfx1152 gfx1153 gfx1200 gfx1201 gfx9-generic gfx9-4-generic gfx10-1-generic gfx10-3-generic gfx11-generic gfx12-generic"}
export GFXLIST

# _gfxlist is used in the build of rocmlibs. Some of the hip math libraries do not support older architectures.
_sep=""

# May use this later to filter the gfxlist. For now rocblas only accepts the subset below
#for _arch in $GFXLIST ; do
#  if [[ $_arch =~ "gfx7" ]] || [[ $_arch =~ "generic" ]]; then
#    continue
#  elif [ $_arch == "gfx90a" ] ; then
#    _gfxlist+="${_sep}gfx90a:xnack-"
#    _gfxlist+=";gfx90a:xnack+"
#  else
#    _gfxlist+=${_sep}$_arch
#  fi
#  _sep=";"
#done

ROCMLIBS_GFXLIST=${ROCMLIBS_GFXLIST:-"gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a;gfx942;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102;gfx1151;gfx1200;gfx1201"}
_gfxlist="$ROCMLIBS_GFXLIST"

# Calculate the number of threads to use for make
COMP_THREADS=1
if [ ! -z `which "getconf"` ]; then
   COMP_THREADS=$(`which "getconf"` _NPROCESSORS_ONLN)
   if [ "$AOMP_PROC" == "ppc64le" ] ; then
      COMP_THREADS=$(( COMP_THREADS / 6))
   fi
   if [ "$AOMP_PROC" == "aarch64" ] ; then
      COMP_THREADS=$(( COMP_THREADS / 4))
   fi
fi
AOMP_JOB_THREADS=${AOMP_JOB_THREADS:-$COMP_THREADS}

if [ "$AOMP_LIMIT_FLANG" == "1" ] ; then
   # Limit build jobs for llvm flang build.
   FLANG_THREADS=1

   # Third column of free is the amount of available memory
   memory_regex="[0-9]+\s+[0-9]+\s+([0-9]+)"
   free_memory=$(free -g)

   if [[ "$free_memory" =~ $memory_regex ]]; then
     memory_size=${BASH_REMATCH[1]}
     # 6 GB estimate per thread for MLIR
     FLANG_THREADS=$((memory_size/6))
   fi

   AOMP_FLANG_THREADS=${AOMP_FLANG_THREADS:-$FLANG_THREADS}
fi

# These are the web sites where the AOMP git repos are pulled from
AOMP_GIT_INTERNAL_IP="git@github-ent"
GITROC="https://github.com/ROCm"
GITGERRIT="ssh://$USER@gerrit-git.amd.com:29418"
GITKHRONOS="https://github.com/KhronosGroup"
GITHWLOC="https://github.com/open-mpi"
GITLIGHTNINGINTERNAL="$AOMP_GIT_INTERNAL_IP:AMD-Lightning-Internal"
GITPROJECT=$GITROCDEV
POWERINFER_REPO="https://github.com/SJTU-IPADS"
if [ "$AOMP_STANDALONE_BUILD" == 1 ]; then
  AOMP_PROJECT_REPO_NAME=${AOMP_PROJECT_REPO_NAME:-llvm-project}
else
  AOMP_PROJECT_REPO_NAME=${AOMP_PROJECT_REPO_NAME:-$(basename $LLVM_PROJECT_ROOT)}
  if [ ! -d "$AOMP_REPOS/$AOMP_PROJECT_REPO_NAME" ]; then
    AOMP_PROJECT_REPO_NAME="../llvm-project"
  fi
fi
AOMP_EXTRAS_REPO_NAME=${AOMP_EXTRAS_REPO_NAME:-aomp-extras}
AOMP_FLANG_REPO_NAME=${AOMP_FLANG_REPO_NAME:-flang}

# Last version of rocm (5.5) that supports the legacy driver (command generation)
# The archives are built during flang-classic.
AOMP_FLANG_CLASSIC_REL=${AOMP_FLANG_CLASSIC_REL:-${AOMP_REPOS}/${AOMP_FLANG_REPO_NAME}/flang-classic}
if [ -d $AOMP_FLANG_CLASSIC_REL/llvm-classic ] ; then
   AOMP_BUILD_FLANG_CLASSIC=1
else
   AOMP_BUILD_FLANG_CLASSIC=0
fi

# The AOMP_XXX_REPO_NAME is the directory name following $AOMP_REPOS.

# External component names for LLVM_EXTERNAL_PROJECTS may have no special
# characters in them.  Therefore, AOMP_xxx_REPO_NAME is different than
# AOMP_xxx_COMPONENT NAME for some components. Recommend that component
# names be short lowercase names.

#  These aomp development repositories
AOMP_REPO_NAME=${AOMP_REPO_NAME:-aomp}

#  These extra repositories are needed but we cannot update them
AOMP_ROCT_REPO_NAME=${AOMP_ROCT_REPO_NAME:-roct-thunk-interface}
AOMP_ROCT_COMPONENT_NAME=${AOMP_ROCT_COMPONENT_NAME:-roct}
AOMP_ROCR_REPO_NAME=${AOMP_ROCR_REPO_NAME:-rocr-runtime}
AOMP_ROCR_COMPONENT_NAME=${AOMP_ROCR_COMPONENT_NAME:-rocr}
AOMP_LIBDEVICE_REPO_NAME=${AOMP_LIBDEVICE_REPO_NAME:-device-libs}
AOMP_LIBDEVICE_COMPONENT_NAME=${AOMP_LIBDEVICE_COMPONENT_NAME:-rocdl}
DEVICELIBS_ROOT=${DEVICELIBS_ROOT:-$AOMP_REPOS/$AOMP_LIBDEVICE_REPO_NAME}
AOMP_COMGR_REPO_NAME=${AOMP_COMGR_REPO_NAME:-comgr}
AOMP_RINFO_REPO_NAME=${AOMP_RINFO_REPO_NAME:-rocminfo}

AOMP_ROCMCMAKE_REPO_NAME=${AOMP_ROCMCMAKE_REPO_NAME:-rocm-cmake}

# These are needed when AOMP_BUILD_DEBUG=1
AOMP_GDB_REPO_NAME=${AOMP_GDB_REPO_NAME:-ROCgdb}
AOMP_DBGAPI_REPO_NAME=${AOMP_DBGAPI_REPO_NAME:-ROCdbgapi}
AOMP_PROF_REPO_NAME=${AOMP_PROF_REPO_NAME:-rocprofiler}
AOMP_PROF_REGISTER_REPO_NAME=${AOMP_PROF_REGISTER_REPO_NAME:-rocprofiler-register}
AOMP_TRACE_REPO_NAME=${AOMP_TRACE_REPO_NAME:-roctracer}

# These are test repositories
GITNEKBONE="https://github.com/AMDComputeLibraries"
AOMP_NEKBONE_REPO_NAME=${AOMP_NEKBONE_REPO_NAME:-Nekbone}
AOMP_APPS_REPO_NAME=${AOMP_APPS_REPO_NAME:-openmpapps}
GITOPENMPVV="https://github.com/OpenMP-Validation-and-Verification"
AOMP_OPENMPVV_REPO_NAME=${AOMP_OPENMPVV_REPO_NAME:-OpenMP_VV}
GITSOLVV="https://github.com/SOLLVE"
AOMP_SOLVV_REPO_NAME=${AOMP_SOLVV_REPO_NAME:-sollve_vv}
AOMP_APPS_REPO_NAME=${AOMP_APPS_REPO_NAME:-openmpapps}
GITLLNL="https://github.com/llnl"
AOMP_RAJA_REPO_NAME=${AOMP_RAJA_REPO_NAME:-raja}
AOMP_RAJAPERF_REPO_NAME=${AOMP_RAJAPERF_REPO_NAME:-RAJAPerf}
GITKOKKOS="https://github.com/kokkos"
AOMP_KOKKOS_REPO_NAME=${AOMP_KOKKOS_REPO_NAME:-kokkos}
AOMP_INTERNAL_IP="gitlab1.amd.com"
GITINTERNAL="http://${AOMP_INTERNAL_IP}/dmcdouga"  # This will change soon
GITOVO="https://github.com/TApplencourt"
AOMP_OVO_REPO_NAME=${AOMP_OVO_REPO_NAME:-OvO}
GITOMPTESTS="https://github.com/doru1004"
AOMP_OMPTESTS_REPO_NAME=${AOMP_OMPTESTS_REPO_NAME:-omptests}
GITQMCPACK="https://github.com/QMCPACK"
AOMP_QMCPACK_REPO_NAME=${AOMP_QMCPACK_REPO_NAME:-qmcpack}
GITOPENLIBM="https://github.com/JuliaMath"
AOMP_OPENLIBM_REPO_NAME=${AOMP_OPENLIBM_REPO_NAME:-openlibm}
GITLLNLGOULASH="https://github.com/LLNL"
AOMP_GOULASH_REPO_NAME=${AOMP_GOULASH_REPO_NAME:-goulash}
AOMP_BABELSTREAM_REPO_NAME=${AOMP_BABELSTREAM_REPO_NAME:-babelstream}
GITUMT="https://github.com/LLNL"
AOMP_UMT_REPO_NAME=${AOMP_UMT_REPO_NAME:-UMT}

#  We dont use this yet, waiting for trunk merge
AOMP_F18_REPO_NAME=${AOMP_F18_REPO_NAME:-f18}

if [ -d $AOMP_REPOS/$AOMP_TRACE_REPO_NAME ] ; then
   pushd $AOMP_REPOS/$AOMP_TRACE_REPO_NAME > /dev/null
   roctracer_branch=`git branch --list | grep "\*" | cut -d" " -f2`
   if [ "$roctracer_branch" == "rocm-4.0.x" ] ; then
      AOMP_PATCH_CONTROL_FILE="${AOMP_REPOS}/${AOMP_REPO_NAME}/bin/patches/patch-control-file_4.0.txt"
   fi
   popd > /dev/null
fi
AOMP_PATCH_CONTROL_FILE=${AOMP_PATCH_CONTROL_FILE:-${AOMP_REPOS}/${AOMP_REPO_NAME}/bin/patches/patch-control-file.txt}

STASH_BEFORE_PULL=${STASH_BEFORE_PULL:-YES}

# It is highly recommendded that developers set SUDO to disable and 
# this have update access to install directory $AOMP. In the future
# we may set this default to disable 
SUDO=${SUDO:-NO}
if [ "$SUDO" == "set" ]  || [ "$SUDO" == "yes" ] || [ "$SUDO" == "YES" ] ; then
   SUDO="sudo"
else
   SUDO=""
fi

# The default for BUILD_AOMP is AOMP_REPOS. 
# The cmake and make builds are actually done in 
# BUILD_AOMP/build, not the actual repos. 
BUILD_AOMP=${BUILD_AOMP:-$AOMP_REPOS}
BUILD_DIR=$BUILD_AOMP

# Here is where we define different behaviors for STANDALONE build
# and the new ROCm integrated build. The default is STANDALONE build.
if [ $AOMP_STANDALONE_BUILD == 1 ] ; then
   # Standalone build gets ROCm components from the AOMP Installation.
   # As a result their is an order to component builds.
   ROCM_DIR=${ROCM_DIR:-$AOMP_INSTALL_DIR}
   # We hardcode the ROCM_VERSION here for standalone builds
else
   # ROCm integrated build gets ROCm pieces from installed ROCm
   ROCM_DIR=${ROCM_DIR:-/opt/rocm}

   # For Integrated ROCm build we do not install to versioned dir and create 
   # a symbolic link.  Instead we install directly to /opt/rocm/aomp
   AOMP_INSTALL_DIR=${AOMP}
fi

if [ "$AOMP_BUILD_SANITIZER" == 1 ]; then
   if [ -f "${LLVM_INSTALL_LOC}/bin/clang" ] ; then
      ASAN_LIB_PATH=$(${LLVM_INSTALL_LOC}/bin/clang --print-runtime-dir)
      if [ ! -d $ASAN_LIB_PATH ] ; then
         # --print-runtime-dir outputs the new lib directory that includes the host target.
         # Workaround this for now until LLVM_ENABLE_PER_TARGET_RUNTIME_DIR is on by default.
         RESOURCE_PATH=$(${LLVM_INSTALL_LOC}/bin/clang --print-resource-dir)
         ASAN_LIB_PATH="$RESOURCE_PATH/lib/linux"
      fi
   else
      # no compiler at $AOMP so set where we expect ASAN_LIP_PATH to be
      ASAN_LIB_PATH=${LLVM_INSTALL_LOC}/lib/clang/${AOMP_MAJOR_VERSION}/lib/linux
   fi
   ASAN_FLAGS="-g -fsanitize=address -shared-libasan -Wl,-rpath=$ASAN_LIB_PATH -L$ASAN_LIB_PATH"
   if [ "$AOMP_STANDALONE_BUILD" == 0 ]; then
     ASAN_FLAGS="$ASAN_FLAGS -I$ROCM_DIR/include -I$ROCM_DIR/include/hsa"
   fi
fi

#  TO use this function set variables patchdir and patchfile
function patchrepo(){
patchdir=$1
if [ "$AOMP_APPLY_ROCM_PATCHES" == 1 ] && [ -d "$patchdir" ] ; then
   patches=""
   cd $patchdir
   if [[ "$2" =~ "postinstall" ]]; then
     getpatchlist $2
   else
     getpatchlist
   fi

   #loop through list of patches to apply
   if [ "$patches" != "" ] ; then
   patchloc=${AOMP_PATCH_CONTROL_FILE%/*}
   echo patchloc=$patchloc
   for patch in $patches; do
     patchfile=$patchloc/$patch
     if [ ! -f $patchfile ] ; then
       echo
       echo "ERROR: Patch $patchfile does not exist"
       echo "       It is referenced in ${AOMP_PATCH_CONTROL_FILE}"
       exit 1
     fi

     echo "Testing patch $patchfile to $patchdir"

     applypatch="yes"
     patch -p1 -t -N --dry-run <$patchfile >/dev/null
     if [ $? != 0 ] ; then
        applypatch="no"
        # Check to see if reverse patch applies cleanly
        patch -p1 -R --dry-run -t <$patchfile >/dev/null
        if [ $? == 0 ] ; then
           echo "patch $patchfile was already applied to $patchdir"
        else
           echo
           echo "ERROR: Patch $patchfile will not apply"
           echo "       cleanly to directory $patchdir"
           echo "       Check if it was already applied."
           echo
           exit 1
        fi
     fi
     if [ "$applypatch" == "yes" ] ; then
        echo "Applying patch $patchfile to $patchdir"
        patch -p1 --no-backup-if-mismatch <$patchfile
     fi
   done
   fi
fi

}

function removepatch(){
patchdir=$1
if [ "$AOMP_APPLY_ROCM_PATCHES" == 1 ] && [ -d "$patchdir" ] ; then
   patches=""
   cd $patchdir
   getpatchlist
   if [ "$patches" != "" ] ; then
      echo "Patchdir $patchdir"
      echo "PATCHES TO REMOVE: $patches"
   fi
   patchloc=${AOMP_PATCH_CONTROL_FILE%/*}
   if [ "$patches" != "" ] ; then
   for patch in $patches; do
     patchfile=$patchloc/$patch
     echo "Testing reverse patch $patchfile to $patchdir"
     reversepatch="yes"
     # Check to see if reverse patch applies cleanly
     patch -p1 -R --dry-run -f <$patchfile  >/dev/null
     if [ $? != 0 ] ; then
        echo "patch $patchfile was NOT applied $patchdir, no patch to reverse"
        reversepatch="no"
     fi
     if [ "$reversepatch" == "yes" ] ; then
        echo "Reversing patch $patchfile to $patchdir"
        patch -p1 -R -f --no-backup-if-mismatch <$patchfile
     fi
   done
   fi
fi
}

function getpatchlist(){
   currdir=$(pwd)
   basedir=$(basename $currdir)
   if [[ "$1" =~ "postinstall" ]]; then
     reporegex="(^$1:\s)"
   else
     reporegex="(^$basedir:\s)"
   fi
   echo FILE: $AOMP_PATCH_CONTROL_FILE
   echo regex $reporegex
   #read patch control file and look for correct patches
   while read -r line; do
   if [[ "$line" =~ $reporegex ]]; then
     #remove basename from list of patches
     patches=${line/"${BASH_REMATCH[1]}"}
     echo "patches: $patches"
     break
   fi
   done < "$AOMP_PATCH_CONTROL_FILE"
}

function setaompgpu (){
  if [[ -e "$AOMP/../../bin/rocm_agent_enumerator" ]]; then
    echo Set AOMP_GPU with ../.. rocm_agent_enumerator.
    detected_gpu=$($AOMP/../../bin/rocm_agent_enumerator | grep -m 1 -E gfx[^0]{1}.{2})
  elif [[ -e "$AOMP/../bin/rocm_agent_enumerator" ]]; then
    echo Set AOMP_GPU with .. rocm_agent_enumerator.
    detected_gpu=$($AOMP/../bin/rocm_agent_enumerator | grep -m 1 -E gfx[^0]{1}.{2})
  elif [[ "$AOMP" =~ "opt" ]]; then
    echo Set AOMP_GPU with rocm_agent_enumerator.
    detected_gpu=$($AOMP/../bin/rocm_agent_enumerator | grep -m 1 -E gfx[^0]{1}.{2})
  else
    echo Set AOMP_GPU with rocm_agent_enumerator.
    if [ -a $AOMP/bin/rocm_agent_enumerator ]; then
      detected_gpu=$($AOMP/bin/rocm_agent_enumerator | grep -m 1 -E gfx[^0]{1}.{2})
    elif [ -a $AOMP/bin/amdgpu-arch ]; then
      detected_gpu=$($AOMP/bin/amdgpu-arch)
      detected_gpu=$(echo $detected_gpu | awk '{print $1,$5}')
      detected_gpu=`echo $detected_gpu | cut -d" " -f1` # pick first found
    elif [ -e $AOMP/../bin/mygpu ]; then
      detected_gpu=$($AOMP/../bin/mygpu)
    else
      echo "Error: in setaompgpu, could not find rocm_agent_enumerator"
      echo "       Set AOMP_GPU, AOMP, or install ROCm"
      exit 1
    fi
  fi

  AOMP_GPU=${AOMP_GPU:-$detected_gpu}
  export AOMP_GPU
  echo AOMP_GPU = $AOMP_GPU
  echo AOMP     = $AOMP

  if [ "$AOMP_GPU" == "" ] || [ "$AOMP_GPU" == "unknown" ]; then
    echo Error: AOMP_GPU not properly set...exiting.
    exit 1
  fi
}

# GPUs which need HSA_XNACK=1 to be set for USM to work.  The canonical place
# this info is stored in these scripts is in test/Makefile.defs (SUPPORTS_USM).
# We use a tiny custom makefile to extract the info.
# Usage:
# if gpu_needs_xnack_for_usm "$GPU"; then ...; fi

function gpu_needs_xnack_for_usm(){
  local needs_usm
  needs_usm=$(make -s -C "$thisdir"/../test -f USM_check.make GPU="$1")
  case "$needs_usm" in
    true)
      return 0
      ;;
    false)
      return 1
      ;;
    *)
      return 2
      ;;
  esac
}

# Return success (0) if ROCR_VISIBLE_DEVICES (or the first GPU if it is unset)
# refers to an APU, as indicated by rocminfo returning
# "Coherent Host Access: TRUE" for the device, else return failure (1).

function is_apu(){
  local device_num=${ROCR_VISIBLE_DEVICES:-0}
  local cha
  cha=$(ROCR_VISIBLE_DEVICES="$device_num" rocminfo | grep -m 1 -F "Coherent Host Access:")
  if [[ "$cha" =~ Coherent\ Host\ Access:[[:blank:]]+(TRUE|FALSE) ]]; then
    case "${BASH_REMATCH[1]}" in
      TRUE)
        return 0
        ;;
      FALSE)
        return 1
        ;;
     esac
  else
    # If we don't see the above pattern in rocminfo output, it's probably not
    # an APU.
    return 1
  fi
}

function help_build_aomp(){
   /bin/cat 2>&1 <<"EOF"

  The build scripts in this directory are used to build AOMP.
  
  Repositories:
     Many repositories are used to build AOMP.  The script clone_aomp.sh will
     clone all the necessary repositories as subdirectories of the directory
     $HOME/git/aomp11. The web repository locations and the required branches
     are set with the file aomp_common_vars
  
  Build all components:
     To build all components, run these commands:

        ./clone_aomp.sh
        unset LD_LIBRARY_PATH
        ./build_aomp.sh

  Component builds:
     Developers can rebuild individual components by running the build script for
     that component. Make sure dependent components are built first. You can run
     build_aomp.sh to build call components in the correct order orsee the README.md
     file in this directory for the required order.

     Each build script can run with no arguments or with a single argument
     "install" or "nocmake".  Running  with no options starts fresh with an empty
     component build directory.  It then runs cmake with the correct cmake options
     then it runs make with a proper -j option.
  
  Optional Arguments 'nocmake' and 'install' :
     The 'nocmake' or 'install' options can only be used after your initial build
     with no options. The 'nocmake' option is intended to restart make after 
     you fix code following a failed build. The 'install' option will run 'make' 
     and 'make install' causing installation into the directory $AOMP_INSTALL_DIR. 
     The 'install' option will also create a symbolic link to directory $AOMP.
  
  Environment Variables:
     You can set environment variables to override behavior of the build scripts
     NAME             DEFAULT             DESCRIPTION
     ----             -------             -----------
     AOMP             $HOME/rocm/aomp     Where the compiler will be installed
     AOMP_REPOS       $HOME/git/aomp11    Location of all aomp repos
     BUILD_TYPE       Release             The CMAKE build type
     BUILD_AOMP       same as AOMP_REPOS  Set a different build location than AOMP_REPOS
     AOMP_USE_NINJA   0                   Use ninja if possible
  
  To build a debug version of the compiler, run this command before the build:
     export BUILD_TYPE=debug
  
  The BUILD_AOMP Envronment Variable:
     The build scripts will always build with cmake and make outside your source git trees.
     By default (without BUILD_AOMP) the build will occur in the "build" subdirectory of
     AOMP_REPOS. For example build_llvm will build in $AOMP_REPOS/build/llvm
  
     The BUILD_AOMP environment variable enables source development outside your git
     repositories. By default, this feature is OFF.  The BUILD_AOMP environment variable 
     can be used if access to your git repositories is very slow or you want to test 
     changes outside of your local git repositories (specified by AOMP_REPOS env var). 
     If BUILD_AOMP is set, your git repositories (specifed by AOMP_REPOS) will be
     replicated to subdirectories of BUILD_AOMP using rsync.  The subsequent build 
     (cmake and make) will occur in subdirectory BUILD_AOMP/build/llvm.
     This replication only happens on your initial build, that is, if you specify no arguments.
     The option 'nocmake' skips replication and then restarts make in the build directory.
     The install option skips replication, skips cmake, runs 'make' and 'make install'. 
     Be careful to always use options nocmake or install if you made local changes in
     BUILD_AOMP or your changes will be lost by a new replica of your git repositories.
  
EOF
   exit 0
}