add rocm 6.3 container for ci (#1429)

azure-pipelines.yml

@@ -26,7 +26,7 @@ variables:
   ubuntu_22_04_tag: alpinedav/ascent-devel:ubuntu-22.04-x86_64-tpls_2024-07-10-sha3a1ef8
   ubuntu_20_04_cuda_11_4_3_tag: alpinedav/ascent-devel:ubuntu-20.04-cuda-11.4.3-x86_64-tpls_2024-07-09-sha3a1ef8
   ubuntu_20_04_cuda_12_1_1_tag: alpinedav/ascent-devel:ubuntu-20.04-cuda-12.1.1-x86_64-tpls_2024-07-09-sha3a1ef8
-  ubuntu_20_04_rocm_6_0_0_tag:  alpinedav/ascent-devel:ubuntu-20.04-rocm-6.0.0--x86_64-build-ascent-tpls_2024-07-10-sha3a1ef8
+  ubuntu_20_04_rocm_6_0_0_tag:  alpinedav/ascent-devel:ubuntu-20.04-rocm-6.3.0-x86_64-build-ascent-tpls_2024-12-09-sha21f334
 
 
 # only build merge target pr to develop

scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.0.0_devel_build_ascent_tpls/build.sh

@@ -6,7 +6,7 @@ set -ev
 
 export REPO_NAME="ascent"
 export TAG_ARCH=`uname -m`
-export TAG_BASE=alpinedav/ascent-devel:ubuntu-20.04-rocm-6.0.0--${TAG_ARCH}-build-ascent-tpls
+export TAG_BASE=alpinedav/ascent-devel:ubuntu-20.04-rocm-6.0.0-${TAG_ARCH}-build-ascent-tpls
 
 date
 

scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/Dockerfile

@@ -0,0 +1,75 @@
+# Copyright (c) Lawrence Livermore National Security, LLC and other Ascent
+# Project developers. See top-level LICENSE AND COPYRIGHT files for dates and
+# other details. No copyright assignment is required to contribute to Ascent.
+
+FROM ubuntu:20.04
+# add sudo to base ubuntu container
+# so we can install additional packages as
+# non-root, but admin default user on azure pipelines
+RUN apt-get update && apt-get -y install sudo
+RUN apt-get update && apt-get -y install wget gnupg
+# tzdata install wants to ask questions, so handled as sep case
+RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata
+
+
+#
+# install rocm (recipe from warpx)
+# https://github.com/ECP-WarpX/WarpX/blob/development/.github/workflows/dependencies/hip.sh
+#
+# Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#ubuntu
+RUN wget -q -O - http://repo.radeon.com/rocm/rocm.gpg.key \
+  | sudo apt-key add -
+
+RUN echo 'deb [arch=amd64] http://repo.radeon.com/rocm/apt/6.3/ ubuntu main' \
+  | sudo tee /etc/apt/sources.list.d/rocm.list
+
+RUN echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \
+  | sudo tee -a /etc/profile.d/rocm.sh
+
+
+
+# install std packages
+# Notes:
+# kmod -- hip requires lsmod
+# liblocale-codes-perl  -- hip uses perl, if locale bad, life isn't good
+# language-pack-en  -- hip uses perl, if locale bad, life isn't good
+# for 6.0, `rocm` - new meta package
+RUN apt-get update && apt-get -y install \
+             binutils \
+             gcc \
+             g++ \
+             gfortran \
+             python \
+             perl \
+             git \
+             git-lfs \
+             curl \
+             wget \
+             tar \
+             unzip \
+             build-essential \
+             libncurses-dev \
+             libssl-dev \
+             libblas-dev \
+             liblapack-dev \
+             liblapacke-dev \
+             zlib1g-dev \
+             libgdbm-dev \
+             libreadline-dev \
+             libsqlite3-dev \
+             libbz2-dev \
+             mpich \
+             libmpich-dev \
+             libnuma-dev \
+             rocm-dev        \
+             rocfft-dev      \
+             rocprim-dev     \
+             rocrand-dev     \
+             hiprand-dev     \
+             openssh-server \
+             language-pack-en \
+             liblocale-codes-perl \
+             vim
+
+#RUN apt-get update && apt-get -y install --no-install-recommends 
+# rocm-hip-sdk, vs all of these packages?

scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel/build.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Copyright (c) Lawrence Livermore National Security, LLC and other Ascent
+# Project developers. See top-level LICENSE AND COPYRIGHT files for dates and
+# other details. No copyright assignment is required to contribute to Ascent.
+set -ev
+
+export TAG_ARCH=`uname -m`
+export TAG_NAME=alpinedav/ascent-devel:ubuntu-20.04-rocm-6.3.0-${TAG_ARCH}
+
+# exec docker build to create image
+echo "docker build -t ${TAG_NAME} ."
+docker build -t ${TAG_NAME} .

scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/Dockerfile

@@ -0,0 +1,20 @@
+# Copyright (c) Lawrence Livermore National Security, LLC and other Ascent
+# Project developers. See top-level LICENSE AND COPYRIGHT files for dates and
+# other details. No copyright assignment is required to contribute to Ascent.
+
+ARG TAG_ARCH="x86_64"
+FROM alpinedav/ascent-devel:ubuntu-20.04-rocm-6.3.0-${TAG_ARCH}
+
+# obtain a copy of ascent source from host env,
+# which we use to call uberenv
+COPY ascent.docker.src.tar.gz /
+# extract 
+RUN tar -xzf ascent.docker.src.tar.gz
+
+# copy spack build script in
+COPY docker_uberenv_build.sh docker_env_setup.sh /
+RUN chmod -R a+x /*.sh
+
+RUN /docker_uberenv_build.sh
+
+RUN /docker_env_setup.sh

scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/build.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Copyright (c) Lawrence Livermore National Security, LLC and other Ascent
+# Project developers. See top-level LICENSE AND COPYRIGHT files for dates and
+# other details. No copyright assignment is required to contribute to Ascent.
+set -ev
+
+export REPO_NAME="ascent"
+export TAG_ARCH=`uname -m`
+export TAG_BASE=alpinedav/ascent-devel:ubuntu-20.04-rocm-6.3.0-${TAG_ARCH}-build-ascent-tpls
+
+date
+
+python3 ../../../docker_build_and_tag.py ${REPO_NAME} ${TAG_ARCH} ${TAG_BASE}
+
+date

scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/docker_env_setup.sh

@@ -0,0 +1,7 @@
+# create some helper scripts
+python ascent/scripts/gen_spack_env_script.py cmake mpi
+echo "git clone --recursive https://github.com/Alpine-DAV/ascent.git" > clone.sh
+chmod +x clone.sh
+
+# delete copy of source from host (ci will fetch new from repo)
+rm -rf ascent

scripts/ci/docker/alpinedav_ubuntu_20.04_rocm_6.3.0_devel_build_ascent_tpls/docker_uberenv_build.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -ev
+
+export CMAKE_ARCH=`uname -m`
+
+if [[ ${CMAKE_ARCH} == "arm64" ]]; then
+  export CMAKE_ARCH="aarch64"
+fi
+
+cmake_install_dir=/cmake-3.23.2-linux-${CMAKE_ARCH}
+if [ ! -d ${cmake_install_dir} ]; then
+  # setup cmake in container
+  curl -L https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-linux-${CMAKE_ARCH}.tar.gz -o cmake-3.23.2-linux-${CMAKE_ARCH}.tar.gz
+  tar -xzf cmake-3.23.2-linux-${CMAKE_ARCH}.tar.gz
+fi
+
+export PATH=$PATH:/${cmake_install_dir}/bin/
+
+# build rocm tpls with helper script
+chmod +x ascent/scripts/build_ascent/build_ascent.sh
+# enable tests is of b/c one of the vtk-m tests fails to link
+env enable_tests=OFF build_ascent=false enable_hip=ON ascent/scripts/build_ascent/build_ascent.sh
+
+############################
+# TODO: get spack working
+############################
+# variants
+# TODO:
+# (+genten) genten currently disabled, wait for genten master to gain cokurt
+# (+dray+raja+umpire) (can't get raja or umpire to build with rocm)
+#export SPACK_SPEC="%clang+mpi+vtkh~dray~raja~umpire+mfem+occa+rocm~genten~python~openmp~fortran"
+# constraints
+#export SPACK_SPEC="${SPACK_SPEC} ^hdf5~mpi ^mfem~rocm ^hypre~rocm ^conduit~fortran"
+# config
+#export SPACK_CONFIG="scripts/uberenv_configs/spack_configs/configs/alpinedav/ubuntu_20.04_rocm_5.1.3_devel/"
+
+#cd ascent && python scripts/uberenv/uberenv.py \
+#     -k \
+#     --spec="${SPACK_SPEC}" \
+#     --spack-config-dir="${SPACK_CONFIG}" \
+#     --prefix="/uberenv_libs"
+
+# cleanup the spack build stuff to free up space
+#/uberenv_libs/spack/bin/spack clean --all
+
+# change perms
+# chmod -R a+rX /uberenv_libs
+
+# back to where we started
+#cd ../