From cf21c8412c4495f20759c6c91b5f458197a71117 Mon Sep 17 00:00:00 2001
From: Brian Han <han12@llnl.gov>
Date: Thu, 18 Apr 2024 08:22:59 -0700
Subject: [PATCH] Try clang@10+cuda

---
 docker/spack.yaml                       | 18 ++++++
 docker/tpl-ubuntu-clang-cuda.Dockerfile | 84 ++++++++++++++-----------
 2 files changed, 67 insertions(+), 35 deletions(-)

diff --git a/docker/spack.yaml b/docker/spack.yaml
index 2f66d317..4ad47e96 100644
--- a/docker/spack.yaml
+++ b/docker/spack.yaml
@@ -26,6 +26,19 @@ spack:
         fc: /usr/bin/gfortran-11
       spec: clang@15
       target: x86_64
+  - compiler:
+      environment: {}
+      extra_rpaths: []
+      flags: {}
+      modules: []
+      operating_system: ubuntu20.04
+      paths:
+        cc: /usr/bin/clang
+        cxx: /usr/bin/clang++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      spec: clang@10
+      target: x86_64
   - compiler:
       environment: {}
       extra_rpaths: []
@@ -145,6 +158,11 @@ spack:
       externals:
       - spec: cmake@3.23.1
         prefix: /usr/local
+    cuda:
+      buildable: False
+      externals:
+      - spec: cuda@11.8.0 +allow-unsupported-compilers
+        prefix: /usr/local/cuda
     flex:
       version: [2.6.4]
       buildable: false
diff --git a/docker/tpl-ubuntu-clang-cuda.Dockerfile b/docker/tpl-ubuntu-clang-cuda.Dockerfile
index 991ef500..75211d75 100644
--- a/docker/tpl-ubuntu-clang-cuda.Dockerfile
+++ b/docker/tpl-ubuntu-clang-cuda.Dockerfile
@@ -24,56 +24,67 @@ RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime && \
         openmpi-bin \
         libopenmpi-dev \
         python3 \
-        clang
+        clang \
+# Additional spack dependencies
+        python3-pip \
+        pkg-config \
+        xz-utils \
+        unzip \
+        libmpfr-dev \
+        lbzip2 \
+        virtualenv
+
+# Install clingo for Spack
+RUN python3 -m pip install --upgrade pip && \
+    python3 -m pip install clingo
 
 RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh
 
-ENV CC=/usr/bin/clang \
-    CXX=/usr/bin/clang++ \
-    MPICC=/usr/bin/mpicc \
-    MPICXX=/usr/bin/mpicxx \
-    MPIEXEC=/usr/bin/mpirun
-ENV OMPI_CC=$CC \
-    OMPI_CXX=$CXX 
-ENV ENABLE_CUDA=ON \
-    CMAKE_CUDA_FLAGS="-restrict -arch sm_70 --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations"
-
 # Installing TPL's
 FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain
 ARG SRC_DIR
 ARG BLD_DIR
 
-ENV FC=/usr/bin/gfortran \
-    MPIFC=/usr/bin/mpifort
-ENV OMPI_FC=$FC
-
 RUN apt-get install -y --no-install-recommends \
     libtbb-dev \
     bc \
     file \
     bison \
     flex \
-    patch
-
-ARG HOST_CONFIG
-
-ARG CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
-ARG CUDA_ARCH=sm_70
-ARG CMAKE_CUDA_COMPILER=$CUDA_TOOLKIT_ROOT_DIR/bin/nvcc
-ARG CMAKE_CUDA_ARCHITECTURES=70
-
-ENV HYPRE_CUDA_SM=70
-ENV CUDA_HOME=$CUDA_TOOLKIT_ROOT_DIR
+    patch \
+# `ca-certificates`  needed by `git` to download GEOS repo.
+    ca-certificates \
+    git
+
+# Clone branch with spack configs
+# TODO decide landing place of spack recipes
+RUN git clone --branch feature/han12/wip_docker_spack_recipes \
+              --depth 1 \
+          --single-branch \
+          https://github.com/GEOS-DEV/GEOS.git
+
+# Run uberenv
+# Have to create install directory first for uberenv
+# -k flag is to ignore SSL errors
+# Remove generated host-config for lvarray
+ RUN --mount=src=.,dst=$SRC_DIR cd GEOS && \
+     mkdir -p ${GEOSX_TPL_DIR} && \
+     git submodule init scripts/uberenv && \
+     git submodule update && \
+     ./scripts/uberenv/uberenv.py \
+       --spec "%clang@10+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers ^caliper@2.10.0~gotcha~sampler~libunwind~libdw~papi" \
+       --spack-env-file=${SRC_DIR}/docker/spack.yaml \
+       --project-json=.uberenv_config.json \
+       --prefix ${GEOSX_TPL_DIR} \
+       -k && \
+# Remove host-config generated for LvArray
+     rm lvarray* && \
+# Rename and copy spack-generated host-config to root directory
+     cp *.cmake /spack-generated.cmake && \
+# Remove extraneous spack files
+     cd ${GEOSX_TPL_DIR} && \
+     rm -rf bin/ build_stage/ misc_cache/ spack/ spack-env/ .spack-db/
 
-RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh \
-    -DENABLE_CUDA=$ENABLE_CUDA \
-    -DENABLE_HYPRE_DEVICE="CUDA" \
-    -DCUDA_TOOLKIT_ROOT_DIR=$CUDA_TOOLKIT_ROOT_DIR \
-    -DCUDA_ARCH=$CUDA_ARCH \
-    -DCMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES \
-    -DCMAKE_CUDA_COMPILER=$CMAKE_CUDA_COMPILER
-WORKDIR $BLD_DIR
-RUN --mount=src=.,dst=$SRC_DIR make
 
 # Extract only TPL's from previous stage
 FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain
@@ -81,6 +92,9 @@ ARG SRC_DIR
 
 COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR
 
+# Extract the generated host-config
+COPY --from=tpl_toolchain /spack-generated.cmake /
+
 RUN apt-get install -y --no-install-recommends \
     openssh-client \
     ca-certificates \