Spack/Singularity

.github/workflows/spack.yml

@@ -15,57 +15,55 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
+        include:  # Include GPU build tests
           - compiler: gcc
-            mpi: openmpi
+            mpi: mpich
+            gpu: none
+
+          - compiler: gcc
+            mpi: mpich
+            gpu: cuda
+
+          - compiler: gcc
+            mpi: mpich
+            gpu: rocm
 
     runs-on: palace_ubuntu-latest_16-core
     steps:
       - uses: actions/checkout@v4
 
-      - name: Configure Open MPI
-        if: matrix.mpi == 'openmpi'
-        run: |
-          sudo apt-get install -y openmpi-bin libopenmpi-dev
-
       - name: Configure MPICH
         if: matrix.mpi == 'mpich'
         run: |
           sudo apt-get install -y mpich libmpich-dev
 
-      - name: Configure Intel MPI
-        if: matrix.mpi == 'intelmpi'
-        uses: mpi4py/setup-mpi@v1
-        with:
-          mpi: ${{ matrix.mpi }}
-
-      - name: Configure Clang compiler
-        if: matrix.compiler == 'clang'
-        run: |
-          sudo apt-get install -y clang lld
-
-      - name: Configure Intel oneAPI compiler
-        if: matrix.compiler == 'intel'
-        run: |
-          sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp \
-                                  intel-oneapi-compiler-fortran
-
       - uses: vsoch/spack-package-action/install@main
 
       - name: Build Palace
         run: |
           # Clean up Android SDK install (confuses Spack MKL link line?)
           sudo rm -rf $ANDROID_HOME
 
-          # Set up Spack to use external packages (MPI, etc.)
+          # Set up Spack to use external packages (MPI, etc.) and local Palace package
+          # recipe
           . /opt/spack/share/spack/setup-env.sh
           spack external find --all
+          spack repo add spack/local
+
+          # Configure GPU variant
+          if [[ "${{ matrix.gpu }}" == 'cuda' ]]; then
+            GPU_VARIANT="+cuda cuda_arch=70"
+          elif [[ "${{ matrix.gpu }}" == 'rocm' ]]; then
+            GPU_VARIANT="+rocm amdgpu_target=gfx900"
+          else
+            GPU_VARIANT=""
+          fi
 
           # Build and install
-          spack repo add spack/local
-          spack dev-build local.palace@develop%${{ matrix.compiler }} ^petsc~hdf5 ^intel-oneapi-mkl
+          spack dev-build local.palace@develop%${{ matrix.compiler }} $GPU_VARIANT ^petsc~hdf5 ^intel-oneapi-mkl
 
       - name: Run tests
+        if: matrix.gpu == 'none'  # Skip tests for GPU builds
         env:
           NUM_PROC_TEST_MAX: '8'
         run: |

singularity/singularity.def

@@ -83,13 +83,15 @@ From: rockylinux:9
         -DBUILD_SHARED_LIBS:BOOL=ON \
         -DPALACE_WITH_64BIT_INT:BOOL=OFF \
         -DPALACE_WITH_OPENMP:BOOL=ON \
+        -DPALACE_WITH_CUDA:BOOL=OFF \
+        -DPALACE_WITH_HIP:BOOL=OFF \
         -DPALACE_WITH_SUPERLU:BOOL=ON \
         -DPALACE_WITH_STRUMPACK:BOOL=ON \
         -DPALACE_WITH_MUMPS:BOOL=ON \
         -DPALACE_WITH_SLEPC:BOOL=ON \
         -DPALACE_WITH_ARPACK:BOOL=ON \
         -DPALACE_WITH_LIBXSMM:BOOL=ON \
-        -DPALACE_WITH_MAGMA:BOOL=OFF \
+        -DPALACE_WITH_MAGMA:BOOL=ON \
         -DPALACE_WITH_GSLIB:BOOL=ON
     make -j"$(nproc)"
     cd .. && rm -rf palace-build

spack/local/packages/palace/package.py

@@ -1,12 +1,12 @@
-# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 from spack.package import *
 
 
-class Palace(CMakePackage):
+class Palace(CMakePackage, CudaPackage, ROCmPackage):
     """3D finite element solver for computational electromagnetics"""
 
     tags = ["cem", "fem", "finite-elements", "hpc", "solver"]
@@ -18,6 +18,8 @@ class Palace(CMakePackage):
 
     version("develop", branch="main")
 
+    # Note: 'cuda' and 'cuda_arch' variants are added by the CudaPackage
+    # Note: 'rocm' and 'amdgpu_target' variants are added by the ROCmPackage
     variant("shared", default=True, description="Build shared libraries")
     variant("int64", default=False, description="Use 64 bit integers")
     variant("openmp", default=False, description="Use OpenMP for shared-memory parallelism")
@@ -97,7 +99,7 @@ class Palace(CMakePackage):
         depends_on("arpack-ng~shared", when="~shared")
 
     with when("+libxsmm"):
-        depends_on("libxsmm@main")
+        depends_on("libxsmm@=main")  # LIBXSMM has a older main-DATE version
         depends_on("libxsmm+shared", when="+shared")
         depends_on("libxsmm~shared", when="~shared")
 
@@ -106,6 +108,25 @@ class Palace(CMakePackage):
         depends_on("magma+shared", when="+shared")
         depends_on("magma~shared", when="~shared")
 
+    # Propagate CUDA architectures/AMD GPU targets down to dependencies (concretization
+    # fails if we try to use == to propagate)
+    with when("+cuda"):
+        for arch in CudaPackage.cuda_arch_values:
+            cuda_variant = f"+cuda cuda_arch={arch}"
+            depends_on(f"hypre{cuda_variant}", when=f"{cuda_variant}")
+            depends_on(f"superlu-dist{cuda_variant}", when=f"+superlu-dist{cuda_variant}")
+            depends_on(f"strumpack{cuda_variant}", when=f"+strumpack{cuda_variant}")
+            depends_on(f"slepc{cuda_variant} ^petsc{cuda_variant}", when=f"+slepc{cuda_variant}")
+            depends_on(f"magma{cuda_variant}", when=f"+magma{cuda_variant}")
+    with when("+rocm"):
+        for arch in ROCmPackage.amdgpu_targets:
+            rocm_variant = f"+rocm amdgpu_target={arch}"
+            depends_on(f"hypre{rocm_variant}", when=f"{rocm_variant}")
+            depends_on(f"superlu-dist{rocm_variant}", when=f"+superlu-dist{rocm_variant}")
+            depends_on(f"strumpack{rocm_variant}", when=f"+strumpack{rocm_variant}")
+            depends_on(f"slepc{rocm_variant} ^petsc{rocm_variant}", when=f"+slepc{rocm_variant}")
+            depends_on(f"magma{rocm_variant}", when=f"+magma{rocm_variant}")
+
     # Palace always builds its own internal MFEM, libCEED, and GSLIB
     conflicts("mfem")
     conflicts("libceed")
@@ -115,6 +136,7 @@ class Palace(CMakePackage):
     conflicts("^hypre+int64", msg="Palace uses HYPRE's mixedint option for 64 bit integers")
     conflicts("^mumps+int64", msg="Palace requires MUMPS without 64 bit integers")
     conflicts("^slepc+arpack", msg="Palace requires SLEPc without ARPACK")
+    conflicts("+cuda+rocm", msg="PALACE_WITH_CUDA is not compatible with PALACE_WITH_HIP")
 
     def cmake_args(self):
         args = [
@@ -132,6 +154,24 @@ def cmake_args(self):
             self.define("PALACE_BUILD_EXTERNAL_DEPS", False),
         ]
 
+        # Handle GPU builds
+        args += [
+            self.define_from_variant("PALACE_WITH_CUDA", "cuda"),
+            self.define_from_variant("PALACE_WITH_HIP", "rocm"),
+        ]
+        if "+cuda" in self.spec and not "none" in self.spec.variants["cuda_arch"].value:
+            args += [
+                self.define(
+                    "CMAKE_CUDA_ARCHITECTURES", ";".join(self.spec.variants["cuda_arch"].value)
+                )
+            ]
+        if "+rocm" in self.spec and not "none" in self.spec.variants["amdgpu_target"].value:
+            args += [
+                self.define(
+                    "CMAKE_HIP_ARCHITECTURES", ";".join(self.spec.variants["amdgpu_target"].value)
+                )
+            ]
+
         # HYPRE is always built with external BLAS/LAPACK
         args += [
             self.define("HYPRE_REQUIRED_PACKAGES", "LAPACK;BLAS"),