diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 350c42c4..f7108b99 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -7,6 +7,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true + +env: + ROOT_IMAGE_TAG_HASH: b5b06ad736cc8f2df227dabf8ff1da1a2f538939 + jobs: build_images: name: ${{ matrix.name }} @@ -20,79 +24,85 @@ jobs: matrix: include: - - name: Pangea 2 (centos 7.6, gcc 8.3.0, open-mpi 2.1.5, mkl 2019.3) - DOCKER_REPOSITORY: geosx/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3 - DOCKERFILE: docker/TotalEnergies/Dockerfile - DOCKER_ROOT_IMAGE: totogaz/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3-no-geosx:0.0.1 - INSTALL_DIR_ROOT: /workrd/SCR/GEOSX/install/gcc8 - - name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119) - DOCKER_REPOSITORY: geosx/pecan-gpu-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119 - DOCKERFILE: docker/TotalEnergies/Dockerfile - DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 - HOST_CONFIG: docker/TotalEnergies/pecan-GPU.cmake - INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/GPU - - name: Pecan CPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5) - DOCKER_REPOSITORY: geosx/pecan-cpu-gcc8.2.0-openmpi4.0.1-mkl2019.5 - DOCKERFILE: docker/TotalEnergies/Dockerfile - DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 - HOST_CONFIG: docker/TotalEnergies/pecan-CPU.cmake - INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/CPU - - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 - DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2 - HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake - INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU - - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.7.1, openblas 0.3.10, zlib 1.2.11) - DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11 - DOCKERFILE: docker/Stanford/Dockerfile - DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2 - HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake - INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU - - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) - DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 - DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_ROOT_IMAGE: ubuntu:20.04 - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" - - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces - DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 - DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" - - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) - DOCKER_ROOT_IMAGE: ubuntu:22.04 - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 - DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" - - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) - DOCKER_ROOT_IMAGE: ubuntu:22.04 - DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 - DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile - DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=12" - - name: Ubuntu (22.04, clang 15.0.7 + gcc 11.4.0, open-mpi 4.1.2) - DOCKER_ROOT_IMAGE: ubuntu:22.04 - DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 - DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile - DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11" - - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) - DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 - DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile - - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.89) - DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89 - DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile - + # - name: Pangea 2 (centos 7.6, gcc 8.3.0, open-mpi 2.1.5, mkl 2019.3) + # DOCKER_REPOSITORY: geosx/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3 + # DOCKERFILE: docker/TotalEnergies/Dockerfile + # DOCKER_ROOT_IMAGE: totogaz/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3-no-geosx:0.0.1 + # INSTALL_DIR_ROOT: /workrd/SCR/GEOSX/install/gcc8 + # - name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119) + # DOCKER_REPOSITORY: geosx/pecan-gpu-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119 + # DOCKERFILE: docker/TotalEnergies/Dockerfile + # DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 + # HOST_CONFIG: docker/TotalEnergies/pecan-GPU.cmake + # INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/GPU + # - name: Pecan CPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5) + # DOCKER_REPOSITORY: geosx/pecan-cpu-gcc8.2.0-openmpi4.0.1-mkl2019.5 + # DOCKERFILE: docker/TotalEnergies/Dockerfile + # DOCKER_ROOT_IMAGE: totogaz/pecan-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119-no-geosx:0.0.1 + # HOST_CONFIG: docker/TotalEnergies/pecan-CPU.cmake + # INSTALL_DIR_ROOT: /shared/data1/Users/j0436735/travis-deployments/CPU + # - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) + # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 + # DOCKERFILE: docker/Stanford/Dockerfile + # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2 + # HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake + # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/CPU + # - name: Sherlock GPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, cuda 11.7.1, openblas 0.3.10, zlib 1.2.11) + # DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11 + # DOCKERFILE: docker/Stanford/Dockerfile + # DOCKER_ROOT_IMAGE: matteofrigo5/sherlock-gcc10.1.0-openmpi4.1.2-cuda11.7.1-openblas0.3.10-zlib1.2.11-no-geosx:0.0.2 + # HOST_CONFIG: docker/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10-cuda11.7.1-sm70.cmake + # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU + # - name: Ubuntu (20.04, gcc 9.4.0, open-mpi 4.0.3) + # DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9 + # DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_ROOT_IMAGE: ubuntu:20.04 + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=9" + # - name: Ubuntu (20.04, gcc 10.5.0, open-mpi 4.0.3) - github codespaces + # DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10 + # DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_ROOT_IMAGE: mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=10" + # - name: Ubuntu (22.04, gcc 11.4.0, open-mpi 4.1.2) + # DOCKER_ROOT_IMAGE: ubuntu:22.04 + # DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11 + # DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=11" + # - name: Ubuntu (22.04, gcc 12.3.0, open-mpi 4.1.2) + # DOCKER_ROOT_IMAGE: ubuntu:22.04 + # DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12 + # DOCKERFILE: docker/tpl-ubuntu-gcc.Dockerfile + # DOCKER_COMPILER_BUILD_ARG: "--build-arg GCC_MAJOR_VERSION=12" + # - name: Ubuntu (22.04, clang 15.0.7 + gcc 11.4.0, open-mpi 4.1.2) + # DOCKER_ROOT_IMAGE: ubuntu:22.04 + # DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15 + # DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile + # DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11" + # - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) + # DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 + # DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile + # - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.89) + # DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89 + # DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile + - name: ubi (8.9, gcc 13.2.1, open-mpi 1.10.7, cuda 12.4.1) + DOCKER_ROOT_IMAGE: geosx/ubi:8.9-cuda12.4 + DOCKER_REPOSITORY: geosx/ubi8.9-gcc13-cuda12.4 + DOCKERFILE: docker/tpl-ubi-gcc-cuda.Dockerfile + CMAKE_CXX_COMPILER: /opt/rh/gcc-toolset-13/root/bin/g++ + CMAKE_C_COMPILER: /opt/rh/gcc-toolset-13/root/bin/gcc + CMAKE_FORTRAN_COMPILER: /opt/rh/gcc-toolset-13/root/bin/gfortran steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4.1.6 with: submodules: true lfs: true - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3.3.0 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3.2.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} @@ -104,10 +114,13 @@ jobs: env: DOCKERFILE: ${{ matrix.DOCKERFILE }} DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }} + CMAKE_CXX_COMPILER: ${{ matrix.CMAKE_CXX_COMPILER }} + CMAKE_C_COMPILER: ${{ matrix.CMAKE_C_COMPILER }} + CMAKE_FORTRAN_COMPILER: ${{ matrix.CMAKE_FORTRAN_COMPILER }} DOCKER_COMPILER_BUILD_ARG: ${{ matrix.DOCKER_COMPILER_BUILD_ARG }} INSTALL_DIR_ROOT: ${{ matrix.INSTALL_DIR_ROOT || '/opt/GEOS' }} HOST_CONFIG: ${{ matrix.HOST_CONFIG || 'host-configs/environment.cmake'}} - DOCKER_ROOT_IMAGE: ${{ matrix.DOCKER_ROOT_IMAGE || 'undefined' }} + DOCKER_ROOT_IMAGE: ${{ matrix.DOCKER_ROOT_IMAGE || 'undefined' }}-${{env.ROOT_IMAGE_TAG_HASH}} PULL_REQUEST_NUMBER: ${{ github.event.number }} BUILD_NUMBER: ${{ github.run_number }} COMMIT: ${{ github.sha }} diff --git a/docker/tpl-ubi-gcc-cuda.Dockerfile b/docker/tpl-ubi-gcc-cuda.Dockerfile new file mode 100644 index 00000000..6608cf38 --- /dev/null +++ b/docker/tpl-ubi-gcc-cuda.Dockerfile @@ -0,0 +1,92 @@ +# NOTE: see docker/tpl-ubuntu-gcc.Dockerfile for detailed comments +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/thirdPartyLibs +ARG BLD_DIR=$TMP_DIR/build +ARG CMAKE_C_COMPILER +ARG CMAKE_CXX_COMPILER +ARG CMAKE_FORTRAN_COMPILER + +FROM geosx/ubi:8.9-cuda12.4-b5b06ad736cc8f2df227dabf8ff1da1a2f538939 AS tpl_toolchain_intersect_geosx_toolchain +ARG SRC_DIR + +ARG INSTALL_DIR +ENV GEOSX_TPL_DIR=$INSTALL_DIR + + +# RUN dnf search blas +# RUN dnf search lapack +RUN dnf search zlib +RUN dnf search openmpi +# Installing dependencies +#RUN dnf -y install tbb +RUN dnf -y install openblas +#RUN dnf -y install lapack-devel +RUN dnf -y install zlib-devel +#RUN dnf -y install openmpi-devel +RUN dnf -y install https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.3-1.src.rpm + + +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh + +ENV CC=$CMAKE_C_COMPILER \ + CXX=$CMAKE_CXX_COMPILER \ + MPICC=/usr/lib64/openmpi/bin/mpicc \ + MPICXX=/usr/lib64/openmpi/bin/mpicxx \ + MPIEXEC=/usr/lib64/openmpi/bin/mpirun +ENV OMPI_CC=$CC \ + OMPI_CXX=$CXX +ENV ENABLE_CUDA=ON \ + CMAKE_CUDA_FLAGS="-restrict -arch sm_70 --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" + +# Installing TPL's +FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain +ARG SRC_DIR +ARG BLD_DIR + +ENV FC=$CMAKE_FORTRAN_COMPILER \ + MPIFC=/usr/lib64/openmpi/bin/mpifort +ENV OMPI_FC=$FC + +RUN dnf install -y \ + tbb-devel \ + bc \ + file \ + bison \ + flex \ + patch + +ARG HOST_CONFIG + +ARG CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda +ARG CUDA_ARCH=sm_70 +ARG CMAKE_CUDA_COMPILER=$CUDA_TOOLKIT_ROOT_DIR/bin/nvcc +ARG CMAKE_CUDA_ARCHITECTURES=70 + +ENV HYPRE_CUDA_SM=70 +ENV CUDA_HOME=$CUDA_TOOLKIT_ROOT_DIR + +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh \ + -DENABLE_CUDA=$ENABLE_CUDA \ + -DENABLE_HYPRE_DEVICE="CUDA" \ + -DCUDA_TOOLKIT_ROOT_DIR=$CUDA_TOOLKIT_ROOT_DIR \ + -DCUDA_ARCH=$CUDA_ARCH \ + -DCMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES \ + -DCMAKE_CUDA_COMPILER=$CMAKE_CUDA_COMPILER +WORKDIR $BLD_DIR +RUN --mount=src=.,dst=$SRC_DIR make + +# Extract only TPL's from previous stage +FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain +ARG SRC_DIR + +COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR +RUN dnf install -y \ + openssh-client \ + ca-certificates \ + texlive \ + graphviz + +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-ninja.sh + +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh +ENV SCCACHE=/opt/sccache/bin/sccache