diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index a62d5f3e..1c01f9ba 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -20,11 +20,11 @@ jobs: matrix: include: - #- name: Pangea 2 (centos 7.6, gcc 8.3.0, open-mpi 2.1.5, mkl 2019.3) - # DOCKER_REPOSITORY: geosx/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3 - # DOCKERFILE: docker/TotalEnergies/Dockerfile - # DOCKER_ROOT_IMAGE: totogaz/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3-no-geosx:0.0.1 - # INSTALL_DIR_ROOT: /workrd/SCR/GEOSX/install/gcc8 + - name: Pangea 2 (centos 7.6, gcc 8.3.0, open-mpi 2.1.5, mkl 2019.3) + DOCKER_REPOSITORY: gseosx/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3 + DOCKERFILE: docker/TotalEnergies/Dockerfile + DOCKER_ROOT_IMAGE: totogaz/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3-no-geosx:0.0.1 + INSTALL_DIR_ROOT: /workrd/SCR/GEOSX/install/gcc8 #- name: Pecan GPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5, cuda 11.5.119) # DOCKER_REPOSITORY: geosx/pecan-gpu-gcc8.2.0-openmpi4.0.1-mkl2019.5-cuda11.5.119 # DOCKERFILE: docker/TotalEnergies/Dockerfile diff --git a/docker/TotalEnergies/Dockerfile b/docker/TotalEnergies/Dockerfile index 929cd081..f06b223b 100644 --- a/docker/TotalEnergies/Dockerfile +++ b/docker/TotalEnergies/Dockerfile @@ -15,7 +15,18 @@ RUN yum install -y \ ca-certificates \ curl \ python3 \ - zlib-devel + zlib-devel \ +# Additional spack dependencies + python3-pip \ + pkgconfig \ + # xz \ + unzip \ + bzip2 \ + && pip3 install virtualenv + +# Install clingo for Spack +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip install clingo RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh @@ -29,19 +40,50 @@ RUN yum install -y \ file \ bison \ flex \ - patch + patch \ +# `ca-certificates` needed by `git` to download GEOS repo. + ca-certificates \ + autoconf \ + automake \ + git -ARG HOST_CONFIG +# Clone branch with spack configs +# TODO decide landing place of spack recipes +RUN git clone --branch feature/han12/wip_docker_spack_recipes \ + --depth 1 \ + --single-branch \ + https://github.com/GEOS-DEV/GEOS.git -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh -WORKDIR $BLD_DIR -RUN --mount=src=.,dst=$SRC_DIR make +# Run uberenv +# Have to create install directory first for uberenv +# -k flag is to ignore SSL errors +# Remove generated host-config for lvarray +RUN --mount=src=.,dst=$SRC_DIR cd GEOS && \ + mkdir -p ${GEOSX_TPL_DIR} && \ + git submodule init scripts/uberenv && \ + git submodule update && \ + ./scripts/uberenv/uberenv.py \ + --spec "%gcc@8.3.0~pygeosx" \ + --spack-env-file=${SRC_DIR}/docker/total-spack.yaml \ + --project-json=.uberenv_config.json \ + --prefix ${GEOSX_TPL_DIR} \ + -k && \ +# Remove host-config generated for LvArray + rm lvarray* && \ +# Rename and copy spack-generated host-config to root directory + cp *.cmake /spack-generated.cmake && \ +# Remove extraneous spack files + cd ${GEOSX_TPL_DIR} && \ + rm -rf bin/ build_stage/ misc_cache/ spack/ spack-env/ .spack-db/ FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain ARG SRC_DIR COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR +# Extract the generated host-config +COPY --from=tpl_toolchain /spack-generated.cmake / + RUN yum -y install \ openssh-clients \ ca-certificates \ diff --git a/docker/total-spack.yaml b/docker/total-spack.yaml new file mode 100644 index 00000000..2b08867c --- /dev/null +++ b/docker/total-spack.yaml @@ -0,0 +1,173 @@ +spack: + config: + install_tree: + root: $spack/.. + projections: + all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + misc_cache: $spack/../misc_cache + test_stage: $spack/../test_stage + build_stage:: + - $spack/../build_stage + + # Regular TPLs do not need views + view: false + + compilers:: + # Pangea 2 compiler + - compiler: + environment: {} + extra_rpaths: [] + flags: + cflags: -pthread + cxxflags: -pthread + modules: [] + operating_system: centos7 + paths: + cc: /data_local/sw/gcc/RHEL7/8.3.0/bin/gcc + cxx: /data_local/sw/gcc/RHEL7/8.3.0/bin/g++ + f77: /data_local/sw/gcc/RHEL7/8.3.0/bin/gfortran + fc: /data_local/sw/gcc/RHEL7/8.3.0/bin/gfortran + spec: gcc@8.3.0 + target: x86_64 + + packages: + all: + target: [x86_64] + compiler: [gcc, clang] + providers: + blas: [intel-mkl] + lapack: [intel-mkl] + mpi: [openmpi] + + + blt: + require: "@git.5a792c1775e7a7628d84dcde31652a689f1df7b5=develop" + + hypre: + require: "@git.d475cdfc63ac59ea9a554493a06e4033b8d6fade" + + chai: + require: "@git.6fe3470ad020303530af2f3dbbfe18826bd3319b=develop" + + umpire: + require: "@git.4bd9b2ded81d3216b3f62e2aad62d0e34fe2c256=develop" + + raja: + require: "@git.e330b2560747d5417cd7bd265fab3fb91d32ecbd=develop" + + camp: + require: "@git.ac34c25b722a06b138bc045d38bfa5e8fa3ec9c5=main" + + uncrustify: + require: "@git.401a4098bce9dcc47e024987403f2d59d9ba7bd2" + + superlu-dist: + require: "@git.0f6efc377df2440c235452d13d28d2c717f832a1" + + autoconf: + version: [2.71] + buildable: false + externals: + - spec: autoconf@2.71 + prefix: /usr + automake: + version: [1.16.5] + buildable: false + externals: + - spec: automake@1.16.5 + prefix: /usr + bison: + version: [3.8.2] + buildable: false + externals: + - spec: bison@3.8.2 + prefix: /usr + cmake: + version: [3.23.1] + buildable: false + externals: + - spec: cmake@3.23.1 + prefix: /usr/local + cuda: + buildable: False + externals: + - spec: cuda@11.8.0 +allow-unsupported-compilers + prefix: /usr/local/cuda + flex: + version: [2.6.4] + buildable: false + externals: + - spec: flex@2.6.4 + prefix: /usr + + # Lock down blas, lapack implementation + intel-mkl: + buildable: False + externals: + # LC + #- spec: intel-mkl@2020.0.166 threads=openmp + # prefix: /usr/tce/packages/mkl/mkl-2020.0/ + + # Pangea 2 + - spec: intel-mkl@2019.3.199 threads=openmp + prefix: /opt/intel/compilers_and_libraries_2019.3.199/linux/mkl/ + blas: + buildable: false + + lapack: + buildable: false + + m4: + buildable: false + externals: + - spec: m4@1.4.18 + prefix: /usr + + # Spack may grab for mpi & we don't want to use them + mpi: + buildable: false + #mpfr: + # buildable: false + # externals: + # - spec: mpfr@4.1.0 + # prefix: /usr + + openmpi: + externals: + - spec: openmpi@2.1.5%gcc@8.3.0 + prefix: /data_local/sw/OpenMPI/RHEL7/2.1.5/gcc/8.3.0 + perl: + buildable: false + externals: + - spec: perl@5.34.0 + prefix: /usr + pkg-config: + buildable: false + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + py-sphinx: + buildable: false + externals: + - spec: py-sphinx@4.3.2 + prefix: /usr + python: + buildable: false + externals: + - spec: python@3.10.12 + prefix: /usr + tar: + buildable: false + externals: + - spec: tar@1.34 + prefix: /usr + unzip: + buildable: false + externals: + - spec: unzip@6.0 + prefix: /usr + xz: + buildable: false + externals: + - spec: xz@5.2.5 + prefix: /usr