From 5497ef8f53dbf304a41f91ef8b88225f3a3dc099 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 24 Aug 2023 14:20:35 -0600 Subject: [PATCH] Update Derecho site config (#720) * Update Derecho site config and according documentation * Use correct compiler paths in configs/sites/derecho/compilers.yaml * Fix esmf build error on Derecho * Bug fix in configs/sites/aws-pcluster/modules.yaml: blacklist ecflow * Update spack-stack documentation for Derecho * Treat derecho like a real cray: use cray compiler wrappers * Update doc/source/PreConfiguredSites.rst for Hercules * Update configs/sites/derecho/compilers.yaml - use Cray compiler wrappers * Add external krb5 to configs/sites/derecho/packages.yaml * Update spack env path for derecho in doc/source/PreConfiguredSites.rst --- configs/common/packages.yaml | 7 +- configs/sites/aws-pcluster/modules.yaml | 2 + configs/sites/derecho/compilers.yaml | 15 +- configs/sites/derecho/modules.yaml | 2 +- configs/sites/derecho/packages.yaml | 178 ++++++++++++------------ doc/source/MaintainersSection.rst | 26 +++- doc/source/PreConfiguredSites.rst | 56 ++++++-- 7 files changed, 174 insertions(+), 112 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 5623a86dc..245d72cff 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -53,11 +53,8 @@ eigen: version: ['3.4.0'] # Attention - when updating the version also check the common modules.yaml - # config and update the projections for lmod/tcl - # Note: Stay away from esmf@8.4.1 ... breaks all sorts of things - #esmf: - # version: ['8.4.1'] - # variants: ~xerces ~pnetcdf +parallelio ~shared + # config and update the projections for lmod/tcl. + # Also, check the acorn and derecho site configs which have esmf modifications. esmf: version: ['8.4.2'] variants: ~xerces ~pnetcdf snapshot=none ~shared +external-parallelio diff --git a/configs/sites/aws-pcluster/modules.yaml b/configs/sites/aws-pcluster/modules.yaml index 11c5514e2..1f6fc0f74 100644 --- a/configs/sites/aws-pcluster/modules.yaml +++ b/configs/sites/aws-pcluster/modules.yaml @@ -6,3 +6,5 @@ modules: include: # List of packages for which we need modules that are blacklisted by default - python + exclude: + - ecflow \ No newline at end of file diff --git a/configs/sites/derecho/compilers.yaml b/configs/sites/derecho/compilers.yaml index 7007a1182..f0a449399 100644 --- a/configs/sites/derecho/compilers.yaml +++ b/configs/sites/derecho/compilers.yaml @@ -2,22 +2,25 @@ compilers:: - compiler: spec: intel@2021.8.0 paths: - cc: /glade/u/apps/derecho/23.06/spack/opt/spack/ncarcompilers/1.0.0/oneapi/2023.0.0/ec7b/bin/icc - cxx: /glade/u/apps/derecho/23.06/spack/opt/spack/ncarcompilers/1.0.0/oneapi/2023.0.0/ec7b/bin/icpc - f77: /glade/u/apps/derecho/23.06/spack/opt/spack/ncarcompilers/1.0.0/oneapi/2023.0.0/ec7b/bin/ifort - fc: /glade/u/apps/derecho/23.06/spack/opt/spack/ncarcompilers/1.0.0/oneapi/2023.0.0/ec7b/bin/ifort + cc: cc + cxx: CC + f77: ftn + fc: ftn flags: {} operating_system: sles15 target: x86_64 modules: + - ncarenv/23.06 - intel-classic/2023.0.0 environment: prepend_path: PATH: '/opt/cray/pe/gcc/12.2.0/bin' CPATH: '/opt/cray/pe/gcc/12.2.0/snos/include' - LD_LIBRARY_PATH: '/glade/u/apps/common/23.04/spack/opt/spack/intel-oneapi-compilers/2023.0.0/compiler/2023.0.0/linux/compiler/lib/intel64_lin:/opt/cray/pe/gcc/default/snos/lib64' + LD_LIBRARY_PATH: '/glade/u/apps/common/23.04/spack/opt/spack/intel-oneapi-compilers/2023.0.0/compiler/2023.0.0/linux/compiler/lib/intel64_lin:/opt/cray/pe/gcc/default/snos/lib:/opt/cray/pe/gcc/default/snos/lib64' set: - I_MPI_ROOT: '/glade/u/apps/derecho/23.06/spack/opt/spack/intel-oneapi-mpi/2021.8.0/oneapi/2023.0.0/mhf4/mpi/2021.8.0' + CRAYPE_LINK_TYPE: 'dynamic' + #set: + # I_MPI_ROOT: '/glade/u/apps/derecho/23.06/spack/opt/spack/intel-oneapi-mpi/2021.8.0/oneapi/2023.0.0/mhf4/mpi/2021.8.0' extra_rpaths: [] # - compiler: # spec: gcc@10.1.0 diff --git a/configs/sites/derecho/modules.yaml b/configs/sites/derecho/modules.yaml index 7728eff92..8264ecf47 100644 --- a/configs/sites/derecho/modules.yaml +++ b/configs/sites/derecho/modules.yaml @@ -3,5 +3,5 @@ modules: enable:: - lmod lmod: - blacklist: + exclude: - ecflow diff --git a/configs/sites/derecho/packages.yaml b/configs/sites/derecho/packages.yaml index 08ccba3ff..2d81fa0d7 100644 --- a/configs/sites/derecho/packages.yaml +++ b/configs/sites/derecho/packages.yaml @@ -2,30 +2,25 @@ packages: all: compiler:: [intel@2021.8.0] providers: - mpi:: [intel-oneapi-mpi@2021.8.0, cray-mpich@8.1.25] + mpi:: [cray-mpich@8.1.25] # intel-oneapi-mpi@2021.8.0, target: [core2] + ### MPI, Python, MKL mpi: buildable: False - intel-oneapi-mpi: - externals: - - spec: intel-oneapi-mpi@2021.8.0%intel@2021.8.0 - prefix: /glade/u/apps/derecho/23.06/spack/opt/spack/intel-oneapi-mpi/2021.8.0/oneapi/2023.0.0/mhf4 - modules: - - intel-mpi/2021.8.0 + #intel-oneapi-mpi: + # externals: + # - spec: intel-oneapi-mpi@2021.8.0%intel@2021.8.0 + # prefix: /glade/u/apps/derecho/23.06/spack/opt/spack/intel-oneapi-mpi/2021.8.0/oneapi/2023.0.0/mhf4 + # modules: + # - intel-mpi/2021.8.0 cray-mpich: externals: - - spec: cray-mpich@8.1.25%intel@2021.8.0 + - spec: cray-mpich@8.1.25%intel@2021.8.0 ~wrappers prefix: /opt/cray/pe/mpich/8.1.25/ofi/intel/19.0 modules: - craype/2.7.20 - - cray-mpich/8.1.25 -# openmpi: -# externals: -# - spec: openmpi@4.1.1%gcc@10.1.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm,tm -# prefix: /glade/u/apps/ch/opt/openmpi/4.1.1/gnu/10.1.0 -# modules: -# - openmpi/4.1.1 + - cray-mpich/8.1.25 python: buildable: False externals: @@ -36,6 +31,34 @@ packages: externals: - spec: py-pip@22.0.4 prefix: /usr + +### Modification of common packages + #esmf: + # variants: ~xerces ~pnetcdf snapshot=none ~shared +external-parallelio esmf_os=Linux esmf_comm=mpich3 + +### All other external packages listed alphabetically + autoconf: + externals: + - spec: autoconf@2.71 + prefix: /glade/u/apps/derecho/23.06/opt + - spec: autoconf@2.69 + prefix: /usr + automake: + externals: + - spec: automake@1.16.5 + prefix: /glade/u/apps/derecho/23.06/opt + - spec: automake@1.15.1 + prefix: /usr + binutils: + externals: + - spec: binutils@2.37.20211103 + prefix: /usr + bison: + externals: + - spec: bison@3.8.2 + prefix: /glade/u/apps/derecho/23.06/opt + - spec: bison@3.0.4 + prefix: /usr cmake: externals: - spec: cmake@3.26.3 @@ -44,45 +67,42 @@ packages: externals: - spec: coreutils@8.32 prefix: /usr - texinfo: + curl: externals: - - spec: texinfo@7.0 + - spec: curl@8.0.1+nghttp2 prefix: /glade/u/apps/derecho/23.06/opt - - spec: texinfo@6.5 + - spec: curl@7.79.1+gssapi+ldap+nghttp2 prefix: /usr - groff: + cvs: externals: - - spec: groff@1.22.4 + - spec: cvs@1.12.13 prefix: /usr - automake: + diffutils: externals: - - spec: automake@1.16.5 - prefix: /glade/u/apps/derecho/23.06/opt - - spec: automake@1.15.1 + - spec: diffutils@3.6 prefix: /usr - gawk: + doxygen: externals: - - spec: gawk@4.2.1 - prefix: /usr - m4: + - spec: doxygen@1.9.6 + prefix: /lustre/desc1/scratch/epicufsrt/contrib/doxygen + ecflow: + buildable: False externals: - - spec: m4@1.4.18 - prefix: /usr - subversion: + - spec: ecflow@5.8.4+ui+static_boost + prefix: /lustre/desc1/scratch/epicufsrt/contrib/ecflow-5.8.4 + modules: + - ecflow/5.8.4 + findutils: externals: - - spec: subversion@1.14.1 + - spec: findutils@4.8.0 prefix: /usr flex: externals: - spec: flex@2.6.4+lex prefix: /glade/u/apps/derecho/23.06/opt - openssl: - externals: - - spec: openssl@1.1.1l - prefix: /usr - pkg-config: + gawk: externals: - - spec: pkg-config@0.29.2 + - spec: gawk@4.2.1 prefix: /usr git: externals: @@ -94,77 +114,61 @@ packages: externals: - spec: git-lfs@3.3.0 prefix: /glade/u/apps/derecho/23.06/opt - meson: + gmake: externals: - - spec: meson@1.1.0 - prefix: /glade/u/apps/derecho/23.06/opt - curl: + - spec: gmake@4.2.1 + prefix: /usr + groff: externals: - - spec: curl@8.0.1+nghttp2 - prefix: /glade/u/apps/derecho/23.06/opt - - spec: curl@7.79.1+gssapi+ldap+nghttp2 + - spec: groff@1.22.4 prefix: /usr + krb5: + externals: + - spec: krb5@1.19.2 + prefix: /usr/lib/mit libtool: externals: - spec: libtool@2.4.7 prefix: /glade/u/apps/derecho/23.06/opt - spec: libtool@2.4.6 prefix: /usr - autoconf: - externals: - - spec: autoconf@2.71 - prefix: /glade/u/apps/derecho/23.06/opt - - spec: autoconf@2.69 - prefix: /usr - gmake: - externals: - - spec: gmake@4.2.1 - prefix: /usr - bison: + m4: externals: - - spec: bison@3.8.2 - prefix: /glade/u/apps/derecho/23.06/opt - - spec: bison@3.0.4 + - spec: m4@1.4.18 prefix: /usr - binutils: + # Automatically detected, but don't use - missing "ninja" dependency + #meson: + # externals: + # - spec: meson@1.1.0 + # prefix: /glade/u/apps/derecho/23.06/opt + mysql: + buildable: False externals: - - spec: binutils@2.37.20211103 - prefix: /usr - cvs: + - spec: mysql@8.0.33 + prefix: /lustre/desc1/scratch/epicufsrt/contrib/mysql-8.0.33 + openssh: externals: - - spec: cvs@1.12.13 + - spec: openssh@8.4p1 prefix: /usr - findutils: + openssl: externals: - - spec: findutils@4.8.0 + - spec: openssl@1.1.1l prefix: /usr - diffutils: + pkg-config: externals: - - spec: diffutils@3.6 + - spec: pkg-config@0.29.2 prefix: /usr - openssh: + subversion: externals: - - spec: openssh@8.4p1 + - spec: subversion@1.14.1 prefix: /usr tar: externals: - spec: tar@1.34 prefix: /usr - boost: - buildable: False - externals: - - spec: boost@1.78.0 - prefix: /lustre/desc1/scratch/epicufsrt/contrib/boost_1_78_0 - mysql: - buildable: False - externals: - - spec: mysql@8.0.33 - prefix: /lustre/desc1/scratch/epicufsrt/contrib/mysql-8.0.33 - doxygen: + texinfo: externals: - - spec: doxygen@1.9.6 - prefix: /lustre/desc1/scratch/epicufsrt/contrib/doxygen - - - - + - spec: texinfo@7.0 + prefix: /glade/u/apps/derecho/23.06/opt + - spec: texinfo@6.5 + prefix: /usr diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst index 3bb74cc17..75112fbf2 100644 --- a/doc/source/MaintainersSection.rst +++ b/doc/source/MaintainersSection.rst @@ -111,7 +111,7 @@ The following instructions are for Discover (see :numref:`Section %s &1 | tee log.cmake .. note:: - Finally, on Gaea C5, one needs to pass the correct ``python3`` executable to the ``cmake`` command: + Further, on Gaea C5, one needs to pass the correct ``python3`` executable to the ``cmake`` command: .. code-block:: console cmake .. -DPython3_EXECUTABLE=`which python3` -DCMAKE_INSTALL_PREFIX=/path/to/ecflow/installation 2>&1 | tee log.cmake +.. note:: + Finally, on Derecho (or any other system with ``gcc@12.2.0``), one needs to patch file ``ecflow-5.8.4/src/ecFlow-5.8.4-Source/ACore/src/Passwd.cpp`` by adding ``#include `` below line ``#include "Passwd.hpp"`` before running ``make``. + .. _MaintainersSection_MySQL: ------------------------------ @@ -466,6 +469,25 @@ openmpi make check make install +.. _MaintainersSection_Derecho: + +------------------------------ +NCAR-Wyoming Derecho +------------------------------ + +ecflow + ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After loading the following modules, follow the instructions in :numref:`Section %s ` to install ``ecflow`` in ``/lustre/desc1/scratch/epicufsrt/contrib/ecflow-5.8.4``. Be sure to follow the extra instructions for Derecho in that section. + +.. code-block:: console + + module purge + export LMOD_TMOD_FIND_FIRST=yes + module load gcc/12.2.0 + module load cmake/3.26.3 + +mysql + ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s ` to install ``mysql`` in ``/lustre/desc1/scratch/epicufsrt/contrib/mysql-8.0.33``. + .. _MaintainersSection_WCOSS2: ------------------------------ diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 5726946e7..6ab209b3e 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -18,15 +18,17 @@ Ready-to-use spack-stack 1.4.1 installations are available on the following, ful +=====================+==================================+=================+=========================================================================================================+===============================+ | **HPC platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Hercules | GCC^*, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230717/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Hercules^** | GCC^*, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230814/envs/unified-env`` | Cam Book / Dom Heinzeller | | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | NASA | Discover | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Casper | Intel | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.4.1/envs/unified-env`` | Dom Heinzeller / ??? | -| NCAR-Wyoming +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Cheyenne | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +| +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ +| NCAR-Wyoming | Cheyenne | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +| +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ +| | Derecho^** | Intel | ``/lustre/desc1/scratch/epicufsrt/contrib/spack-stack/spack-stack-dev-20230814/envs/unified-en2`` | Mark Potts / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Hang Lei / Alex Richert | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -40,7 +42,7 @@ Ready-to-use spack-stack 1.4.1 installations are available on the following, ful +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-intel-2021.4.0-hdf5-1.14.0`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | Intel (test) | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-dev-20230628/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | +| | Narwhal^** | Intel (test) | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-dev-20230628/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | | U.S. Navy (HPCMP) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -65,6 +67,8 @@ Ready-to-use spack-stack 1.4.1 installations are available on the following, ful ^* This system uses a different wgrib2 version 3.1.1 than the default 2.0.8. +^** This system uses a different version of spack-stack (mostly newer), which is usually the case when we onboard a new system in between spack-stack releases. + For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. .. _Preconfigured_Sites_Tier1: @@ -122,28 +126,28 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-dev-20230717`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-dev-20230814`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230717/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230814/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.9.0 module load stack-intel-oneapi-mpi/2021.9.0 module load stack-python/3.9.14 module available -For ``spack-stack-dev-20230717`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-dev-20230814`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230717/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230814/envs/unified-env/install/modulefiles/Core module load stack-gcc/11.3.1 module load stack-openmpi/4.1.5 module load stack-python/3.9.14 module available .. note:: - The recent update to ``spack-stack-dev-20230717`` was required on Hercules due to a bug in the Intel compilers used in ``spack-stack-1.4.1``. + The recent update to ``spack-stack-dev-20230814`` was required on Hercules due to a bug in the Intel compilers used in ``spack-stack-1.4.1``. .. _Preconfigured_Sites_Discover: @@ -356,7 +360,7 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.4.1`` with Intel, load the following modules after loading miniconda and ecflow. +For ``spack-stack-1.4.1`` with Intel, load the following modules after loading miniconda, ecflow and mysql. .. code-block:: console @@ -366,7 +370,7 @@ For ``spack-stack-1.4.1`` with Intel, load the following modules after loading m module load stack-python/3.9.12 module available -For ``spack-stack-1.4.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.4.1`` with GNU, load the following modules after loading miniconda, ecflow and mysql. .. code-block:: console @@ -376,6 +380,36 @@ For ``spack-stack-1.4.1`` with GNU, load the following modules after loading min module load stack-python/3.9.12 module available +.. _Preconfigured_Sites_Derecho: + +-------------------- +NCAR-Wyoming Derecho +-------------------- + +The following is required for building new spack environments and for using spack to build and run software. + +.. code-block:: console + + module purge + # ignore that the sticky module ncarenv/... is not unloaded + export LMOD_TMOD_FIND_FIRST=yes + module use /lustre/desc1/scratch/epicufsrt/contrib/modulefiles + module load ecflow/5.8.4 + module load mysql/8.0.33 + +For ``spack-stack-develop`` as of 2023/08/14 with Intel, load the following modules after loading ecflow and mysql: + +.. code-block:: console + + module use /lustre/desc1/scratch/epicufsrt/contrib/spack-stack/spack-stack-dev-20230814/envs/unified-en2/install/modulefiles/Core + module load stack-intel/2021.8.0 + module load stack-cray-mpich/8.1.25 + module load stack-python/3.10.10 + module available + +.. note:: + CISL restricts the amount of memory available for processes on the login nodes. For example, it is impossible to compile JEDI with even one task (``make -j1``) with the Intel compiles in release mode (``-O2``). We therefore recommend compiling on compute nodes using interactive jobs, if possible. + .. _Preconfigured_Sites_Acorn: -------------------------------