Update Derecho site config (JCSDA#720)

* Update Derecho site config and according documentation * Use correct compiler paths in configs/sites/derecho/compilers.yaml * Fix esmf build error on Derecho * Bug fix in configs/sites/aws-pcluster/modules.yaml: blacklist ecflow * Update spack-stack documentation for Derecho * Treat derecho like a real cray: use cray compiler wrappers * Update doc/source/PreConfiguredSites.rst for Hercules * Update configs/sites/derecho/compilers.yaml - use Cray compiler wrappers * Add external krb5 to configs/sites/derecho/packages.yaml * Update spack env path for derecho in doc/source/PreConfiguredSites.rst
AlexanderRichert-NOAA · Aug 24, 2023 · 5497ef8 · 5497ef8
1 parent 915c330
commit 5497ef8
Show file tree

Hide file tree

Showing 7 changed files with 174 additions and 112 deletions.
diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml
@@ -53,11 +53,8 @@
     eigen:
       version: ['3.4.0']
     # Attention - when updating the version also check the common modules.yaml
-    # config and update the projections for lmod/tcl
-    # Note: Stay away from [email protected] ... breaks all sorts of things
-    #esmf:
-    #  version: ['8.4.1']
-    #  variants: ~xerces ~pnetcdf +parallelio ~shared
+    # config and update the projections for lmod/tcl.
+    # Also, check the acorn and derecho site configs which have esmf modifications.
     esmf:
       version: ['8.4.2']
       variants: ~xerces ~pnetcdf snapshot=none ~shared +external-parallelio

diff --git a/configs/sites/aws-pcluster/modules.yaml b/configs/sites/aws-pcluster/modules.yaml
@@ -6,3 +6,5 @@ modules:
       include:
       # List of packages for which we need modules that are blacklisted by default
       - python
+      exclude:
+      - ecflow
diff --git a/configs/sites/derecho/compilers.yaml b/configs/sites/derecho/compilers.yaml
@@ -2,22 +2,25 @@ compilers::
   - compiler:
       spec: [email protected]
       paths:
-        cc: /glade/u/apps/derecho/23.06/spack/opt/spack/ncarcompilers/1.0.0/oneapi/2023.0.0/ec7b/bin/icc
-        cxx: /glade/u/apps/derecho/23.06/spack/opt/spack/ncarcompilers/1.0.0/oneapi/2023.0.0/ec7b/bin/icpc
-        f77: /glade/u/apps/derecho/23.06/spack/opt/spack/ncarcompilers/1.0.0/oneapi/2023.0.0/ec7b/bin/ifort
-        fc: /glade/u/apps/derecho/23.06/spack/opt/spack/ncarcompilers/1.0.0/oneapi/2023.0.0/ec7b/bin/ifort
+        cc: cc
+        cxx: CC
+        f77: ftn
+        fc: ftn
       flags: {}
       operating_system: sles15
       target: x86_64
       modules:
+      - ncarenv/23.06
       - intel-classic/2023.0.0
       environment:
         prepend_path:
           PATH: '/opt/cray/pe/gcc/12.2.0/bin'
           CPATH: '/opt/cray/pe/gcc/12.2.0/snos/include'
-          LD_LIBRARY_PATH: '/glade/u/apps/common/23.04/spack/opt/spack/intel-oneapi-compilers/2023.0.0/compiler/2023.0.0/linux/compiler/lib/intel64_lin:/opt/cray/pe/gcc/default/snos/lib64'
+          LD_LIBRARY_PATH: '/glade/u/apps/common/23.04/spack/opt/spack/intel-oneapi-compilers/2023.0.0/compiler/2023.0.0/linux/compiler/lib/intel64_lin:/opt/cray/pe/gcc/default/snos/lib:/opt/cray/pe/gcc/default/snos/lib64'
         set:
-          I_MPI_ROOT: '/glade/u/apps/derecho/23.06/spack/opt/spack/intel-oneapi-mpi/2021.8.0/oneapi/2023.0.0/mhf4/mpi/2021.8.0'
+          CRAYPE_LINK_TYPE: 'dynamic'
+        #set:
+        #  I_MPI_ROOT: '/glade/u/apps/derecho/23.06/spack/opt/spack/intel-oneapi-mpi/2021.8.0/oneapi/2023.0.0/mhf4/mpi/2021.8.0'
       extra_rpaths: []
 # - compiler:
 #     spec: [email protected]

diff --git a/configs/sites/derecho/modules.yaml b/configs/sites/derecho/modules.yaml
@@ -3,5 +3,5 @@ modules:
     enable::
     - lmod
     lmod:
-      blacklist:
+      exclude:
       - ecflow
diff --git a/configs/sites/derecho/packages.yaml b/configs/sites/derecho/packages.yaml
@@ -2,30 +2,25 @@ packages:
   all:
     compiler:: [[email protected]]
     providers:
-      mpi:: [[email protected], [email protected]]
+      mpi:: [[email protected]] # [email protected], 
     target: [core2]
+
 ### MPI, Python, MKL
   mpi:
     buildable: False
-  intel-oneapi-mpi:
-    externals:
-    - spec: [email protected]%[email protected]   
-      prefix: /glade/u/apps/derecho/23.06/spack/opt/spack/intel-oneapi-mpi/2021.8.0/oneapi/2023.0.0/mhf4
-      modules:
-      - intel-mpi/2021.8.0  
+  #intel-oneapi-mpi:
+  #  externals:
+  #  - spec: [email protected]%[email protected]
+  #    prefix: /glade/u/apps/derecho/23.06/spack/opt/spack/intel-oneapi-mpi/2021.8.0/oneapi/2023.0.0/mhf4
+  #    modules:
+  #    - intel-mpi/2021.8.0
   cray-mpich:
     externals:
-    - spec: [email protected]%[email protected]   
+    - spec: [email protected]%[email protected] ~wrappers
       prefix: /opt/cray/pe/mpich/8.1.25/ofi/intel/19.0
       modules:
       - craype/2.7.20
-      - cray-mpich/8.1.25 
-# openmpi:
-#   externals:
-#   - spec: [email protected]%[email protected]~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm,tm
-#     prefix: /glade/u/apps/ch/opt/openmpi/4.1.1/gnu/10.1.0
-#     modules:
-#     - openmpi/4.1.1
+      - cray-mpich/8.1.25
   python:
     buildable: False
     externals:
@@ -36,6 +31,34 @@ packages:
     externals:
     - spec: [email protected]
       prefix: /usr
+
+### Modification of common packages
+  #esmf:
+  #  variants: ~xerces ~pnetcdf snapshot=none ~shared +external-parallelio esmf_os=Linux esmf_comm=mpich3
+
+### All other external packages listed alphabetically
+  autoconf:
+    externals:
+    - spec: [email protected]
+      prefix: /glade/u/apps/derecho/23.06/opt
+    - spec: [email protected]
+      prefix: /usr
+  automake:
+    externals:
+    - spec: [email protected]
+      prefix: /glade/u/apps/derecho/23.06/opt
+    - spec: [email protected]
+      prefix: /usr
+  binutils:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  bison:
+    externals:
+    - spec: [email protected]
+      prefix: /glade/u/apps/derecho/23.06/opt
+    - spec: [email protected]
+      prefix: /usr
   cmake:
     externals:
     - spec: [email protected]
@@ -44,45 +67,42 @@ packages:
     externals:
     - spec: [email protected]
       prefix: /usr
-  texinfo:
+  curl:
     externals:
-    - spec: [email protected]
+    - spec: [email protected]+nghttp2
       prefix: /glade/u/apps/derecho/23.06/opt
-    - spec: [email protected]
+    - spec: [email protected]+gssapi+ldap+nghttp2
       prefix: /usr
-  groff:
+  cvs:
     externals:
-    - spec: groff@1.22.4
+    - spec: cvs@1.12.13
       prefix: /usr
-  automake:
+  diffutils:
     externals:
-    - spec: [email protected]
-      prefix: /glade/u/apps/derecho/23.06/opt
-    - spec: [email protected]
+    - spec: [email protected]
       prefix: /usr
-  gawk:
+  doxygen:
     externals:
-    - spec: [email protected]
-      prefix: /usr
-  m4:
+    - spec: [email protected]
+      prefix: /lustre/desc1/scratch/epicufsrt/contrib/doxygen
+  ecflow:
+    buildable: False
     externals:
-    - spec: [email protected]
-      prefix: /usr
-  subversion:
+    - spec: [email protected]+ui+static_boost
+      prefix: /lustre/desc1/scratch/epicufsrt/contrib/ecflow-5.8.4
+      modules:
+      - ecflow/5.8.4
+  findutils:
     externals:
-    - spec: [email protected]
+    - spec: [email protected]
       prefix: /usr
   flex:
     externals:
     - spec: [email protected]+lex
       prefix: /glade/u/apps/derecho/23.06/opt
-  openssl:
-    externals:
-    - spec: [email protected]
-      prefix: /usr
-  pkg-config:
+  gawk:
     externals:
-    - spec: [email protected]
+    - spec: [email protected]
       prefix: /usr
   git:
     externals:
@@ -94,77 +114,61 @@ packages:
     externals:
     - spec: [email protected]
       prefix: /glade/u/apps/derecho/23.06/opt
-  meson:
+  gmake:
     externals:
-    - spec: [email protected]
-      prefix: /glade/u/apps/derecho/23.06/opt
-  curl:
+    - spec: [email protected]
+      prefix: /usr
+  groff:
     externals:
-    - spec: [email protected]+nghttp2
-      prefix: /glade/u/apps/derecho/23.06/opt
-    - spec: [email protected]+gssapi+ldap+nghttp2
+    - spec: [email protected]
       prefix: /usr
+  krb5:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/lib/mit
   libtool:
     externals:
     - spec: [email protected]
       prefix: /glade/u/apps/derecho/23.06/opt
     - spec: [email protected]
       prefix: /usr
-  autoconf:
-    externals:
-    - spec: [email protected]
-      prefix: /glade/u/apps/derecho/23.06/opt
-    - spec: [email protected]
-      prefix: /usr
-  gmake:
-    externals:
-    - spec: [email protected]
-      prefix: /usr
-  bison:
+  m4:
     externals:
-    - spec: [email protected]
-      prefix: /glade/u/apps/derecho/23.06/opt
-    - spec: [email protected]
+    - spec: [email protected]
       prefix: /usr
-  binutils:
+  # Automatically detected, but don't use - missing "ninja" dependency
+  #meson:
+  #  externals:
+  #  - spec: [email protected]
+  #    prefix: /glade/u/apps/derecho/23.06/opt
+  mysql:
+    buildable: False
     externals:
-    - spec: [email protected]
-      prefix: /usr
-  cvs:
+    - spec: [email protected]
+      prefix: /lustre/desc1/scratch/epicufsrt/contrib/mysql-8.0.33
+  openssh:
     externals:
-    - spec: [email protected]
+    - spec: [email protected]
       prefix: /usr
-  findutils:
+  openssl:
     externals:
-    - spec: [email protected]
+    - spec: [email protected]
       prefix: /usr
-  diffutils:
+  pkg-config:
     externals:
-    - spec: [email protected]
+    - spec: [email protected]
       prefix: /usr
-  openssh:
+  subversion:
     externals:
-    - spec: [email protected]
+    - spec: [email protected]
       prefix: /usr
   tar:
     externals:
     - spec: [email protected]
       prefix: /usr
-  boost:
-    buildable: False
-    externals:
-    - spec: [email protected]
-      prefix: /lustre/desc1/scratch/epicufsrt/contrib/boost_1_78_0
-  mysql:
-    buildable: False
-    externals:
-    - spec: [email protected] 
-      prefix: /lustre/desc1/scratch/epicufsrt/contrib/mysql-8.0.33
-  doxygen:
+  texinfo:
     externals:
-    - spec: [email protected]
-      prefix: /lustre/desc1/scratch/epicufsrt/contrib/doxygen
-
-
-
-
+    - spec: [email protected]
+      prefix: /glade/u/apps/derecho/23.06/opt
+    - spec: [email protected]
+      prefix: /usr
diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst
@@ -111,7 +111,7 @@ The following instructions are for Discover (see :numref:`Section %s <Maintainer
 Create modulefile ``/lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles/ecflow/5.8.4`` from template ``doc/modulefile_templates/ecflow`` and update ``ECFLOW_PATH`` in this file.
 
 .. note::
-   For certain Cray systems, for example NRL's Narwhal or NOAA's Gaea C4/C5, the following modifications are necessary: After extracting the ecflow tarball, edit ``ecFlow-5.8.4-Source/build_scripts/boost_build.sh`` and remove the following lines:
+   For Cray systems, for example NRL's Narwhal, NOAA's Gaea C4/C5, or NCAR's Derecho, the following modifications are necessary: After extracting the ecflow tarball, edit ``ecFlow-5.8.4-Source/build_scripts/boost_build.sh`` and remove the following lines:
 
 .. code-block:: console
 
@@ -130,12 +130,15 @@ Create modulefile ``/lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles/ecflow/5.8
    CC=gcc CXX=g++ FC=gfortran cmake .. -DCMAKE_INSTALL_PREFIX=/path/to/ecflow/installation 2>&1 | tee log.cmake
 
 .. note::
-   Finally, on Gaea C5, one needs to pass the correct ``python3`` executable to the ``cmake`` command:
+   Further, on Gaea C5, one needs to pass the correct ``python3`` executable to the ``cmake`` command:
 
 .. code-block:: console
 
    cmake .. -DPython3_EXECUTABLE=`which python3` -DCMAKE_INSTALL_PREFIX=/path/to/ecflow/installation 2>&1 | tee log.cmake
 
+.. note::
+   Finally, on Derecho (or any other system with ``[email protected]``), one needs to patch file ``ecflow-5.8.4/src/ecFlow-5.8.4-Source/ACore/src/Passwd.cpp`` by adding ``#include <ctime>`` below line ``#include "Passwd.hpp"`` before running ``make``.
+
 ..  _MaintainersSection_MySQL:
 
 ------------------------------
@@ -466,6 +469,25 @@ openmpi
    make check
    make install
 
+.. _MaintainersSection_Derecho:
+
+------------------------------
+NCAR-Wyoming Derecho
+------------------------------
+
+ecflow
+  ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After loading the following modules, follow the instructions in :numref:`Section %s <MaintainersSection_ecFlow>` to install ``ecflow`` in ``/lustre/desc1/scratch/epicufsrt/contrib/ecflow-5.8.4``. Be sure to follow the extra instructions for Derecho in that section.
+
+.. code-block:: console
+
+   module purge
+   export LMOD_TMOD_FIND_FIRST=yes
+   module load gcc/12.2.0
+   module load cmake/3.26.3
+
+mysql
+  ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s <MaintainersSection_MySQL>` to install ``mysql`` in ``/lustre/desc1/scratch/epicufsrt/contrib/mysql-8.0.33``.
+
 .. _MaintainersSection_WCOSS2:
 
 ------------------------------