fix(InputOutput): address issue appending processor id for new intel …

.gitlab-ci.yml

@@ -1,3 +1,26 @@
+# Deploy MODFLOW 6 to HPC systems.
+#
+# Expects shared environment variables...
+#
+#  - GIT_REMOTE, name of the mf6 remote to pull from
+#  - GIT_BRANCH, name of the mf6 branch to pull from
+#  - MODULE_SCRIPT, module update script path (relative to mf6 proj root)
+#  - SSH_KNOWN_HOSTS, content of ~/.ssh/known_hosts
+#  - SSH_USERNAME, SSH username to login with
+#  - SSH_PRIVATE_KEY, SSH private key for user
+#
+# ...and environment-specific variables:
+#
+#  - DENALI_HOSTNAME, SSH hostname of cluster
+#  - DENALI_SLURM_ACCOUNT, slurm account for jobs
+#  - DENALI_SLURM_RESERVATION, slurm reservation for jobs
+#  - DENALI_MODULES_PATH, path to root of module system
+#  - DENALI_MF6_PREV_VERSION, version of modulefile to use as template
+#  - DENALI_MF6_PROJ_ROOT, path to modflow6 project root
+#  - DENALI_BUILD_SCRIPT, relpath of script to build mf6
+#
+# ...and likewise for HOVENWEEP_* or other systems
+
 image: ubuntu:20.04
 workflow:
   rules:
@@ -9,23 +32,35 @@ before_script:
   - eval $(ssh-agent -s)
   # set private key permissions
   - chmod 400 "$SSH_PRIVATE_KEY"
-  # add key to the agent store
+  # add keys to the agent store
   - ssh-add "$SSH_PRIVATE_KEY"
   # configure known hosts
   - mkdir -p ~/.ssh
   - cp "$SSH_KNOWN_HOSTS" ~/.ssh/known_hosts
   - chmod 644 ~/.ssh/known_hosts
 deploy_denali:
-  stage: deploy
-  script: |
-    # fetch and checkout latest
-    ssh -l "$SSH_USERNAME" "$SSH_HOSTNAME" "cd $MF6_PROJ_ROOT && git fetch $GIT_REMOTE && git checkout $GIT_BRANCH && git pull"
-    echo "Updated repository $MF6_PROJ_ROOT"
-    # submit a job to build mf6
-    jobid=$(ssh -l "$SSH_USERNAME" "$SSH_HOSTNAME" "sbatch --account=$SLURM_ACCOUNT --reservation=$SLURM_RESERVATION --parsable -D $MF6_PROJ_ROOT/.hpc $MF6_PROJ_ROOT/$BUILD_SCRIPT" | tail -n 1)
-    echo "Submitted build job $jobid"
-    # submit a job to update the mf6 module
-    jobid=$(ssh -l "$SSH_USERNAME" "$SSH_HOSTNAME" "sbatch --export=ALL,INTEL_VERSION=$INTEL_VERSION,MF6_PROJ_ROOT=$MF6_PROJ_ROOT,MODULES_PATH=$MODULES_PATH --account=$SLURM_ACCOUNT --reservation=$SLURM_RESERVATION --parsable -D $MF6_PROJ_ROOT/.hpc -d afterok:$jobid $MF6_PROJ_ROOT/$MODULE_SCRIPT" | tail -n 1)
-    echo "Submitted module update job $jobid"
   environment: denali
-
+  script: "$CI_PROJECT_DIR/.hpc/deploy.sh"
+  stage: deploy
+  variables:
+    SSH_USERNAME: $SSH_USERNAME
+    SSH_HOSTNAME: $DENALI_HOSTNAME
+    SLURM_ACCOUNT: $DENALI_SLURM_ACCOUNT
+    SLURM_RESERVATION: $DENALI_SLURM_RESERVATION
+    MODULES_PATH: $DENALI_MODULES_PATH
+    MF6_PROJ_ROOT: $DENALI_MF6_PROJ_ROOT
+    MF6_PREV_VERSION: $DENALI_MF6_PREV_VERSION
+    BUILD_SCRIPT: $DENALI_BUILD_SCRIPT
+deploy_hovenweep:
+  environment: hovenweep
+  script: "$CI_PROJECT_DIR/.hpc/deploy.sh"
+  stage: deploy
+  variables:
+    SSH_USERNAME: $SSH_USERNAME
+    SSH_HOSTNAME: $HOVENWEEP_HOSTNAME
+    SLURM_ACCOUNT: $HOVENWEEP_SLURM_ACCOUNT
+    SLURM_RESERVATION: $HOVENWEEP_SLURM_RESERVATION
+    MODULES_PATH: $HOVENWEEP_MODULES_PATH
+    MF6_PROJ_ROOT: $HOVENWEEP_MF6_PROJ_ROOT
+    MF6_PREV_VERSION: $HOVENWEEP_MF6_PREV_VERSION
+    BUILD_SCRIPT: $HOVENWEEP_BUILD_SCRIPT

.hpc/BUILD.md

@@ -1,59 +1,30 @@
 
 # Building MODFLOW 6 on HPC systems
 
-## SLURM job
+_On Denali_ 
 
 ```
 sbatch --reservation=dev cray-meson-build.slurm.batch
 ```
 
-## Interactive job
+_Hovenweep_
 
 ```
-module switch PrgEnv-${PE_ENV,,} PrgEnv-intel
-module load cray-petsc meson ninja
-export PKG_CONFIG_PATH=/opt/cray/pe/mpt/7.7.19/gni/mpich-intel/16.0/lib/pkgconfig:/opt/cray/pe/petsc/3.14.5.0/real/INTEL/19.1/x86_skylake/lib/pkgconfig:$PKG_CONFIG_PATH
-
-srun --reservation=dev --account=impd --pty bash
-
-meson setup builddir -Ddebug=false --prefix=$(pwd) --libdir=bin --bindir=bin -Dcray=true -Ddebug=false --wipe
-meson install -C builddir
+sbatch cray-hovenweep-meson-build.slurm.batch
 ```
 
-# Installing a new version of MODFLOW 6 on HPC systems
-
-
-After building the new version (`6.x.x`) of MODFLOW 6 using a SLURM or interactive job you will need to install the new version and create a new module file.
-
-## Install a new version of MODFLOW 6
-
-Create a directory for the new version in `/home/software/denali/contrib/impd/apps/modflow/`
-
-Since MODFLOW 6 is currently only being compiled with INTEL (version 19.1.0.166), make the following subdirectories in the `/home/software/denali/contrib/impd/apps/modflow/6.x.x` directory you just created.
-
-1. `/INTEL/19.1.0.166/bin`
-2. `/INTEL/19.1.0.166/lib`
-
-Copy `mf6` and `zbud6` to the `/INTEL/19.1.0.166/bin` and `libmf6.so` to the `/INTEL/19.1.0.166/lib` subdirectories using
 
-```
-rsync ../bin/mf6 /home/software/denali/contrib/impd/apps/modflow/6.x.x/INTEL/19.1.0.166/bin/
-```
-
-```
-rsync ../bin/zbud6 /home/software/denali/contrib/impd/apps/modflow/6.x.x/INTEL/19.1.0.166/bin/
-```
+## Create a module file for a new version of MODFLOW 6
 
+On _Denali_ make a copy of an existing module file using
 ```
-rsync ../bin/libmf6.so /home/software/denali/contrib/impd/apps/modflow/6.x.x/INTEL/19.1.0.166/lib/
+rsync /home/software/denali/contrib/impd/modulefiles/modflow/6.5.0.dev0 /home/software/denali/contrib/impd/modulefiles/modflow/6.x.x
 ```
-## Create a module file for a new version of MODFLOW 6
-
-Make a copy of an existing module file using
+On _Hovenweep_ make a copy of an existing module file using
 ```
-rsync /home/software/denali/contrib/impd/modulefiles/modflow/6.4.2 /home/software/denali/contrib/impd/modulefiles/modflow/6.x.x
+rsync /home/software/hovenweep/contrib/impd/modulefiles/modflow/6.5.0.dev0 /home/software/denali/contrib/impd/modulefiles/modflow/6.x.x
 ```
 
-Edit `product_version` in the new module file from `6.4.2` to `6.x.x`.
+Edit `product_version` in the new module file from `6.5.0.dev0` to `6.x.x` on both systems.
 
 

.hpc/cray-hovenweep-meson-build.slurm.batch

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+#SBATCH --job-name=hovenweep-build
+#SBATCH --nodes=1
+#SBATCH --ntasks=2
+#SBATCH --account=impd
+#SBATCH --time=00:10:00
+#SBATCH --output=slurm-%j.out
+#SBATCH --chdir=../
+
+set -euxo pipefail
+
+# load appropriate modules
+module switch PrgEnv-${PE_ENV,,} PrgEnv-intel
+module load petsc/3.15.5
+export PKG_CONFIG_PATH=$CRAY_MPICH_DIR/lib/pkgconfig:$PKG_CONFIG_PATH
+
+# list loaded modules
+module list
+
+# define the project root (expected to be cwd)
+MODFLOW6ROOT=$(pwd)
+
+# define the version
+VERSION=$(cat "$MODFLOW6ROOT/version.txt") 
+echo "MODFLOW 6 version: $VERSION"
+
+# define paths relative to the root directory
+BUILDDIR=$MODFLOW6ROOT/$PE_ENV-$VERSION
+BINDIR=$BUILDDIR/src
+TESTDIR=$MODFLOW6ROOT/.mf6minsim
+
+# define the installation location
+PREFIX=/home/software/hovenweep/contrib/impd/apps/modflow/$VERSION/$PE_ENV/2023.2.0
+
+# build MODFLOW 6
+CC=cc CXX=CC F77=ftn F90=ftn FC=ftn meson setup $BUILDDIR --prefix=$PREFIX --bindir=bin --libdir=lib -Dcray=true -Ddebug=false --wipe
+meson compile -C $BUILDDIR
+
+# install MODFLOW 6
+meson install -C $BUILDDIR
+
+# test MODFLOW 6 build
+cd $TESTDIR
+
+# serial run
+$BINDIR/mf6
+
+# parallel run
+srun $BINDIR/mf6 -p

.hpc/cray-meson-build.slurm.batch

@@ -1,10 +1,14 @@
 #!/bin/bash
 
-#SBATCH --job-name=meson-MODFLOW-build
+#SBATCH --job-name=denali-build
 #SBATCH --nodes=1
 #SBATCH --ntasks=2
+#SBATCH --account=impd
 #SBATCH --time=00:10:00
 #SBATCH --output=slurm-%j.out
+#SBATCH --chdir=../
+
+set -euxo pipefail
 
 # load appropriate modules
 module switch PrgEnv-${PE_ENV,,} PrgEnv-intel
@@ -14,17 +18,27 @@ export PKG_CONFIG_PATH=/opt/cray/pe/mpt/7.7.19/gni/mpich-intel/16.0/lib/pkgconfi
 # list loaded modules
 module list
 
-# move to root directory
-cd ..
+# define the project root (expected to be cwd)
+MODFLOW6ROOT=$(pwd)
+
+# define the version
+VERSION=$(cat "$MODFLOW6ROOT/version.txt") 
+echo "MODFLOW 6 version: $VERSION"
 
 # define paths relative to the root directory
-MODFLOW6ROOT=$(pwd)
-BINDIR=$MODFLOW6ROOT/bin
+BUILDDIR=$MODFLOW6ROOT/$PE_ENV-$VERSION
+BINDIR=$BUILDDIR/src
 TESTDIR=$MODFLOW6ROOT/.mf6minsim
 
+# define the installation location
+PREFIX=/home/software/denali/contrib/impd/apps/modflow/$VERSION/$PE_ENV/19.1.0.166
+
 # build MODFLOW 6
-CC=cc CXX=CC F77=ftn F90=ftn FC=ftn meson setup builddir --prefix=$(pwd) --bindir=bin --libdir=bin -Dcray=true -Ddebug=false --wipe
-meson install -C builddir
+CC=cc CXX=CC F77=ftn F90=ftn FC=ftn meson setup $BUILDDIR --prefix=$PREFIX --bindir=bin --libdir=lib -Dcray=true -Ddebug=false
+meson compile -C $BUILDDIR
+
+# install MODFLOW 6
+meson install -C $BUILDDIR
 
 # test MODFLOW 6 build
 cd $TESTDIR
@@ -33,4 +47,4 @@ cd $TESTDIR
 $BINDIR/mf6
 
 # parallel run
-srun $BINDIR/mf6 -p
+srun $BINDIR/mf6 -p

.hpc/deploy.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+# fetch and checkout latest
+ssh -l "$SSH_USERNAME" "$SSH_HOSTNAME" "cd $MF6_PROJ_ROOT && git fetch $GIT_REMOTE && git checkout $GIT_BRANCH && git pull"
+echo "Updated repository $MF6_PROJ_ROOT"
+# submit a job to build mf6
+jobid=$(ssh -l "$SSH_USERNAME" "$SSH_HOSTNAME" "sbatch --account=$SLURM_ACCOUNT --reservation=$SLURM_RESERVATION --parsable -D $MF6_PROJ_ROOT $MF6_PROJ_ROOT/$BUILD_SCRIPT" | tail -n 1)
+echo "Submitted build job $jobid"
+# submit a job to update the mf6 module
+jobid=$(ssh -l "$SSH_USERNAME" "$SSH_HOSTNAME" "sbatch --export=ALL,MF6_PREV_VERSION=$MF6_PREV_VERSION,MF6_PROJ_ROOT=$MF6_PROJ_ROOT,MODULES_PATH=$MODULES_PATH --account=$SLURM_ACCOUNT --reservation=$SLURM_RESERVATION --parsable -D $MF6_PROJ_ROOT -d afterok:$jobid $MF6_PROJ_ROOT/$MODULE_SCRIPT" | tail -n 1)
+echo "Submitted module update job $jobid"

.hpc/update-module.slurm.batch

@@ -3,33 +3,29 @@
 #SBATCH --job-name=update-mf6-module
 #SBATCH --nodes=1
 #SBATCH --ntasks=1
+#SBATCH --account=impd
 #SBATCH --time=00:05:00
 #SBATCH --output=slurm-%j.out
+#SBATCH --chdir=../
 
-# variables
-MF6_BINDIR="$MF6_PROJ_ROOT/bin"
-MF6_VERSION=$(cat "$MF6_PROJ_ROOT/version.txt")
-MF6_MODULEFILE_PATH="$MODULES_PATH/modulefiles/modflow/$MF6_VERSION"
-MF6_MODULES_PATH="$MODULES_PATH/apps/modflow"
-MF6_MODULE_PATH="$MF6_MODULES_PATH/$MF6_VERSION"
-
-if [ -d "$MF6_MODULE_PATH" ]; then
-  echo "Updating module modflow/$MF6_VERSION"
-else
-  echo "Creating module modflow/$MF6_VERSION"
+set -euxo pipefail
 
-  mkdir -p "$MF6_MODULE_PATH/$INTEL_VERSION/bin"
-  mkdir -p "$MF6_MODULE_PATH/$INTEL_VERSION/lib"
-  echo "Created bin/lib dirs in $MF6_MODULE_PATH/$INTEL_VERSION/"
-
-  rsync "$MODULES_PATH/modulefiles/modflow/6.4.2" "$MF6_MODULEFILE_PATH"
-  sed -i -e "s/6.4.2/$MF6_VERSION/g" "$MF6_MODULEFILE_PATH"
-  echo "Created module file"
-fi
+# this script expects cwd to be mf6 project root, with env vars...
+# - MF6_PREV_VERSION, mf6 modulefile version to use as a template
+# - MODULES_PATH, the base path of the module system
+MF6_PROJ_ROOT=$(pwd)
+MF6_BINDIR="$MF6_PROJ_ROOT/bin"
+MF6_LOCAL_VERSION=$(cat "$MF6_PROJ_ROOT/version.txt")
 
-# copy binaries from build dir
-rsync "$MF6_BINDIR/mf6" "$MF6_MODULE_PATH/$INTEL_VERSION/bin/"
-rsync "$MF6_BINDIR/zbud6" "$MF6_MODULE_PATH/$INTEL_VERSION/bin/"
-rsync "$MF6_BINDIR/libmf6.so" "$MF6_MODULE_PATH/$INTEL_VERSION/lib/"
-echo "Copied binaries to $MF6_MODULE_PATH/$INTEL_VERSION lib/bin dirs"
+# ...and assumes the module system is laid out as follows
+MF6_MODULEFILE_PATH="$MODULES_PATH/modulefiles/modflow/$MF6_LOCAL_VERSION"
+MF6_MODULES_PATH="$MODULES_PATH/apps/modflow"
+MF6_MODULE_PATH="$MF6_MODULES_PATH/$MF6_LOCAL_VERSION"
 
+# create mf6 modulefile if needed (the build script
+# will have already created the module directory)
+if [ ! -f "$MF6_MODULEFILE_PATH" ]; then
+  rsync "$MODULES_PATH/modulefiles/modflow/$MF6_PREV_VERSION" "$MF6_MODULEFILE_PATH"
+  sed -i -e "s/$MF6_PREV_VERSION/$MF6_LOCAL_VERSION/g" "$MF6_MODULEFILE_PATH"
+  echo "Created module file: $MF6_MODULEFILE_PATH"
+fi