diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6e5e6b8ee3d..c518b77c314 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,26 @@ +# Deploy MODFLOW 6 to HPC systems. +# +# Expects shared environment variables... +# +# - GIT_REMOTE, name of the mf6 remote to pull from +# - GIT_BRANCH, name of the mf6 branch to pull from +# - MODULE_SCRIPT, module update script path (relative to mf6 proj root) +# - SSH_KNOWN_HOSTS, content of ~/.ssh/known_hosts +# - SSH_USERNAME, SSH username to login with +# - SSH_PRIVATE_KEY, SSH private key for user +# +# ...and environment-specific variables: +# +# - DENALI_HOSTNAME, SSH hostname of cluster +# - DENALI_SLURM_ACCOUNT, slurm account for jobs +# - DENALI_SLURM_RESERVATION, slurm reservation for jobs +# - DENALI_MODULES_PATH, path to root of module system +# - DENALI_MF6_PREV_VERSION, version of modulefile to use as template +# - DENALI_MF6_PROJ_ROOT, path to modflow6 project root +# - DENALI_BUILD_SCRIPT, relpath of script to build mf6 +# +# ...and likewise for HOVENWEEP_* or other systems + image: ubuntu:20.04 workflow: rules: @@ -9,23 +32,35 @@ before_script: - eval $(ssh-agent -s) # set private key permissions - chmod 400 "$SSH_PRIVATE_KEY" - # add key to the agent store + # add keys to the agent store - ssh-add "$SSH_PRIVATE_KEY" # configure known hosts - mkdir -p ~/.ssh - cp "$SSH_KNOWN_HOSTS" ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts deploy_denali: - stage: deploy - script: | - # fetch and checkout latest - ssh -l "$SSH_USERNAME" "$SSH_HOSTNAME" "cd $MF6_PROJ_ROOT && git fetch $GIT_REMOTE && git checkout $GIT_BRANCH && git pull" - echo "Updated repository $MF6_PROJ_ROOT" - # submit a job to build mf6 - jobid=$(ssh -l "$SSH_USERNAME" "$SSH_HOSTNAME" "sbatch --account=$SLURM_ACCOUNT --reservation=$SLURM_RESERVATION --parsable -D $MF6_PROJ_ROOT/.hpc $MF6_PROJ_ROOT/$BUILD_SCRIPT" | tail -n 1) - echo "Submitted build job $jobid" - # submit a job to update the mf6 module - jobid=$(ssh -l "$SSH_USERNAME" "$SSH_HOSTNAME" "sbatch --export=ALL,INTEL_VERSION=$INTEL_VERSION,MF6_PROJ_ROOT=$MF6_PROJ_ROOT,MODULES_PATH=$MODULES_PATH --account=$SLURM_ACCOUNT --reservation=$SLURM_RESERVATION --parsable -D $MF6_PROJ_ROOT/.hpc -d afterok:$jobid $MF6_PROJ_ROOT/$MODULE_SCRIPT" | tail -n 1) - echo "Submitted module update job $jobid" environment: denali - + script: "$CI_PROJECT_DIR/.hpc/deploy.sh" + stage: deploy + variables: + SSH_USERNAME: $SSH_USERNAME + SSH_HOSTNAME: $DENALI_HOSTNAME + SLURM_ACCOUNT: $DENALI_SLURM_ACCOUNT + SLURM_RESERVATION: $DENALI_SLURM_RESERVATION + MODULES_PATH: $DENALI_MODULES_PATH + MF6_PROJ_ROOT: $DENALI_MF6_PROJ_ROOT + MF6_PREV_VERSION: $DENALI_MF6_PREV_VERSION + BUILD_SCRIPT: $DENALI_BUILD_SCRIPT +deploy_hovenweep: + environment: hovenweep + script: "$CI_PROJECT_DIR/.hpc/deploy.sh" + stage: deploy + variables: + SSH_USERNAME: $SSH_USERNAME + SSH_HOSTNAME: $HOVENWEEP_HOSTNAME + SLURM_ACCOUNT: $HOVENWEEP_SLURM_ACCOUNT + SLURM_RESERVATION: $HOVENWEEP_SLURM_RESERVATION + MODULES_PATH: $HOVENWEEP_MODULES_PATH + MF6_PROJ_ROOT: $HOVENWEEP_MF6_PROJ_ROOT + MF6_PREV_VERSION: $HOVENWEEP_MF6_PREV_VERSION + BUILD_SCRIPT: $HOVENWEEP_BUILD_SCRIPT \ No newline at end of file diff --git a/.hpc/BUILD.md b/.hpc/BUILD.md index f5dc39f1977..9f9b0ac8374 100644 --- a/.hpc/BUILD.md +++ b/.hpc/BUILD.md @@ -1,59 +1,30 @@ # Building MODFLOW 6 on HPC systems -## SLURM job +_On Denali_ ``` sbatch --reservation=dev cray-meson-build.slurm.batch ``` -## Interactive job +_Hovenweep_ ``` -module switch PrgEnv-${PE_ENV,,} PrgEnv-intel -module load cray-petsc meson ninja -export PKG_CONFIG_PATH=/opt/cray/pe/mpt/7.7.19/gni/mpich-intel/16.0/lib/pkgconfig:/opt/cray/pe/petsc/3.14.5.0/real/INTEL/19.1/x86_skylake/lib/pkgconfig:$PKG_CONFIG_PATH - -srun --reservation=dev --account=impd --pty bash - -meson setup builddir -Ddebug=false --prefix=$(pwd) --libdir=bin --bindir=bin -Dcray=true -Ddebug=false --wipe -meson install -C builddir +sbatch cray-hovenweep-meson-build.slurm.batch ``` -# Installing a new version of MODFLOW 6 on HPC systems - - -After building the new version (`6.x.x`) of MODFLOW 6 using a SLURM or interactive job you will need to install the new version and create a new module file. - -## Install a new version of MODFLOW 6 - -Create a directory for the new version in `/home/software/denali/contrib/impd/apps/modflow/` - -Since MODFLOW 6 is currently only being compiled with INTEL (version 19.1.0.166), make the following subdirectories in the `/home/software/denali/contrib/impd/apps/modflow/6.x.x` directory you just created. - -1. `/INTEL/19.1.0.166/bin` -2. `/INTEL/19.1.0.166/lib` - -Copy `mf6` and `zbud6` to the `/INTEL/19.1.0.166/bin` and `libmf6.so` to the `/INTEL/19.1.0.166/lib` subdirectories using -``` -rsync ../bin/mf6 /home/software/denali/contrib/impd/apps/modflow/6.x.x/INTEL/19.1.0.166/bin/ -``` - -``` -rsync ../bin/zbud6 /home/software/denali/contrib/impd/apps/modflow/6.x.x/INTEL/19.1.0.166/bin/ -``` +## Create a module file for a new version of MODFLOW 6 +On _Denali_ make a copy of an existing module file using ``` -rsync ../bin/libmf6.so /home/software/denali/contrib/impd/apps/modflow/6.x.x/INTEL/19.1.0.166/lib/ +rsync /home/software/denali/contrib/impd/modulefiles/modflow/6.5.0.dev0 /home/software/denali/contrib/impd/modulefiles/modflow/6.x.x ``` -## Create a module file for a new version of MODFLOW 6 - -Make a copy of an existing module file using +On _Hovenweep_ make a copy of an existing module file using ``` -rsync /home/software/denali/contrib/impd/modulefiles/modflow/6.4.2 /home/software/denali/contrib/impd/modulefiles/modflow/6.x.x +rsync /home/software/hovenweep/contrib/impd/modulefiles/modflow/6.5.0.dev0 /home/software/denali/contrib/impd/modulefiles/modflow/6.x.x ``` -Edit `product_version` in the new module file from `6.4.2` to `6.x.x`. +Edit `product_version` in the new module file from `6.5.0.dev0` to `6.x.x` on both systems. diff --git a/.hpc/cray-hovenweep-meson-build.slurm.batch b/.hpc/cray-hovenweep-meson-build.slurm.batch new file mode 100644 index 00000000000..a1020f54cf0 --- /dev/null +++ b/.hpc/cray-hovenweep-meson-build.slurm.batch @@ -0,0 +1,50 @@ +#!/bin/bash + +#SBATCH --job-name=hovenweep-build +#SBATCH --nodes=1 +#SBATCH --ntasks=2 +#SBATCH --account=impd +#SBATCH --time=00:10:00 +#SBATCH --output=slurm-%j.out +#SBATCH --chdir=../ + +set -euxo pipefail + +# load appropriate modules +module switch PrgEnv-${PE_ENV,,} PrgEnv-intel +module load petsc/3.15.5 +export PKG_CONFIG_PATH=$CRAY_MPICH_DIR/lib/pkgconfig:$PKG_CONFIG_PATH + +# list loaded modules +module list + +# define the project root (expected to be cwd) +MODFLOW6ROOT=$(pwd) + +# define the version +VERSION=$(cat "$MODFLOW6ROOT/version.txt") +echo "MODFLOW 6 version: $VERSION" + +# define paths relative to the root directory +BUILDDIR=$MODFLOW6ROOT/$PE_ENV-$VERSION +BINDIR=$BUILDDIR/src +TESTDIR=$MODFLOW6ROOT/.mf6minsim + +# define the installation location +PREFIX=/home/software/hovenweep/contrib/impd/apps/modflow/$VERSION/$PE_ENV/2023.2.0 + +# build MODFLOW 6 +CC=cc CXX=CC F77=ftn F90=ftn FC=ftn meson setup $BUILDDIR --prefix=$PREFIX --bindir=bin --libdir=lib -Dcray=true -Ddebug=false --wipe +meson compile -C $BUILDDIR + +# install MODFLOW 6 +meson install -C $BUILDDIR + +# test MODFLOW 6 build +cd $TESTDIR + +# serial run +$BINDIR/mf6 + +# parallel run +srun $BINDIR/mf6 -p \ No newline at end of file diff --git a/.hpc/cray-meson-build.slurm.batch b/.hpc/cray-meson-build.slurm.batch index 08d0e4ed89c..1f62d43f6f3 100644 --- a/.hpc/cray-meson-build.slurm.batch +++ b/.hpc/cray-meson-build.slurm.batch @@ -1,10 +1,14 @@ #!/bin/bash -#SBATCH --job-name=meson-MODFLOW-build +#SBATCH --job-name=denali-build #SBATCH --nodes=1 #SBATCH --ntasks=2 +#SBATCH --account=impd #SBATCH --time=00:10:00 #SBATCH --output=slurm-%j.out +#SBATCH --chdir=../ + +set -euxo pipefail # load appropriate modules module switch PrgEnv-${PE_ENV,,} PrgEnv-intel @@ -14,17 +18,27 @@ export PKG_CONFIG_PATH=/opt/cray/pe/mpt/7.7.19/gni/mpich-intel/16.0/lib/pkgconfi # list loaded modules module list -# move to root directory -cd .. +# define the project root (expected to be cwd) +MODFLOW6ROOT=$(pwd) + +# define the version +VERSION=$(cat "$MODFLOW6ROOT/version.txt") +echo "MODFLOW 6 version: $VERSION" # define paths relative to the root directory -MODFLOW6ROOT=$(pwd) -BINDIR=$MODFLOW6ROOT/bin +BUILDDIR=$MODFLOW6ROOT/$PE_ENV-$VERSION +BINDIR=$BUILDDIR/src TESTDIR=$MODFLOW6ROOT/.mf6minsim +# define the installation location +PREFIX=/home/software/denali/contrib/impd/apps/modflow/$VERSION/$PE_ENV/19.1.0.166 + # build MODFLOW 6 -CC=cc CXX=CC F77=ftn F90=ftn FC=ftn meson setup builddir --prefix=$(pwd) --bindir=bin --libdir=bin -Dcray=true -Ddebug=false --wipe -meson install -C builddir +CC=cc CXX=CC F77=ftn F90=ftn FC=ftn meson setup $BUILDDIR --prefix=$PREFIX --bindir=bin --libdir=lib -Dcray=true -Ddebug=false +meson compile -C $BUILDDIR + +# install MODFLOW 6 +meson install -C $BUILDDIR # test MODFLOW 6 build cd $TESTDIR @@ -33,4 +47,4 @@ cd $TESTDIR $BINDIR/mf6 # parallel run -srun $BINDIR/mf6 -p +srun $BINDIR/mf6 -p \ No newline at end of file diff --git a/.hpc/deploy.sh b/.hpc/deploy.sh new file mode 100644 index 00000000000..b4edee4791b --- /dev/null +++ b/.hpc/deploy.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -euxo pipefail + +# fetch and checkout latest +ssh -l "$SSH_USERNAME" "$SSH_HOSTNAME" "cd $MF6_PROJ_ROOT && git fetch $GIT_REMOTE && git checkout $GIT_BRANCH && git pull" +echo "Updated repository $MF6_PROJ_ROOT" +# submit a job to build mf6 +jobid=$(ssh -l "$SSH_USERNAME" "$SSH_HOSTNAME" "sbatch --account=$SLURM_ACCOUNT --reservation=$SLURM_RESERVATION --parsable -D $MF6_PROJ_ROOT $MF6_PROJ_ROOT/$BUILD_SCRIPT" | tail -n 1) +echo "Submitted build job $jobid" +# submit a job to update the mf6 module +jobid=$(ssh -l "$SSH_USERNAME" "$SSH_HOSTNAME" "sbatch --export=ALL,MF6_PREV_VERSION=$MF6_PREV_VERSION,MF6_PROJ_ROOT=$MF6_PROJ_ROOT,MODULES_PATH=$MODULES_PATH --account=$SLURM_ACCOUNT --reservation=$SLURM_RESERVATION --parsable -D $MF6_PROJ_ROOT -d afterok:$jobid $MF6_PROJ_ROOT/$MODULE_SCRIPT" | tail -n 1) +echo "Submitted module update job $jobid" \ No newline at end of file diff --git a/.hpc/update-module.slurm.batch b/.hpc/update-module.slurm.batch index cb6ac1ed18f..00135103525 100644 --- a/.hpc/update-module.slurm.batch +++ b/.hpc/update-module.slurm.batch @@ -3,33 +3,29 @@ #SBATCH --job-name=update-mf6-module #SBATCH --nodes=1 #SBATCH --ntasks=1 +#SBATCH --account=impd #SBATCH --time=00:05:00 #SBATCH --output=slurm-%j.out +#SBATCH --chdir=../ -# variables -MF6_BINDIR="$MF6_PROJ_ROOT/bin" -MF6_VERSION=$(cat "$MF6_PROJ_ROOT/version.txt") -MF6_MODULEFILE_PATH="$MODULES_PATH/modulefiles/modflow/$MF6_VERSION" -MF6_MODULES_PATH="$MODULES_PATH/apps/modflow" -MF6_MODULE_PATH="$MF6_MODULES_PATH/$MF6_VERSION" - -if [ -d "$MF6_MODULE_PATH" ]; then - echo "Updating module modflow/$MF6_VERSION" -else - echo "Creating module modflow/$MF6_VERSION" +set -euxo pipefail - mkdir -p "$MF6_MODULE_PATH/$INTEL_VERSION/bin" - mkdir -p "$MF6_MODULE_PATH/$INTEL_VERSION/lib" - echo "Created bin/lib dirs in $MF6_MODULE_PATH/$INTEL_VERSION/" - - rsync "$MODULES_PATH/modulefiles/modflow/6.4.2" "$MF6_MODULEFILE_PATH" - sed -i -e "s/6.4.2/$MF6_VERSION/g" "$MF6_MODULEFILE_PATH" - echo "Created module file" -fi +# this script expects cwd to be mf6 project root, with env vars... +# - MF6_PREV_VERSION, mf6 modulefile version to use as a template +# - MODULES_PATH, the base path of the module system +MF6_PROJ_ROOT=$(pwd) +MF6_BINDIR="$MF6_PROJ_ROOT/bin" +MF6_LOCAL_VERSION=$(cat "$MF6_PROJ_ROOT/version.txt") -# copy binaries from build dir -rsync "$MF6_BINDIR/mf6" "$MF6_MODULE_PATH/$INTEL_VERSION/bin/" -rsync "$MF6_BINDIR/zbud6" "$MF6_MODULE_PATH/$INTEL_VERSION/bin/" -rsync "$MF6_BINDIR/libmf6.so" "$MF6_MODULE_PATH/$INTEL_VERSION/lib/" -echo "Copied binaries to $MF6_MODULE_PATH/$INTEL_VERSION lib/bin dirs" +# ...and assumes the module system is laid out as follows +MF6_MODULEFILE_PATH="$MODULES_PATH/modulefiles/modflow/$MF6_LOCAL_VERSION" +MF6_MODULES_PATH="$MODULES_PATH/apps/modflow" +MF6_MODULE_PATH="$MF6_MODULES_PATH/$MF6_LOCAL_VERSION" +# create mf6 modulefile if needed (the build script +# will have already created the module directory) +if [ ! -f "$MF6_MODULEFILE_PATH" ]; then + rsync "$MODULES_PATH/modulefiles/modflow/$MF6_PREV_VERSION" "$MF6_MODULEFILE_PATH" + sed -i -e "s/$MF6_PREV_VERSION/$MF6_LOCAL_VERSION/g" "$MF6_MODULEFILE_PATH" + echo "Created module file: $MF6_MODULEFILE_PATH" +fi \ No newline at end of file diff --git a/PARALLEL.md b/PARALLEL.md index 8229ae1cbbb..3d1e8f11ebc 100644 --- a/PARALLEL.md +++ b/PARALLEL.md @@ -5,7 +5,7 @@ This document describes how to set up your build environment for developing and --- **DISCLAIMER** -*Expectations on platform compatibility* +*Expectations on platform compatibility* The serial version of the MODFLOW 6 program has had no external dependencies and is traditionally available for a variety of platforms (Windows, GNU/linux, macOS) and compatible with the mainstream Fortran compilers (gfortran, ifort). The parallel version comes with dependencies on third party components, most notably the MPI and PETSc libraries. While the goal is a continued support of the above mentioned configurations, this has become more challenging and can generally not be guaranteed. To assist developers as well as end users who are planning to compile the code themselves, a list of successfully tested build configurations will be included in this document. @@ -19,7 +19,7 @@ The design philosophy has been to maintain MODFLOW as a single codebase and have ## Prerequisites -The parallel version of MODFLOW 6 requires the the Message Passing Interface (MPI) and the Portable, Extensible Toolkit for Scientific Computation (PETSc - pronounced PET-see (/ˈpɛt-siː/)) libraries. +The parallel version of MODFLOW 6 requires the Message Passing Interface (MPI) and the Portable, Extensible Toolkit for Scientific Computation (PETSc - pronounced PET-see (/ˈpɛt-siː/)) libraries. ### MPI @@ -36,7 +36,7 @@ In addition to compiling, the MPI toolset is also required to run a parallel sim ### PETSc -The PETSc library is a suite of data structures and routines for the scalable (parallel) solution of scientific applications modeled by partial differential equations: +The PETSc library is a suite of data structures and routines for the scalable (parallel) solution of scientific applications modeled by partial differential equations: https://petsc.org/release/ @@ -44,38 +44,41 @@ The PETSc library (version 3.16 or higher) is used by MODFLOW for its parallel l ## Compiling MPI and PETSC from source -The PETSc website gives details on a large number of configurations, depending on the target platform/OS, and many different ways to configure/make/install the library: https://petsc.org/release/install/. Building on Windows is notoriously challenging and discouraged by the PETSc development team. On Linux, however, PETSc can be installed (configure/make/install) by executing the following command +The PETSc website gives details on a large number of configurations, depending on the target platform/OS, and many different ways to configure/make/install the library: https://petsc.org/release/install/. Building on Windows is notoriously challenging and discouraged by the PETSc development team. On Linux, however, PETSc can be installed (configure/make/install) by executing the following command + ``` $ ./configure --download-openmpi --download-fblaslapack $ make all ``` -in a terminal open in the root directory of your PETSc download - +in a terminal open in the root directory of your PETSc download ## Using a package manager to install MPI and PETSc Use of a package manager can simplify the process of building the parallel version of MODFLOW 6. ### MacOS + [OpenMPI](https://formulae.brew.sh/formula/open-mpi) and [PETSc](https://formulae.brew.sh/formula/petsc) are available on Homebrew for Intel and Apple Silicon (M1). Both of these depend on [gcc 13.1.0](https://formulae.brew.sh/formula/gcc). [pkg-config](https://formulae.brew.sh/formula/pkg-config) should also be installed from Homebrew, if not already installed, so that Meson will be able to resolve the installation location of MPI and PETSc. ### Ubuntu -OpenMPI and PETSc are available for a variety of Ubuntu versions using the Advanced Packaging Tool (apt). + +OpenMPI and PETSc are available for a variety of Ubuntu versions using the Advanced Packaging Tool (apt). ### Windows -??? +Under evaluation. ## Using pkg-config to check your PETSc installation -Eventually, the MODFLOW build process has to resolve the installation location of all external dependencies. The pkg-config tool (https://en.wikipedia.org/wiki/Pkg-config) can be used to take care of that. +Eventually, the MODFLOW build process has to resolve the installation location of all external dependencies. The pkg-config tool (https://en.wikipedia.org/wiki/Pkg-config) can be used to take care of that. ``` pkg-config --libs petsc ``` If PETSc was build from source, you can check the contents of the folder + ``` $PETSC_DIR/$PETSC_ARCH/lib/pkgconfig/ ``` @@ -84,7 +87,6 @@ and confirm that there are one or more `*.pc` files in there. A similar `pkgconf To connect everything, both of these folder paths have to be added to the `PKG_CONFIG_PATH` variable so that the `pkg-config` executable can resolve the installed libraries. - ## Building the parallel version of MODFLOW 6 The primary build system for MODFLOW is Meson (https://mesonbuild.com/). The `meson.build` script takes an additional argument to activate a parallel build of the software. E.g for building and installing a parallel release version: @@ -95,11 +97,12 @@ meson setup builddir -Ddebug=false -Dparallel=true \ meson install -C builddir meson test --verbose --no-rebuild -C builddir ``` + Note that changing the option flags in the `meson setup` command requires the flag `--reconfigure` to reconfigure the build directory. If the `PKG_CONFIG_PATH` was set as described above, the linking to PETSc and MPI is done automatically. -It's always a good idea to check your `mf6pro` executable to confirm that it is successfully linked against the external dependencies. You can use the command line tools `ldd` (Linux), `otool` (macOS), or `Dependencies.exe` (Windows, https://github.com/lucasg/Dependencies) to do that. In the list of dependencies, you should be able to identify `libpetsc` and `libmpi` for parallel builds. +It's always a good idea to check your parallel MODFLOW executable to confirm that it is successfully linked against the external dependencies. You can use the command line tools `ldd` (Linux), `otool` (macOS), or `Dependencies.exe` (Windows, https://github.com/lucasg/Dependencies) to do that. In the list of dependencies, you should be able to identify `libpetsc` and `libmpi` for parallel builds. -The other build systems in the MODFLOW project (MS Visual Studio, `pymake`, `Makefile`) continue to be supported for *serial* builds only. `pymake` uses the `excludefiles.txt` to ignore those files that can only be build when MPI and PETSc are present on the system. In MS Visual Studio these same files are included in the solution but not in the build process. +The other build systems in the MODFLOW project (MS Visual Studio, `pymake`, `Makefile`) continue to be supported for *serial* builds only. `pymake` uses the `excludefiles.txt` to ignore those files that can only be build when MPI and PETSc are present on the system. In MS Visual Studio these same files are included in the solution but not in the build process. --- @@ -111,8 +114,7 @@ Parallel MODFLOW was designed to have all third party functionality (MPI and PET --- - -## Testing the parallel of MODFLOW 6 +## Testing the parallel of MODFLOW 6 Parallel MODFLOW can be tested using the same test framework as the serial program, with just a few modifications. To run a test inside the `autotest` folder in parallel mode, make sure to add a marker `@pytest.mark.parallel` so that the test is only executed in the Continuous Integration when running a configuration with a parallel build of MODFLOW. @@ -121,6 +123,7 @@ The `TestSimulation` object that is being run from the framework should be confi ``` $ pytest -s --parallel test_par_gwf01.py ``` + Running without the `--parallel` flag will simply skip the test. ## Debugging @@ -130,11 +133,13 @@ The most straightforward way to debug a parallel simulation is to start a run an ``` -wait_dbg ``` + telling MODFLOW to pause immediately after startup. This will give you time to attach one or multiple debuggers to the processes. Then start the parallel program, for example on two cores: ``` mpiexec -np 2 mf6 -p ``` + In the process explorer you should now see 2 processes called `mf6` or `mf6.exe`. On the prompt where the command was executed, MODFLOW waits for input: ``` @@ -165,7 +170,8 @@ In VSCode parallel debugging is easiest done by duplicating the development envi ] } ``` -After building parallel MODFLOW, press `Ctrl+Shift+p` to execute *Workspaces: Duplicate As Workspace in New Window*. This will open a second VSCode window, identical to the first. Starting the debug process and selecting *"Attach to ..."* pop ups a process selection window with the processes started from the `mpiexec` command described above. Select both, each from their own instance of the VSCode program. Now you can put breakpoints in the code, "Hit enter to continue" on the command prompt, and step through the parallel processes side-by-side. + +After building parallel MODFLOW, press `Ctrl+Shift+p` to execute *Workspaces: Duplicate As Workspace in New Window*. This will open a second VSCode window, identical to the first. Starting the debug process and selecting *"Attach to ..."* opens a process selection window with the processes started from the `mpiexec` command described above. Select both, each from their own instance of the VSCode program. Now you can put breakpoints in the code, "Hit enter to continue" on the command prompt, and step through the parallel processes side-by-side. --- **TIP** @@ -174,19 +180,23 @@ Make sure that you work with gdb versions >= 10. We have found that earlier vers --- - ## Compatibility Parallel MODFLOW has been built successfully with the following configurations: -| Operating System | Toolchain | MPI | PETSc | Package Manager | -|-----------------------|-------------|---------------|--------|-----------------| -| MS Windows | ? | ? | ? | NA | -| WSL2 (Ubuntu 20.04.5) | gcc 9.4.0 | OpenMPI 4.0.3 | 3.18.2 | NA | -| Ubuntu 22.04 | gcc 9.5.0 | OpenMPI 4.1.4 | 3.18.5 | NA | -| Ubuntu 23.04 | gcc 13 | OpenMPI 4.1.4 | 3.18.1 | apt | -| macOS 12.6.3 | gcc 9.5.0 | OpenMPI 4.1.4 | 3.18.5 | NA | -| macOS 12.6.6 | gcc 13.1.0 | OpenMPI 4.1.5 | 3.19.1 | Homebrew | +| Operating System | Toolchain | MPI | PETSc | Package Manager | +|-------------------------------------|---------------------------|-------------------|---------------------|-----------------| +| MS Windows | ? | ? | ? | NA | +| WSL2 (Ubuntu 20.04.5) | gcc 9.4.0 | OpenMPI 4.0.3 | 3.18.2 | NA | +| macOS 12.6.3 | gcc 9.5.0 | OpenMPI 4.1.4 | 3.18.5 | NA | +| macOS 12.6.6 | gcc 13.1.0 | OpenMPI 4.1.5 | 3.19.1 | Homebrew | +| Ubuntu 22.04 | gcc 9.5.0 | OpenMPI 4.1.4 | 3.18.5 | NA | +| Ubuntu 22.04 ARM64 | gcc 11.4.0 | OpenMPI 4.1.5 | 3.19.3 | apt | +| Ubuntu 22.04 ARM64 | gcc 9.5.0 | MPICH 3.4.1 | 3.15.5 | NA | +| Ubuntu 22.04 ARM64 | gcc 12.3.0 | MPICH 4.1.1 | 3.19.6 | NA | +| Ubuntu 22.04 ARM64 | gcc 12.3.0 | MPICH 4.1.1 | 3.20.0 | NA | +| SUSE Linux Enterprise Server 15 SP2 | intel 19.1.0.166 20191121 | CRAY-MPICH 7.7.19 | CRAY-PETSC 3.14.5.0 | NA | +| Red Hat Enterprise Linux 8.7 | intel 2021.10.0 20230609 | CRAY-MPICH 8.1.26 | 3.15.5 | NA | The most up-to-date configurations are available in the GitHub CI script: `.github/workflows/ci.yml` under the task `parallel_test`. These are being tested upon every change to the `develop` branch of MODFLOW. @@ -194,4 +204,46 @@ To improve support, we kindly ask you to share your experience with building and ## Known issues -tbd \ No newline at end of file +### Building PETSc on Ubuntu 22.04 with MPICH and GNU compilers + +Versions of PETSc that use MPICH 3.4 (v3.14, v3.15, v3.16)) must be built with gcc-9 or earlier. Versions of PETSc that use MPICH 4.1 (v3.17 or newer) can be built with newer versions of the gcc compiler (gcc-11, gcc-12, etc.). + +Meson does not correctly load the Fortran compiler flags from the `mpich.pc` package configuration file in the `$PETSC_DIR/$PETSC_ARCH/lib/pkgconfig` directory. To overcome this issue, make a copy of `mpich.pc` and name it `mpichfort.pc`. Then determine the appropriate Fortran flags using + +``` +$PETSC_DIR/$PETSC_ARCH/bin/mpifort -show +``` + +which will return something like + +``` +$ linux-real-gcc12.3.0-3.20.0/bin/mpifort -show +gfortran -fPIC -ffree-line-length-none -ffree-line-length-0 -Wno-lto-type-mismatch -O2 -fallow-argument-mismatch -I/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/include -I/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/include -L/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/lib -lmpifort -Wl,-rpath -Wl,/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/lib -Wl,--enable-new-dtags -lmpi +``` + +Copy the returned Fortran flags and replace the `Libs:` and `Cflags:` attributes in the `mpichfort.pc` file. Also modify the `Name:` attribute to `mpichfort`. The modified `mpichfort.pc` file should look something like + +``` +# this gives access to the mpich header files +prefix=/media/psf/Development/petsc/linux-real-gcc9.5.0-3.20.0 +exec_prefix=${prefix} +libdir=/media/psf/Development/petsc/linux-real-gcc9.5.0-3.20.0/lib +includedir=${prefix}/include + +Name: mpichfort +Description: High Performance and portable MPI +Version: 4.1.2 +URL: http://www.mcs.anl.gov/research/projects/mpich +Requires: +Libs: -fPIC -ffree-line-length-none -ffree-line-length-0 -Wno-lto-type-mismatch -O2 -fallow-argument-mismatch -I/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/include -I/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/include -L/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/lib -lmpifort -Wl,-rpath -Wl,/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/lib -Wl,--enable-new-dtags -lmpi +Cflags: -fPIC -ffree-line-length-none -ffree-line-length-0 -Wno-lto-type-mismatch -O2 -fallow-argument-mismatch -I/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/include -I/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/include -L/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/lib -lmpifort -Wl,-rpath -Wl,/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/lib -Wl,--enable-new-dtags -lmpi + +# pkg-config does not understand Cxxflags, etc. So we allow users to +# query them using the --variable option + +cxxflags= -Wno-lto-type-mismatch -Wno-psabi -O2 -std=gnu++17 -fPIC -I${includedir} +fflags=-fPIC -ffree-line-length-none -ffree-line-length-0 -Wno-lto-type-mismatch -O2 -I${includedir} +fcflags=-fPIC -ffree-line-length-none -ffree-line-length-0 -Wno-lto-type-mismatch -O2 -I${includedir} +``` + +The `/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/include` and `/media/psf/Development/petsc/linux-real-gcc12.3.0-3.20.0/lib` entries in the `Libs:` and `Cflags:` attributes can be replaced with `${includedir}` and `${libdir}`, respectively, to simplify `mpichfort.pc`. \ No newline at end of file diff --git a/meson.build b/meson.build index 634cafa8e87..79376248f3f 100644 --- a/meson.build +++ b/meson.build @@ -155,9 +155,8 @@ if is_parallel_build # find mpi if is_mpich - mpi = dependency('mpich', required : true) - mpifort = dependency(mpifort_name, required : mpi.found()) - dependencies += [ mpi, mpifort ] + mpifort = dependency(mpifort_name, required : true) + dependencies += mpifort else mpi = dependency('mpi', language : 'fortran', required : true) dependencies += mpi diff --git a/src/Utilities/InputOutput.f90 b/src/Utilities/InputOutput.f90 index 7759c30f1bd..824529fa87d 100644 --- a/src/Utilities/InputOutput.f90 +++ b/src/Utilities/InputOutput.f90 @@ -272,6 +272,7 @@ subroutine append_processor_id(name, proc_id) integer(I4B), intent(in) :: proc_id !< processor id ! -- local variables character(len=linelength) :: name_local + character(len=linelength) :: name_processor character(len=linelength) :: extension_local integer(I4B) :: ipos0 integer(I4B) :: ipos1 @@ -286,8 +287,9 @@ subroutine append_processor_id(name, proc_id) ipos0 = ipos1 extension_local = '' end if - write(name, '(a,a,i0,a)') & + write(name_processor, '(a,a,i0,a)') & name(1:ipos0-1), '.p', proc_id, trim(adjustl(extension_local)) + name = name_processor ! ! -- return return