feat(netcdf): support model ugrid layered mesh exports (#1868)

* update definitions and add netcdf utility input

* add netcdf export source

* integrate export and update build

* add netcdf tests and update ci

* install xugrid/xarray in ci test

* update prt dis dfns

* install netcdf4 in ci env

* update msvs build

* preliminary support for variable chunking and compression

* remove ncf6 and netcdf from docs for now

* update memorylist to memorystore

* rebuild makefiles

* exclude netcdf dependent files from make build

* remove unintended definition file

* update msvs build

* rebuild makefiles

* update netcdf transport dispersion test to remove packagedata keyword

* update pixi version in CI extended test

---------

Co-authored-by: mjreno <[email protected]>

.github/actions/test-extended/action.yml

@@ -0,0 +1,104 @@
+name: Test Extended MF6
+description: Build and test Extended MODFLOW 6
+runs:
+  using: "composite"
+  steps:
+
+    - name: Setup GNU Fortran
+      uses: fortran-lang/setup-fortran@v1
+      with:
+        compiler: gcc
+        version: 13
+
+    - name: Checkout PETSc
+      uses: actions/checkout@v4
+      with:
+        repository: petsc/petsc
+        path: petsc
+        ref: release
+
+    - name: Configure environment
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        echo "PKG_CONFIG_PATH=$GITHUB_WORKSPACE/petsc/linux-gnu/lib/pkgconfig" >> $GITHUB_ENV
+        echo "$GITHUB_WORKSPACE/petsc/linux-gnu/bin" >> $GITHUB_PATH
+
+    - name: Configure environment
+      if: runner.os == 'macOS'
+      shell: bash
+      run: |
+        echo "PKG_CONFIG_PATH=$GITHUB_WORKSPACE/petsc/arch-darwin-gcc/lib/pkgconfig" >> $GITHUB_ENV
+        echo "$GITHUB_WORKSPACE/petsc/arch-darwin-gcc/bin" >> $GITHUB_PATH
+
+    - name: Configure PETSc
+      if: runner.os == 'Linux'
+      shell: bash
+      working-directory: petsc
+      run: |
+        sudo wget -P $GITHUB_WORKSPACE/petsc https://download.open-mpi.org/release/open-mpi/v5.0/openmpi-5.0.2.tar.gz
+        sudo ./configure PETSC_ARCH=linux-gnu --download-fblaslapack --download-openmpi=$GITHUB_WORKSPACE/petsc/openmpi-5.0.2.tar.gz
+        sudo make all
+
+    - name: Configure PETSc
+      if: runner.os == 'macOS'
+      shell: bash
+      working-directory: petsc
+      run: |
+        sudo ./configure PETSC_DIR="$GITHUB_WORKSPACE/petsc" PETSC_ARCH=arch-darwin-gcc --download-fblaslapack --download-openmpi --with-debugging=0
+        sudo make all
+
+    - name: Install netcdf
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install build-essential \
+          libnetcdf-dev \
+          libnetcdff-dev \
+          netcdf-bin
+        nc-config --all
+        nf-config --all
+
+    - name: Install netcdf
+      if: runner.os == 'macOS'
+      shell: bash
+      run: |
+        bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+        brew install netcdf-fortran
+        brew install gcc@13
+        nc-config --all
+        nf-config
+
+    - name: Build modflow6
+      shell: bash
+      working-directory: modflow6
+      run: |
+        pixi run setup -Dparallel=true builddir
+        pixi run build builddir
+        pixi run test builddir
+
+    - name: Show Meson logs
+      if: failure()
+      shell: bash
+      working-directory: modflow6
+      run: cat builddir/meson-logs/meson-log.txt
+
+    - name: Update flopy
+      shell: bash
+      working-directory: modflow6
+      run: pixi run update-flopy
+
+    - name: Get executables
+      shell: bash
+      working-directory: modflow6
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: pixi run get-exes
+
+    - name: Test programs
+      shell: bash
+      working-directory: modflow6
+      env:
+        REPOS_PATH: ${{ github.workspace }}
+      run: pixi run autotest --parallel --netcdf -k "test_par or test_netcdf"

.github/common/check_format.py

@@ -24,7 +24,9 @@
 ]
 
 # exclude these files from checks
-excludefiles = []
+excludefiles = [
+    PROJ_ROOT / "src" / "Idm" / "gwf-stoidm.f90"
+]
 
 # commands
 fprettify = "fprettify -c .fprettify.yaml"

.github/workflows/ci.yml

@@ -476,3 +476,49 @@ jobs:
         with:
           name: failed-${{ matrix.os }}
           path: modflow6/autotest/.failed
+
+  extended_test:
+    name: Extended executable testing
+    needs:
+      - lint
+      - build
+      - smoke_test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-22.04, macos-12 ]
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+
+      - name: Checkout modflow6
+        uses: actions/checkout@v4
+        with:
+          path: modflow6
+
+      - name: Setup pixi
+        uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.24.2
+          manifest-path: "modflow6/pixi.toml"
+
+      - name: Custom pixi install
+        working-directory: modflow6
+        run: pixi run install
+
+      - name: Install additional netcdf testing packages
+        working-directory: modflow6
+        run: pixi run pip install xugrid xarray netcdf4
+
+      - name: Test Extended MF6
+        uses: ./modflow6/.github/actions/test-extended
+
+      - name: Upload failed test output
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: failed-${{ matrix.os }}
+          path: modflow6/autotest/.failed

autotest/conftest.py

@@ -119,6 +119,12 @@ def pytest_addoption(parser):
         default=False,
         help="include parallel test cases",
     )
+    parser.addoption(
+        "--netcdf",
+        action="store_true",
+        default=False,
+        help="include netcdf test cases",
+    )
 
 
 def pytest_collection_modifyitems(config, items):
@@ -129,3 +135,11 @@ def pytest_collection_modifyitems(config, items):
     for item in items:
         if "parallel" in item.keywords:
             item.add_marker(skip_parallel)
+
+    if config.getoption("--netcdf"):
+        # --netcdf given in cli: do not skip netcdf tests
+        return
+    skip_netcdf = pytest.mark.skip(reason="need --netcdf option to run")
+    for item in items:
+        if "netcdf" in item.keywords:
+            item.add_marker(skip_netcdf)

autotest/pytest.ini

@@ -15,8 +15,9 @@ markers =
     lak: tests for lake package
     maw: tests for multi-aquifer well package
     parallel: test relying on a parallel (MPI+PETSc) build
+    netcdf: test relying on a netcdf (NetCDF-FORTRAN) build
 filterwarnings =
     # from python-dateutil, used by arrow, jupyter_client, matplotlib, pandas
     ignore:datetime.datetime.utcfromtimestamp  
     # from pandas, see https://github.com/pandas-dev/pandas/issues/54466
-    ignore:\n.*Pyarrow  
+    ignore:\n.*Pyarrow