ci(netcdf): windows extended build and tests (#2037)

* baseline netcdf windows tests * add windows extended build and test * idm update post rebase * release extended instead of parallel * build release netcdf-fortran libraries * remove temporary artifact upload --------- Co-authored-by: mjreno <[email protected]> Co-authored-by: mjr-deltares <[email protected]>
MODFLOW-USGS · Nov 5, 2024 · d12df58 · d12df58
1 parent c88c436
commit d12df58
Show file tree

Hide file tree

Showing 25 changed files with 454 additions and 85 deletions.
diff --git a/.github/actions/build-extended-win/action.yml b/.github/actions/build-extended-win/action.yml
@@ -0,0 +1,23 @@
+name: Build Extended MF6 (Windows)
+description: Build Extended MODFLOW 6 on Windows
+runs:
+  using: "composite"
+  steps:
+
+    - name: Setup PETSc
+      uses: ./modflow6/.github/actions/build-petsc-win
+
+    - name: Setup NetCDF
+      uses: ./modflow6/.github/actions/build-netcdf-win
+
+    - name: Build modflow6
+      shell: cmd
+      run: |
+        unix2dos -n "%GITHUB_WORKSPACE%\modflow6\.github\common\compile_modflow6_extended.bat" "%TEMP%\compile_modflow6_extended.bat"
+        "%ONEAPI_ROOT%\setvars.bat" intel64 vs2022 && "%TEMP%\compile_modflow6_extended.bat"
+
+    - name: Show Meson logs
+      if: failure()
+      shell: bash
+      working-directory: modflow6
+      run: cat builddir/meson-logs/meson-log.txt
diff --git a/.github/actions/build-netcdf-mf6/action.yml b/.github/actions/build-netcdf-mf6/action.yml
@@ -0,0 +1,19 @@
+name: Build NetCDF MF6 (Windows)
+description: Build NetCDF MODFLOW 6 on Windows
+runs:
+  using: "composite"
+  steps:
+
+    - name: Build modflow6
+      shell: cmd
+      run: |
+        unix2dos -n "%GITHUB_WORKSPACE%\modflow6\.github\common\compile_modflow6_netcdf.bat" "%TEMP%\compile_modflow6_netcdf.bat"
+        "%ONEAPI_ROOT%\setvars.bat" intel64 vs2022 && "%TEMP%\compile_modflow6_netcdf.bat"
+
+    - name: Show Meson logs
+      if: failure()
+      shell: bash
+      working-directory: modflow6
+      run: |
+        cat builddir/meson-logs/meson-log.txt
+        cat builddir/build.ninja
diff --git a/.github/actions/build-netcdf-win/action.yml b/.github/actions/build-netcdf-win/action.yml
@@ -0,0 +1,43 @@
+name: Build NetCDF (Windows)
+description: Build NetCDF on Windows
+runs:
+  using: "composite"
+  steps:
+
+    - name: Convert line endings
+      shell: cmd
+      run: |
+        unix2dos -n "%GITHUB_WORKSPACE%\modflow6\.github\common\compile_netcdf.bat" "%TEMP%\compile_netcdf.bat"
+
+    - name: Install latest cmake and ninja
+      uses: lukka/get-cmake@latest
+      with:
+        cmakeVersion: latest
+        ninjaVersion: latest
+
+    - name: Setup oneAPI
+      uses: ./modflow6/.github/actions/setup-par-oneapi
+
+    - name: Setup 7-zip
+      uses: milliewalky/setup-7-zip@v1
+
+    - name: Download NetCDF-C
+      shell: bash
+      run: |
+        mkdir -p netcdf/netCDF4.9.2-NC4-64
+        cd netcdf/netCDF4.9.2-NC4-64
+        curl https://downloads.unidata.ucar.edu/netcdf-c/4.9.2/netCDF4.9.2-NC4-64.exe -O -J
+        7z x netCDF4.9.2-NC4-64.exe -aou
+
+    - name: Download NetCDF-Fortran
+      shell: bash
+      run: |
+        mkdir -p netcdf/netcdf-fortran-4.6.1/build
+        cd netcdf/netcdf-fortran-4.6.1
+        curl https://downloads.unidata.ucar.edu/netcdf-fortran/4.6.1/netcdf-fortran-4.6.1.zip -O -J
+        unzip netcdf-fortran-4.6.1.zip
+
+    - name: Build NetCDF
+      shell: cmd
+      run: |
+        "%ONEAPI_ROOT%\setvars.bat" intel64 vs2022 && "%TEMP%\compile_netcdf.bat"
diff --git a/.github/actions/test-extended-win/action.yml b/.github/actions/test-extended-win/action.yml
@@ -0,0 +1,78 @@
+name: Test extended MF6 (Windows)
+description: Build and test extended MODFLOW 6 on Windows
+runs:
+  using: "composite"
+  steps:
+
+    - name: Build MF6 parallel
+      uses: ./modflow6/.github/actions/build-extended-win
+
+    - name: Update flopy
+      working-directory: modflow6
+      shell: cmd
+      run: pixi run update-flopy
+
+    - name: Get executables
+      working-directory: modflow6/autotest
+      shell: cmd
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: pixi run get-exes
+
+    - name: Convert unix2dos
+      shell: cmd
+      run: |
+        unix2dos -n "%GITHUB_WORKSPACE%\modflow6\.github\common\test_modflow6_parallel.bat" "%TEMP%\test_modflow6_parallel.bat"
+        unix2dos -n "%GITHUB_WORKSPACE%\modflow6\.github\common\test_modflow6_serial.bat" "%TEMP%\test_modflow6_serial.bat"
+        unix2dos -n "%GITHUB_WORKSPACE%\modflow6\.github\common\test_modflow6_netcdf.bat" "%TEMP%\test_modflow6_netcdf.bat"
+
+    - name: Test programs (parallel)
+      if: github.ref_name != 'master'
+      shell: cmd
+      env:
+        REPOS_PATH: ${{ github.workspace }}
+      run: |
+        "%ONEAPI_ROOT%\setvars.bat" intel64 vs2022 && "%TEMP%\test_modflow6_parallel.bat"
+
+    - name: Test programs (parallel)
+      if: github.ref_name == 'master'
+      shell: cmd
+      env:
+        REPOS_PATH: ${{ github.workspace }}
+        MARKERS: not developmode
+      run: |
+        "%ONEAPI_ROOT%\setvars.bat" intel64 vs2022 && "%TEMP%\test_modflow6_parallel.bat"
+
+    - name: Test programs (netcdf)
+      if: github.ref_name != 'master'
+      shell: cmd
+      env:
+        REPOS_PATH: ${{ github.workspace }}
+      run: |
+        "%ONEAPI_ROOT%\setvars.bat" intel64 vs2022 && "%TEMP%\test_modflow6_netcdf.bat"
+
+    - name: Test programs (netcdf)
+      if: github.ref_name == 'master'
+      shell: cmd
+      env:
+        REPOS_PATH: ${{ github.workspace }}
+        MARKERS: not developmode
+      run: |
+        "%ONEAPI_ROOT%\setvars.bat" intel64 vs2022 && "%TEMP%\test_modflow6_netcdf.bat"
+
+    - name: Test programs (serial)
+      if: github.ref_name != 'master'
+      shell: cmd
+      env:
+        REPOS_PATH: ${{ github.workspace }}
+      run: |
+        "%ONEAPI_ROOT%\setvars.bat" intel64 vs2022 && "%TEMP%\test_modflow6_serial.bat"
+
+    - name: Test programs (serial)
+      if: github.ref_name == 'master'
+      shell: cmd
+      env:
+        REPOS_PATH: ${{ github.workspace }}
+        MARKERS: not developmode
+      run: |
+        "%ONEAPI_ROOT%\setvars.bat" intel64 vs2022 && "%TEMP%\test_modflow6_serial.bat"
diff --git a/.github/actions/test-netcdf-win/action.yml b/.github/actions/test-netcdf-win/action.yml
@@ -0,0 +1,45 @@
+name: Test NetCDF MF6 (Windows)
+description: Build and test NetCDF MODFLOW 6 on Windows
+runs:
+  using: "composite"
+  steps:
+
+    - name: Build MF6 NetCDF
+      uses: ./modflow6/.github/actions/build-netcdf-win
+
+    - name: Build MF6 NetCDF
+      uses: ./modflow6/.github/actions/build-netcdf-mf6
+
+    - name: Update flopy
+      working-directory: modflow6
+      shell: cmd
+      run: pixi run update-flopy
+
+    - name: Get executables
+      working-directory: modflow6/autotest
+      shell: cmd
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: pixi run get-exes
+
+    - name: Convert unix2dos
+      shell: cmd
+      run: |
+        unix2dos -n "%GITHUB_WORKSPACE%\modflow6\.github\common\test_modflow6_netcdf.bat" "%TEMP%\test_modflow6_netcdf.bat"
+
+    - name: Test programs
+      if: github.ref_name != 'master'
+      shell: cmd
+      env:
+        REPOS_PATH: ${{ github.workspace }}
+      run: |
+        "%ONEAPI_ROOT%\setvars.bat" intel64 vs2022 && "%TEMP%\test_modflow6_netcdf.bat"
+
+    - name: Test programs
+      if: github.ref_name == 'master'
+      shell: cmd
+      env:
+        REPOS_PATH: ${{ github.workspace }}
+        MARKERS: not developmode
+      run: |
+        "%ONEAPI_ROOT%\setvars.bat" intel64 vs2022 && "%TEMP%\test_modflow6_netcdf.bat"
diff --git a/.github/actions/test-netcdf/action.yml b/.github/actions/test-netcdf/action.yml
@@ -0,0 +1,64 @@
+name: Test NetCDF MF6
+description: Build and test NetCDF MODFLOW 6
+runs:
+  using: "composite"
+  steps:
+
+    - name: Setup GNU Fortran
+      uses: fortran-lang/setup-fortran@v1
+      with:
+        compiler: gcc
+        version: 13
+
+    - name: Install netcdf
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install build-essential \
+          libnetcdf-dev \
+          libnetcdff-dev \
+          netcdf-bin
+        nc-config --all
+        nf-config --all
+
+    - name: Install netcdf
+      if: runner.os == 'macOS'
+      shell: bash
+      run: |
+        brew install netcdf-fortran
+        nc-config --all
+        nf-config
+
+    - name: Build modflow6
+      shell: bash
+      working-directory: modflow6
+      run: |
+        pixi run setup -Dnetcdf=true builddir
+        pixi run build builddir
+        pixi run test builddir
+
+    - name: Show Meson logs
+      if: failure()
+      shell: bash
+      working-directory: modflow6
+      run: cat builddir/meson-logs/meson-log.txt
+
+    - name: Update flopy
+      shell: bash
+      working-directory: modflow6
+      run: pixi run update-flopy
+
+    - name: Get executables
+      shell: bash
+      working-directory: modflow6
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: pixi run get-exes
+
+    - name: Test programs
+      shell: bash
+      working-directory: modflow6
+      env:
+        REPOS_PATH: ${{ github.workspace }}
+      run: pixi run autotest --netcdf -k "test_netcdf"
diff --git a/.github/common/compile_modflow6_extended.bat b/.github/common/compile_modflow6_extended.bat
@@ -0,0 +1,5 @@
+set FC=ifort
+cd "%GITHUB_WORKSPACE%\modflow6"
+pixi run setup -Dextended=true builddir
+pixi run build builddir
+pixi run test builddir
diff --git a/.github/common/compile_modflow6_netcdf.bat b/.github/common/compile_modflow6_netcdf.bat
@@ -0,0 +1,4 @@
+set FC=ifort
+cd "%GITHUB_WORKSPACE%\modflow6"
+pixi run setup -Dnetcdf=true builddir
+pixi run build builddir
diff --git a/.github/common/compile_netcdf.bat b/.github/common/compile_netcdf.bat
@@ -0,0 +1,11 @@
+cd "%GITHUB_WORKSPACE%\netcdf\netcdf-fortran-4.6.1\build"
+
+:: build/install static libs
+cmake --fresh -G Ninja -DCMAKE_Fortran_COMPILER="C:/Program Files (x86)/Intel/oneAPI/compiler/latest/bin/ifort.exe" -DCMAKE_BUILD_TYPE=Release -DNETCDF_C_LIBRARY="%GITHUB_WORKSPACE%/netcdf/netCDF4.9.2-NC4-64/lib/netcdf.lib" -DNETCDF_C_INCLUDE_DIR="%GITHUB_WORKSPACE%/netcdf/netCDF4.9.2-NC4-64/include" -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX="%GITHUB_WORKSPACE%/netcdf/netcdf-fortran-4.6.1/build" ../netcdf-fortran-4.6.1
+cmake --build .
+cmake –install .
+
+:: build/install shared libs
+cmake --fresh -G Ninja -DCMAKE_Fortran_COMPILER="C:/Program Files (x86)/Intel/oneAPI/compiler/latest/bin/ifort.exe" -DCMAKE_BUILD_TYPE=Release -DNETCDF_C_LIBRARY="%GITHUB_WORKSPACE%/netcdf/netCDF4.9.2-NC4-64/lib/netcdf.lib" -DNETCDF_C_INCLUDE_DIR="%GITHUB_WORKSPACE%/netcdf/netCDF4.9.2-NC4-64/include" -DBUILD_SHARED_LIBS=1 -DCMAKE_INSTALL_PREFIX="%GITHUB_WORKSPACE%/netcdf/netcdf-fortran-4.6.1/build" ../netcdf-fortran-4.6.1
+cmake --build .
+cmake –install .
diff --git a/.github/common/test_modflow6_netcdf.bat b/.github/common/test_modflow6_netcdf.bat
@@ -0,0 +1,3 @@
+cd "%GITHUB_WORKSPACE%\modflow6\autotest"
+ldd ..\bin\mf6
+pixi run pytest -v --durations=0 --keep-failed .failed --netcdf -k "test_netcdf" -m "%MARKERS%"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -550,8 +550,68 @@ jobs:
           name: failed-${{ matrix.os }}
           path: modflow6/autotest/.failed
 
+  netcdf_test:
+    name: NetCDF testing
+    needs:
+      - lint
+      - build
+      - smoke_test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-22.04, macos-13, windows-2022 ]
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+
+      - name: Checkout MF6
+        uses: actions/checkout@v4
+        with:
+          path: modflow6
+
+      - name: Setup pixi
+        uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.24.2
+          manifest-path: "modflow6/pixi.toml"
+
+      - name: Custom pixi install
+        working-directory: modflow6
+        run: pixi run install
+
+      - name: Install additional netcdf testing packages
+        working-directory: modflow6
+        run: pixi run pip install xugrid xarray netcdf4
+
+      - name: Set LDFLAGS (macOS)
+        if: matrix.os == 'macos-13'
+        run: |
+          os_ver=$(sw_vers -productVersion | cut -d'.' -f1)
+          if (( "$os_ver" > 12 )); then
+            ldflags="$LDFLAGS -Wl,-ld_classic"
+            echo "LDFLAGS=$ldflags" >> $GITHUB_ENV
+          fi
+
+      - name: Test NetCDF MF6
+        if: runner.os != 'Windows'
+        uses: ./modflow6/.github/actions/test-netcdf
+
+      - name: Test NetCDF MF6
+        if: runner.os == 'Windows'
+        uses: ./modflow6/.github/actions/test-netcdf-win
+
+      - name: Upload failed test output
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: failed-${{ matrix.os }}
+          path: modflow6/autotest/.failed
+
   extended_test:
-    name: Extended executable testing
+    name: Extended testing
     needs:
       - lint
       - build
@@ -560,7 +620,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, macos-13]
+        os: [ ubuntu-22.04, macos-13, windows-2022 ]
     defaults:
       run:
         shell: bash
@@ -595,8 +655,13 @@ jobs:
           fi
 
       - name: Test extended MF6
+        if: runner.os != 'Windows'
         uses: ./modflow6/.github/actions/test-extended
 
+      - name: Test extended MF6
+        if: runner.os == 'Windows'
+        uses: ./modflow6/.github/actions/test-extended-win
+
       - name: Upload failed test output
         if: failure()
         uses: actions/upload-artifact@v4