update MOM6, ocean_BGC hashes and model configs, add NWA12.CFC test, and refactor NWA12.COBALT test

.github/workflows/NWA12-ci.yaml

@@ -27,16 +27,23 @@ jobs:
         path: ${{ github.run_id }}/NWA12_CHECK
         submodules: recursive
 
-#    - uses: jitterbit/get-changed-files@v1
-#      id: abc
-#      with:
-#        format: space-delimited
-#        token: ${{ secrets.GITHUB_TOKEN }}     
-
     - name: Build MOM6SIS2
       run: |
         cd ${{ env.TEST_DIR }}/NWA12_CHECK/builds
-        ./linux-build.bash -m gaea -p ncrc5.intel23 -t repro -f mom6sis2
+        jobid=$(sbatch --parsable ci_build_driver.sh | awk -F';' '{print $1}' | cut -f1)
+        #
+        sleep 1
+        while :; do
+            job_status=$(squeue -h -j "$jobid" -o "%T" 2>/dev/null)
+            if [ -z "$job_status" ]; then
+               echo "Job with ID $jobid is not found or completed."
+               break
+            else
+               echo "Job with ID $jobid is still running."
+               echo "Job Status: $job_status"
+            fi
+            sleep 60  # Adjust the sleep duration as needed
+        done
         #
         check_file="${{ env.TEST_DIR }}/NWA12_CHECK/builds/build/gaea-ncrc5.intel23/ocean_ice/repro/MOM6SIS2"
         if [ -f "$check_file" ]; then
@@ -52,7 +59,7 @@ jobs:
     strategy:
       max-parallel: 2
       matrix:
-        case: ["NWA12.COBALT"]
+        case: ["NWA12.CFC"] #, "NWA12.COBALT"]
     steps:
       - name: Run Experiment ${{ matrix.case }}
         run: |
@@ -69,7 +76,7 @@ jobs:
                  echo "Job with ID $jobid is still running."
                  echo "Job Status: $job_status"
               fi
-              sleep 30  # Adjust the sleep duration as needed
+              sleep 60  # Adjust the sleep duration as needed
           done
           # 
           expected_string="All restart files are identical, PASS"

.github/workflows/mom6_cobalt_1D.yaml

@@ -10,6 +10,10 @@ jobs:
   build:
     runs-on: ubuntu-latest
 
+    container:
+      image: clouden90/1d_mom6_cobalt:base
+      options: --user=root
+
     env:
       PR_NUMBER: ${{ github.event.number }}
 
@@ -19,37 +23,29 @@ jobs:
       with:
         submodules: recursive
 
-#    - uses: jitterbit/get-changed-files@v1
-#      id: abc
-#      with:
-#        format: space-delimited
-#        token: ${{ secrets.GITHUB_TOKEN }}        
-
-#    - name: Login to DockerHub Registry
-#      run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
-
-    - name: Build mom6_obgc 1D Docker Image
+    - name: Build mom6-sis2-cobalt
       run: |
-        echo "maximize disk space on git-runner ..."
-        df -h 
-        sudo rm -rf /usr/share/dotnet
-        sudo rm -rf /opt/ghc
-        sudo rm -rf "/usr/local/share/boost"
-        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-        df -h
-        echo "docker image build ..."
-        docker build --file "./ci/docker/Dockerfile.ci" -t mom6_obgc_1d:develop .
+        apt update; apt install -y wget
+        echo "build mom6-sis2-cobalt ..."
+        cd ./builds
+        ./linux-build.bash -m docker -p linux-gnu -t debug -f mom6sis2
+
+    - name: Download 1d model dataset
+      run: | 
+       cd ./exps
+       wget -q ftp.gfdl.noaa.gov:/pub/Yi-cheng.Teng/1d_datasets.tar.gz && tar -zxvf 1d_datasets.tar.gz && rm -rf 1d_datasets.tar.gz
 
-    - name: Check mom6_obgc 1D Docker Image build
+    - name: Run 1D toy case and check repro across restarts
       run: | 
-       docker images 
-       if [[ $(docker image inspect mom6_obgc_1d:develop) ]]; then
-            echo "mom6_obgc_1d:develop generated ..."          
-       else
-            echo "mom6_obgc_1d:develop was not generated ..."
-            exit 1
-       fi
+       cd exps/OM4.single_column.COBALT
+       #mpirun --allow-run-as-root -np 1 ../../builds/build/docker-linux-gnu/ocean_ice/prod/MOM6SIS2
+       ./driver.sh 
 
-#    - name: Docker Logout
-#      run: |
-#       docker logout   
+    - name: Check ref
+      run: | 
+       cd exps/OM4.single_column.COBALT
+       echo "new 48hrs ocean.stats: "
+       cat ./RESTART_48hrs/ocean.stats
+       echo "baseline 48 hrs ocean.stats: "
+       cat ./ref/ocean.stats
+       diff -q ref/ocean.stats ./RESTART_48hrs/ocean.stats > /dev/null || { echo "Error: ocean.stats are different."; echo "Plz update exps/OM4.single_column.COBALT/ref/ocean.stats with the following: "; cat ./RESTART_48hrs/ocean.stats; exit 1; }

.gitmodules

@@ -13,10 +13,6 @@
 [submodule "src/MOM6"]
 	path = src/MOM6
 	url = https://github.com/NOAA-CEFI-Regional-Ocean-Modeling/MOM6.git
-	branch = stable/cefi
-[submodule "src/ocean_BGC"]
-	path = src/ocean_BGC
-	url = https://github.com/NOAA-CEFI-Regional-Ocean-Modeling/ocean_BGC.git
 	branch = dev/cefi
 [submodule "src/FMS"]
 	path = src/FMS
@@ -42,3 +38,7 @@
 	path = src/Icepack
 	url = https://github.com/CICE-Consortium/Icepack.git
 	branch = Icepack1.3.3
+[submodule "src/ocean_BGC"]
+	path = src/ocean_BGC
+	url = https://github.com/NOAA-CEFI-Regional-Ocean-Modeling/ocean_BGC.git
+	branch = dev/cefi

builds/ci_build_driver.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#SBATCH --nodes=1
+#SBATCH --time=60
+#SBATCH --job-name="MOM6SIS2_ci_build"
+#SBATCH --output=MOM6SIS2_ci_build_o.%j
+#SBATCH --error=MOM6SIS2_ci_build_e.%j
+#SBATCH --qos=debug
+#SBATCH --partition=batch
+#SBATCH --clusters=c5
+#SBATCH --account=cefi
+
+#
+
+#
+[ -d "build" ] && rm -rf build
+
+#
+echo "Build started:  " `date`
+
+#
+./linux-build.bash -m gaea -p ncrc5.intel23 -t repro -f fms1_mom6sis2
+
+#
+echo "Build ended:  " `date`

builds/docker/linux-gnu.mk

@@ -92,7 +92,7 @@ FFLAGS := -fcray-pointer -fdefault-double-8 -fdefault-real-8 -Waliasing -ffree-l
 # Flags based on perforance target (production (OPT), reproduction (REPRO), or debug (DEBUG)
 FFLAGS_OPT = -O3
 FFLAGS_REPRO = -O2 -fbounds-check
-FFLAGS_DEBUG = -O0 -g -W -fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow
+FFLAGS_DEBUG = -O0 -g -W -fbacktrace #-fbounds-check -fbacktrace -ffpe-trap=invalid,zero,overflow
 
 # Flags to add additional build options
 FFLAGS_OPENMP = -fopenmp

builds/linux-build.bash

@@ -48,14 +48,6 @@ source $rootdir/$machine_name/$platform.env
 
 makeflags="NETCDF=3"
 
-if [[ "$target" =~ "openmp" ]] ; then 
-   makeflags="$makeflags OPENMP=1" 
-fi
-
-if [[ "$target" =~ "openacc" ]] ; then 
-   makeflags="$makeflags OPENACC=1" 
-fi
-
 if [[ $target =~ "repro" ]] ; then
    makeflags="$makeflags REPRO=1"
 fi
@@ -76,22 +68,23 @@ srcdir=$abs_rootdir/../src
 
 sed -i 's/static pid_t gettid(void)/pid_t gettid(void)/g' $srcdir/FMS/affinity/affinity.c
 
+if [[ $flavor == "mom6sis2" ]] ; then
+
 mkdir -p build/$machine_name-$platform/shared/$target
-pushd build/$machine_name-$platform/shared/$target   
-rm -f path_names                       
+pushd build/$machine_name-$platform/shared/$target
+rm -f path_names
 $srcdir/mkmf/bin/list_paths $srcdir/FMS/{affinity,amip_interp,column_diagnostics,diag_integral,drifters,horiz_interp,memutils,sat_vapor_pres,topography,astronomy,constants,diag_manager,field_manager,include,monin_obukhov,platform,tracer_manager,axis_utils,coupler,fms,fms2_io,interpolator,mosaic,mosaic2,random_numbers,time_interp,tridiagonal,block_control,data_override,exchange,mpp,time_manager,string_utils,parser}/ $srcdir/FMS/libFMS.F90
 $srcdir/mkmf/bin/mkmf -t $abs_rootdir/$machine_name/$platform.mk -p libfms.a -c "-Duse_libMPI -Duse_netCDF -DMAXFIELDMETHODS_=800" path_names
 
-make $makeflags libfms.a         
+make $makeflags libfms.a
 
 if [ $? -ne 0 ]; then
    echo "Could not build the FMS library!"
    exit 1
 fi
 
 popd
-
-if [[ $flavor =~ "mom6sis2" ]] ; then
+
     mkdir -p build/$machine_name-$platform/ocean_ice/$target
     pushd build/$machine_name-$platform/ocean_ice/$target
     rm -f path_names
@@ -107,20 +100,42 @@ fi
 
     $srcdir/mkmf/bin/mkmf -t $abs_rootdir/$machine_name/$platform.mk -o "-I../../shared/$target" -p MOM6SIS2 -l "-L../../shared/$target -lfms $linker_options" -c "$compiler_options" path_names
 
-if [[ "$target" =~ "cobaltACC" ]] ; then 
-    sed -e 's/-c\(.*\)COBALT/-acc -ta=nvidia:managed -Minfo=accel -c \1COBALT/' -i Makefile
-    sed -e 's/-lfms/-lfms -acc/' -i Makefile
-fi
+    make $makeflags MOM6SIS2
 
-if [[ "$target" =~ "cobaltOMP" ]] ; then 
-    sed -e 's/-c\(.*\)COBALT/-mp -c \1COBALT/' -i Makefile
+elif [[ $flavor == "fms1_mom6sis2" ]]; then
+
+    echo "build mom6sis2 with FMS1 cap" 
+
+mkdir -p build/$machine_name-$platform/shared/$target
+pushd build/$machine_name-$platform/shared/$target
+rm -f path_names
+$srcdir/mkmf/bin/list_paths $srcdir/FMS/{affinity,amip_interp,column_diagnostics,diag_integral,drifters,horiz_interp,memutils,sat_vapor_pres,topography,astronomy,constants,diag_manager,field_manager,include,monin_obukhov,platform,tracer_manager,axis_utils,coupler,fms,fms2_io,interpolator,mosaic,mosaic2,random_numbers,time_interp,tridiagonal,block_control,data_override,exchange,mpp,time_manager,string_utils,parser}/ $srcdir/FMS/libFMS.F90
+$srcdir/mkmf/bin/mkmf -t $abs_rootdir/$machine_name/$platform.mk -p libfms.a -c "-Duse_deprecated_io -Duse_libMPI -Duse_netCDF -DMAXFIELDMETHODS_=800" path_names
+
+make $makeflags libfms.a
+
+if [ $? -ne 0 ]; then
+   echo "Could not build the FMS library!"
+   exit 1
 fi
 
-if [[ "$target" =~ "cobaltOMPGPU" ]] ; then 
-    sed -e 's/-c\(.*\)COBALT/-mp=gpu -gpu=managed -Minfo=accel -c \1COBALT/' -i Makefile
-    sed -e 's/-lfms/-lfms -mp=gpu -gpu=managed/' -i Makefile
+popd
+
+    mkdir -p build/$machine_name-$platform/ocean_ice/$target
+    pushd build/$machine_name-$platform/ocean_ice/$target
+    rm -f path_names
+    $srcdir/mkmf/bin/list_paths $srcdir/MOM6/{config_src/infra/FMS1,config_src/memory/dynamic_symmetric,config_src/drivers/FMS_cap,config_src/external/ODA_hooks,config_src/external/database_comms,config_src/external/drifters,config_src/external/stochastic_physics,pkg/GSW-Fortran/{modules,toolbox}/,src/{*,*/*}/} $srcdir/SIS2/{config_src/dynamic_symmetric,config_src/external/Icepack_interfaces,src} $srcdir/icebergs/src $srcdir/FMS/{coupler,include}/ $srcdir/{ocean_BGC/generic_tracers,ocean_BGC/mocsy/src}/ $srcdir/{atmos_null,ice_param,land_null,coupler/shared/,coupler/full/}/
+
+
+compiler_options='-DINTERNAL_FILE_NML -DMAX_FIELDS_=600 -DNOT_SET_AFFINITY -D_USE_MOM6_DIAG -D_USE_GENERIC_TRACER -DUSE_PRECISION=2 -D_USE_LEGACY_LAND_ -Duse_AM3_physics'
+linker_options=''
+if [[ "$target" =~ "stdpar" ]] ; then
+    compiler_options="$compiler_options -stdpar -Minfo=accel"
+    linker_options="$linker_options -stdpar "
 fi
 
+    $srcdir/mkmf/bin/mkmf -t $abs_rootdir/$machine_name/$platform.mk -o "-I../../shared/$target" -p MOM6SIS2 -l "-L../../shared/$target -lfms $linker_options" -c "$compiler_options" path_names
+
     make $makeflags MOM6SIS2
 
 else 

ci/docker/Dockerfile.ci

@@ -27,7 +27,7 @@ WORKDIR /opt/MOM6_OBGC_examples/exps/OM4.single_column.COBALT
 RUN mpirun -np 1 /opt/MOM6_OBGC_examples/builds/build/docker-linux-gnu/ocean_ice/prod/MOM6SIS2
 
 # create ref folder
-RUN mkdir ref
+RUN [ ! -d "ref" ] && mkdir -p "ref"
 RUN cp /opt/MOM6_OBGC_examples/src/ocean_BGC/.github/ref/OM4.single_column.COBALT.p4/docker-linux-gnu/* ./ref/
 
 # check

exps/NWA12.CFC/INPUT/ERA5_lp_1995_padded.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ERA5_lp_1995_padded.nc

exps/NWA12.CFC/INPUT/ERA5_msl_1995_padded.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ERA5_msl_1995_padded.nc

exps/NWA12.CFC/INPUT/ERA5_sf_1995_padded.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ERA5_sf_1995_padded.nc

exps/NWA12.CFC/INPUT/ERA5_sphum_1995_padded.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ERA5_sphum_1995_padded.nc

exps/NWA12.CFC/INPUT/ERA5_ssrd_1995_padded.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ERA5_ssrd_1995_padded.nc

exps/NWA12.CFC/INPUT/ERA5_strd_1995_padded.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ERA5_strd_1995_padded.nc

exps/NWA12.CFC/INPUT/ERA5_t2m_1995_padded.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ERA5_t2m_1995_padded.nc

exps/NWA12.CFC/INPUT/ERA5_u10_1995_padded.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ERA5_u10_1995_padded.nc

exps/NWA12.CFC/INPUT/ERA5_v10_1995_padded.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ERA5_v10_1995_padded.nc

exps/NWA12.CFC/INPUT/atmos_mosaic_tile1Xland_mosaic_tile1.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/atmos_mosaic_tile1Xland_mosaic_tile1.nc

exps/NWA12.CFC/INPUT/atmos_mosaic_tile1Xocean_mosaic_tile1.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/atmos_mosaic_tile1Xocean_mosaic_tile1.nc

exps/NWA12.CFC/INPUT/cfc.bc.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/cfc.bc.nc

exps/NWA12.CFC/INPUT/cfc11_obc_1995.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/cfc11_obc_1995.nc

exps/NWA12.CFC/INPUT/cfc12_obc_1995.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/cfc12_obc_1995.nc

exps/NWA12.CFC/INPUT/cfc_ic_1995-01-01.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/cfc_ic_1995-01-01.nc

exps/NWA12.CFC/INPUT/glofas_runoff_1995.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/glofas_runoff_1995.nc

exps/NWA12.CFC/INPUT/grid_spec.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/grid_spec.nc

exps/NWA12.CFC/INPUT/ic_1995-01-01.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ic_1995-01-01.nc

exps/NWA12.CFC/INPUT/land_mask.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/land_mask.nc

exps/NWA12.CFC/INPUT/land_mosaic_tile1Xocean_mosaic_tile1.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/land_mosaic_tile1Xocean_mosaic_tile1.nc

exps/NWA12.CFC/INPUT/ocean_hgrid.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ocean_hgrid.nc

exps/NWA12.CFC/INPUT/ocean_mask.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ocean_mask.nc

exps/NWA12.CFC/INPUT/ocean_mosaic.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ocean_mosaic.nc

exps/NWA12.CFC/INPUT/ocean_static_no_mask_table.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ocean_static_no_mask_table.nc

exps/NWA12.CFC/INPUT/ocean_topog.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/ocean_topog.nc

exps/NWA12.CFC/INPUT/seawifs-clim-1997-2010.nwa12.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/seawifs-clim-1997-2010.nwa12.nc

exps/NWA12.CFC/INPUT/so_001.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/so_001.nc

exps/NWA12.CFC/INPUT/so_002.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/so_002.nc

exps/NWA12.CFC/INPUT/so_003.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/so_003.nc

exps/NWA12.CFC/INPUT/thetao_001.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/thetao_001.nc

exps/NWA12.CFC/INPUT/thetao_002.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/thetao_002.nc

exps/NWA12.CFC/INPUT/thetao_003.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/thetao_003.nc

exps/NWA12.CFC/INPUT/tu_001.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/tu_001.nc

exps/NWA12.CFC/INPUT/tu_002.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/tu_002.nc

exps/NWA12.CFC/INPUT/tu_003.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/tu_003.nc

exps/NWA12.CFC/INPUT/tz_001.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/tz_001.nc

exps/NWA12.CFC/INPUT/tz_002.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/tz_002.nc

exps/NWA12.CFC/INPUT/tz_003.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/tz_003.nc

exps/NWA12.CFC/INPUT/uv_001.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/uv_001.nc

exps/NWA12.CFC/INPUT/uv_002.nc

@@ -0,0 +1 @@
+../../datasets/nwa12_cfc_input/uv_002.nc