Add GHA and RT scripts (#1)

* Add CI and RT scripts * Rename RT.yaml * Rm RT.xml and mod ci bash script * mod RT.yaml * Update create_run_xml_from_template.bash * Modify workdir name
NOAA-CEFI-Regional-Ocean-Modeling · Jan 10, 2024 · f9453f9 · f9453f9
1 parent 04205c6
commit f9453f9
Show file tree

Hide file tree

Showing 6 changed files with 1,085 additions and 0 deletions.
diff --git a/.github/workflows/NWA12_RT.yaml b/.github/workflows/NWA12_RT.yaml
@@ -0,0 +1,34 @@
+# This is a basic workflow to help you get started with Actions
+
+name: NWA12_RT
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the "main" branch
+  push:
+    branches: [ "dev/cefi" ]
+  pull_request:
+    branches: [ "dev/cefi" ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: self-hosted
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      # Run create_run_xml_from_template.bash 
+      - name: create and run FRE xml
+        run: |
+          pwd
+          cd ci
+          bash create_run_xml_from_template.bash 
diff --git a/.github/workflows/mom6_cobalt_1D.yaml → .github/workflows/mom6_cobalt_1D._yaml b/.github/workflows/mom6_cobalt_1D.yaml → .github/workflows/mom6_cobalt_1D._yaml
diff --git a/ci/compile_ocean_ice_cobalt.xml b/ci/compile_ocean_ice_cobalt.xml
@@ -0,0 +1,121 @@
+<freInclude xmlns:xi="http://www.w3.org/2003/XInclude">
+   <experiment name="MOM6_SIS2_GENERIC_4P_compile_symm">
+      <description>
+Make the executable for cefi ocean-ice-cobalt experiments.
+      </description>
+    <component name="fms" paths="FMS">
+      <source versionControl="git" root="https://github.com/NOAA-GFDL">
+        <codeBase version="$(FMS_TAG)"> FMS.git </codeBase>
+        <csh><![CDATA[
+              pushd FMS
+              git checkout $(FMS_GIT_HASH)
+              popd
+        ]]></csh>
+      </source>
+      <compile>
+            <cppDefs>-Duse_libMPI -Duse_netCDF $(F2003_FLAGS) -DMAXFIELDMETHODS_=600</cppDefs>
+      </compile>
+    </component>
+
+    <component name="mom6" requires="fms" paths="MOM6/config_src/memory/dynamic_symmetric MOM6/config_src/drivers/FMS_cap MOM6/src/*/ MOM6/src/*/*/ MOM6/config_src/external/ODA_hooks MOM6/config_src/external/stochastic_physics MOM6/config_src/external/drifters MOM6/config_src/external/database_comms ocean_BGC/generic_tracers ocean_BGC/mocsy/src MOM6/pkg/GSW-Fortran/modules MOM6/pkg/GSW-Fortran/toolbox MOM6/config_src/infra/FMS1">
+      <description domainName="Ocean" communityName="GFDL-MOM6" communityVersion="dev/cefi" communityGrid="regional"/>
+       <source versionControl="git" root="$(MOM6_SUBMODULE_URL)">
+        <codeBase version="dev/cefi"> MOM6.git </codeBase>
+        <csh><![CDATA[
+               pushd MOM6
+               git checkout $(MOM6_GIT_HASH)
+	       popd
+	       git clone $(OBGC_SUBMODULE_URL)/ocean_BGC.git
+	       pushd ocean_BGC
+               git checkout $(OBGC_GIT_HASH)
+               git submodule update --init --recursive
+	       popd	
+       ]]></csh>
+      </source>
+      <compile>
+        <cppDefs><![CDATA[ $(F2003_FLAGS) -DMAX_FIELDS_=100 -DNOT_SET_AFFINITY -D_USE_MOM6_DIAG -D_USE_GENERIC_TRACER  -DUSE_PRECISION=2 -D_FILE_VERSION="'"`git-version-string $<`"'" ]]></cppDefs>
+      </compile>
+    </component>
+
+    <component name="sis2" paths="SIS2/config_src/dynamic_symmetric SIS2/config_src/external/Icepack_interfaces SIS2/src icebergs ice_param" requires="fms mom6" includeDir="$root/$(LIBS_ROOT)/src/MOM6/src/framework" >
+         <source versionControl="git" root="https://github.com/NOAA-GFDL">
+	   <codeBase version="dev/gfdl"> SIS2.git </codeBase> 
+           <csh><![CDATA[
+               pushd SIS2
+               git checkout $(SIS2_GIT_HASH)
+               popd
+         ]]></csh>
+         </source>
+         <compile>
+            <cppDefs><![CDATA[$(F2003_FLAGS) -D_FILE_VERSION="'"`git-version-string $<`"'"]]></cppDefs>
+         </compile>
+      </component>
+
+      <component name="icebergs" paths="/dev/null" requires="coupler">
+         <description domainName="NA" communityName="NA" communityVersion="NA" communityGrid="NA"/>
+         <source versionControl="git" root="https://github.com/NOAA-GFDL">
+           <codeBase version="dev/gfdl"> icebergs.git </codeBase>
+           <csh><![CDATA[
+               pushd icebergs
+               git checkout $(ICEBERGS_GIT_HASH)
+               popd
+         ]]></csh>
+         </source>
+      </component>
+
+      <component name="ice_param" paths="/dev/null" requires="coupler">
+         <description domainName="NA" communityName="NA" communityVersion="NA" communityGrid="NA"/>
+         <source versionControl="git" root="https://github.com/NOAA-GFDL">
+            <codeBase version="master"> ice_param.git </codeBase>
+            <csh><![CDATA[
+               pushd ice_param
+               git checkout $(ICEPARAM_GIT_HASH)
+               popd
+          ]]></csh>
+         </source>
+      </component>
+
+      <component name="land_null" paths="land_null" requires="fms">
+         <description domainName="NA" communityName="NA" communityVersion="NA" communityGrid="NA"/>
+         <source versionControl="git" root="https://github.com/NOAA-GFDL">
+            <codeBase version="$(NULL_TAG)"> land_null.git </codeBase>
+            <csh><![CDATA[
+               pushd land_null
+               git checkout $(LAND_GIT_HASH)
+               popd
+          ]]></csh>
+         </source>
+      </component>
+
+      <component name="atmos_null" paths="atmos_null" requires="fms">
+         <source versionControl="git" root="https://github.com/NOAA-GFDL">
+           <codeBase version="$(NULL_TAG)"> atmos_null.git </codeBase>
+           <csh><![CDATA[
+              pushd atmos_null
+              git checkout $(ATMOS_GIT_HASH)
+              popd
+         ]]></csh>
+         </source>
+         <compile>
+            <cppDefs>$(F2003_FLAGS) </cppDefs>
+        </compile>
+      </component>
+
+      <!--component name="coupler" paths="coupler/full coupler/shared" requires="fms land_null atmos_null sis2 mom6" -->
+      <component name="coupler" paths="coupler/full coupler/shared" requires="fms land_null atmos_null sis2 mom6" >
+        <description domainName="FMS Coupler" communityName="coupler" communityVersion="$(RELEASE)" communityGrid=""/>  
+        <source versionControl="git" root="https://github.com/NOAA-GFDL">
+          <codeBase version="$(COUPLER_TAG)"> coupler.git </codeBase>
+          <csh><![CDATA[
+             pushd coupler
+             git checkout $(COUPLER_GIT_HASH)
+             popd
+        ]]></csh>
+        </source>
+        <compile>
+            <cppDefs><![CDATA[$(F2003_FLAGS) -D_USE_LEGACY_LAND_ -Duse_AM3_physics -D_FILE_VERSION="'"`git-version-string $<`"'"]]></cppDefs>
+        </compile>
+      </component>
+   </experiment>
+
+</freInclude>
diff --git a/ci/create_run_xml_from_template.bash b/ci/create_run_xml_from_template.bash
@@ -0,0 +1,162 @@
+#!/bin/bash
+
+# --- get current date ---
+CURRENT_DATE=$(date "+%Y%m%d%H%M")
+
+# --- submodules from MOM6_OBGC_examples ---
+
+submodules="FMS MOM6 ocean_BGC SIS2 atmos_null coupler ice_param icebergs land_null"
+
+# --- get the remote URL for ocean_BGC  & MOM6 repos
+pushd ../ 
+git config --file .gitmodules --get submodule.src/ocean_BGC.url
+export OBGC_SUBMODULE_URL=$(git config --file .gitmodules --get submodule.src/ocean_BGC.url | sed 's|/ocean_BGC\.git$||')
+git config --file .gitmodules --get submodule.src/MOM6.url
+export MOM6_SUBMODULE_URL=$(git config --file .gitmodules --get submodule.src/MOM6.url | sed 's|/MOM6\.git$||')
+popd
+
+# --- get the hash for the various repos
+if [ -f ocean_ice_cobalt_experiments.xml ]; then rm -rf ocean_ice_cobalt_experiments.xml ; fi
+pushd ../
+for mod in $submodules ; do
+  export hash_${mod}=$( git submodule status | grep $mod | awk '{print $1}' | cut -c 2-10 )
+done
+popd
+
+echo "Current date: " $CURRENT_DATE
+echo "MOM6 tag: " $hash_MOM6
+echo "ocean_BGC tag: " $hash_ocean_BGC
+echo "SIS2 tag: " $hash_SIS2
+echo "ICEBREGS tag: " $hash_icebergs
+echo "ICE_PARAM tag: " $hash_ice_param
+echo "COUPLER tag: " $hash_coupler
+echo "FMS tag: " $hash_FMS
+echo "ATMOS tag: " $hash_atmos_null
+echo "LAND tag: " $hash_land_null
+echo "MOM6_SUBMODULE_URL: " $MOM6_SUBMODULE_URL
+echo "OBGC_MOM6_SUBMODULE_URL: " $OBGC_SUBMODULE_URL
+
+# --- replace the hashes in the xml template
+
+cat ocean_ice_cobalt_experiments.template.xml | sed -e "s/<FMS_GIT_HASH>/$hash_FMS/g" \
+                                                    -e "s/<COUPLER_GIT_HASH>/$hash_coupler/g" \
+                                                    -e "s/<SIS2_GIT_HASH>/$hash_SIS2/g" \
+                                                    -e "s/<ICEBERGS_GIT_HASH>/$hash_icebergs/g" \
+                                                    -e "s/<ICEPARAM_GIT_HASH>/$hash_ice_param/g" \
+                                                    -e "s/<MOM6_GIT_HASH>/$hash_MOM6/g" \
+                                                    -e "s/<OBGC_GIT_HASH>/$hash_ocean_BGC/g" \
+                                                    -e "s/<LAND_GIT_HASH>/$hash_land_null/g" \
+                                                    -e "s/<ATMOS_GIT_HASH>/$hash_atmos_null/g" \
+						    -e "s|<MOM6_SUBMODULE_URL>|$MOM6_SUBMODULE_URL|g" \
+						    -e "s|<OBGC_SUBMODULE_URL>|$OBGC_SUBMODULE_URL|g" \
+					            -e "s/<CURRENT_DATE>/$CURRENT_DATE/g" \
+                                                    > ocean_ice_cobalt_experiments.xml
+
+
+# -- clean up tmp folders or runs from previos runs
+echo "remove MOM6_OBGC_examples folder"
+rm -rf MOM6_OBGC_examples
+
+workflow_directory="/lustre/f2/dev/Yi-cheng.Teng/github/cefi_NWA12_regression_${CURRENT_DATE}"
+# Check if the directory exists
+if [ -d "$workflow_directory" ]; then
+    # If it exists, remove it
+    echo "Removing workflow directory: $workflow_directory"
+    rm -rf "$workflow_directory"
+else
+    echo "Workflow directory does not exist: $workflow_directory"
+fi
+
+# Now compile mom6-sis2-cobalt 
+module use -a /ncrc/home2/fms/local/modulefiles
+module load fre/bronx-20
+module use -a /ncrc/home2/fms/local/modulefiles
+module load fre/bronx-20
+echo "run fremake and submit compile job"
+fremake -f -F -x ocean_ice_cobalt_experiments.xml -p ncrc5.intel22 -t repro MOM6_SIS2_GENERIC_4P_compile_symm
+jobid=$(sbatch --parsable /lustre/f2/dev/Yi-cheng.Teng/github/cefi_NWA12_regression_${CURRENT_DATE}/MOM6_SIS2_GENERIC_4P_compile_symm/ncrc5.intel22-repro/exec/compile_MOM6_SIS2_GENERIC_4P_compile_symm.csh | awk -F';' '{print $1}' | cut -f1)
+echo "Submitted Slurm job with ID: $jobid"
+
+# Check the status of the job in a loop
+sleep 1
+while :; do
+    # Check the status of the job
+    job_status=$(squeue -h -j "$jobid" -o "%T" 2>/dev/null)
+
+    if [ -z "$job_status" ]; then
+        echo "Job with ID $jobid is not found or completed."
+        break
+    else
+        echo "Job with ID $jobid is still running."
+        echo "Job Status: $job_status"
+    fi
+
+    # Sleep for a short duration before checking again
+    sleep 60  # Adjust the sleep duration as needed
+done
+
+
+# check if fms_MOM6_SIS2_GENERIC_4P_compile_symm.x create successfully or not
+executable_file="/lustre/f2/dev/Yi-cheng.Teng/github/cefi_NWA12_regression_${CURRENT_DATE}/MOM6_SIS2_GENERIC_4P_compile_symm/ncrc5.intel22-repro/exec/fms_MOM6_SIS2_GENERIC_4P_compile_symm.x"
+if [ -f "$executable_file" ]; then
+    echo "Executable file created successfully: $executable_file"
+    # Rest of your script...
+else
+    echo "Executable file not created within the specified duration."
+    echo "Please check /lustre/f2/dev/Yi-cheng.Teng/github/cefi_NWA12_regression_${CURRENT_DATE}/MOM6_SIS2_GENERIC_4P_compile_symm/ncrc5.intel22-repro/exec/compile_MOM6_SIS2_GENERIC_4P_compile_symm.csh.o$jobid"
+    exit 1
+fi
+
+# run frerun and submit a RT test for NWA12-RT case
+echo "run frerun and submit a NWA12-RT case"
+frerun --notransfer -o -x ocean_ice_cobalt_experiments.xml -p ncrc5.intel22 -q debug -r NWA12_RT -t repro NWA12_COBALT_V1
+rt_jobid=$(sbatch --parsable /lustre/f2/dev/Yi-cheng.Teng/github/cefi_NWA12_regression_${CURRENT_DATE}/NWA12_COBALT_V1/ncrc5.intel22-repro/scripts/run/NWA12_COBALT_V1_1x0m2d_1646x1o | awk -F';' '{print $1}' | cut -f1)
+echo "Submitted RT job with ID: $rt_jobid"
+
+# Check the status of the job in a loop
+sleep 1
+while :; do
+    # Check the status of the job
+    job_status2=$(squeue -h -j "$rt_jobid" -o "%T" 2>/dev/null)
+
+    if [ -z "$job_status2" ]; then
+        echo "Job with ID $rt_jobid is not found or completed."
+        break
+    else
+        echo "Job with ID $rt_jobid is still running."
+        echo "Job Status: $job_status2"
+    fi
+
+    # Sleep for a short duration before checking again
+    sleep 300  # Adjust the sleep duration as needed
+done
+
+# check if restart create successfully or not
+check_file="/lustre/f2/scratch/Yi-cheng.Teng/github/cefi_NWA12_regression_${CURRENT_DATE}/NWA12_COBALT_V1/ncrc5.intel22-repro/archive/1x0m2d_1646x1o/restart/19930103.tar.ok"
+if [ -f "$check_file" ]; then
+    echo "restart files exist successfully: $check_file"
+else
+    echo "NWA12 RT is not done within the specified duration."
+    echo "Please check /lustre/f2/scratch/Yi-cheng.Teng/github/cefi_NWA12_regression_${CURRENT_DATE}/NWA12_COBALT_V1/ncrc5.intel22-repro/stdout/run/NWA12_COBALT_V1_1x0m2d_1646x1o.o$rt_jobid"
+    exit 10
+fi
+
+# check with references
+if [ -f check.log ]; then rm -rf check.log ; fi
+frecheck -x ocean_ice_cobalt_experiments.xml -p ncrc5.intel22 -r NWA12_RT -t repro NWA12_COBALT_V1 > check.log
+
+# String to check
+expected_string="REFERENTIALLY   PASSED: NWA12_COBALT_V1"
+
+# Check if the string exists in the file
+if grep -qF "$expected_string" check.log; then
+    echo "PASSED: RT results are identical."
+else
+    echo "FAIL: check the check.log"
+    cat check.log
+    exit 100
+fi
+
+# Final clean-up
+rm -rf /lustre/f2/scratch/Yi-cheng.Teng/work/github/cefi_NWA12_regression_${CURRENT_DATE}
+rm -rf /lustre/f2/dev/Yi-cheng.Teng/github/cefi_NWA12_regression_${CURRENT_DATE}