[production/AQM.v7] Enhance EE2 compliance (#987)

* update J-job scripts * change emission dir name to be consistent with operational forecast * update PARMaqm_utils for 4 J-job scripts and auto_build.sh * update hash number of AQM-utils * update auto_build_exec.sh * update auto_build_exec.sh * update var_defns.sh.nco.statis and aqm_nco_config.sh * update source_util_funcs.sh * update COMINm1 and COMINm2 in the J-Job scripts * udate exscript for forecast * update J-job and excripts to move emission from COMaqm to FIXaqm * update var_defns.sh.nco.static * update var_defns.sh.nco.static * fix nexus job failure associated with var_defns.sh.nco.static * update Externals.cfg
ufs-community · Dec 8, 2023 · c069c2a · c069c2a
1 parent f07d952
commit c069c2a
Show file tree

Hide file tree

Showing 24 changed files with 1,205 additions and 59 deletions.
diff --git a/jobs/JAQM_BIAS_CORRECTION_O3 b/jobs/JAQM_BIAS_CORRECTION_O3
@@ -38,8 +38,8 @@ export FIXaqm=${FIXaqm:-${HOMEaqm}/fix}
 export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 export DCOMINairnow="${DCOMINairnow:-${DCOMROOT}}"
@@ -83,6 +83,8 @@ if [ "${cyc}" = "06" ] || [ "${cyc}" = "12" ]; then
   fi
 fi
 
+export PARMaqm_utils="${PARMaqm_utils:-${HOMEaqm}/parm/aqm_utils}"
+
 TMP_STDAY=`${NDATE} -8760 ${PDY}${cyc} | cut -c1-8`  # 1 year back
 export BC_STDAY=${BC_STDAY:-${TMP_STDAY}}
 #
@@ -91,7 +93,6 @@ export BC_STDAY=${BC_STDAY:-${TMP_STDAY}}
 # bles.
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_bias_correction_o3.sh
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_BIAS_CORRECTION_PM25 b/jobs/JAQM_BIAS_CORRECTION_PM25
@@ -40,8 +40,8 @@ export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 export COMINgfs="${COMINgfs:-$(compath.py ${envir}/gfs/${gfs_ver})}"
 
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 export DCOMINairnow="${DCOMINairnow:-${DCOMROOT}}"
@@ -84,6 +84,8 @@ if [ "${cyc}" = "06" ] || [ "${cyc}" = "12" ]; then
   fi
 fi
 
+export PARMaqm_utils="${PARMaqm_utils:-${HOMEaqm}/parm/aqm_utils}"
+
 TMP_STDAY=`${NDATE} -8760 ${PDY}${cyc} | cut -c1-8`  # 1 year back
 export BC_STDAY=${BC_STDAY:-${TMP_STDAY}}
 
@@ -93,7 +95,6 @@ env
 # bles.
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_bias_correction_pm25.sh
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_FIRE_EMISSION b/jobs/JAQM_FIRE_EMISSION
@@ -38,7 +38,8 @@ export DCOMROOT=${DCOMROOT:-/lfs/h1/ops/prod/dcom}
 export DCOMINfire="${DCOMINfire:-${DCOMROOT}/${PDY}/rave}"
 
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 #-----------------------------------------------------------------------
@@ -92,7 +93,6 @@ mkdir -p ${FIRE_EMISSION_STAGING_DIR}
 # Call the ex-script for this J-job and pass to it the necessary variables.
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_fire_emission.sh
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_FORECAST b/jobs/JAQM_FORECAST
@@ -39,8 +39,8 @@ export FIXaqm=${FIXaqm:-${HOMEaqm}/fix}
 export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 #-----------------------------------------------------------------------
@@ -82,7 +82,6 @@ export INPUT_DATA="${INPUT_DATA:-${COMIN}/${cyc}}"
 # bles.
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_forecast.sh
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_ICS b/jobs/JAQM_ICS
@@ -38,8 +38,8 @@ export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 
 export COMINgfs="${COMINgfs:-$(compath.py ${envir}/gfs/${gfs_ver})}"
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 #-----------------------------------------------------------------------
@@ -85,7 +85,6 @@ DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_AQM_ICS}"
 # Call the ex-script for this J-job and pass to it the necessary variables.
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_ics.sh
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_LBCS b/jobs/JAQM_LBCS
@@ -38,8 +38,8 @@ export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 
 export COMINgfs="${COMINgfs:-$(compath.py ${envir}/gfs/${gfs_ver})}"
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 export COMINgefs="${COMINgefs:-$(compath.py ${envir}/gefs/${gefs_ver})}"
@@ -88,7 +88,6 @@ DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_AQM_LBCS}"
 # Call the ex-script for this J-job and pass to it the necessary variables.
 #-----------------------------------------------------------------------
 #
-startmsg 
 ${HOMEaqm}/scripts/exaqm_lbcs.sh
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_MAKE_ICS b/jobs/JAQM_MAKE_ICS
@@ -38,8 +38,8 @@ export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 
 export COMINgfs="${COMINgfs:-$(compath.py ${envir}/gfs/${gfs_ver})}"
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 #-----------------------------------------------------------------------
@@ -99,7 +99,6 @@ fi
 # Call the ex-script for this J-job and pass to it the necessary variables.
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_make_ics.sh 
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_MAKE_LBCS b/jobs/JAQM_MAKE_LBCS
@@ -39,8 +39,8 @@ export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 
 export COMINgfs="${COMINgfs:-$(compath.py ${envir}/gfs/${gfs_ver})}"
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 #-----------------------------------------------------------------------
@@ -102,7 +102,6 @@ fi
 # Call the ex-script for this J-job and pass to it the necessary variables.
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_make_lbcs.sh 
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_NEXUS_EMISSION b/jobs/JAQM_NEXUS_EMISSION
@@ -41,12 +41,12 @@ export PARMaqm=${PARMaqm:-${HOMEaqm}/parm}
 export FIXaqm=${FIXaqm:-${HOMEaqm}/fix}
 export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 
-export COMINemis="${COMINemis:-${COMaqm}/emissions}"
+export FIXemis="${FIXemis:-${FIXaqm}/emission}"
 export COMINgfs="${COMINgfs:-$(compath.py ${envir}/gfs/${gfs_ver})}"
 
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 #-----------------------------------------------------------------------
@@ -104,7 +104,6 @@ fi
 # Call the ex-script for this J-job.
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_nexus_emission.sh
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_NEXUS_POST_SPLIT b/jobs/JAQM_NEXUS_POST_SPLIT
@@ -42,11 +42,12 @@ export FIXaqm=${FIXaqm:-${HOMEaqm}/fix}
 export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 
 export COMINgfs="${COMINgfs:-$(compath.py ${envir}/gfs/${gfs_ver})}"
-export COMINemis="${COMINemis:-${COMaqm}/emissions}"
+export FIXemis="${FIXemis:-${FIXaqm}/emission}"
+
 
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 #-----------------------------------------------------------------------
@@ -93,7 +94,6 @@ DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_NEXUS_POST_SPLIT}"
 #
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_nexus_post_split.sh
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_POINT_SOURCE b/jobs/JAQM_POINT_SOURCE
@@ -34,11 +34,11 @@ export FIXaqm=${FIXaqm:-${HOMEaqm}/fix}
 export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 
 export COMINgfs="${COMINgfs:-$(compath.py ${envir}/gfs/${gfs_ver})}"
-export COMINemispt="${COMINemispt:-${COMaqm}/emissions/nei2016v1-pt}"
+export FIXemispt="${FIXemispt:-${FIXaqm}/emission/nei2016v1-pt}"
 
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 #-----------------------------------------------------------------------
@@ -82,7 +82,6 @@ DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_POINT_SOURCE}"
 # Call the ex-script for this J-job.
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_point_source.sh
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_POST b/jobs/JAQM_POST
@@ -40,8 +40,8 @@ export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 
 export COMINgfs="${COMINgfs:-$(compath.py ${envir}/gfs/${gfs_ver})}"
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 #-----------------------------------------------------------------------
@@ -91,7 +91,6 @@ fi
 # bles.
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_post.sh 
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_POST_STAT_O3 b/jobs/JAQM_POST_STAT_O3
@@ -37,8 +37,8 @@ export FIXaqm=${FIXaqm:-${HOMEaqm}/fix}
 export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 #-----------------------------------------------------------------------
@@ -76,13 +76,14 @@ if [ "${cyc}" = "06" ] || [ "${cyc}" = "12" ]; then
      mkdir -p "${PCOM}"
   fi
 fi
+
+export PARMaqm_utils="${PARMaqm_utils:-${HOMEaqm}/parm/aqm_utils}"
 #
 #-----------------------------------------------------------------------
 # Call the ex-script for this J-job and pass to it the necessary varia-
 # bles.
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_post_stat_o3.sh
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_POST_STAT_PM25 b/jobs/JAQM_POST_STAT_PM25
@@ -38,8 +38,8 @@ export FIXaqm=${FIXaqm:-${HOMEaqm}/fix}
 export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 #-----------------------------------------------------------------------
@@ -77,13 +77,15 @@ if [ "${cyc}" = "06" ] || [ "${cyc}" = "12" ]; then
       mkdir -p "${PCOM}"
   fi
 fi
+
+
+export PARMaqm_utils="${PARMaqm_utils:-${HOMEaqm}/parm/aqm_utils}"
 #
 #-----------------------------------------------------------------------
 # Call the ex-script for this J-job and pass to it the necessary varia-
 # bles.
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_post_stat_pm25.sh
 export err=$?; err_chk
 #

diff --git a/jobs/JAQM_PRE_POST_STAT b/jobs/JAQM_PRE_POST_STAT
@@ -34,8 +34,8 @@ export FIXaqm=${FIXaqm:-${HOMEaqm}/fix}
 export UTILaqm=${UTILaqm:-${HOMEaqm}/util}
 
 export COMIN="${COMIN:-${COMaqm}/${RUN}.${PDY}}"
-export COMINm1="${COMIN:-${COMaqm}/${RUN}.${PDYm1}}"
-export COMINm2="${COMIN:-${COMaqm}/${RUN}.${PDYm2}}"
+export COMINm1="${COMaqm}/${RUN}.${PDYm1}"
+export COMINm2="${COMaqm}/${RUN}.${PDYm2}"
 export COMOUT="${COMOUT:-${COMaqm}/${RUN}.${PDY}/${cyc}}"
 
 #-----------------------------------------------------------------------
@@ -72,7 +72,6 @@ DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}/tmp_PRE_POST_STAT}"
 # bles.
 #-----------------------------------------------------------------------
 #
-startmsg
 ${HOMEaqm}/scripts/exaqm_pre_post_stat.sh
 export err=$?; err_chk
 #

diff --git a/parm/config/aqm_nco_config.sh b/parm/config/aqm_nco_config.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
 set -eux
 
 # This utility is to replace configuration temperate with production setting before running ecflow workflow

diff --git a/parm/config/var_defns.sh.nco.static b/parm/config/var_defns.sh.nco.static
@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # [metadata]
 description='config for Online-CMAQ, AQM_NA_13km, real-time, NCO mode on WCOSS2'
 version='1.0'
@@ -53,11 +55,6 @@ TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS=''
 TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS=''
 TEST_VX_FCST_INPUT_BASEDIR=''
 FIXgsm='@HOMEaqm@/fix/fix_am'
-FIXaer='@HOMEaqm@/fix/fix_aer'
-FIXlut='@HOMEaqm@/fix/fix_lut'
-FIXorg='@HOMEaqm@/fix/fix_orog'
-FIXsfc='@HOMEaqm@/fix/fix_sfc_climo'
-FIXshp='@HOMEaqm@/fix/ufs/NaturalEarth'
 EXTRN_MDL_DATA_STORES='hpss aws nomads'
 
 # [workflow]
@@ -947,7 +944,7 @@ AQM_LBCS_FILES='am4_bndy.c793.2019<MM>.v1.nc'
 COMINgefs='@COMINgefs@'
 AQM_GEFS_FILE_PREFIX='geaer'
 AQM_GEFS_FILE_CYC=''
-COMINemis='@COMaqm@/emissions'
+FIXemis='@HOMEaqm@/fix/emission'
 FIXaqmconfig='@HOMEaqm@/fix/aqm/epa/data'
 FIXaqmfire='@HOMEaqm@/fix/fire'
 FIXaqmbio='@HOMEaqm@/fix/bio'
@@ -958,7 +955,7 @@ FIXaqmnexus='@HOMEaqm@/fix/nexus'
 NEXUS_GRID_FN='grid_spec_793.nc'
 NUM_SPLIT_NEXUS='6'
 NEXUS_GFS_SFC_ARCHV_DIR='/NCEPPROD/hpssprod/runhistory'
-COMINemispt='@COMaqm@/emissions/nei2016v1-pt'
+FIXemispt='@HOMEaqm@/fix/emission/nei2016v1-pt'
 DCOMINairnow='/lfs/h1/ops/prod/dcom'
 COMINbicor='@COMaqm@'
 COMOUTbicor='@COMaqm@'