[develop] Transition the var_defns bash file to YAML. (ufs-community#…

…1098)

Use YAML for the configuration language at run time.

---------

Co-authored-by: Michael Lueken <[email protected]>
Co-authored-by: Michael Kavulich <[email protected]>
Co-authored-by: michael.lueken <[email protected]>

.cicd/scripts/wrapper_srw_ftest.sh

@@ -24,7 +24,8 @@ fi
 if [[ "${SRW_PLATFORM}" == gaea ]]; then
     sed -i '15i #SBATCH --clusters=c5' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh
     sed -i 's|qos=batch|qos=normal|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh
-    sed -i 's|${JOBSdir}/JREGIONAL_RUN_POST|$USHdir/load_modules_run_task.sh "run_post" ${JOBSdir}/JREGIONAL_RUN_POST|g' ${WORKSPACE}/${SRW_PLATFORM}/ush/wrappers/run_post.sh
+    sed -i 's|00:30:00|00:45:00|g' ${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/${workflow_cmd}_srw_ftest.sh
+    sed -i 's|${JOBSdir}/JREGIONAL_RUN_POST|$USHdir/load_modules_run_task.sh "gaea" "run_post" ${JOBSdir}/JREGIONAL_RUN_POST|g' ${WORKSPACE}/${SRW_PLATFORM}/ush/wrappers/run_post.sh
 fi
 
 if [[ "${SRW_PLATFORM}" == hera ]]; then

aqm_environment.yml

@@ -9,5 +9,5 @@ dependencies:
   - pylint=2.17*
   - pytest=7.2*
   - scipy=1.10.*
-  - uwtools=2.1*
+  - uwtools=2.3*
   - xarray=2022.11.*

doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst

@@ -293,7 +293,7 @@ WORKFLOW Configuration Parameters
 
 If non-default parameters are selected for the variables in this section, they should be added to the ``workflow:`` section of the ``config.yaml`` file. 
 
-``WORKFLOW_ID``: (Default: ``!nowtimestamp ''``)
+``WORKFLOW_ID``: (Default: ``''``)
    Unique ID for the workflow run that will be set in ``setup.py``.
 
 ``RELATIVE_LINK_FLAG``: (Default: "--relative")
@@ -458,8 +458,8 @@ This section contains files and paths to files that are staged in the experiment
 ``WFLOW_XML_FN``: (Default: "FV3LAM_wflow.xml")
    Name of the Rocoto workflow XML file that the experiment generation script creates. This file defines the workflow for the experiment.
 
-``GLOBAL_VAR_DEFNS_FN``: (Default: "var_defns.sh")
-   Name of the file (a shell script) containing definitions of the primary and secondary experiment variables (parameters). This file is sourced by many scripts (e.g., the J-job scripts corresponding to each workflow task) in order to make all the experiment variables available in those scripts. The primary variables are defined in the default configuration file (``config_defaults.yaml``) and in the user configuration file (``config.yaml``). The secondary experiment variables are generated by the experiment generation script. 
+``GLOBAL_VAR_DEFNS_FN``: (Default: "var_defns.yaml")
+   Name of the auto-generated experiment configuration file. It contains the primary experiment variables defined in this default configuration script and in the user-specified configuration as well as secondary experiment variables generated by the experiment generation script from machine files and other settings. This file is the primary source of information used in the scripts at run time.
 
 ``ROCOTO_YAML_FN``: (Default: "rocoto_defns.yaml")
    Name of the YAML file containing the YAML workflow definition from which the Rocoto XML file is created.

environment.yml

@@ -5,4 +5,4 @@ channels:
 dependencies:
   - pylint=2.17*
   - pytest=7.2*
-  - uwtools=2.2*
+  - uwtools=2.3*

jobs/JREGIONAL_CHECK_POST_OUTPUT

@@ -3,7 +3,22 @@
 #
 #-----------------------------------------------------------------------
 #
+# The J-Job script for checking the post output.
 #
+# Run-time environment variables:
+#
+#    CDATE
+#    ENSMEM_INDX
+#    GLOBAL_VAR_DEFNS_FP
+#
+# Experiment variables
+#
+#  user:
+#    SCRIPTSdir
+#    USHdir
+#
+#  workflow:
+#    EXPTDIR
 #
 #-----------------------------------------------------------------------
 #
@@ -16,7 +31,9 @@
 #-----------------------------------------------------------------------
 #
 . $USHdir/source_util_funcs.sh
-source_config_for_task "" ${GLOBAL_VAR_DEFNS_FP}
+for sect in user nco workflow ; do
+  source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect}
+done
 . $USHdir/job_preamble.sh
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_GET_EXTRN_MDL_FILES

@@ -3,20 +3,48 @@
 #
 #-----------------------------------------------------------------------
 #
-# This script gets either from the system directory or from mass store
-# (HPSS) the files generated by the external model (specified by the 
-# variable EXTRN_MDL_NAME) for either the initial conditions (ICs) or the
-# lateral boundary conditions (LBCs).  Which of these we are considering
-# depends on the value of the variable ICS_OR_LBCS, which should be defined
-# in the environment (when calling this script from a rocoto workflow, 
-# the workflow should define this variable, e.g. using rocoto's <envar> 
-# tag).
-#
-# Note that when we refer to ICs, we are referring to not only the atmospheric 
-# fields at the initial time but also various surface fields (which are 
-# for now time-independent) as well as the 0-th forecast hour LBCs.  Also, 
-# when we refer to LBCs, we are referring to the LBCs excluding the one 
-# at the 0-th hour.  
+# The J-Job script for getting the model files that will be used for
+# either initial conditions or lateral boundary conditions for the
+# experiment.
+#
+# Run-time environment variables:
+#
+#    CDATE
+#    COMIN
+#    cyc
+#    DATA
+#    EXTRN_MDL_STAGING_DIR
+#    GLOBAL_VAR_DEFNS_FP
+#    ICS_OR_LBCS
+#    PDY
+#
+# Experiment variables
+#
+#  user:
+#    SCRIPTSdir
+#    USHdir
+#
+#  workflow:
+#    EXPTDIR
+#
+#  task_get_extrn_lbcs:
+#    EXTRN_MDL_FILES_LBCS
+#    EXTRN_MDL_LBCS_OFFSET_HRS
+#    EXTRN_MDL_NAME_LBCS
+#    EXTRN_MDL_SOURCE_BASEDIR_LBCS
+#    EXTRN_MDL_SYSBASEDIR_LBCS
+#    FV3GFS_FILE_FMT_LBCS
+#    LBC_SPEC_INTVL_HRS
+#    USE_USER_STAGED_EXTRN_FILES
+#
+#  task_get_extrn_ics:
+#    EXTRN_MDL_FILES_ICS
+#    EXTRN_MDL_ICS_OFFSET_HRS
+#    EXTRN_MDL_NAME_ICS
+#    EXTRN_MDL_SOURCE_BASEDIR_ICS
+#    EXTRN_MDL_SYSBASEDIR_ICS
+#    FV3GFS_FILE_FMT_ICS
+#    USE_USER_STAGED_EXTRN_FILES
 #
 #-----------------------------------------------------------------------
 #
@@ -29,8 +57,12 @@
 #-----------------------------------------------------------------------
 #
 . $USHdir/source_util_funcs.sh
-source_config_for_task "task_get_extrn_ics|task_get_extrn_lbcs" ${GLOBAL_VAR_DEFNS_FP}
-. $USHdir/job_preamble.sh "TRUE"
+for sect in user nco workflow task_get_extrn_lbcs task_get_extrn_ics ; do
+  source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect}
+done
+. $USHdir/job_preamble.sh
+
+
 #
 #-----------------------------------------------------------------------
 #

jobs/JREGIONAL_GET_VERIF_OBS

@@ -3,7 +3,19 @@
 #
 #-----------------------------------------------------------------------
 #
-# This script checks, pulls, and stages observation data for model verification.
+# The J-Job script that checks, pulls, and stages observation data for
+# model verification.
+#
+# Run-time environment variables:
+#
+#    CDATE
+#    GLOBAL_VAR_DEFNS_FP
+#
+# Experiment variables
+#
+#  user:
+#    SCRIPTSdir
+#    USHdir
 #
 #-----------------------------------------------------------------------
 #
@@ -16,7 +28,9 @@
 #-----------------------------------------------------------------------
 #
 . $USHdir/source_util_funcs.sh
-source_config_for_task " " ${GLOBAL_VAR_DEFNS_FP}
+for sect in user nco workflow ; do
+  source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect}
+done
 . $USHdir/job_preamble.sh
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_INTEGRATION_TEST

@@ -1,5 +1,31 @@
 #!/bin/bash
 
+
+#
+#-----------------------------------------------------------------------
+#
+# This J-Job script runs a set of tests at the end of WE2E tests.
+#
+# Run-time environment variables:
+#
+#    GLOBAL_VAR_DEFNS_FP
+#    CDATE
+#    FCST_DIR
+#    SLASH_ENSMEM_SUBDIR
+#
+# Experiment variables
+#
+#  user:
+#    RUN_ENV
+#    SCRIPTSdir
+#    USHdir
+#
+#  workflow:
+#    FCST_LEN_HRS
+#
+#-----------------------------------------------------------------------
+#
+
 #
 #-----------------------------------------------------------------------
 #
@@ -8,8 +34,11 @@
 #-----------------------------------------------------------------------
 #
 . $USHdir/source_util_funcs.sh
-source_config_for_task "task_integration_test|task_run_fcst" ${GLOBAL_VAR_DEFNS_FP}
+for sect in user nco workflow ; do
+  source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect}
+done
 . $USHdir/job_preamble.sh
+
 #
 #-----------------------------------------------------------------------
 #

jobs/JREGIONAL_MAKE_GRID

@@ -3,97 +3,25 @@
 #
 #-----------------------------------------------------------------------
 #
-# This script generates grid and orography files in NetCDF format that
-# are required as inputs for running the FV3-LAM model (i.e. the FV3 mo-
-# del on a regional domain).  It in turn calls three other scripts whose
-# file names are specified in the variables grid_gen_scr, orog_gen_scr,
-# and orog_fltr_scr and then calls the executable defined in the varia-
-# ble shave_exec.  These scripts/executable perform the following tasks:
-#
-# 1) grid_gen_scr:
-#
-#    This script generates grid files that will be used by subsequent
-#    preprocessing steps.  It places its output in the directory defined 
-#    by GRID_DIR.  Note that:
-#
-#    a) This script creates grid files for each of the 7 tiles of the
-#       cubed sphere grid (where tiles 1 through 6 cover the globe, and
-#       tile 7 is the regional grid located somewhere within tile 6)
-#       even though the forecast will be performed only on tile 7.
-#
-#    b) The tile 7 grid file that this script creates includes a halo,
-#       i.e. a layer of cells beyond the boundary of tile 7).  The width
-#       of this halo (i.e. the number of cells in the halo in the direc-
-#       tion perpendicular to the boundary of the tile) must be made
-#       large enough such that the "shave" steps later below (which take
-#       this file as input and generate grid files with thinner halos)
-#       have a wide enough starting halo to work with.  More specifical-
-#       ly, the FV3-LAM model needs as inputs two grid files: one with a
-#       halo that is 3 cells and another with a halo that is 4 cells 
-#       wide.  Thus, the halo in the grid file that the grid_gen_scr 
-#       script generates must be greater than 4 since otherwise, the
-#       shave steps would shave off cells from within the interior of
-#       tile 7.  We will let NHW denote the width of the halo in the
-#       grid file generated by grid_gen_scr.  The "n" in this variable
-#       name denotes number of cells, the "h" is used to indicate that
-#       it refers to a halo region, the "w" is used to indicate that it
-#       refers to a wide halo (i.e. wider than the 3-cell and 4-cell ha-
-#       los that the FV3-LAM model requires as inputs, and the "T7" is
-#       used to indicate that the cell count is on tile 7.
-#
-# 2) orog_gen_scr:
-#
-#    This script generates the orography file.  It places its output in
-#    the directory defined by OROG_DIR.  Note that:
-#
-#    a) This script generates an orography file only on tile 7.
-#
-#    b) This orography file contains a halo of the same width (NHW)
-#       as the grid file for tile 7 generated by the grid_gen_scr script
-#       in the previous step.
-#
-# 3) orog_fltr_scr:
-#
-#    This script generates a filtered version of the orography file ge-
-#    nerated by the script orog_gen_scr.  This script places its output
-#    in the temporary directory defined in WORKDIR_FLTR.  Note that:
-#
-#    a) The filtered orography file generated by this script contains a
-#       halo of the same width (NHW) as the (unfiltered) orography file
-#       generated by script orog_gen_scr (and the grid file generated by 
-#       grid_gen_scr).
-#
-#    b) In analogy with the input grid files, the FV3-LAM model needs as
-#       input two (filtered) orography files -- one with no halo cells
-#       and another with 3.  These are obtained later below by "shaving"
-#       off layers of halo cells from the (filtered) orography file ge-
-#       nerated in this step.
-#
-# 4) shave_exec:
-#
-#    This "shave" executable is called 4 times to generate 4 files from
-#    the tile 7 grid file generated by grid_gen_scr and the tile 7 fil-
-#    tered orography file generated by orog_fltr_scr (both of which have
-#    a halo of width NHW cells).  The 4 output files are placed in the
-#    temporary directory defined in WORKDIR_SHVE.  More specifically:
-#
-#    a) shave_exec is called to shave the halo in the tile 7 grid file
-#       generated by grid_gen_scr down to a width of 3 cells and store
-#       the result in a new grid file in WORKDIR_SHVE.
-#
-#    b) shave_exec is called to shave the halo in the tile 7 grid file
-#       generated by grid_gen_scr down to a width of 4 cells and store
-#       the result in a new grid file in WORKDIR_SHVE.
-#
-#    c) shave_exec is called to shave the halo in the tile 7 filtered
-#       orography file generated by orog_fltr_scr down to a width of 0
-#       cells (i.e. no halo) and store the result in a new filtered oro-
-#       graphy file in WORKDIR_SHVE.
-#
-#    d) shave_exec is called to shave the halo in the tile 7 filtered
-#       orography file generated by orog_fltr_scr down to a width of 4
-#       cells and store the result in a new filtered orography file in
-#       WORKDIR_SHVE.
+# The J-Job that generates input NetCDF grid files for running the
+# regional configuration of FV3 
+#
+# Run-time environment variables:
+#
+#    GLOBAL_VAR_DEFNS_FP
+#    DATA
+#
+# Experiment variables
+#
+#  user:
+#    USHdir
+#    SCRIPTSdir
+#
+#  workflow:
+#    PREEXISTING_DIR_METHOD
+#
+#  task_make_grid:
+#    GRID_DIR
 #
 #-----------------------------------------------------------------------
 #
@@ -106,7 +34,9 @@
 #-----------------------------------------------------------------------
 #
 . $USHdir/source_util_funcs.sh
-source_config_for_task "task_make_grid" ${GLOBAL_VAR_DEFNS_FP}
+for sect in user nco workflow task_make_grid ; do
+  source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect}
+done
 . $USHdir/job_preamble.sh
 #
 #-----------------------------------------------------------------------