[develop] Make get_obs tasks day-dependent in workflow; other improvements and bug fixes

doc/Makefile

@@ -24,4 +24,4 @@ linkcheck:
 # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
 
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) -w $(BUILDDIR)/warnings.log
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) -w $(BUILDDIR)/warnings.log

doc/TechDocs/ush/eval_metplus_timestr_tmpl.rst

@@ -0,0 +1,7 @@
+eval\_metplus\_timestr\_tmpl module
+===================================
+
+.. automodule:: eval_metplus_timestr_tmpl
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/TechDocs/ush/get_obs.rst

@@ -0,0 +1,7 @@
+get\_obs module
+===============
+
+.. automodule:: get_obs
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/TechDocs/ush/modules.rst

@@ -12,18 +12,21 @@ ush
    create_diag_table_file
    create_model_configure_file
    create_ufs_configure_file
+   eval_metplus_timestr_tmpl
    generate_FV3LAM_wflow
    get_crontab_contents
+   get_obs
    link_fix
    mrms_pull_topofhour
    python_utils
    retrieve_data
    run_srw_tests
-   set_cycle_dates
+   set_cycle_and_obs_timeinfo
    set_fv3nml_ens_stoch_seeds
    set_fv3nml_sfc_climo_filenames
    set_gridparams_ESGgrid
    set_gridparams_GFDLgrid
+   set_leadhrs
    set_predef_grid_params
    setup
    update_input_nml

doc/TechDocs/ush/set_cycle_and_obs_timeinfo.rst

@@ -0,0 +1,7 @@
+set\_cycle\_and\_obs\_timeinfo module
+=====================================
+
+.. automodule:: set_cycle_and_obs_timeinfo
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/TechDocs/ush/set_leadhrs.rst

@@ -0,0 +1,7 @@
+set\_leadhrs module
+===================
+
+.. automodule:: set_leadhrs
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/UsersGuide/Reference/Glossary.rst

@@ -26,7 +26,7 @@ Glossary
 
    chgres_cube
        The preprocessing software used to create initial and boundary condition files to 
-       “cold start” the forecast model. It is part of :term:`UFS_UTILS`.
+       "cold start" the forecast model. It is part of :term:`UFS_UTILS`.
 
    CIN
       Convective Inhibition.
@@ -48,6 +48,9 @@ Glossary
    Component Repository
       A :term:`repository` that contains, at a minimum, source code for a single component.
 
+   composite reflectivity
+      `Composite reflectivity <https://glossary.ametsoc.org/wiki/Composite_reflectivity>`__ is a display or mapping of the maximum radar reflectivity factor at any altitude as a function of position on the ground.
+
    Container
       `Docker <https://www.docker.com/resources/what-container>`__ describes a container as "a standard unit of software that packages up code and all its dependencies so the application runs quickly and reliably from one computing environment to another."
 
@@ -70,6 +73,9 @@ Glossary
    data assimilation
       Data assimilation is the process of combining observations, model data, and error statistics to achieve the best estimate of the state of a system. One of the major sources of error in weather and climate forecasts is uncertainty related to the initial conditions that are used to generate future predictions. Even the most precise instruments have a small range of unavoidable measurement error, which means that tiny measurement errors (e.g., related to atmospheric conditions and instrument location) can compound over time. These small differences result in very similar forecasts in the short term (i.e., minutes, hours), but they cause widely divergent forecasts in the long term. Errors in weather and climate forecasts can also arise because models are imperfect representations of reality. Data assimilation systems seek to mitigate these problems by combining the most timely observational data with a "first guess" of the atmospheric state (usually a previous forecast) and other sources of data to provide a "best guess" analysis of the atmospheric state to start a weather or climate simulation. When combined with an "ensemble" of model runs (many forecasts with slightly different conditions), data assimilation helps predict a range of possible atmospheric states, giving an overall measure of uncertainty in a given forecast.
 
+   DTC
+      The `Developmental Testbed Center <https://dtcenter.org>`__ is a distributed facility where the NWP community can test and evaluate new models and techniques for use in research and operations.
+
    dycore
    dynamical core
       Global atmospheric model based on fluid dynamics principles, including Euler's equations of motion.
@@ -87,7 +93,7 @@ Glossary
       Extended Schmidt Gnomonic (ESG) grid. The ESG grid uses the map projection developed by Jim Purser of NOAA :term:`EMC` (:cite:t:`Purser_2020`). 
 
    ESMF
-      `Earth System Modeling Framework <https://earthsystemmodeling.org/docs/release/latest/ESMF_usrdoc/>`__. The ESMF defines itself as “a suite of software tools for developing high-performance, multi-component Earth science modeling applications.” 
+      `Earth System Modeling Framework <https://earthsystemmodeling.org/docs/release/latest/ESMF_usrdoc/>`__. The ESMF defines itself as "a suite of software tools for developing high-performance, multi-component Earth science modeling applications."
 
    ex-scripts
       Scripting layer (contained in ``ufs-srweather-app/scripts/``) that should be called by a :term:`J-job <J-jobs>` for each workflow componentto run a specific task or sub-task in the workflow. The different scripting layers are described in detail in the :nco:`NCO Implementation Standards document <ImplementationStandards.v11.0.0.pdf>`
@@ -152,6 +158,9 @@ Glossary
    MERRA2
       The `Modern-Era Retrospective analysis for Research and Applications, Version 2 <https://gmao.gsfc.nasa.gov/reanalysis/MERRA-2/>`__ provides satellite observation data back to 1980. According to NASA, "It was introduced to replace the original MERRA dataset because of the advances made in the assimilation system that enable assimilation of modern hyperspectral radiance and microwave observations, along with GPS-Radio Occultation datasets. It also uses NASA's ozone profile observations that began in late 2004. Additional advances in both the GEOS model and the GSI assimilation system are included in MERRA-2. Spatial resolution remains about the same (about 50 km in the latitudinal direction) as in MERRA."
 
+   MET
+      The `Model Evaluation Tools <https://dtcenter.org/community-code/model-evaluation-tools-met>`__ is a highly-configurable, state-of-the-art suite of verification tools developed at the :term:`DTC`.
+
    MPI
       MPI stands for Message Passing Interface. An MPI is a standardized communication system used in parallel programming. It establishes portable and efficient syntax for the exchange of messages and data between multiple processors that are used by a single computer program. An MPI is required for high-performance computing (HPC) systems.
 
@@ -215,7 +224,7 @@ Glossary
       The branch of physical geography dealing with mountains.
 
    Parameterizations
-      Simplified functions that approximate the effects of small-scale processes (e.g., microphysics, gravity wave drag) that cannot be explicitly resolved by a model grid’s representation of the earth.
+      Simplified functions that approximate the effects of small-scale processes (e.g., microphysics, gravity wave drag) that cannot be explicitly resolved by a model grid's representation of the earth.
 
    RAP
       `Rapid Refresh <https://rapidrefresh.noaa.gov/>`__. The continental-scale NOAA hourly-updated assimilation/modeling system operational at :term:`NCEP`. RAP covers North America and is comprised primarily of a numerical forecast model and an analysis/assimilation system to initialize that model. RAP is complemented by the higher-resolution 3km High-Resolution Rapid Refresh (:term:`HRRR`) model.

doc/conf.py

@@ -253,7 +253,7 @@ def setup(app):
 # -- Options for autodoc extension ---------------------------------------
 
 autodoc_mock_imports = ["f90nml","cartopy","mpl_toolkits.basemap","fill_jinja_template",
-   "matplotlib","numpy","uwtools","mpl_toolkits",
+   "matplotlib","numpy","uwtools","mpl_toolkits","metplus",
    ]
 
 logger = logging.getLogger(__name__)
@@ -310,3 +310,11 @@ def warn_undocumented_members(app, what, name, obj, options, lines):
             'fire-ug': ('https://fire-behavior.readthedocs.io/en/latest/%s', '%s'),
             }
 
+# Define environment variables that need to exist when running the top-level code in python
+# files (outside of functions, classes, etc.).
+#
+# METPLUS_ROOT just needs to exist in the environment; its value is not important since for
+# the purpose of building the documentation, METplus is loaded by including "metplus" in
+# the autodoc_mock_imports list above, not via use of the METPLUS_ROOT environment variable.
+os.environ["METPLUS_ROOT"] = ""
+

jobs/JREGIONAL_GET_VERIF_OBS

@@ -69,7 +69,6 @@ This is the J-job script for the task that checks, pulls, and stages
 observation data for verification purposes.
 ========================================================================"
 
-#
 #
 #-----------------------------------------------------------------------
 #

modulefiles/tasks/derecho/get_obs.local.lua

@@ -0,0 +1 @@
+load("run_vx.local")

modulefiles/tasks/gaea/get_obs.local.lua

@@ -0,0 +1 @@
+load("run_vx.local")

modulefiles/tasks/hera/get_obs.local.lua

@@ -1,3 +1,2 @@
 load("hpss")
-unload("python")
-load("python_srw")
+load("run_vx.local")

modulefiles/tasks/hercules/get_obs.local.lua

@@ -0,0 +1 @@
+load("run_vx.local")

modulefiles/tasks/jet/get_obs.local.lua

@@ -1,3 +1,2 @@
 load("hpss")
-unload("python")
-load("python_srw")
+load("run_vx.local")

modulefiles/tasks/noaacloud/get_obs.local.lua

@@ -0,0 +1 @@
+load("run_vx.local")

modulefiles/tasks/orion/get_obs.local.lua

@@ -0,0 +1 @@
+load("run_vx.local")

parm/data_locations.yml

@@ -319,18 +319,15 @@ CCPA_obs:
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
     archive_internal_dir:
-      - "./00"
-      - "./06"
-      - "./12"
-      - "./18"
+      - "./{hh}"
     archive_file_names:
       - "com2_ccpa_prod_ccpa.{yyyy}{mm}{dd}.tar"
       - "gpfs_dell1_nco_ops_com_ccpa_prod_ccpa.{yyyy}{mm}{dd}.tar"
       - "com_ccpa_prod_ccpa.{yyyy}{mm}{dd}.tar"
       - "com_ccpa_v4.2_ccpa.{yyyy}{mm}{dd}.tar"
     file_names:
       obs:
-        - "ccpa.t{hh}z.01h.hrap.conus.gb2"
+        - "ccpa.t*z.01h.hrap.conus.gb2"
 
 MRMS_obs:
   hpss:
@@ -365,11 +362,13 @@ NDAS_obs:
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
       - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}
+      - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd}        
     archive_file_names:
       - "com2_nam_prod_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
       - "gpfs_dell1_nco_ops_com_nam_prod_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
       - "com_nam_prod_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
       - "com_obsproc_v1.1_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
+      - "com_obsproc_v1.2_nam.{yyyy}{mm}{dd}{hh}.bufr.tar"
     file_names:
       obs:
         - "./nam.t{hh}z.prepbufr.tm*.nr"
@@ -386,6 +385,6 @@ NOHRSC_obs:
       - "dcom_{yyyy}{mm}{dd}.tar"
     file_names:
       obs:
-        - "sfav2_CONUS_*h_{yyyy}{mm}{dd}{hh}_grid184.grb2"
+        - "sfav2_CONUS_6h_{yyyy}{mm}{dd}*_grid184.grb2"
     archive_internal_dir:
       - ./wgrbbul/nohrsc_snowfall/

parm/metplus/EnsembleStat.conf

@@ -31,7 +31,7 @@ INIT_INCREMENT = 3600
 # List of forecast leads to process for each run time (init or valid)
 # In hours if units are not specified
 # If unset, defaults to 0 (don't loop through forecast leads)
-LEAD_SEQ = {{fhr_list}}
+LEAD_SEQ = {{vx_leadhr_list}}
 #
 # Order of loops to process data - Options are times, processes
 # Not relevant if only one item is in the PROCESS_LIST
@@ -87,7 +87,7 @@ OBS_FILE_WINDOW_BEGIN = -300
 OBS_FILE_WINDOW_END = 300
 OBS_{{METPLUS_TOOL_NAME}}_WINDOW_BEGIN = 0
 OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = 0
-{%- elif input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- elif input_field_group in ['SFC', 'UPA'] %}
 OBS_WINDOW_BEGIN = -1799
 OBS_WINDOW_END = 1800
 OBS_{{METPLUS_TOOL_NAME}}_WINDOW_BEGIN = {OBS_WINDOW_BEGIN}
@@ -105,7 +105,7 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END}
 # ens.vld_thresh value in the MET config file
 {{METPLUS_TOOL_NAME}}_ENS_VLD_THRESH = 1.0
 
-{%- if input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- if input_field_group in ['SFC', 'UPA'] %}
 
 {{METPLUS_TOOL_NAME}}_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA
 #{{METPLUS_TOOL_NAME}}_OBS_QUALITY_EXC =
@@ -118,7 +118,7 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END}
 {{METPLUS_TOOL_NAME}}_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt
 {%- elif input_field_group in ['REFC', 'RETOP'] %}
 {{METPLUS_TOOL_NAME}}_MET_OBS_ERR_TABLE =
-{%- elif input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- elif input_field_group in ['SFC', 'UPA'] %}
 {{METPLUS_TOOL_NAME}}_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt
 {%- endif %}
 
@@ -135,7 +135,7 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END}
   {%- set comment_or_null = '' %}
   {%- set regrid_to_grid = 'FCST' %}
   {%- set regrid_method = 'BUDGET' %}
-{%- elif input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- elif input_field_group in ['SFC', 'UPA'] %}
   {%- set comment_or_null = '#' %}
   {%- set regrid_to_grid = 'NONE' %}
   {%- set regrid_method = 'BILIN' %}
@@ -160,8 +160,8 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END}
 {{METPLUS_TOOL_NAME}}_DUPLICATE_FLAG = NONE
 {{METPLUS_TOOL_NAME}}_SKIP_CONST = TRUE
 {{METPLUS_TOOL_NAME}}_OBS_ERROR_FLAG = FALSE
-{%- elif input_field_group in ['ADPSFC', 'ADPUPA'] %}
-{{METPLUS_TOOL_NAME}}_MESSAGE_TYPE = {{input_field_group}}
+{%- elif input_field_group in ['SFC', 'UPA'] %}
+{{METPLUS_TOOL_NAME}}_MESSAGE_TYPE = {{fieldname_in_met_filedir_names}}
 {{METPLUS_TOOL_NAME}}_DUPLICATE_FLAG = NONE
 {{METPLUS_TOOL_NAME}}_SKIP_CONST = FALSE
 {{METPLUS_TOOL_NAME}}_OBS_ERROR_FLAG = FALSE
@@ -197,7 +197,7 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END}
   {%- set comment_or_null = '' %}
 {%- elif input_field_group in ['REFC', 'RETOP'] %}
   {%- set comment_or_null = '' %}
-{%- elif input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- elif input_field_group in ['SFC', 'UPA'] %}
   {%- set comment_or_null = '#' %}
 {%- endif %}
 {{comment_or_null}}{{METPLUS_TOOL_NAME}}_CLIMO_CDF_BINS = 1
@@ -207,7 +207,7 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END}
 {{METPLUS_TOOL_NAME}}_MASK_GRID =
 {%- elif input_field_group in ['REFC', 'RETOP'] %}
 {{METPLUS_TOOL_NAME}}_MASK_GRID = FULL
-{%- elif input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- elif input_field_group in ['SFC', 'UPA'] %}
 {{METPLUS_TOOL_NAME}}_MASK_GRID =
 {%- endif %}
 
@@ -483,7 +483,7 @@ FCST_VAR{{ns.var_count}}_OPTIONS = convert(x) = x * 3.28084 * 0.001; ;; Convert
 {{opts_indent}}ens_phist_bin_size = 0.05;
         {%- endif %}
 
-      {%- elif input_field_group == 'ADPSFC' %}
+      {%- elif input_field_group == 'SFC' %}
 
         {%- if field_fcst == 'HGT' %}
 FCST_VAR{{ns.var_count}}_OPTIONS = GRIB_lvl_typ = 215;
@@ -499,7 +499,7 @@ FCST_VAR{{ns.var_count}}_OPTIONS = interp = { type = [ { method = NEAREST; width
 FCST_VAR{{ns.var_count}}_OPTIONS = GRIB2_pdt = 0; ;; Derive instantaneous 10-m wind from U/V components, overriding max 10-m wind.
         {%- endif %}
 
-      {%- elif input_field_group == 'ADPUPA' %}
+      {%- elif input_field_group == 'UPA' %}
 
         {%- if field_fcst == 'CAPE' %}
 FCST_VAR{{ns.var_count}}_OPTIONS = cnt_thresh = [ >0 ];
@@ -516,15 +516,8 @@ PcpCombine tool.  In that file, the field name consists of the observation
 field name here (field_obs) with the accumulation period appended to it
 (separated by an underscore), so we must do the same here to get an exact
 match.
-
-Note:
-Turns out for ASNOW, PcpCombine is not run for obs, so we exclude that
-from the "if" clause here (so it goes into the "else").  For workflow
-behavior uniformity between APCP and ASNOW, consider running PcpCombine
-for ASNOW observations as well (just as it's run for APCP observations).
-      {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 #}
-      {%- if (input_field_group in ['APCP']) %}
+      {%- if (input_field_group in ['APCP', 'ASNOW']) %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}_{{accum_hh}}
       {%- else %}
 OBS_VAR{{ns.var_count}}_NAME = {{field_obs}}
@@ -615,15 +608,15 @@ OBS_VAR{{ns.var_count}}_OPTIONS = censor_thresh = lt-20;
 {{opts_indent}}ens_phist_bin_size = 0.05;
         {%- endif %}
 
-      {%- elif input_field_group == 'ADPSFC' %}
+      {%- elif input_field_group == 'SFC' %}
 
         {%- if field_obs in ['DPT', 'TMP', 'WIND'] %}
 OBS_VAR{{ns.var_count}}_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; }
         {%- elif field_obs == 'CEILING' %}
 OBS_VAR{{ns.var_count}}_OPTIONS = GRIB_lvl_typ = 215
         {%- endif %}
 
-      {%- elif input_field_group == 'ADPUPA' %}
+      {%- elif input_field_group == 'UPA' %}
 
         {%- if field_obs in ['DPT', 'HGT', 'TMP', 'WIND'] %}
 OBS_VAR{{ns.var_count}}_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; }
@@ -656,15 +649,15 @@ OUTPUT_BASE = {{output_base}}
 #
 # Point observation input directory for {{MetplusToolName}}.
 #
-{%- if input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- if input_field_group in ['SFC', 'UPA'] %}
 OBS_{{METPLUS_TOOL_NAME}}_POINT_INPUT_DIR = {{obs_input_dir}}
 {%- else %}
 OBS_{{METPLUS_TOOL_NAME}}_POINT_INPUT_DIR =
 {%- endif %}
 #
 # Grid observation input directory for {{MetplusToolName}}.
 #
-{%- if input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- if input_field_group in ['SFC', 'UPA'] %}
 OBS_{{METPLUS_TOOL_NAME}}_GRID_INPUT_DIR =
 {%- else %}
 OBS_{{METPLUS_TOOL_NAME}}_GRID_INPUT_DIR = {{obs_input_dir}}
@@ -698,7 +691,7 @@ STAGING_DIR = {{staging_dir}}
 # Template for point observation input to {{MetplusToolName}} relative to
 # OBS_{{METPLUS_TOOL_NAME}}_POINT_INPUT_DIR.
 #
-{%- if input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- if input_field_group in ['SFC', 'UPA'] %}
 OBS_{{METPLUS_TOOL_NAME}}_POINT_INPUT_TEMPLATE = {{obs_input_fn_template}}
 {%- else %}
 OBS_{{METPLUS_TOOL_NAME}}_POINT_INPUT_TEMPLATE =
@@ -707,7 +700,7 @@ OBS_{{METPLUS_TOOL_NAME}}_POINT_INPUT_TEMPLATE =
 # Template for gridded observation input to {{MetplusToolName}} relative to
 # OBS_{{METPLUS_TOOL_NAME}}_GRID_INPUT_DIR.
 #
-{%- if input_field_group in ['ADPSFC', 'ADPUPA'] %}
+{%- if input_field_group in ['SFC', 'UPA'] %}
 OBS_{{METPLUS_TOOL_NAME}}_GRID_INPUT_TEMPLATE =
 {%- else %}
 OBS_{{METPLUS_TOOL_NAME}}_GRID_INPUT_TEMPLATE = {{obs_input_fn_template}}