From 5fc6a96cdcb8e670cc76d75ff3bf8a04604b37ab Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Mon, 2 Dec 2024 14:34:27 -0700 Subject: [PATCH] Further changes to enable the capability to verify against a benchmark forecast instead of obs. Details below. * Remove other changes (besides the metatasks that were removed in the previous commit) that were part of the attempt to loop over various field groups to obtain a meta-metatask. This includes uncommenting commented-out parts of the verify_*.yaml files. * Introduce new metatasks (metatask_GridStat_SFC_UPA_all_mems and metatask_GridStat_SFC_UPA_ensmeanprob) for using GridStat for the SFC and UPA field groups when the verification is being performed against a benchmark forecast (as opposed to against obs). * In setup.py, change the way certain vx (metatasks) are removed from the workflow so that the metatasks are grouped by field group, not observation type. This makes it more convenient to include/exclude the vx metatasks since they are also field-group based, not obtype based. * Bug fix in the way the variable FIELDNAME_IN_MET_FILEDIR_NAMES is set in exregional_run_met_gridstat_or_pointstat_vx.sh in the case in which the verification is being performed against a benchmark forecast (as opposed to against obs). --- parm/wflow/verify_det.yaml | 84 ++++++--- parm/wflow/verify_ens.yaml | 68 ++++--- parm/wflow/verify_pre.yaml | 48 ++--- ...gional_run_met_gridstat_or_pointstat_vx.sh | 25 +-- ush/setup.py | 170 +++++++++--------- 5 files changed, 229 insertions(+), 166 deletions(-) diff --git a/parm/wflow/verify_det.yaml b/parm/wflow/verify_det.yaml index d1a124e55..cb3298c77 100644 --- a/parm/wflow/verify_det.yaml +++ b/parm/wflow/verify_det.yaml @@ -21,6 +21,7 @@ default_task_verify_det: &default_task_verify_det queue: '&QUEUE_DEFAULT;' walltime: 00:30:00 + metatask_GridStat_APCP_all_accums_all_mems: var: ACCUM_HH: '&ACCUM_HH_APCP;' @@ -54,6 +55,7 @@ metatask_GridStat_APCP_all_accums_all_mems: attrs: task: run_MET_PcpCombine_APCP#ACCUM_HH#h_fcst_#mem_name# + metatask_GridStat_ASNOW_all_accums_all_mems: var: ACCUM_HH: '&ACCUM_HH_ASNOW;' @@ -87,14 +89,15 @@ metatask_GridStat_ASNOW_all_accums_all_mems: attrs: task: run_MET_PcpCombine_ASNOW#ACCUM_HH#h_fcst_#mem_name# + metatask_GridStat_REFC_RETOP_all_mems: var: mem_indx: '&ENSMEM_INDICES;' mem_name: '&ENSMEM_NAMES;' metatask_GridStat_REFC_RETOP_#mem_name#: var: - FIELD_GROUP: '{%- for var in verification.VX_GRIDTOGRID_FIELD_GROUPS %} - {%- if var in ["REFC", "RETOP", "SFC", "UPA"] %} + FIELD_GROUP: '{%- for var in verification.VX_FIELD_GROUPS %} + {%- if var in ["REFC", "RETOP"] %} {{- "%s " % var }} {%- endif %} {%- endfor %}' @@ -108,8 +111,7 @@ metatask_GridStat_REFC_RETOP_all_mems: METPLUSTOOLNAME: 'GRIDSTAT' ENSMEM_INDX: "#mem_indx#" ENSMEM_NAME: "#mem_name#" - #FCST_LEVEL: 'L0' - FCST_LEVEL: 'all' + FCST_LEVEL: 'L0' FCST_THRESH: 'all' walltime: 02:00:00 dependency: @@ -138,13 +140,45 @@ metatask_GridStat_REFC_RETOP_all_mems: age: 00:00:00:30 text: !cycstr '{{ workflow.EXPTDIR }}/@Y@m@d@H/post_files_exist_#mem_name#.txt' + +metatask_GridStat_SFC_UPA_all_mems: + var: + mem_indx: '&ENSMEM_INDICES;' + mem_name: '&ENSMEM_NAMES;' + metatask_GridStat_SFC_UPA_#mem_name#: + var: + FIELD_GROUP: '{%- for var in verification.VX_FIELD_GROUPS %} + {%- if var in ["SFC", "UPA"] %} + {{- "%s " % var }} + {%- endif %} + {%- endfor %}' + task_run_MET_GridStat_vx_#FIELD_GROUP#_#mem_name#: + <<: *default_task_verify_det + command: '&LOAD_MODULES_RUN_TASK; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX"' + envars: + <<: *default_vars + OBS_DIR: '&NDAS_OBS_DIR;' + FIELD_GROUP: '#FIELD_GROUP#' + METPLUSTOOLNAME: 'GRIDSTAT' + ENSMEM_INDX: "#mem_indx#" + ENSMEM_NAME: "#mem_name#" + FCST_LEVEL: 'all' + FCST_THRESH: 'all' + walltime: 02:00:00 + dependency: + datadep_post_files_exist: + attrs: + age: 00:00:00:30 + text: !cycstr '{{ workflow.EXPTDIR }}/@Y@m@d@H/post_files_exist_#mem_name#.txt' + + metatask_PointStat_SFC_UPA_all_mems: var: mem_indx: '&ENSMEM_INDICES;' mem_name: '&ENSMEM_NAMES;' metatask_PointStat_SFC_UPA_#mem_name#: var: - FIELD_GROUP: '{%- for var in verification.VX_GRIDTOPOINT_FIELD_GROUPS %} + FIELD_GROUP: '{%- for var in verification.VX_FIELD_GROUPS %} {%- if var in ["SFC", "UPA"] %} {{- "%s " % var }} {%- endif %} @@ -163,26 +197,26 @@ metatask_PointStat_SFC_UPA_all_mems: FCST_THRESH: 'all' walltime: 01:00:00 dependency: -# and: -# datadep_all_pb2nc_obs_ndas_complete: -# attrs: -# age: 00:00:00:30 -# # Check that the flag files that indicate that the Pb2NC tasks are -# # complete are all present before launching any PointStat task. -# text: '{%- set num_obs_days = workflow.OBS_DAYS_ALL_CYCLES_INST|length %} -# {%- set indent = " " %} -# {%- set indent_p2 = indent + " " %} -# {%- for n in range(0, num_obs_days) %} -# {%- set yyyymmdd = workflow.OBS_DAYS_ALL_CYCLES_INST[n] %} -# {%- if n == 0 %} -# {{- workflow.WFLOW_FLAG_FILES_DIR ~ "/run_met_pb2nc_obs_ndas_" ~ yyyymmdd ~ "_complete.txt" }} -# {%- else %} -# {{- indent ~ "\n" }} -# {{- indent ~ "\n" }} -# {{- indent_p2 ~ workflow.WFLOW_FLAG_FILES_DIR ~ "/run_met_pb2nc_obs_ndas_" ~ yyyymmdd ~ "_complete.txt" }} -# {%- endif %} -# {%- if n != num_obs_days-1 %} {{- "\n" }} {%- endif %} -# {%- endfor %}' + and: + datadep_all_pb2nc_obs_ndas_complete: + attrs: + age: 00:00:00:30 + # Check that the flag files that indicate that the Pb2NC tasks are + # complete are all present before launching any PointStat task. + text: '{%- set num_obs_days = workflow.OBS_DAYS_ALL_CYCLES_INST|length %} + {%- set indent = " " %} + {%- set indent_p2 = indent + " " %} + {%- for n in range(0, num_obs_days) %} + {%- set yyyymmdd = workflow.OBS_DAYS_ALL_CYCLES_INST[n] %} + {%- if n == 0 %} + {{- workflow.WFLOW_FLAG_FILES_DIR ~ "/run_met_pb2nc_obs_ndas_" ~ yyyymmdd ~ "_complete.txt" }} + {%- else %} + {{- indent ~ "\n" }} + {{- indent ~ "\n" }} + {{- indent_p2 ~ workflow.WFLOW_FLAG_FILES_DIR ~ "/run_met_pb2nc_obs_ndas_" ~ yyyymmdd ~ "_complete.txt" }} + {%- endif %} + {%- if n != num_obs_days-1 %} {{- "\n" }} {%- endif %} + {%- endfor %}' datadep_post_files_exist: attrs: age: 00:00:00:30 diff --git a/parm/wflow/verify_ens.yaml b/parm/wflow/verify_ens.yaml index cf558bee4..841f6b9ab 100644 --- a/parm/wflow/verify_ens.yaml +++ b/parm/wflow/verify_ens.yaml @@ -177,26 +177,26 @@ metatask_GenEnsProd_EnsembleStat_SFC_UPA: METPLUSTOOLNAME: 'ENSEMBLESTAT' walltime: 01:00:00 dependency: -# and: -# datadep_all_pb2nc_obs_ndas_complete: &all_pb2nc_obs_ndas_complete -# attrs: -# age: 00:00:00:30 -# # Check that the flag files that indicate that the Pb2NC tasks are -# # complete are all present before launching any EnsembleStat task. -# text: '{%- set num_obs_days = workflow.OBS_DAYS_ALL_CYCLES_INST|length %} -# {%- set indent = " " %} -# {%- set indent_p2 = indent + " " %} -# {%- for n in range(0, num_obs_days) %} -# {%- set yyyymmdd = workflow.OBS_DAYS_ALL_CYCLES_INST[n] %} -# {%- if n == 0 %} -# {{- workflow.WFLOW_FLAG_FILES_DIR ~ "/run_met_pb2nc_obs_ndas_" ~ yyyymmdd ~ "_complete.txt" }} -# {%- else %} -# {{- indent ~ "\n" }} -# {{- indent ~ "\n" }} -# {{- indent_p2 ~ workflow.WFLOW_FLAG_FILES_DIR ~ "/run_met_pb2nc_obs_ndas_" ~ yyyymmdd ~ "_complete.txt" }} -# {%- endif %} -# {%- if n != num_obs_days-1 %} {{- "\n" }} {%- endif %} -# {%- endfor %}' + and: + datadep_all_pb2nc_obs_ndas_complete: &all_pb2nc_obs_ndas_complete + attrs: + age: 00:00:00:30 + # Check that the flag files that indicate that the Pb2NC tasks are + # complete are all present before launching any EnsembleStat task. + text: '{%- set num_obs_days = workflow.OBS_DAYS_ALL_CYCLES_INST|length %} + {%- set indent = " " %} + {%- set indent_p2 = indent + " " %} + {%- for n in range(0, num_obs_days) %} + {%- set yyyymmdd = workflow.OBS_DAYS_ALL_CYCLES_INST[n] %} + {%- if n == 0 %} + {{- workflow.WFLOW_FLAG_FILES_DIR ~ "/run_met_pb2nc_obs_ndas_" ~ yyyymmdd ~ "_complete.txt" }} + {%- else %} + {{- indent ~ "\n" }} + {{- indent ~ "\n" }} + {{- indent_p2 ~ workflow.WFLOW_FLAG_FILES_DIR ~ "/run_met_pb2nc_obs_ndas_" ~ yyyymmdd ~ "_complete.txt" }} + {%- endif %} + {%- if n != num_obs_days-1 %} {{- "\n" }} {%- endif %} + {%- endfor %}' taskdep_genensprod: attrs: task: run_MET_GenEnsProd_vx_#FIELD_GROUP# @@ -281,6 +281,34 @@ metatask_GridStat_REFC_RETOP_ensprob: attrs: task: run_MET_GenEnsProd_vx_#FIELD_GROUP# + +metatask_GridStat_SFC_UPA_ensmeanprob: + var: + stat: MEAN PROB + statlc: mean prob + metatask_GridStat_SFC_UPA_ens#statlc#: + var: + FIELD_GROUP: '{%- for var in verification.VX_FIELD_GROUPS %} + {%- if var in ["SFC", "UPA"] %} + {{- "%s " % var }} + {%- endif %} + {%- endfor %}' + task_run_MET_GridStat_vx_#FIELD_GROUP#_ens#statlc#: + <<: *default_task_verify_ens + command: '&LOAD_MODULES_RUN_TASK; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_GRIDSTAT_OR_POINTSTAT_VX_ENS#stat#"' + envars: + <<: *default_vars + OBS_DIR: '&NDAS_OBS_DIR;' + FIELD_GROUP: '#FIELD_GROUP#' + METPLUSTOOLNAME: 'GRIDSTAT' + FCST_LEVEL: 'all' + FCST_THRESH: 'all' + dependency: + taskdep_genensprod: + attrs: + task: run_MET_GenEnsProd_vx_#FIELD_GROUP# + + metatask_PointStat_SFC_UPA_ensmeanprob: var: stat: MEAN PROB diff --git a/parm/wflow/verify_pre.yaml b/parm/wflow/verify_pre.yaml index 416b5aa9b..574f88b9e 100644 --- a/parm/wflow/verify_pre.yaml +++ b/parm/wflow/verify_pre.yaml @@ -71,30 +71,30 @@ task_get_obs_ndas: partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' walltime: 02:00:00 -#task_run_MET_Pb2nc_obs_NDAS: -# <<: *default_task_verify_pre -# attrs: -# cycledefs: cycledefs_obs_days_inst -# maxtries: '2' -# command: '&LOAD_MODULES_RUN_TASK; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PB2NC_OBS"' -# envars: -# <<: *default_vars -# FIELD_GROUP: 'SFC' -# FCST_OR_OBS: OBS -# OBS_DIR: '&NDAS_OBS_DIR;' -# METPLUSTOOLNAME: 'PB2NC' -# dependency: -# and: -# datadep: -# text: "&NDAS_OBS_DIR;" -# or: -# not: -# taskvalid: -# attrs: -# task: get_obs_ndas -# taskdep: -# attrs: -# task: get_obs_ndas +task_run_MET_Pb2nc_obs_NDAS: + <<: *default_task_verify_pre + attrs: + cycledefs: cycledefs_obs_days_inst + maxtries: '2' + command: '&LOAD_MODULES_RUN_TASK; "run_vx" "&JOBSdir;/JREGIONAL_RUN_MET_PB2NC_OBS"' + envars: + <<: *default_vars + FIELD_GROUP: 'SFC' + FCST_OR_OBS: OBS + OBS_DIR: '&NDAS_OBS_DIR;' + METPLUSTOOLNAME: 'PB2NC' + dependency: + and: + datadep: + text: "&NDAS_OBS_DIR;" + or: + not: + taskvalid: + attrs: + task: get_obs_ndas + taskdep: + attrs: + task: get_obs_ndas metatask_PcpCombine_APCP_all_accums_obs_CCPA: var: diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index 513145e5c..e505d97ca 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -89,25 +89,18 @@ CDATE="${PDY}${cyc}" # #----------------------------------------------------------------------- # -FIELDNAME_IN_MET_FILEDIR_NAMES="" +# Note that ACCUM_HH will not be defined for the REFC, RETOP, SFC, and +# UPA field groups. +set_vx_params \ + field_group="${FIELD_GROUP}" \ + accum_hh="${ACCUM_HH:-}" \ + outvarname_obtype="obtype" \ + outvarname_grid_or_point="grid_or_point" \ + outvarname_fieldname_in_MET_filedir_names="FIELDNAME_IN_MET_FILEDIR_NAMES" if [ $(boolify "${VX_VERIFY_AGAINST_BENCHMARK_FCST}") = "TRUE" ]; then - obtype="${VX_BENCHMARK_FCST_MODEL_NAME}" grid_or_point="grid" - FIELDNAME_IN_MET_FILEDIR_NAMES="${VX_BENCHMARK_FCST_MODEL_NAME}" - -else - -# Note that ACCUM_HH will not be defined for the REFC, RETOP, SFC, and -# UPA field groups. - set_vx_params \ - field_group="${FIELD_GROUP}" \ - accum_hh="${ACCUM_HH:-}" \ - outvarname_obtype="obtype" \ - outvarname_grid_or_point="grid_or_point" \ - outvarname_fieldname_in_MET_filedir_names="FIELDNAME_IN_MET_FILEDIR_NAMES" - fi # #----------------------------------------------------------------------- @@ -215,8 +208,6 @@ elif [ "${grid_or_point}" = "point" ]; then OBS_INPUT_DIR="${vx_output_basedir}/metprd/Pb2nc_obs" OBS_INPUT_FN_TEMPLATE="${OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT}" -OBS_INPUT_DIR="${NDAS_OBS_DIR}" -OBS_INPUT_FN_TEMPLATE="${OBS_NDAS_FN_TEMPLATES[1]}" FCST_INPUT_DIR="${vx_fcst_input_basedir}" FCST_INPUT_FN_TEMPLATE="${FCST_SUBDIR_TEMPLATE:+${FCST_SUBDIR_TEMPLATE}/}${FCST_FN_TEMPLATE}" diff --git a/ush/setup.py b/ush/setup.py index 9b7a7e22e..e0b6ec4ff 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -743,119 +743,129 @@ def _remove_tag(tasks, tag): # # ----------------------------------------------------------------------- # - vx_field_groups_all_by_obtype = {} - vx_metatasks_all_by_obtype = {} + vx_metatasks_all_possible_by_fg = {} - vx_field_groups_all_by_obtype["CCPA"] = ["APCP"] - vx_metatasks_all_by_obtype["CCPA"] \ - = ["task_get_obs_ccpa", + vx_metatasks_all_possible_by_fg["APCP"] \ + = ["metatask_check_post_output_all_mems", + "task_get_obs_ccpa", "metatask_PcpCombine_APCP_all_accums_obs_CCPA", "metatask_PcpCombine_APCP_all_accums_all_mems", "metatask_GridStat_APCP_all_accums_all_mems", "metatask_GenEnsProd_EnsembleStat_APCP_all_accums", "metatask_GridStat_APCP_all_accums_ensmeanprob"] - vx_field_groups_all_by_obtype["NOHRSC"] = ["ASNOW"] - vx_metatasks_all_by_obtype["NOHRSC"] \ - = ["task_get_obs_nohrsc", + vx_metatasks_all_possible_by_fg["ASNOW"] \ + = ["metatask_check_post_output_all_mems", + "task_get_obs_nohrsc", "metatask_PcpCombine_ASNOW_all_accums_obs_NOHRSC", "metatask_PcpCombine_ASNOW_all_accums_all_mems", "metatask_GridStat_ASNOW_all_accums_all_mems", "metatask_GenEnsProd_EnsembleStat_ASNOW_all_accums", "metatask_GridStat_ASNOW_all_accums_ensmeanprob"] - vx_field_groups_all_by_obtype["MRMS"] = ["REFC", "RETOP"] - vx_metatasks_all_by_obtype["MRMS"] \ - = ["task_get_obs_mrms", + vx_metatasks_all_possible_by_fg["REFC"] \ + = ["metatask_check_post_output_all_mems", + "task_get_obs_mrms", "metatask_GridStat_REFC_RETOP_all_mems", "metatask_GenEnsProd_EnsembleStat_REFC_RETOP", "metatask_GridStat_REFC_RETOP_ensprob"] - vx_field_groups_all_by_obtype["NDAS"] = ["SFC", "UPA"] - vx_metatasks_all_by_obtype["NDAS"] \ - = ["task_get_obs_ndas", + vx_metatasks_all_possible_by_fg["RETOP"] \ + = vx_metatasks_all_possible_by_fg["REFC"] + + vx_metatasks_all_possible_by_fg["SFC"] \ + = ["metatask_check_post_output_all_mems", + "task_get_obs_ndas", "task_run_MET_Pb2nc_obs_NDAS", + "metatask_GridStat_SFC_UPA_all_mems", "metatask_PointStat_SFC_UPA_all_mems", "metatask_GenEnsProd_EnsembleStat_SFC_UPA", + "metatask_GridStat_SFC_UPA_ensmeanprob", "metatask_PointStat_SFC_UPA_ensmeanprob"] - # If there are no field groups specified for verification, remove those - # tasks that are common to all observation types. - vx_field_groups = vx_config["VX_FIELD_GROUPS"] - if not vx_field_groups: - metatask = "metatask_check_post_output_all_mems" - rocoto_config['tasks'].pop(metatask) - - # If for a given obs type none of its field groups are specified for - # verification, remove all vx metatasks for that obs type. - for obtype in vx_field_groups_all_by_obtype: - vx_field_groups_crnt_obtype = list(set(vx_field_groups) & set(vx_field_groups_all_by_obtype[obtype])) - if not vx_field_groups_crnt_obtype: - for metatask in vx_metatasks_all_by_obtype[obtype]: - if metatask in rocoto_config['tasks']: - logging.info(dedent( - f""" - Removing verification (meta)task - "{metatask}" - from workflow since no field groups from observation type "{obtype}" are - specified for verification.""" - )) - rocoto_config['tasks'].pop(metatask) + vx_metatasks_all_possible_by_fg["UPA"] \ + = vx_metatasks_all_possible_by_fg["SFC"] # # ----------------------------------------------------------------------- # + # # + # ----------------------------------------------------------------------- + # + vx_metatasks_all_possible \ + = [metatask \ + for fg, metatasks_for_fg in vx_metatasks_all_possible_by_fg.items() \ + for metatask in metatasks_for_fg] + vx_metatasks_all_possible = set(vx_metatasks_all_possible) # # ----------------------------------------------------------------------- # + # Invert dictionary to get a new dictionary that, for each vx (meta)task + # (the keys), contains a list of its associated field groups (the values). + # + # ----------------------------------------------------------------------- + # + vx_fieldgroups_by_metatask = {} + for metatask in sorted(list(vx_metatasks_all_possible)): + fg_list = [] + for fg, metatasks in vx_metatasks_all_possible_by_fg.items(): + if metatask in metatasks: fg_list.append(fg) + vx_fieldgroups_by_metatask[metatask] = fg_list + # + # ----------------------------------------------------------------------- + # + # + # + # ----------------------------------------------------------------------- + # + vx_field_groups = vx_config["VX_FIELD_GROUPS"] + vx_metatasks_to_include_by_fg \ + = {fg: metatasks for fg, metatasks in vx_metatasks_all_possible_by_fg.items() \ + if fg in vx_field_groups} + + # For the SFC and UPA field groups, whether we use grid-to-grid or grid- + # to-point verification depends on whether we are verifyfing against obs + # or another (benchmark) forecast. vx_config = expt_config["verification"] vx_verify_against_benchmark_fcst = vx_config["VX_VERIFY_AGAINST_BENCHMARK_FCST"] - if vx_verify_against_benchmark_fcst: - valid_gridtogrid_field_groups = [ "APCP", "REFC", "RETOP", "SFC", "UPA" ] + vx_metatasks_to_exclude_from_SFC_UPA \ + = ["task_get_obs_ndas", + "task_run_MET_Pb2nc_obs_NDAS", + "metatask_PointStat_SFC_UPA_all_mems", + "metatask_PointStat_SFC_UPA_ensmeanprob"] else: - valid_gridtogrid_field_groups = [ "APCP", "REFC", "RETOP" ] - - vx_gridtogrid_field_groups = [] - vx_gridtopoint_field_groups = [] - for fg in vx_field_groups: - if fg in valid_gridtogrid_field_groups: - vx_gridtogrid_field_groups.append(fg) - else: - vx_gridtopoint_field_groups.append(fg) - - vx_config['VX_GRIDTOGRID_FIELD_GROUPS'] = vx_gridtogrid_field_groups - vx_config['VX_GRIDTOPOINT_FIELD_GROUPS'] = vx_gridtopoint_field_groups - expt_config["verification"] = vx_config - - print(f"") - print(f"PPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP") - print(f"{vx_verify_against_benchmark_fcst = }") - print(f"{vx_gridtogrid_field_groups = }") - print(f"{vx_gridtopoint_field_groups = }") - #bbbbbbbbbbb - - wflow_metatasks = rocoto_config['tasks'] - - if not vx_gridtogrid_field_groups: - metatasks_to_remove \ - = ['metatask_GridStat_REFC_RETOP_all_mems', - 'metatask_GenEnsProd_EnsembleStat_REFC_RETOP', - 'metatask_GridStat_REFC_RETOP_ensprob'] - for metatask in metatasks_to_remove: - if metatask in wflow_metatasks.keys(): - wflow_metatasks.pop(metatask) - - if not vx_gridtopoint_field_groups: - metatasks_to_remove \ - = ['metatask_PointStat_SFC_UPA_all_mems', - 'metatask_GenEnsProd_EnsembleStat_SFC_UPA', - 'metatask_PointStat_SFC_UPA_ensmeanprob'] - for metatask in metatasks_to_remove: - if metatask in wflow_metatasks.keys(): - wflow_metatasks.pop(metatask) - - rocoto_config['tasks'] = wflow_metatasks + vx_metatasks_to_exclude_from_SFC_UPA \ + = ["metatask_GridStat_SFC_UPA_all_mems", + "metatask_GridStat_SFC_UPA_ensmeanprob"] + + for fg in ['SFC', 'UPA']: + if fg in vx_metatasks_to_include_by_fg.copy(): + vx_metatasks_to_include_by_fg[fg] \ + = [metatask for metatask in vx_metatasks_to_include_by_fg[fg] \ + if metatask not in vx_metatasks_to_exclude_from_SFC_UPA] + + vx_metatasks_to_include \ + = [metatask \ + for fg, metatasks_for_fg in vx_metatasks_to_include_by_fg.items() \ + for metatask in metatasks_for_fg] + vx_metatasks_to_include = set(vx_metatasks_to_include) + + vx_metatasks_to_exclude = vx_metatasks_all_possible.difference(vx_metatasks_to_include) + + for metatask in vx_metatasks_to_exclude: + if metatask in rocoto_config['tasks']: + logging.info(dedent( + f""" + Removing verification (meta)task + "{metatask}" + from workflow since the list of field groups to verify, i.e. + {vx_field_groups = } + does not include any of the field groups associated with this (meta)task, + which are + {vx_fieldgroups_by_metatask[metatask]}""" + )) + rocoto_config['tasks'].pop(metatask) # # ----------------------------------------------------------------------- #