From 982cf620451a6e55ac0176cd9dc879172d4869de Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 4 Dec 2024 19:31:21 +0000 Subject: [PATCH] Per #3006, starting to tweak config options. Saving progress while it's successfully compiling --- data/config/PairStatConfig_default | 107 +++++++----------- src/tools/core/pair_stat/pair_stat.cc | 16 +-- .../core/pair_stat/pair_stat_conf_info.cc | 41 +------ .../core/pair_stat/pair_stat_conf_info.h | 11 +- 4 files changed, 50 insertions(+), 125 deletions(-) diff --git a/data/config/PairStatConfig_default b/data/config/PairStatConfig_default index f7cb77cd8..1f0a1cced 100644 --- a/data/config/PairStatConfig_default +++ b/data/config/PairStatConfig_default @@ -14,49 +14,21 @@ model = ""; // // Output description to be written -// May be set separately in each "pairs" entry +// Non-empty string overrides input data values +// May be set separately in each "obs.pairs" entry // -desc = "NA"; +desc = ""; //////////////////////////////////////////////////////////////////////////////// -// -// May be set separately in each "pairs" entry -// -censor_thresh = []; -censor_val = []; -mpr_column = []; -mpr_thresh = []; -cat_thresh = [ NA ]; -cnt_thresh = [ NA ]; -cnt_logic = UNION; -wind_thresh = [ NA ]; -wind_logic = UNION; -eclv_points = 0.05; -hss_ec_value = NA; -rank_corr_flag = FALSE; - // // Forecast and observation pairs to be verified -// Parse "pairs" like we'd parse "field", supporting things like set_attr_name // - -pairs = [ - { fcst_name = , obs_name = , filters }, - { }, - -] - - fcst = { pairs = [ { - name = "TMP"; level = "P100-1000"... P100, P200, XXX P1100 // IODA forecast variable name or comma-separated list of FCST_VAR names - filter_str_inc = [ { name = "FCST_LEVEL"; value = "P100,P200,P300"; } ]; - filter_str_exc = [ { name = ""; value = ""; } ]; - filter_thresh = [ { name = ""; thresh = NA; } ]; - cat_thresh = [ >273, >283 ]; - cnt_thresh = [ NA, >273 ]; + name = "TMP"; + level = [ "Z2" ]; } ]; } @@ -64,12 +36,8 @@ fcst = { obs = { pairs = [ { - name = "TMP"; // IODA observation variable name or comma-separated list of OBS_VAR names - filter_str_inc = [ { name = "OBS_LEVEL"; value = "Z2"; } ]; - filter_str_exc = [ { name = ""; value = ""; } ]; - filter_thresh = [ { name = ""; thresh = NA; } ]; - cat_thresh = [ >273, >283 ]; - cnt_thresh = [ NA, >273 ]; + name = "TMP"; + level = [ "Z2" ]; } ]; } @@ -77,21 +45,36 @@ obs = { //////////////////////////////////////////////////////////////////////////////// // -// Point observation filtering options -// May be set separately in each "obs.field" entry +// Forecast and observation data censoring, thresholding, and filtering options +// May be set separately in each "fcst.pairs" or "obs.pairs" entry // -message_type = [ "ADPUPA" ]; -sid_inc = []; -sid_exc = []; -obs_quality_inc = []; -obs_quality_exc = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; +censor_thresh = []; +censor_val = []; +cat_thresh = [ NA ]; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; //////////////////////////////////////////////////////////////////////////////// // +// Matched pairs filtering and control options +// May be set separately in each "obs.pairs" entry +// +mpr_column = []; +mpr_thresh = []; +mpr_str_inc = []; // TODO: new for string inclusion +mpr_str_exc = []; // TODO: new for string exclusion +mpr_summary = NONE; // TODO: new for multiple pairs in the time window +eclv_points = 0.05; +hss_ec_value = NA; +rank_corr_flag = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology mean data // May be set separately in the "fcst" and "obs" dictionaries // climo_mean = { @@ -99,13 +82,6 @@ climo_mean = { file_name = []; field = []; - regrid = { - method = NEAREST; - width = 1; - vld_thresh = 0.5; - shape = SQUARE; - } - time_interp_method = DW_MEAN; day_interval = 31; hour_interval = 6; @@ -122,7 +98,7 @@ climo_stdev = { // // Climatology distribution settings -// May be set separately in each "obs.field" entry +// May be set separately in each "obs.pairs" entry // climo_cdf = { cdf_bins = 1; @@ -137,7 +113,7 @@ climo_cdf = { // Land/Sea mask // For LANDSF message types, only use forecast grid points where land = TRUE. // For WATERSF message types, only use forecast grid points where land = FALSE. -// land_mask.flag may be set separately in each "obs.field" entry. +// land_mask.flag may be set separately in each "obs.pairs" entry. // land_mask = { flag = FALSE; @@ -154,7 +130,7 @@ land_mask = { // For the observations kept, when interpolating forecast data to the // observation location, only use forecast grid points where the topo - station // difference meets the interp_fcst_thresh threshold. -// topo_mask.flag may be set separately in each "obs.field" entry. +// topo_mask.flag may be set separately in each "obs.pairs" entry. // topo_mask = { flag = FALSE; @@ -169,7 +145,7 @@ topo_mask = { // // Point observation time window -// May be set separately in each "obs.field" entry +// May be set separately in each "obs.pairs" entry // obs_window = { beg = -5400; @@ -180,7 +156,7 @@ obs_window = { // // Verification masking regions -// May be set separately in each "obs.field" entry +// May be set separately in each "obs.pairs" entry // mask = { grid = [ "FULL" ]; @@ -193,7 +169,7 @@ mask = { // // Confidence interval settings -// May be set separately in each "obs.field" entry +// May be set separately in each "obs.pairs" entry // ci_alpha = [ 0.05 ]; @@ -216,7 +192,7 @@ seeps_p1_thresh = >=0.1&&<=0.85; // // Statistical output types -// May be set separately in each "obs.field" entry +// May be set separately in each "obs.pairs" entry // output_flag = { fho = NONE; @@ -234,9 +210,6 @@ output_flag = { pstd = NONE; pjc = NONE; prc = NONE; - ecnt = NONE; // Only for HiRA. - orank = NONE; // Only for HiRA. - rps = NONE; // Only for HiRA. eclv = NONE; mpr = NONE; seeps = NONE; @@ -246,7 +219,7 @@ output_flag = { //////////////////////////////////////////////////////////////////////////////// point_weight_flag = NONE; - + tmp_dir = "/tmp"; output_prefix = ""; version = "V12.0.0"; diff --git a/src/tools/core/pair_stat/pair_stat.cc b/src/tools/core/pair_stat/pair_stat.cc index 51b426fdd..43a18179a 100644 --- a/src/tools/core/pair_stat/pair_stat.cc +++ b/src/tools/core/pair_stat/pair_stat.cc @@ -57,8 +57,6 @@ using namespace netCDF; //////////////////////////////////////////////////////////////////////// -#define BUFFER_SIZE (DEF_NC_BUFFER_SIZE/2) - static void process_command_line(int, char **); static void setup_first_pass(const DataPlane &, const Grid &); @@ -693,7 +691,6 @@ void process_ioda_pairs(const ConcatString &file_name) { #endif // Perform GRIB table lookups, if needed - if(!use_var_id) conf_info.process_grib_codes(); is_vgrd = is_ugrd = false; int hdr_count = met_point_obs->get_hdr_cnt(); @@ -982,8 +979,7 @@ void process_scores() { write_mpr_row(shc, pd_ptr, conf_info.vx_opt[i_vx].output_flag[i_mpr], stat_at, i_stat_row, - txt_at[i_mpr], i_txt_row[i_mpr], - conf_info.obtype_as_group_val_flag); + txt_at[i_mpr], i_txt_row[i_mpr], false); // Reset the obtype column shc.set_obtype(conf_info.vx_opt[i_vx].msg_typ[i_msg_typ].c_str()); @@ -998,8 +994,7 @@ void process_scores() { write_seeps_mpr_row(shc, pd_ptr, conf_info.vx_opt[i_vx].output_flag[i_seeps_mpr], stat_at, i_stat_row, - txt_at[i_seeps_mpr], i_txt_row[i_seeps_mpr], - conf_info.obtype_as_group_val_flag); + txt_at[i_seeps_mpr], i_txt_row[i_seeps_mpr], false); // Reset the obtype column shc.set_obtype(conf_info.vx_opt[i_vx].msg_typ[i_msg_typ].c_str()); @@ -1848,8 +1843,7 @@ void do_hira_ens(int i_vx, const PairDataPoint *pd_ptr) { write_orank_row(shc, &hira_pd, conf_info.vx_opt[i_vx].output_flag[i_orank], stat_at, i_stat_row, - txt_at[i_orank], i_txt_row[i_orank], - conf_info.obtype_as_group_val_flag); + txt_at[i_orank], i_txt_row[i_orank], false); // Reset the obtype column shc.set_obtype(pd_ptr->msg_typ.c_str()); @@ -2044,9 +2038,7 @@ void do_hira_prob(int i_vx, const PairDataPoint *pd_ptr) { write_mpr_row(shc, &hira_pd, conf_info.vx_opt[i_vx].output_flag[i_mpr], stat_at, i_stat_row, - txt_at[i_mpr], i_txt_row[i_mpr], - conf_info.obtype_as_group_val_flag, - false); + txt_at[i_mpr], i_txt_row[i_mpr], false, false); // Reset the obtype column shc.set_obtype(pd_ptr->msg_typ.c_str()); diff --git a/src/tools/core/pair_stat/pair_stat_conf_info.cc b/src/tools/core/pair_stat/pair_stat_conf_info.cc index f2a6aff41..afc904423 100644 --- a/src/tools/core/pair_stat/pair_stat_conf_info.cc +++ b/src/tools/core/pair_stat/pair_stat_conf_info.cc @@ -79,9 +79,6 @@ PairStatConfInfo::~PairStatConfInfo() { void PairStatConfInfo::init_from_scratch() { - // Initialize pointers - vx_opt = (PairStatVxOpt *) nullptr; - clear(); return; @@ -93,13 +90,12 @@ void PairStatConfInfo::clear() { // Initialize values model.clear(); - grib_codes_set = false; + vx_opt.clear(); land_mask.clear(); topo_dp.clear(); topo_use_obs_thresh.clear(); topo_interp_fcst_thresh.clear(); msg_typ_group_map.clear(); - obtype_as_group_val_flag = false; mask_area_map.clear(); mask_sid_map.clear(); point_weight_flag = PointWeightType::None; @@ -109,9 +105,6 @@ void PairStatConfInfo::clear() { seeps_climo_name.clear(); seeps_p1_thresh.clear(); - // Deallocate memory - if(vx_opt) { delete [] vx_opt; vx_opt = (PairStatVxOpt *) nullptr; } - // Set count to zero n_vx = 0; @@ -163,10 +156,6 @@ void PairStatConfInfo::process_config(PairsFormat ftype) { // Conf: message_type_group_map msg_typ_group_map = parse_conf_message_type_group_map(&conf); - // Conf: obtype_as_group_val_flag - obtype_as_group_val_flag = - conf.lookup_bool(conf_key_obtype_as_group_val_flag); - // Conf: fcst.field and obs.field fdict = conf.lookup_array(conf_key_fcst_field); odict = conf.lookup_array(conf_key_obs_field); @@ -187,7 +176,7 @@ void PairStatConfInfo::process_config(PairsFormat ftype) { // Allocate memory for the verification task options n_vx = n_fvx; - vx_opt = new PairStatVxOpt [n_vx]; + vx_opt.resize(n_vx); // Check for consistent number of climatology fields check_climo_n_vx(fdict, n_vx); @@ -304,32 +293,6 @@ void PairStatConfInfo::process_config(PairsFormat ftype) { //////////////////////////////////////////////////////////////////////// -void PairStatConfInfo::process_grib_codes() { - - // Only needs to be set once - if(grib_codes_set) return; - - mlog << Debug(3) << "Processing each \"" << conf_key_obs_field - << "\" name as a GRIB code abbreviation since the point " - << "observations are specified as GRIB codes.\n"; - - Dictionary *odict = conf.lookup_array(conf_key_obs_field); - Dictionary i_odict; - - // Add the GRIB code by parsing each observation dictionary - for(int i=0; iadd_grib_code(i_odict); - } - - // Flag to prevent processing more than once - grib_codes_set = true; - - return; -} - -//////////////////////////////////////////////////////////////////////// - void PairStatConfInfo::process_flags() { int i, j; bool output_ascii_flag = false; diff --git a/src/tools/core/pair_stat/pair_stat_conf_info.h b/src/tools/core/pair_stat/pair_stat_conf_info.h index c24434b03..34dd66946 100644 --- a/src/tools/core/pair_stat/pair_stat_conf_info.h +++ b/src/tools/core/pair_stat/pair_stat_conf_info.h @@ -242,8 +242,7 @@ class PairStatConfInfo { // Store data parsed from the Pair-Stat configuration object ConcatString model; // Model name - PairStatVxOpt * vx_opt; // Array of vx task options [n_vx] (allocated) - bool grib_codes_set; + std::vector vx_opt; // Vector of vx options [n_vx] // Land/sea mask and topography info for data filtering MaskPlane land_mask; @@ -251,15 +250,14 @@ class PairStatConfInfo { SingleThresh topo_use_obs_thresh; SingleThresh topo_interp_fcst_thresh; - // Message type groups that should be processed together + // Message type groups std::map msg_typ_group_map; - bool obtype_as_group_val_flag; // Mapping of mask names to DataPlanes - std::map mask_area_map; + std::map mask_area_map; // Mapping of mask names to Station ID lists - std::map mask_sid_map; + std::map mask_sid_map; PointWeightType point_weight_flag; // Point weighting flag @@ -280,7 +278,6 @@ class PairStatConfInfo { void read_config(const StringArray &); void process_config(PairsFormat); - void process_grib_codes(); void process_flags(); void process_masks(); void process_geog();