From 2c13fda5851509d9ce8a26c7dd2a7630dac9e943 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 4 Nov 2024 20:20:06 +0000 Subject: [PATCH 01/14] Per #2966, add new solar_time() function to the vx_solar library. --- src/libcode/vx_solar/solar.cc | 43 +++++++++++++++++++++++++++++++++++ src/libcode/vx_solar/solar.h | 19 ++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/src/libcode/vx_solar/solar.cc b/src/libcode/vx_solar/solar.cc index 2d6f6a5280..3ca09df709 100644 --- a/src/libcode/vx_solar/solar.cc +++ b/src/libcode/vx_solar/solar.cc @@ -60,6 +60,49 @@ return; //////////////////////////////////////////////////////////////////////// +double solar_time(unixtime gmt, double lon) + +{ + + // + // right ascension and declination + // + +double Ra; +double Dec; + +solar_radec(gmt, Ra, Dec); + + // + // local hour angle + // + +double lha = gmt_to_gmst(gmt) - lon - Ra; + + // + // rescale angle to -180 to 180 + // + +lha -= 360.0*floor((lha + 180.0)/360.0); + + // + // rescale local hour angle to decimal hours of the solar day + // + +double solar_hr = (lha + 180.0)/360.0 * 24; + + // + // done + // + +return solar_hr; + +} + + +//////////////////////////////////////////////////////////////////////// + + void dh_to_aa(double lat, double Dec, double lha, double & alt, double & azi) { diff --git a/src/libcode/vx_solar/solar.h b/src/libcode/vx_solar/solar.h index a78876e7ff..f4c717d879 100644 --- a/src/libcode/vx_solar/solar.h +++ b/src/libcode/vx_solar/solar.h @@ -46,6 +46,25 @@ extern void solar_altaz(unixtime gmt, double lat, double lon, double & alt, doub //////////////////////////////////////////////////////////////////////// +extern double solar_time(unixtime gmt, double lon); + + // + // calculates the solar time for the given longitude. + // + // + // Input: gmt, greenwich mean time expressed as unix time + // + // lon, longitude (degrees) of given location (+ west, - east) + // + // + // Output: decimal hours f the solar day in range [0, 24), + // where 12 is solar noon + // + + +//////////////////////////////////////////////////////////////////////// + + extern void solar_radec(unixtime gmt, double & Ra, double & Dec); // From 4b469d618cf9d473a5477149233bd51e21c21af8 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 4 Nov 2024 20:21:54 +0000 Subject: [PATCH 02/14] Per #2966, add support for new solar_time masking type. Also make log messages for consistent and eliminate the warning about -thresh not being specified becuase its fine to not specify a threshold. --- src/tools/other/gen_vx_mask/gen_vx_mask.cc | 262 +++++++++++++-------- src/tools/other/gen_vx_mask/gen_vx_mask.h | 27 ++- 2 files changed, 184 insertions(+), 105 deletions(-) diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.cc b/src/tools/other/gen_vx_mask/gen_vx_mask.cc index 83fe7cc568..61fbc4ede0 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.cc +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.cc @@ -31,6 +31,7 @@ // 013 07/06/22 Howard Soh METplus-Internal #19 Rename main to met_main // 014 09/28/22 Prestopnik MET #2227 Remove namespace std and netCDF from header files // 015 05/03/23 Halley Gotway MET #1060 Support multiple shapes +// 016 11/04/24 Halley Gotway MET #2966 Add solar time option. // //////////////////////////////////////////////////////////////////////// @@ -160,8 +161,8 @@ void process_command_line(int argc, char **argv) { mlog << Error << "\n" << program_name << " -> " << "the -type command line requirement must be set to a specific masking type!\n" << "\t\t \"poly\", \"box\", \"circle\", \"track\", \"grid\", " - << "\"data\", \"solar_alt\", \"solar_azi\", \"lat\", \"lon\" " - << "or \"shape\"" << "\n\n"; + << "\"data\", \"solar_alt\", \"solar_azi\", \"solar_time\", " + << "\"lat\", \"lon\", or \"shape\"\n\n"; exit(1); } @@ -325,6 +326,13 @@ void process_mask_file(DataPlane &dp) { exit(1); } + // Report the threshold + if(is_thresh_masktype(mask_type)) { + mlog << Debug(2) + << masktype_to_description(mask_type) + << " Threshold:\t" << thresh.get_str() << "\n"; + } + // Initialize the masking field, if needed if(dp.is_empty()) dp.set_size(grid.nx(), grid.ny()); @@ -361,6 +369,7 @@ void process_mask_file(DataPlane &dp) { case MaskType::Solar_Alt: case MaskType::Solar_Azi: + case MaskType::Solar_Time: apply_solar_mask(dp); break; @@ -693,13 +702,15 @@ void apply_poly_mask(DataPlane & dp) { if(complement) { mlog << Debug(3) - << "Applying complement of polyline mask.\n"; + << "Applying complement of the " + << masktype_to_string(mask_type) << " mask.\n"; } // List number of points inside the mask mlog << Debug(3) - << "Polyline Masking:\t" << n_in << " of " << grid.nx() * grid.ny() - << " points inside\n"; + << masktype_to_description(mask_type) + << " Masking:\t" << n_in << " of " + << grid.nxy() << " points inside\n"; return; } @@ -739,13 +750,15 @@ void apply_poly_xy_mask(DataPlane & dp) { if(complement) { mlog << Debug(3) - << "Applying complement of polyline XY mask.\n"; + << "Applying complement of the " + << masktype_to_string(mask_type) << " mask.\n"; } // List number of points inside the mask mlog << Debug(3) - << "Polyline XY Masking:\t" << n_in << " of " << grid.nx() * grid.ny() - << " points inside\n"; + << masktype_to_description(mask_type) + << " Masking:\t" << n_in << " of " + << grid.nxy() << " points inside\n"; return; } @@ -814,13 +827,15 @@ void apply_box_mask(DataPlane &dp) { if(complement) { mlog << Debug(3) - << "Applying complement of box mask.\n"; + << "Applying complement of the " + << masktype_to_string(mask_type) << " mask.\n"; } + // List number of points inside the mask mlog << Debug(3) - << "Box Masking (" << height << " x " << width << "):\t" - << n_in << " of " << grid.nx() * grid.ny() - << " points inside\n"; + << masktype_to_description(mask_type) + << " Masking (" << height << " x " << width << "):\t" + << n_in << " of " << grid.nxy() << " points inside\n"; return; } @@ -837,10 +852,10 @@ void apply_circle_mask(DataPlane &dp) { // Check for no threshold if(thresh.get_type() == thresh_na) { - mlog << Warning - << "\napply_circle_mask() -> since \"-thresh\" was not used " - << "to specify a threshold in kilometers for circle masking, " - << "the minimum distance to the points will be written.\n\n"; + mlog << Debug(3) + << "Since \"-thresh\" was not used to specify a threshold " + << "in kilometers for circle masking, the minimum distance " + << "to the points will be written.\n"; } // For each grid point, compute mimumum distance to polyline points @@ -881,14 +896,16 @@ void apply_circle_mask(DataPlane &dp) { if(thresh.get_type() != thresh_na && complement) { mlog << Debug(3) - << "Applying complement of circle mask.\n"; + << "Applying complement of the " + << masktype_to_string(mask_type) << " mask.\n"; } // List the number of points inside the mask if(thresh.get_type() != thresh_na) { mlog << Debug(3) - << "Circle Masking:\t" << n_in << " of " << grid.nx() * grid.ny() - << " points inside\n"; + << masktype_to_description(mask_type) + << " Masking:\t" << n_in << " of " + << grid.nxy() << " points inside\n"; } // Otherwise, list the min/max distances computed else { @@ -896,7 +913,8 @@ void apply_circle_mask(DataPlane &dp) { double dmax; dp.data_range(dmin, dmax); mlog << Debug(3) - << "Circle Masking:\tDistances ranging from " + << masktype_to_description(mask_type) + << " Masking:\tDistances ranging from " << dmin << " km to " << dmax << " km\n"; } @@ -915,10 +933,10 @@ void apply_track_mask(DataPlane &dp) { // Check for no threshold if(thresh.get_type() == thresh_na) { - mlog << Warning - << "\napply_track_mask() -> since \"-thresh\" was not used " - << "to specify a threshold for track masking, the minimum " - << "distance to the track will be written.\n\n"; + mlog << Debug(3) + << "Since \"-thresh\" was not used to specify a threshold " + << "in kilometers for track masking, the minimum distance " + << "to the track will be written.\n"; } // For each grid point, compute mimumum distance to track @@ -962,21 +980,24 @@ void apply_track_mask(DataPlane &dp) { if(thresh.get_type() != thresh_na && complement) { mlog << Debug(3) - << "Applying complement of track mask.\n"; + << "Applying complement of the " + << masktype_to_string(mask_type) << " mask.\n"; } // List the number of points inside the mask if(thresh.get_type() != thresh_na) { mlog << Debug(3) - << "Track Masking:\t\t" << n_in << " of " << grid.nx() * grid.ny() - << " points inside\n"; + << masktype_to_description(mask_type) + << " Masking:\t" << n_in << " of " + << grid.nxy() << " points inside\n"; } // Otherwise, list the min/max distances computed else { double dmin, dmax; dp.data_range(dmin, dmax); mlog << Debug(3) - << "Track Masking:\t\tDistances ranging from " + << masktype_to_description(mask_type) + << " Masking:\tDistances ranging from " << dmin << " km to " << dmax << " km\n"; } @@ -1022,12 +1043,15 @@ void apply_grid_mask(DataPlane &dp) { if(complement) { mlog << Debug(3) - << "Applying complement of grid mask.\n"; + << "Applying complement of the " + << masktype_to_string(mask_type) << " mask.\n"; } + // List number of points inside the mask mlog << Debug(3) - << "Grid Masking:\t\t" << n_in << " of " << grid.nx() * grid.ny() - << " points inside\n"; + << masktype_to_description(mask_type) + << " Masking:\t" << n_in << " of " + << grid.nxy() << " points inside\n"; return; } @@ -1040,15 +1064,16 @@ void apply_data_mask(DataPlane &dp) { // Nothing to do without a threshold if(thresh.get_type() == thresh_na) { + mlog << Debug(3) + << "Since \"-thresh\" was not used to specify a threshold " + << "in kilometers for data masking, the raw data values " + << "will be written.\n"; double dmin, dmax; dp.data_range(dmin, dmax); mlog << Debug(3) - << "Data Masking:\t\tValues ranging from " - << dmin << " km to " << dmax << " km\n"; - mlog << Warning - << "\napply_data_mask() -> since \"-thresh\" was not used " - << "to specify a threshold for data masking, the raw data " - << "values will be written.\n\n"; + << masktype_to_description(mask_type) + << " Masking:\tValues ranging from " + << dmin << " to " << dmax << "\n"; return; } @@ -1083,10 +1108,11 @@ void apply_data_mask(DataPlane &dp) { } // end for y } // end for x - // List the number of points inside the mask + // List number of points inside the mask mlog << Debug(3) - << "Data Masking:\t\t" << n_in << " of " << grid.nx() * grid.ny() - << " points inside\n"; + << masktype_to_description(mask_type) + << " Masking:\t" << n_in << " of " + << grid.nxy() << " points inside\n"; return; } @@ -1100,10 +1126,10 @@ void apply_solar_mask(DataPlane &dp) { // Check for no threshold if(thresh.get_type() == thresh_na) { - mlog << Warning - << "\napply_solar_mask() -> since \"-thresh\" was not used " + mlog << Debug(3) + << "Since \"-thresh\" was not used to specify a threshold, " << "the raw " << masktype_to_string(mask_type) - << " values will be written.\n\n"; + << " values will be written.\n"; } // Compute solar value for each grid point Lat/Lon @@ -1114,9 +1140,15 @@ void apply_solar_mask(DataPlane &dp) { grid.xy_to_latlon(x, y, lat, lon); lon -= 360.0*floor((lon + 180.0)/360.0); + // Compute the solar time in hours + if(mask_type == MaskType::Solar_Time) { + v = solar_time(solar_ut, lon); + } // Compute the solar altitude and azimuth - solar_altaz(solar_ut, lat, lon, alt, azi); - v = (mask_type == MaskType::Solar_Alt ? alt : azi); + else { + solar_altaz(solar_ut, lat, lon, alt, azi); + v = (mask_type == MaskType::Solar_Alt ? alt : azi); + } // Apply threshold, if specified if(thresh.get_type() != thresh_na) { @@ -1143,23 +1175,21 @@ void apply_solar_mask(DataPlane &dp) { << masktype_to_string(mask_type) << " mask.\n"; } - const char *mask_str = (mask_type == MaskType::Solar_Alt ? - "Altitude" : "Azimuth"); - // List the number of points inside the mask if(thresh.get_type() != thresh_na) { mlog << Debug(3) - << "Solar " << mask_str << " Masking:\t\t" - << n_in << " of " << grid.nx() * grid.ny() - << " points inside\n"; + << masktype_to_description(mask_type) + << " Masking:\t" << n_in << " of " + << grid.nxy() << " points inside\n"; } // Otherwise, list the min/max distances computed else { double dmin, dmax; dp.data_range(dmin, dmax); mlog << Debug(3) - << "Solar " << mask_str << " Masking:\t\t" - << "Values ranging from " << dmin << " to " << dmax << "\n"; + << masktype_to_description(mask_type) + << " Masking:\tValues ranging from " + << dmin << " to " << dmax << "\n"; } return; @@ -1174,10 +1204,10 @@ void apply_lat_lon_mask(DataPlane &dp) { // Check for no threshold if(thresh.get_type() == thresh_na) { - mlog << Warning - << "\napply_lat_lon_mask() -> since \"-thresh\" was not used " + mlog << Debug(3) + << "Since \"-thresh\" was not used to specify a threshold, " << "the raw " << masktype_to_string(mask_type) - << " values will be written.\n\n"; + << " values will be written.\n"; } // Compute Lat/Lon value for each grid point @@ -1214,21 +1244,20 @@ void apply_lat_lon_mask(DataPlane &dp) { << masktype_to_string(mask_type) << " mask.\n"; } - const char *mask_str = (mask_type == MaskType::Lat ? - "Latitude" : "Longitude"); - // List the number of points inside the mask if(thresh.get_type() != thresh_na) { mlog << Debug(3) - << mask_str << " Masking:\t\t" << n_in << " of " - << grid.nx() * grid.ny()<< " points inside\n"; + << masktype_to_description(mask_type) + << " Masking:\t" << n_in << " of " + << grid.nxy() << " points inside\n"; } // Otherwise, list the min/max distances computed else { double dmin, dmax; dp.data_range(dmin, dmax); mlog << Debug(3) - << mask_str << " Masking:\t\tValues ranging from " + << masktype_to_description(mask_type) + << " Masking:\tValues ranging from " << dmin << " to " << dmax << "\n"; } @@ -1280,13 +1309,15 @@ void apply_shape_mask(DataPlane & dp) { if(complement) { mlog << Debug(3) - << "Applying complement of the shapefile mask.\n"; + << "Applying complement of the " + << masktype_to_string(mask_type) << " mask.\n"; } // List number of points inside the mask mlog << Debug(3) - << "Shape Masking:\t\t" << n_in << " of " << grid.nx() * grid.ny() - << " points inside\n"; + << masktype_to_description(mask_type) + << " Masking:\t" << n_in << " of " + << grid.nxy() << " points inside\n"; return; } @@ -1365,7 +1396,7 @@ DataPlane combine(const DataPlane &dp_data, const DataPlane &dp_mask, mlog << Debug(3) << "Mask " << setlogic_to_string(logic) << (logic == SetLogic::Intersection ? ":\t" : ":\t\t") - << n_in << " of " << grid.nx() * grid.ny() + << n_in << " of " << grid.nxy() << " points inside\n"; } @@ -1470,7 +1501,22 @@ void write_netcdf(const DataPlane &dp) { //////////////////////////////////////////////////////////////////////// bool is_solar_masktype(MaskType t) { - return(t == MaskType::Solar_Alt || t == MaskType::Solar_Azi); + return(t == MaskType::Solar_Alt || + t == MaskType::Solar_Azi || + t == MaskType::Solar_Time); +} + +//////////////////////////////////////////////////////////////////////// + +bool is_thresh_masktype(MaskType t) { //circle,track,data,solar,lat,lon + return(t == MaskType::Circle || + t == MaskType::Track || + t == MaskType::Data || + t == MaskType::Solar_Alt || + t == MaskType::Solar_Azi || + t == MaskType::Solar_Time || + t == MaskType::Lat || + t == MaskType::Lon); } //////////////////////////////////////////////////////////////////////// @@ -1478,18 +1524,19 @@ bool is_solar_masktype(MaskType t) { MaskType string_to_masktype(const char *s) { MaskType t = MaskType::None; - if(strcasecmp(s, "poly") == 0) t = MaskType::Poly; - else if(strcasecmp(s, "poly_xy") == 0) t = MaskType::Poly_XY; - else if(strcasecmp(s, "box") == 0) t = MaskType::Box; - else if(strcasecmp(s, "circle") == 0) t = MaskType::Circle; - else if(strcasecmp(s, "track") == 0) t = MaskType::Track; - else if(strcasecmp(s, "grid") == 0) t = MaskType::Grid; - else if(strcasecmp(s, "data") == 0) t = MaskType::Data; - else if(strcasecmp(s, "solar_alt") == 0) t = MaskType::Solar_Alt; - else if(strcasecmp(s, "solar_azi") == 0) t = MaskType::Solar_Azi; - else if(strcasecmp(s, "lat") == 0) t = MaskType::Lat; - else if(strcasecmp(s, "lon") == 0) t = MaskType::Lon; - else if(strcasecmp(s, "shape") == 0) t = MaskType::Shape; + if(strcasecmp(s, "poly") == 0) t = MaskType::Poly; + else if(strcasecmp(s, "poly_xy") == 0) t = MaskType::Poly_XY; + else if(strcasecmp(s, "box") == 0) t = MaskType::Box; + else if(strcasecmp(s, "circle") == 0) t = MaskType::Circle; + else if(strcasecmp(s, "track") == 0) t = MaskType::Track; + else if(strcasecmp(s, "grid") == 0) t = MaskType::Grid; + else if(strcasecmp(s, "data") == 0) t = MaskType::Data; + else if(strcasecmp(s, "solar_alt") == 0) t = MaskType::Solar_Alt; + else if(strcasecmp(s, "solar_azi") == 0) t = MaskType::Solar_Azi; + else if(strcasecmp(s, "solar_time") == 0) t = MaskType::Solar_Time; + else if(strcasecmp(s, "lat") == 0) t = MaskType::Lat; + else if(strcasecmp(s, "lon") == 0) t = MaskType::Lon; + else if(strcasecmp(s, "shape") == 0) t = MaskType::Shape; else { mlog << Error << "\nstring_to_masktype() -> " << "unsupported masking type \"" << s << "\"\n\n"; @@ -1505,20 +1552,47 @@ const char * masktype_to_string(const MaskType t) { const char *s = (const char *) nullptr; switch(t) { - case MaskType::Poly: s = "poly"; break; - case MaskType::Poly_XY: s = "poly_xy"; break; - case MaskType::Box: s = "box"; break; - case MaskType::Circle: s = "circle"; break; - case MaskType::Track: s = "track"; break; - case MaskType::Grid: s = "grid"; break; - case MaskType::Data: s = "data"; break; - case MaskType::Solar_Alt: s = "solar_alt"; break; - case MaskType::Solar_Azi: s = "solar_azi"; break; - case MaskType::Lat: s = "lat"; break; - case MaskType::Lon: s = "lon"; break; - case MaskType::Shape: s = "shape"; break; - case MaskType::None: s = na_str; break; - default: s = (const char *) nullptr; break; + case MaskType::Poly: s = "poly"; break; + case MaskType::Poly_XY: s = "poly_xy"; break; + case MaskType::Box: s = "box"; break; + case MaskType::Circle: s = "circle"; break; + case MaskType::Track: s = "track"; break; + case MaskType::Grid: s = "grid"; break; + case MaskType::Data: s = "data"; break; + case MaskType::Solar_Alt: s = "solar_alt"; break; + case MaskType::Solar_Azi: s = "solar_azi"; break; + case MaskType::Solar_Time: s = "solar_time"; break; + case MaskType::Lat: s = "lat"; break; + case MaskType::Lon: s = "lon"; break; + case MaskType::Shape: s = "shape"; break; + case MaskType::None: s = na_str; break; + default: s = (const char *) nullptr; break; + } + + return s; +} + +//////////////////////////////////////////////////////////////////////// + +const char * masktype_to_description(const MaskType t) { + const char *s = (const char *) nullptr; + + switch(t) { + case MaskType::Poly: s = "Polyline"; break; + case MaskType::Poly_XY: s = "Polyline XY"; break; + case MaskType::Box: s = "Box"; break; + case MaskType::Circle: s = "Circle"; break; + case MaskType::Track: s = "Track"; break; + case MaskType::Grid: s = "Grrid"; break; + case MaskType::Data: s = "Data"; break; + case MaskType::Solar_Alt: s = "Solar Altitude"; break; + case MaskType::Solar_Azi: s = "Solar Azimuth"; break; + case MaskType::Solar_Time: s = "Solar Time"; break; + case MaskType::Lat: s = "Latitude"; break; + case MaskType::Lon: s = "Longitude"; break; + case MaskType::Shape: s = "Shapefile"; break; + case MaskType::None: s = na_str; break; + default: s = (const char *) nullptr; break; } return s; @@ -1604,6 +1678,8 @@ void usage() { << "\"mask_field\".\n" << "\t\t For \"solar_alt\" and \"solar_azi\" masking, " << "threshold the computed solar values.\n" + << "\t\t For \"solar_time\" masking, " + << "threshold the decimal hours of the solar day.\n" << "\t\t For \"lat\" and \"lon\" masking, threshold the " << "latitude and longitude values.\n" diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.h b/src/tools/other/gen_vx_mask/gen_vx_mask.h index 962693f0c4..b1fbbf3ef6 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.h +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.h @@ -49,31 +49,34 @@ static const char *program_name = "gen_vx_mask"; enum class MaskType { - Poly, // Polyline masking in lat/lon space - Poly_XY, // Polyline masking in grid x/y space + Poly, // Polyline masking in lat/lon space + Poly_XY, // Polyline masking in grid x/y space - Box, // Box masking type - Circle, // Circle masking region + Box, // Box masking type + Circle, // Circle masking region - Track, // Track masking region - Grid, // Grid masking type - Data, // Data masking type + Track, // Track masking region + Grid, // Grid masking type + Data, // Data masking type - Solar_Alt, // Solar altitude masking type - Solar_Azi, // Solar azimuth masking type + Solar_Alt, // Solar altitude masking type + Solar_Azi, // Solar azimuth masking type + Solar_Time, // Solar time masking type - Lat, // Latitude masking type - Lon, // Longitude masking type + Lat, // Latitude masking type + Lon, // Longitude masking type - Shape, // Shapefile + Shape, // Shapefile None }; extern bool is_solar_masktype(MaskType); +extern bool is_thresh_masktype(MaskType); extern MaskType string_to_masktype(const char *); extern const char * masktype_to_string(MaskType); +extern const char * masktype_to_description(MaskType); //////////////////////////////////////////////////////////////////////// // From 2328400437203c1158c00f5a64b48feaf3ed8dd7 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 4 Nov 2024 20:52:39 +0000 Subject: [PATCH 03/14] Per #2966, add a units attribute to the output NetCDF mask variable. --- src/tools/other/gen_vx_mask/gen_vx_mask.cc | 43 +++++++++++++--------- src/tools/other/gen_vx_mask/gen_vx_mask.h | 1 + 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.cc b/src/tools/other/gen_vx_mask/gen_vx_mask.cc index 61fbc4ede0..126b16c484 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.cc +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.cc @@ -458,6 +458,9 @@ void get_data_plane(const ConcatString &file_name, << mtddf_ptr->filename() << "\" with data ranging from " << dmin << " to " << dmax << ".\n"; + // Store the units string if no threhsold was specified + if(thresh.get_type() == thresh_na) units_cs = vi_ptr->units(); + // Clean up if(vi_ptr) { delete vi_ptr; vi_ptr = (VarInfo *) nullptr; } @@ -853,9 +856,10 @@ void apply_circle_mask(DataPlane &dp) { // Check for no threshold if(thresh.get_type() == thresh_na) { mlog << Debug(3) - << "Since \"-thresh\" was not used to specify a threshold " - << "in kilometers for circle masking, the minimum distance " - << "to the points will be written.\n"; + << "Write the minimum distance in kilometers to the " + << "nearest point for " << masktype_to_description(mask_type) + << " masking since no \"-thresh\" specified.\n"; + units_cs = "km"; } // For each grid point, compute mimumum distance to polyline points @@ -934,9 +938,10 @@ void apply_track_mask(DataPlane &dp) { // Check for no threshold if(thresh.get_type() == thresh_na) { mlog << Debug(3) - << "Since \"-thresh\" was not used to specify a threshold " - << "in kilometers for track masking, the minimum distance " - << "to the track will be written.\n"; + << "Write the minimum distance in kilometers to the " + << "nearest point for " << masktype_to_description(mask_type) + << " masking since no \"-thresh\" specified.\n"; + units_cs = "km"; } // For each grid point, compute mimumum distance to track @@ -1065,14 +1070,14 @@ void apply_data_mask(DataPlane &dp) { // Nothing to do without a threshold if(thresh.get_type() == thresh_na) { mlog << Debug(3) - << "Since \"-thresh\" was not used to specify a threshold " - << "in kilometers for data masking, the raw data values " - << "will be written.\n"; + << "Write the raw inputs values for " + << masktype_to_description(mask_type) + << " masking since no \"-thresh\" specified.\n"; double dmin, dmax; dp.data_range(dmin, dmax); mlog << Debug(3) << masktype_to_description(mask_type) - << " Masking:\tValues ranging from " + << " Masking:\t\tValues ranging from " << dmin << " to " << dmax << "\n"; return; } @@ -1127,9 +1132,11 @@ void apply_solar_mask(DataPlane &dp) { // Check for no threshold if(thresh.get_type() == thresh_na) { mlog << Debug(3) - << "Since \"-thresh\" was not used to specify a threshold, " - << "the raw " << masktype_to_string(mask_type) - << " values will be written.\n"; + << "Write the raw " + << masktype_to_description(mask_type) + << " values since no \"-thresh\" specified.\n"; + units_cs = (mask_type == MaskType::Solar_Time ? + "hr" : "deg"); } // Compute solar value for each grid point Lat/Lon @@ -1205,9 +1212,10 @@ void apply_lat_lon_mask(DataPlane &dp) { // Check for no threshold if(thresh.get_type() == thresh_na) { mlog << Debug(3) - << "Since \"-thresh\" was not used to specify a threshold, " - << "the raw " << masktype_to_string(mask_type) - << " values will be written.\n"; + << "Write the raw " + << masktype_to_description(mask_type) + << " values since no \"-thresh\" specified.\n"; + units_cs = "deg"; } // Compute Lat/Lon value for each grid point @@ -1409,7 +1417,7 @@ void write_netcdf(const DataPlane &dp) { int n; ConcatString cs; - NcFile *f_out = (NcFile *) nullptr; + NcFile *f_out = (NcFile *) nullptr; NcDim lat_dim; NcDim lon_dim; NcVar mask_var; @@ -1455,6 +1463,7 @@ void write_netcdf(const DataPlane &dp) { mask_var = add_var(f_out, string(mask_name), ncFloat, lat_dim, lon_dim, deflate_level); cs << cs_erase << mask_name << " masking region"; add_att(&mask_var, "long_name", string(cs)); + add_att(&mask_var, "units", string(units_cs)); add_att(&mask_var, "_FillValue", bad_data_float); cs << cs_erase << masktype_to_string(mask_type); if(thresh.get_type() != thresh_na) cs << thresh.get_str(); diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.h b/src/tools/other/gen_vx_mask/gen_vx_mask.h index b1fbbf3ef6..e927d5c7a9 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.h +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.h @@ -108,6 +108,7 @@ static int width = bad_data_double; static double mask_val = default_mask_val; static ConcatString mask_name; static unixtime solar_ut = (unixtime) 0; +static ConcatString units_cs("flag"); static std::map shape_str_map; static NumArray shape_numbers; From ea9840d7750532ca0e76f037cf912517c0e502b4 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 4 Nov 2024 21:03:34 +0000 Subject: [PATCH 04/14] Per #2966, modify solar azimuth and altitude strings to make the log messages align well. --- src/tools/other/gen_vx_mask/gen_vx_mask.cc | 28 +++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.cc b/src/tools/other/gen_vx_mask/gen_vx_mask.cc index 126b16c484..dfd7785c14 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.cc +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.cc @@ -1587,20 +1587,20 @@ const char * masktype_to_description(const MaskType t) { const char *s = (const char *) nullptr; switch(t) { - case MaskType::Poly: s = "Polyline"; break; - case MaskType::Poly_XY: s = "Polyline XY"; break; - case MaskType::Box: s = "Box"; break; - case MaskType::Circle: s = "Circle"; break; - case MaskType::Track: s = "Track"; break; - case MaskType::Grid: s = "Grrid"; break; - case MaskType::Data: s = "Data"; break; - case MaskType::Solar_Alt: s = "Solar Altitude"; break; - case MaskType::Solar_Azi: s = "Solar Azimuth"; break; - case MaskType::Solar_Time: s = "Solar Time"; break; - case MaskType::Lat: s = "Latitude"; break; - case MaskType::Lon: s = "Longitude"; break; - case MaskType::Shape: s = "Shapefile"; break; - case MaskType::None: s = na_str; break; + case MaskType::Poly: s = "Polyline"; break; + case MaskType::Poly_XY: s = "Polyline XY"; break; + case MaskType::Box: s = "Box"; break; + case MaskType::Circle: s = "Circle"; break; + case MaskType::Track: s = "Track"; break; + case MaskType::Grid: s = "Grid"; break; + case MaskType::Data: s = "Data"; break; + case MaskType::Solar_Alt: s = "Solar Alt"; break; + case MaskType::Solar_Azi: s = "Solar Azi"; break; + case MaskType::Solar_Time: s = "Solar Time"; break; + case MaskType::Lat: s = "Latitude"; break; + case MaskType::Lon: s = "Longitude"; break; + case MaskType::Shape: s = "Shapefile"; break; + case MaskType::None: s = na_str; break; default: s = (const char *) nullptr; break; } From 0ed3fe2349030136567c821672e50cb43476b4c3 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 4 Nov 2024 21:04:12 +0000 Subject: [PATCH 05/14] Per #2966, add gen_vx_mask unit test to demonstrate the solar_time masking type. --- internal/test_unit/xml/unit_gen_vx_mask.xml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/internal/test_unit/xml/unit_gen_vx_mask.xml b/internal/test_unit/xml/unit_gen_vx_mask.xml index ca66cea64c..24e838961d 100644 --- a/internal/test_unit/xml/unit_gen_vx_mask.xml +++ b/internal/test_unit/xml/unit_gen_vx_mask.xml @@ -438,6 +438,23 @@ + + + + + + &MET_BIN;/gen_vx_mask + \ + 'G004' \ + '20050808_12' \ + &OUTPUT_DIR;/gen_vx_mask/SOLAR_TIME_raw.nc \ + -type solar_time -v 2 + + + &OUTPUT_DIR;/gen_vx_mask/SOLAR_TIME_raw.nc + + + From 2a32e453082233d30d335bb057a4294fc5deaaea Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 4 Nov 2024 21:15:24 +0000 Subject: [PATCH 06/14] Per #2966, add documentation about the -solar_time option --- docs/Users_Guide/masking.rst | 14 +++++++++----- src/tools/other/gen_vx_mask/gen_vx_mask.cc | 4 ++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/Users_Guide/masking.rst b/docs/Users_Guide/masking.rst index 5dd8fe72d8..d4b5744281 100644 --- a/docs/Users_Guide/masking.rst +++ b/docs/Users_Guide/masking.rst @@ -51,7 +51,7 @@ Required Arguments for gen_vx_mask • For "grid" and "data" masking, specify a gridded data file. -• For "solar_alt" and "solar_azi" masking, specify a gridded data file or a time string in YYYYMMDD[_HH[MMSS]] format. +• For "solar_alt", "solar_azi", and "solar_time" masking, specify a gridded data file or a time string in YYYYMMDD[_HH[MMSS]] UTC format. • For "lat" and "lon" masking, no "mask_file" needed, simply repeat the path for "input_file". @@ -78,7 +78,9 @@ Optional Arguments for gen_vx_mask • For "data" masking, threshold the values of "mask_field". -• For "solar_alt" and "solar_azi" masking, threshold the computed solar values. +• For "solar_alt" and "solar_azi" masking, threshold the computed solar degrees. + +• For "solar_time" masking, threshold the decimal solar hours of the day. • For "lat" and "lon" masking, threshold the latitude and longitude values. @@ -118,11 +120,13 @@ The Gen-Vx-Mask tool supports the following types of masking region definition s 7. Data (**data**) masking reads an input gridded data file, extracts the field specified using the **-mask_field** command line option, thresholds the data using the **-thresh** command line option, and selects grid points which meet that threshold criteria. The option is useful when thresholding topography to define a mask based on elevation or when threshold land use to extract a particular category. -8. Solar altitude (**solar_alt**) and solar azimuth (**solar_azi**) masking computes the solar altitude and azimuth values at each grid point for the time defined by the **mask_file** setting. **mask_file** may either be set to an explicit time string in YYYYMMDD[_HH[MMSS]] format or to a gridded data file. If set to a gridded data file, the **-mask_field** command line option specifies the field of data whose valid time should be used. If the **-thresh** command line option is not used, the raw solar altitude or azimuth value for each grid point will be written to the output. If it is used, the resulting binary mask field will be written. This option is useful when defining a day/night mask. +8. Solar altitude (**solar_alt**) and solar azimuth (**solar_azi**) masking computes the solar altitude and azimuth values in degrees at each grid point for the time defined by the **mask_file** setting. **mask_file** may either be set to an explicit time string in YYYYMMDD[_HH[MMSS]] UTC format or to a gridded data file. If set to a gridded data file, the **-mask_field** command line option specifies the field of data whose valid time should be used. If the **-thresh** command line option is not used, the raw solar altitude or azimuth degrees for each grid point will be written to the output. If it is used, the resulting binary mask field will be written. This option is useful when defining a day/night mask. + +9. Solar time (**solar_time**) masking computes the solar time in decimal hours at each grid point for the for the time defined by the **mask_file** setting, as described above. The solar hours of the day range from 0 to 24, with a value of 12 indicating solar noon. Note that solar time is based only on longitude. If the **-thresh** command line option is not used, the raw solar time hours will be written to the output. -9. Latitude (**lat**) and longitude (**lon**) masking computes the latitude and longitude value at each grid point. This logic only requires the definition of the grid, specified by the **input_file**. Technically, the **mask_file** is not needed, but a value must be specified for the command line to parse correctly. Users are advised to simply repeat the **input_file** setting twice. If the **-thresh** command line option is not used, the raw latitude or longitude values for each grid point will be written to the output. This option is useful when defining latitude or longitude bands over which to compute statistics. +10. Latitude (**lat**) and longitude (**lon**) masking computes the latitude and longitude value at each grid point. This logic only requires the definition of the grid, specified by the **input_file**. Technically, the **mask_file** is not needed, but a value must be specified for the command line to parse correctly. Users are advised to simply repeat the **input_file** setting twice. If the **-thresh** command line option is not used, the raw latitude or longitude values for each grid point will be written to the output. This option is useful when defining latitude or longitude bands over which to compute statistics. -10. Shapefile (**shape**) masking uses closed polygons taken from an ESRI shapefile to define the masking region. Gen-Vx-Mask reads the shapefile with the ".shp" suffix and extracts the latitude and longitudes of the vertices. The shapefile must consist of closed polygons rather than polylines, points, or any of the other data types that shapefiles support. When the **-shape_str** command line option is used, Gen-Vx-Mask also reads metadata from the corresponding dBASE file with the ".dbf" suffix. +11. Shapefile (**shape**) masking uses closed polygons taken from an ESRI shapefile to define the masking region. Gen-Vx-Mask reads the shapefile with the ".shp" suffix and extracts the latitude and longitudes of the vertices. The shapefile must consist of closed polygons rather than polylines, points, or any of the other data types that shapefiles support. When the **-shape_str** command line option is used, Gen-Vx-Mask also reads metadata from the corresponding dBASE file with the ".dbf" suffix. Shapefiles usually contain more than one polygon, and the user must select which of these shapes should be used. The **-shapeno n** and **-shape_str name string** command line options enable the user to select one or more polygons from the shapefile. For **-shape n**, **n** is a comma-separated list of integer shape indices to be used. Note that these values are zero-based. So the first polygon in the shapefile is shape number 0, the second polygon in the shapefile is shape number 1, etc. For example, **-shapeno 0,1,2** uses the first three shapes in the shapefile. When multiple shapes are specified, the mask is defined as their union. So all grid points falling inside at least one of the specified shapes are included in the mask. diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.cc b/src/tools/other/gen_vx_mask/gen_vx_mask.cc index dfd7785c14..6fb302e943 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.cc +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.cc @@ -1686,9 +1686,9 @@ void usage() { << "\t\t For \"data\" masking, threshold the values of " << "\"mask_field\".\n" << "\t\t For \"solar_alt\" and \"solar_azi\" masking, " - << "threshold the computed solar values.\n" + << "threshold the solar values in degrees.\n" << "\t\t For \"solar_time\" masking, " - << "threshold the decimal hours of the solar day.\n" + << "threshold the solar time in decimal hours.\n" << "\t\t For \"lat\" and \"lon\" masking, threshold the " << "latitude and longitude values.\n" From 87348010e052e88dadd8bd79d6e58be36307f0ed Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 4 Nov 2024 17:04:02 -0700 Subject: [PATCH 07/14] Per #2966, reduce SonarQube code smells in gen_vx_mask --- src/tools/other/gen_vx_mask/gen_vx_mask.cc | 47 +++++++++++----------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.cc b/src/tools/other/gen_vx_mask/gen_vx_mask.cc index 6fb302e943..3a706a1121 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.cc +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.cc @@ -72,18 +72,20 @@ using namespace netCDF; //////////////////////////////////////////////////////////////////////// int met_main(int argc, char *argv[]) { - static DataPlane dp_data, dp_mask, dp_out; // Process the command line arguments process_command_line(argc, argv); // Process the input grid + static DataPlane dp_data; process_input_grid(dp_data); // Process the mask file + static DataPlane dp_mask; process_mask_file(dp_mask); // Apply combination logic if the current mask is binary + static DataPlane dp_out; if(mask_type == MaskType::Poly || mask_type == MaskType::Poly_XY || mask_type == MaskType::Shape || @@ -105,13 +107,13 @@ int met_main(int argc, char *argv[]) { //////////////////////////////////////////////////////////////////////// -const string get_tool_name() { +string get_tool_name() { return "gen_vx_mask"; } //////////////////////////////////////////////////////////////////////// -void process_command_line(int argc, char **argv) { +static void process_command_line(int argc, char **argv) { CommandLine cline; // Check for zero arguments @@ -160,9 +162,8 @@ void process_command_line(int argc, char **argv) { if(mask_type == MaskType::None) { mlog << Error << "\n" << program_name << " -> " << "the -type command line requirement must be set to a specific masking type!\n" - << "\t\t \"poly\", \"box\", \"circle\", \"track\", \"grid\", " - << "\"data\", \"solar_alt\", \"solar_azi\", \"solar_time\", " - << "\"lat\", \"lon\", or \"shape\"\n\n"; + << "\t\t poly, box, circle, track, grid, data, solar_alt, " + << "solar_azi, solar_time, lat, lon, or shape\n\n"; exit(1); } @@ -176,7 +177,7 @@ void process_command_line(int argc, char **argv) { //////////////////////////////////////////////////////////////////////// -void process_input_grid(DataPlane &dp) { +static void process_input_grid(DataPlane &dp) { if (!build_grid_by_grid_string(input_gridname, grid, "process_input_grid", false)) { // Extract the grid from a gridded data file @@ -203,7 +204,7 @@ void process_input_grid(DataPlane &dp) { //////////////////////////////////////////////////////////////////////// -void process_mask_file(DataPlane &dp) { +static void process_mask_file(DataPlane &dp) { // Initialize solar_ut = (unixtime) 0; @@ -227,7 +228,7 @@ void process_mask_file(DataPlane &dp) { else if(mask_type == MaskType::Shape) { // If -shape_str was specified, find the matching records - if(shape_str_map.size() > 0) get_shapefile_strings(); + if(!shape_str_map.empty()) get_shapefile_strings(); // Get the records specified by -shapeno and -shape_str get_shapefile_records(); @@ -257,9 +258,10 @@ void process_mask_file(DataPlane &dp) { << unix_to_yyyymmdd_hhmmss(solar_ut) << "\n"; } - // Nothing to do for Lat/Lon masking types + // For Lat/Lon masking types else if(mask_type == MaskType::Lat || mask_type == MaskType::Lon) { + // Nothing to do for Lat/Lon masking types } // Otherwise, process the mask file as a named grid, grid specification @@ -393,10 +395,10 @@ void process_mask_file(DataPlane &dp) { //////////////////////////////////////////////////////////////////////// -void get_data_plane(const ConcatString &file_name, - const ConcatString &config_str, - bool read_gen_vx_mask_output, - DataPlane &dp, Grid &dp_grid) { +static void get_data_plane(const ConcatString &file_name, + const ConcatString &config_str, + bool read_gen_vx_mask_output, + DataPlane &dp, Grid &dp_grid) { ConcatString local_cs = config_str; GrdFileType ftype = FileType_None; @@ -404,7 +406,7 @@ void get_data_plane(const ConcatString &file_name, MetConfig local_config = global_config; // Parse non-empty config strings - if(local_cs.length() > 0) { + if(!local_cs.empty()) { local_config.read_string(local_cs.c_str()); ftype = parse_conf_file_type(&local_config); } @@ -420,15 +422,14 @@ void get_data_plane(const ConcatString &file_name, // Read gen_vx_mask output from a previous run if(read_gen_vx_mask_output && - local_cs.length() == 0 && - mtddf_ptr->file_type() == FileType_NcMet) { - if(get_gen_vx_mask_config_str((MetNcMetDataFile *) mtddf_ptr, local_cs)) { - local_config.read_string(local_cs.c_str()); - } + local_cs.empty() && + mtddf_ptr->file_type() == FileType_NcMet && + get_gen_vx_mask_config_str((MetNcMetDataFile *) mtddf_ptr, local_cs)) { + local_config.read_string(local_cs.c_str()); } // Read data plane, if requested - if(local_cs.length() > 0) { + if(!local_cs.empty()) { // Allocate new VarInfo object VarInfo *vi_ptr = VarInfoFactory::new_var_info(mtddf_ptr->file_type()); @@ -477,8 +478,8 @@ void get_data_plane(const ConcatString &file_name, //////////////////////////////////////////////////////////////////////// -bool get_gen_vx_mask_config_str(MetNcMetDataFile *mnmdf_ptr, - ConcatString &config_str) { +static bool get_gen_vx_mask_config_str(MetNcMetDataFile *mnmdf_ptr, + ConcatString &config_str) { bool status = false; ConcatString tool; From e865e98698bec80284dd2c533c300a80d0dc9804 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 5 Nov 2024 15:27:29 +0000 Subject: [PATCH 08/14] Per #2966, reduce SonarQube findings --- src/tools/other/gen_vx_mask/gen_vx_mask.cc | 216 +++++++++++---------- 1 file changed, 113 insertions(+), 103 deletions(-) diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.cc b/src/tools/other/gen_vx_mask/gen_vx_mask.cc index 3a706a1121..ffa7b121cf 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.cc +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.cc @@ -501,8 +501,9 @@ static bool get_gen_vx_mask_config_str(MetNcMetDataFile *mnmdf_ptr, // Read the first non-lat/lon variable config_str << cs_erase - << "'name=\"" << mnmdf_ptr->MetNc->Var[i].name - << "\"; level=\"(*,*)\";'"; + << R"('name=")" + << mnmdf_ptr->MetNc->Var[i].name + << R"_("; level="(*,*)";')_"; status = true; break; } @@ -512,7 +513,7 @@ static bool get_gen_vx_mask_config_str(MetNcMetDataFile *mnmdf_ptr, //////////////////////////////////////////////////////////////////////// -void get_shapefile_strings() { +static void get_shapefile_strings() { DbfFile f; StringArray rec_names; StringArray rec_values; @@ -543,13 +544,10 @@ void get_shapefile_strings() { << ").\n"; // Check that the attributes requested actually exist - map::const_iterator it; - for(it = shape_str_map.begin(); - it != shape_str_map.end(); it++) { - - if(!rec_names.has(it->first)) { + for(const auto& pair: shape_str_map) { + if(!rec_names.has(pair.first)) { mlog << Warning << "\nget_shapefile_strings() -> " - << "the \"-shape_str\" name \"" << it->first + << "the \"-shape_str\" name \"" << pair.first << "\" is not in the list of " << rec_names.n() << " shapefile attributes and will be ignored:\n" << write_css(rec_names) << "\n\n"; @@ -676,7 +674,7 @@ bool is_shape_str_match(const int i_shape, const StringArray &names, const Strin //////////////////////////////////////////////////////////////////////// -void apply_poly_mask(DataPlane & dp) { +static void apply_poly_mask(DataPlane & dp) { int n_in = 0; bool inside; double lat; @@ -721,10 +719,11 @@ void apply_poly_mask(DataPlane & dp) { //////////////////////////////////////////////////////////////////////// -void apply_poly_xy_mask(DataPlane & dp) { +static void apply_poly_xy_mask(DataPlane & dp) { int n_in = 0; bool inside; - double x_dbl, y_dbl; + double x_dbl; + double y_dbl; GridClosedPoly poly_xy; // Convert MaskPoly Lat/Lon coordinates to Grid X/Y @@ -769,15 +768,15 @@ void apply_poly_xy_mask(DataPlane & dp) { //////////////////////////////////////////////////////////////////////// -void apply_box_mask(DataPlane &dp) { - int i, x_ll, y_ll, x, y, n_in; - double cen_x, cen_y; - bool inside; +static void apply_box_mask(DataPlane &dp) { + int n_in = 0; + double cen_x; + double cen_y; // Process the height and width if(is_bad_data(height) && is_bad_data(width)) { mlog << Error << "\napply_box_mask() -> " - << "the \"-height\" and/or \"-width\" options must be " + << R"(the "-height" and/or "-width" options must be )" << "specified in grid units for box masking.\n\n"; exit(1); } @@ -789,19 +788,21 @@ void apply_box_mask(DataPlane &dp) { } // Process each lat/lon point - for(i=0; i= dp.nx()) continue; - for(y=y_ll; y= dp.ny()) continue; // Set the mask @@ -812,16 +813,16 @@ void apply_box_mask(DataPlane &dp) { } // end for i // Loop through the field, handle the complement, and count up points - for(x=0,n_in=0; x= 0 && mask_x < grid_mask.nx() && - mask_y >= 0 && mask_y < grid_mask.ny()); + bool inside = (mask_x >= 0 && mask_x < grid_mask.nx() && + mask_y >= 0 && mask_y < grid_mask.ny()); // Apply the complement if(complement) inside = !inside; // Increment count - n_in += inside; + if(inside) n_in++; // Store the current mask value dp.set(inside, x, y); @@ -1064,17 +1067,18 @@ void apply_grid_mask(DataPlane &dp) { //////////////////////////////////////////////////////////////////////// -void apply_data_mask(DataPlane &dp) { +static void apply_data_mask(DataPlane &dp) { int n_in = 0; - bool check; // Nothing to do without a threshold if(thresh.get_type() == thresh_na) { mlog << Debug(3) << "Write the raw inputs values for " << masktype_to_description(mask_type) - << " masking since no \"-thresh\" specified.\n"; - double dmin, dmax; + << R"( masking since no "-thresh" specified.)" + << "\n"; + double dmin; + double dmax; dp.data_range(dmin, dmax); mlog << Debug(3) << masktype_to_description(mask_type) @@ -1100,13 +1104,13 @@ void apply_data_mask(DataPlane &dp) { for(int y=0; y poly_list; - vector::const_iterator rec_it; - for(rec_it = shape_recs.begin(); - rec_it != shape_recs.end(); ++rec_it) { - poly.set(*rec_it, grid); + for(const auto& rec: shape_recs) { + poly.set(rec, grid); poly_list.push_back(poly); } @@ -1293,12 +1305,10 @@ void apply_shape_mask(DataPlane & dp) { for(int x=0; x<(grid.nx()); x++) { for(int y=0; y<(grid.ny()); y++) { - vector::const_iterator poly_it; - for(poly_it = poly_list.begin(); - poly_it != poly_list.end(); ++poly_it) { + for(const auto& poly: poly_list) { // Check if point is inside - status = poly_it->is_inside(x, y); + status = poly.is_inside(x, y); // Break after the first match if(status) break; @@ -1333,10 +1343,10 @@ void apply_shape_mask(DataPlane & dp) { //////////////////////////////////////////////////////////////////////// -DataPlane combine(const DataPlane &dp_data, const DataPlane &dp_mask, - SetLogic logic) { +static DataPlane combine(const DataPlane &dp_data, + const DataPlane &dp_mask, + SetLogic logic) { int n_in = 0; - bool v_data, v_mask; double v; DataPlane dp; @@ -1363,8 +1373,8 @@ DataPlane combine(const DataPlane &dp_data, const DataPlane &dp_mask, for(int y=0; y " << "the -type command line requirement can only be used once!\n" @@ -1745,79 +1755,79 @@ void set_type(const StringArray & a) { //////////////////////////////////////////////////////////////////////// -void set_input_field(const StringArray & a) { +static void set_input_field(const StringArray & a) { input_field_str = a[0]; } //////////////////////////////////////////////////////////////////////// -void set_mask_field(const StringArray & a) { +static void set_mask_field(const StringArray & a) { mask_field_str = a[0]; } //////////////////////////////////////////////////////////////////////// -void set_complement(const StringArray & a) { +static void set_complement(const StringArray &) { complement = true; } //////////////////////////////////////////////////////////////////////// -void set_union(const StringArray & a) { +static void set_union(const StringArray &) { set_logic = SetLogic::Union; } //////////////////////////////////////////////////////////////////////// -void set_intersection(const StringArray & a) { +static void set_intersection(const StringArray &) { set_logic = SetLogic::Intersection; } //////////////////////////////////////////////////////////////////////// -void set_symdiff(const StringArray & a) { +static void set_symdiff(const StringArray &) { set_logic = SetLogic::SymDiff; } //////////////////////////////////////////////////////////////////////// -void set_thresh(const StringArray & a) { +static void set_thresh(const StringArray & a) { thresh.set(a[0].c_str()); } //////////////////////////////////////////////////////////////////////// -void set_height(const StringArray & a) { +static void set_height(const StringArray & a) { height = atoi(a[0].c_str()); } //////////////////////////////////////////////////////////////////////// -void set_width(const StringArray & a) { +static void set_width(const StringArray & a) { width = atoi(a[0].c_str()); } //////////////////////////////////////////////////////////////////////// -void set_value(const StringArray & a) { +static void set_value(const StringArray & a) { mask_val = atof(a[0].c_str()); } //////////////////////////////////////////////////////////////////////// -void set_name(const StringArray & a) { +static void set_name(const StringArray & a) { mask_name = a[0]; } //////////////////////////////////////////////////////////////////////// -void set_compress(const StringArray & a) { +static void set_compress(const StringArray & a) { compress_level = atoi(a[0].c_str()); } //////////////////////////////////////////////////////////////////////// -void set_shapeno(const StringArray & a) { +static void set_shapeno(const StringArray & a) { NumArray cur_na; cur_na.add_css(a[0].c_str()); @@ -1839,7 +1849,7 @@ void set_shapeno(const StringArray & a) { //////////////////////////////////////////////////////////////////////// -void set_shape_str(const StringArray & a) { +static void set_shape_str(const StringArray & a) { StringArray sa; // Comma-separated list of matching strings, ignoring case From a64930ffb43114e385076fe18bbcd76abb72b325 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 5 Nov 2024 18:25:44 +0000 Subject: [PATCH 09/14] Per #2966, support multiple mask types with the same mask field being supported in a single run. Still need to update the user's guide. --- internal/test_unit/xml/unit_gen_vx_mask.xml | 10 +- src/tools/other/gen_vx_mask/gen_vx_mask.cc | 182 +++++++++++++------- src/tools/other/gen_vx_mask/gen_vx_mask.h | 14 +- 3 files changed, 133 insertions(+), 73 deletions(-) diff --git a/internal/test_unit/xml/unit_gen_vx_mask.xml b/internal/test_unit/xml/unit_gen_vx_mask.xml index 24e838961d..a3dd56a2c0 100644 --- a/internal/test_unit/xml/unit_gen_vx_mask.xml +++ b/internal/test_unit/xml/unit_gen_vx_mask.xml @@ -447,11 +447,15 @@ \ 'G004' \ '20050808_12' \ - &OUTPUT_DIR;/gen_vx_mask/SOLAR_TIME_raw.nc \ - -type solar_time -v 2 + &OUTPUT_DIR;/gen_vx_mask/SOLAR_MIDNIGHT_NH.nc \ + -type solar_time,solar_alt,lat \ + -thresh 'ge21||le3,le0,ge0' \ + -intersection \ + -name SOLAR_MIDNIGHT_NH \ + -v 3 - &OUTPUT_DIR;/gen_vx_mask/SOLAR_TIME_raw.nc + &OUTPUT_DIR;/gen_vx_mask/SOLAR_MIDNIGHT_NH.nc diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.cc b/src/tools/other/gen_vx_mask/gen_vx_mask.cc index ffa7b121cf..55f202d8e2 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.cc +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.cc @@ -77,22 +77,64 @@ int met_main(int argc, char *argv[]) { process_command_line(argc, argv); // Process the input grid - static DataPlane dp_data; - process_input_grid(dp_data); + static DataPlane dp_input; + process_input_grid(dp_input); - // Process the mask file static DataPlane dp_mask; - process_mask_file(dp_mask); - // Apply combination logic if the current mask is binary + // Process each -type setting + for(int i=0; i i) { + mask_field_str = mask_field_opts[i]; + } + else { + mask_field_str.clear(); + } + + // Set the current threshold + if(thresh_opts.n() == 1) { + thresh = thresh_opts[0]; + } + else if(thresh_opts.n() > i) { + thresh = thresh_opts[i]; + } + else { + thresh.clear(); + } + + // Build mask type description string + if(i>0) mask_type_desc_cs << " " << setlogic_to_abbr(set_logic) << " "; + mask_type_desc_cs << masktype_to_string(mask_type); + if(thresh.get_type() != thresh_na) mask_type_desc_cs << thresh.get_str(); + + // Process the mask file + static DataPlane dp_cur; + process_mask_file(dp_cur); + + // Combine with prior masks + if(dp_mask.nxy() == 0) dp_mask = dp_cur; + else dp_mask = combine(dp_mask, dp_cur, set_logic); + + } // end for i + + // Combine the input data with the current binary mask static DataPlane dp_out; - if(mask_type == MaskType::Poly || - mask_type == MaskType::Poly_XY || - mask_type == MaskType::Shape || - mask_type == MaskType::Box || - mask_type == MaskType::Grid || - thresh.get_type() != thresh_na) { - dp_out = combine(dp_data, dp_mask, set_logic); + if(have_input_data && + (mask_type == MaskType::Poly || + mask_type == MaskType::Poly_XY || + mask_type == MaskType::Shape || + mask_type == MaskType::Box || + mask_type == MaskType::Grid || + thresh.get_type() != thresh_na)) { + dp_out = combine(dp_input, dp_mask, set_logic); } // Otherwise, pass through the distance or raw values else { @@ -158,13 +200,13 @@ static void process_command_line(int argc, char **argv) { mask_filename = cline[1]; out_filename = cline[2]; - // Check for the mask type (from -type string) - if(mask_type == MaskType::None) { - mlog << Error << "\n" << program_name << " -> " - << "the -type command line requirement must be set to a specific masking type!\n" - << "\t\t poly, box, circle, track, grid, data, solar_alt, " - << "solar_azi, solar_time, lat, lon, or shape\n\n"; - exit(1); + // Check for at least one mask type + if(mask_type_opts.empty()) { + mlog << Error << "\n" << program_name << " -> " + << "the -type command line option must be used at least once!\n" + << "\t\t poly, box, circle, track, grid, data, solar_alt, " + << "solar_azi, solar_time, lat, lon, or shape\n\n"; + exit(1); } // List the input files @@ -179,7 +221,9 @@ static void process_command_line(int argc, char **argv) { static void process_input_grid(DataPlane &dp) { - if (!build_grid_by_grid_string(input_gridname, grid, "process_input_grid", false)) { + // Read grid string + if(!build_grid_by_grid_string(input_gridname, grid, "process_input_grid", false)) { + // Extract the grid from a gridded data file mlog << Debug(3) << "Use input grid defined by file \"" << input_gridname @@ -191,6 +235,7 @@ static void process_input_grid(DataPlane &dp) { // If not yet set, fill the input data plane with zeros if(dp.is_empty()) { + have_input_data = false; dp.set_size(grid.nx(), grid.ny()); dp.set_constant(0.0); } @@ -309,8 +354,8 @@ static void process_mask_file(DataPlane &dp) { if(mask_field_str.empty()) { mlog << Error << "\nprocess_mask_file() -> " << "use \"-mask_field\" to specify the data whose valid " - << "time should be used for \"solar_alt\" and " - << "\"solar_azi\" masking.\n\n"; + << "time should be used for \"solar_alt\", \"solar_azi\", " + << "and \"solar_time\" masking.\n\n"; exit(1); } solar_ut = dp.valid(); @@ -478,7 +523,7 @@ static void get_data_plane(const ConcatString &file_name, //////////////////////////////////////////////////////////////////////// -static bool get_gen_vx_mask_config_str(MetNcMetDataFile *mnmdf_ptr, +static bool get_gen_vx_mask_config_str(const MetNcMetDataFile *mnmdf_ptr, ConcatString &config_str) { bool status = false; ConcatString tool; @@ -547,7 +592,7 @@ static void get_shapefile_strings() { for(const auto& pair: shape_str_map) { if(!rec_names.has(pair.first)) { mlog << Warning << "\nget_shapefile_strings() -> " - << "the \"-shape_str\" name \"" << pair.first + << R"(the "-shape_str" name ")" << pair.first << "\" is not in the list of " << rec_names.n() << " shapefile attributes and will be ignored:\n" << write_css(rec_names) << "\n\n"; @@ -1296,8 +1341,8 @@ static void apply_shape_mask(DataPlane & dp) { // Load the shapes GridClosedPolyArray poly; vector poly_list; - for(const auto& rec: shape_recs) { - poly.set(rec, grid); + for(const auto& cur_rec: shape_recs) { + poly.set(cur_rec, grid); poly_list.push_back(poly); } @@ -1305,10 +1350,10 @@ static void apply_shape_mask(DataPlane & dp) { for(int x=0; x<(grid.nx()); x++) { for(int y=0; y<(grid.ny()); y++) { - for(const auto& poly: poly_list) { + for(const auto& cur_poly: poly_list) { // Check if point is inside - status = poly.is_inside(x, y); + status = cur_poly.is_inside(x, y); // Break after the first match if(status) break; @@ -1463,7 +1508,11 @@ static void write_netcdf(const DataPlane &dp) { mask_name = poly_mask.name(); } else { - mask_name << masktype_to_string(mask_type) << "_mask"; + for(int i=0; i0) mask_name << "_"; + mask_name << masktype_to_string(mask_type_opts[i]); + } + mask_name << "_mask"; } } @@ -1476,9 +1525,7 @@ static void write_netcdf(const DataPlane &dp) { add_att(&mask_var, "long_name", string(cs)); add_att(&mask_var, "units", string(units_cs)); add_att(&mask_var, "_FillValue", bad_data_float); - cs << cs_erase << masktype_to_string(mask_type); - if(thresh.get_type() != thresh_na) cs << thresh.get_str(); - add_att(&mask_var, "mask_type", string(cs)); + add_att(&mask_var, "mask_type", mask_type_desc_cs); // Write the solar time if(is_solar_masktype(mask_type)) { @@ -1620,7 +1667,7 @@ const char * masktype_to_description(const MaskType t) { //////////////////////////////////////////////////////////////////////// -static void usage() { +__attribute__((noreturn)) static void usage() { cout << "\n*** Model Evaluation Tools (MET" << met_version << ") ***\n\n" @@ -1658,11 +1705,11 @@ static void usage() { << "\t\t For \"grid\" masking, specify a named grid, the " << "path to a gridded data file, or an explicit grid " << "specification.\n" - << "\t\t For \"data\" masking specify a gridded data file.\n" - << "\t\t For \"solar_alt\" and \"solar_azi\" masking, " - << "specify a gridded data file or a timestring in " + << "\t\t For \"data\" masking, specify a gridded data file.\n" + << "\t\t For \"solar_alt\", \"solar_azi\", and \"solar_time\" " + << "masking, specify a gridded data file or a timestring in " << "YYYYMMDD[_HH[MMSS]] format.\n" - << "\t\t For \"lat\" and \"lon\" masking, no \"mask_file\" " + << "\t\t For \"lat\" and \"lon\" masking, no \"mask_file\" is " << "needed, simply repeat \"input_grid\".\n" << "\t\t For \"shape\" masking, specify a shapefile " << "(suffix \".shp\").\n" @@ -1670,51 +1717,55 @@ static void usage() { << "\t\t\"out_file\" is the output NetCDF mask file to be " << "written (required).\n" - << "\t\t\"-type string\" specify the masking type " + << "\t\t\"-type string\" is a comma-separated list of masking types " << "(required).\n" << "\t\t \"poly\", \"poly_xy\", \"box\", \"circle\", \"track\", " - << "\"grid\", \"data\", \"solar_alt\", \"solar_azi\", \"lat\", " - << "\"lon\" or \"shape\"\n" + << "\"grid\", \"data\", \"solar_alt\", \"solar_azi\", \"solar_time\", " + << "\"lat\", \"lon\" or \"shape\"\n" + << "\t\t Use multiple times for multiple mask types.\n" - << "\t\t\"-input_field string\" reads existing mask data from " - << "the \"input_grid\" gridded data file (optional).\n" + << "\t\t\"-input_field string\" initializes the \"input_grid\" with " + << "values from this field (optional).\n" << "\t\t\"-mask_field string\" (optional).\n" << "\t\t For \"data\" masking, define the field from " << "\"mask_file\" to be used.\n" + << "\t\t Use multiple times for multiple mask types.\n" << "\t\t\"-complement\" computes the complement of the current " << "mask (optional).\n" << "\t\t\"-union | -intersection | -symdiff\" specify how " - << "to combine the \"input_field\" data with the current mask " + << "to combine the \"input_field\" with the current mask " << "(optional).\n" - << "\t\t\"-thresh string\" defines the threshold to be applied " - << "(optional).\n" + << "\t\t\"-thresh string\" is a comma-separated list of thresholds " + << "to be applied (optional).\n" << "\t\t For \"circle\" and \"track\" masking, threshold the " << "distance (km).\n" << "\t\t For \"data\" masking, threshold the values of " << "\"mask_field\".\n" << "\t\t For \"solar_alt\" and \"solar_azi\" masking, " - << "threshold the solar values in degrees.\n" + << "threshold the solar values (deg).\n" << "\t\t For \"solar_time\" masking, " - << "threshold the solar time in decimal hours.\n" + << "threshold the solar time (hr).\n" << "\t\t For \"lat\" and \"lon\" masking, threshold the " - << "latitude and longitude values.\n" + << "latitude and longitude values (deg).\n" + << "\t\t Use multiple times for multiple mask types.\n" << "\t\t\"-height n\" and \"-width n\" (optional).\n" - << "\t\t For \"box\" masking, specify these dimensions in grid " - << "units.\n" + << "\t\t For \"box\" masking, specify the dimensions (grid " + << "units).\n" << "\t\t\"-shapeno n\" (optional).\n" - << "\t\t For \"shape\" masking, specify the integer shape " - << "number(s) (0-based) to be used as a comma-separated list.\n" + << "\t\t For \"shape\" masking, specify a comma-separated list " + << "of 0-based integer shape number(s).\n" << "\t\t\"-shape_str name string\" (optional).\n" << "\t\t For \"shape\" masking, specify the shape(s) to be used " << "as a named attribute followed by a comma-separated list of " - << "matching strings. If used multiple times, only shapes matching " + << "matching strings.\n" + << "\t\t If used multiple times, only shapes matching " << "all named attributes will be used.\n" << "\t\t\"-value n\" overrides the default output mask data " @@ -1723,7 +1774,7 @@ static void usage() { << "\t\t\"-name string\" specifies the output variable name " << "for the mask (optional).\n" - << "\t\t\"-log file\" outputs log messages to the specified " + << "\t\t\"-log file\" writes log messages to the specified " << "file (optional).\n" << "\t\t\"-v level\" overrides the default level of logging (" @@ -1741,16 +1792,11 @@ static void usage() { //////////////////////////////////////////////////////////////////////// static void set_type(const StringArray & a) { - if(type_is_set) { - mlog << Error << "\n" << program_name << " -> " - << "the -type command line requirement can only be used once!\n" - << "To apply multiple masks, run this tool multiple times " - << "using the output of one run as the input to the next." - << "\n\n"; - exit(1); + StringArray sa; + sa.parse_css(a[0]); + for(int i=0; i mask_type_opts; +static MaskType mask_type; +static ConcatString mask_type_desc_cs; // Optional arguments -static ConcatString input_field_str, mask_field_str; +static bool have_input_data = true; +static ConcatString input_field_str; +static StringArray mask_field_opts; +static ConcatString mask_field_str; static SetLogic set_logic = SetLogic::None; static bool complement = false; +static ThreshArray thresh_opts; static SingleThresh thresh; static int height = bad_data_double; static int width = bad_data_double; @@ -132,7 +136,7 @@ static void process_mask_file(DataPlane &dp); static void get_data_plane(const ConcatString &file_name, const ConcatString &config_str, bool, DataPlane &dp, Grid &dp_grid); -static bool get_gen_vx_mask_config_str(MetNcMetDataFile *, +static bool get_gen_vx_mask_config_str(const MetNcMetDataFile *, ConcatString &); static void get_shapefile_strings(); static void get_shapefile_records(); From 42ff3cf8c0615296d03b92e5dc9ce7f48c113438 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 5 Nov 2024 21:38:53 +0000 Subject: [PATCH 10/14] Per #2966, add UTC --- src/tools/other/gen_vx_mask/gen_vx_mask.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.cc b/src/tools/other/gen_vx_mask/gen_vx_mask.cc index 55f202d8e2..29df66bb01 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.cc +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.cc @@ -1708,7 +1708,7 @@ __attribute__((noreturn)) static void usage() { << "\t\t For \"data\" masking, specify a gridded data file.\n" << "\t\t For \"solar_alt\", \"solar_azi\", and \"solar_time\" " << "masking, specify a gridded data file or a timestring in " - << "YYYYMMDD[_HH[MMSS]] format.\n" + << "YYYYMMDD[_HH[MMSS]] UTC format.\n" << "\t\t For \"lat\" and \"lon\" masking, no \"mask_file\" is " << "needed, simply repeat \"input_grid\".\n" << "\t\t For \"shape\" masking, specify a shapefile " From b1268a88a033cdc680eae427d6144bc3d1c5c084 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 5 Nov 2024 14:45:12 -0700 Subject: [PATCH 11/14] Per #2966, update gen_vx_mask docs about supporting multiple -type options in a single run --- docs/Users_Guide/masking.rst | 113 +++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 44 deletions(-) diff --git a/docs/Users_Guide/masking.rst b/docs/Users_Guide/masking.rst index d4b5744281..41c12c89b2 100644 --- a/docs/Users_Guide/masking.rst +++ b/docs/Users_Guide/masking.rst @@ -1,15 +1,15 @@ .. _masking: -******************************************* -Regional Verification using Spatial Masking -******************************************* +*************** +Spatial Masking +*************** -Verification over a particular region or area of interest may be performed using "masking". Defining a masking region is simply selecting the desired set of grid points to be used. The Gen-Vx-Mask tool automates this process and replaces the Gen-Poly-Mask and Gen-Circle-Mask tools from previous releases. It may be run to create a bitmap verification masking region to be used by many of the statistical tools. This tool enables the user to generate a masking region once for a domain and apply it to many cases. It has been enhanced to support additional types of masking region definition (e.g. tropical-cyclone track over water only). An iterative approach may be used to define complex areas by combining multiple masking regions together. +Verification over a particular region or area of interest may be performed using "masking". Defining a masking region is simply selecting the desired set of grid points to be used. The Gen-Vx-Mask tool automates this process and replaces the Gen-Poly-Mask and Gen-Circle-Mask tools from previous releases. It may be run to create a bitmap verification masking region to be used by many of the statistical tools. This tool enables the user to generate a masking region once for a domain and apply it to many cases. It supports multiple methods for defining regional spatial masks, as described below. In addition, Gen-Vx-Mask can be run iteratively, passing the output from one run as input to the next, to combine multiple masking regions and define a complex area of interest. Gen-Vx-Mask Tool ================ -The Gen-Vx-Mask tool may be run to create a bitmap verification masking region to be used by the MET statistics tools. This tool enables the user to generate a masking region once for a domain and apply it to many cases. While the MET statistics tools can define some masking regions on the fly using polylines, doing so can be slow, especially for complex polylines containing hundreds of vertices. Using the Gen-Vx-Mask tool to create a bitmap masking region before running the other MET tools will make them run more efficiently. +The Gen-Vx-Mask tool may be run to create a bitmap verification masking region to be used by the MET statistics tools. This tool enables the user to generate a masking region once for a domain and apply it to many cases. While the MET statistics tools can define some masking regions on the fly using pre-defined grids and polylines, doing so can be slow, especially for complex polylines containing hundreds of vertices. Using the Gen-Vx-Mask tool to create a bitmap masking region before running the other MET tools will make them run more efficiently. gen_vx_mask Usage ----------------- @@ -38,63 +38,65 @@ The usage statement for the Gen-Vx-Mask tool is shown below: [-v level] [-compress level] -gen_vx_mask has four required arguments and can take optional ones. Note that **-type string** (masking type) was previously optional but is now required. +gen_vx_mask has four required arguments and can take optional ones. Note that **-type string** (masking type) was optional in prior versions but is now required. Required Arguments for gen_vx_mask ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -1. The **input_grid** argument is a named grid, the path to a gridded data file, or an explicit grid specification string (see :numref:`App_B-grid_specification_strings`) which defines the grid for which a mask is to be defined. If set to a gen_vx_mask output file, automatically read mask data as the **input_field**. +1. The **input_grid** is a named grid, the path to a gridded data file, or an explicit grid specification string (see :numref:`App_B-grid_specification_strings`) which defines the grid for which a mask is to be defined. If set to a gen_vx_mask output file, automatically read mask data as the **input_field**. -2. The **mask_file** argument defines the masking information, see below. +2. The **mask_file** defines the masking information, see below. • For "poly", "poly_xy", "box", "circle", and "track" masking, specify an ASCII Lat/Lon file. Refer to :ref:`Types_of_masking_gen_vx_mask` for details on how to construct the ASCII Lat/Lon file for each type of mask. -• For "grid" and "data" masking, specify a gridded data file. +• For "grid" masking, specify a named grid, the path to a gridded data file, or an explicit grid specification. + +• For "data" masking, specify a gridded data file. • For "solar_alt", "solar_azi", and "solar_time" masking, specify a gridded data file or a time string in YYYYMMDD[_HH[MMSS]] UTC format. -• For "lat" and "lon" masking, no "mask_file" needed, simply repeat the path for "input_file". +• For "lat" and "lon" masking, no "mask_file" is needed, simply repeat "input_grid". + +• For "shape" masking, specify a shapefile (suffix ".shp"). -• For "shape" masking, specify an ESRI shapefile (.shp). +3. The **out_file** is the output NetCDF mask file to be written. -3. The **out_file** argument is the output NetCDF mask file to be written. +4. The **-type string** is a comma-separated list of masking types to be applied. The application will print an error message and exit if "-type string" is not specified at least once on the command line. Use multiple times for multiple mask types. See the description of supported types below. -4. The **-type string** is required to set the masking type. The application will give an error message and exit if "-type string" is not specified on the command line. See the description of supported types below. - Optional Arguments for gen_vx_mask ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -5. The **-input_field string** option can be used to read existing mask data from "input_file". +5. The **-input_field string** option initializes the "input_grid" with values from this field. -6. The **-mask_field string** option can be used to define the field from "mask_file" to be used for "data" masking. +6. The **-mask_field string** option defines the field from "mask_file" to be used for "data" masking. Use multiple times for multiple mask types. 7. The **-complement** option can be used to compute the complement of the area defined by "mask_file". -8. The **-union | -intersection | -symdiff** option can be used to specify how to combine the masks from "input_file" and "mask_file". +8. The **-union | -intersection | -symdiff** options specify how to combine the "input_field" with the current mask. -9. The **-thresh string** option can be used to define the threshold to be applied. +9. The **-thresh string** option is a comma-separated list of thresholds to be applied. Use multiple times for multiple mask types. • For "circle" and "track" masking, threshold the distance (km). • For "data" masking, threshold the values of "mask_field". -• For "solar_alt" and "solar_azi" masking, threshold the computed solar degrees. +• For "solar_alt" and "solar_azi" masking, threshold the computed solar values (deg). -• For "solar_time" masking, threshold the decimal solar hours of the day. +• For "solar_time" masking, threshold the solar time (hr). -• For "lat" and "lon" masking, threshold the latitude and longitude values. +• For "lat" and "lon" masking, threshold the latitude and longitude values (deg). -10. The **-height n** and **-width n** options set the size in grid units for "box" masking. +10. The **-height n** and **-width n** options specify the dimensions in grid units for "box" masking. 11. The **-shapeno n** option is only used for shapefile masking. See the description of shapefile masking below. 12. The **-shape_str name string** option is only used for shapefile masking. See the description of shapefile masking below. -13. The **-value n** option can be used to override the default output mask data value (1). +13. The **-value n** option overrides the default output mask data value (1). -14. The **-name string** option can be used to specify the output variable name for the mask. +14. The **-name string** option specifies the output variable name for the mask. -15. The **-log file** option directs output and errors to the specified log file. All messages will be written to that file as well as standard out and error. Thus, users can save the messages without having to redirect the output on the command line. The default behavior is no log file. +15. The **-log file** option writes log messages to the specified log file. All messages will be written to that file as well as standard out and error. Thus, users can save the messages without having to redirect the output on the command line. The default behavior is no log file. 16. The **-v level** option indicates the desired level of verbosity. The value of "level" will override the default setting of 2. Setting the verbosity to 0 will make the tool run with no log messages, while increasing the verbosity will increase the amount of logging. @@ -136,48 +138,71 @@ The polyline, polyline XY, box, circle, and track masking methods all read an AS The Gen-Vx-Mask tool performs three main steps, described below. -1. Determine the **input_field** and grid definition. +1. Determine the input grid definition. -• Read the **input_file** to determine the grid over which the mask should be defined. +• Read the **input_grid** to determine the grid over which the mask should be defined. -• By default, initialize the **input_field** at each grid point to a value of zero. +• By default, initialize the input field value at each grid point to zero. -• If the **-input_field** option was specified, initialize the **input_field** at each grid point to the value of that field. +• If the **-input_field** option was specified, initialize each input field value using the values from that field. -• If the **input_file** is the output from a previous run of Gen-Vx-Mask, automatically initialize each grid point with the **input_field** value. +• If the **input_grid** is the output from a previous run of Gen-Vx-Mask, automatically initialize each input field value with the previously-generated mask value. -2. Determine the **mask_field**. +2. Determine the current masking region. -• Read the **mask_file**, process it based on the **-type** setting (as described above), and define the **mask_field** value for each grid point to specify whether or not it is included in the mask. +• For each **-type** mask type option requested, process the **mask_file** setting. + +• Read the **mask_file**, process it based on the **-type** setting (as described above), and define the masking region value for each grid point to specify whether or not it is included in the mask. • By default, store the mask value as 1 unless the **-value** option was specified to override that default value. -• If the **-complement** option was specified, the opposite of the masking area is selected. +• If the **-complement** option was specified, select the opposite of the masking area. + +• Apply logic to combine the newly generated masking region with those defined by previous **-type** mask type options to create a **mask_field**. -3. Apply logic to combine the **input_field** and **mask_field** and write the **out_file**. +3. Apply logic to combine the input field and current masking region and write the **out_file**. -• By default, the output value at each grid point is set to the value of **mask_field** if included in the mask, or the value of **input_field** if not included. +• By default, the output value at each grid point is set to the value of current masking region if included in the mask, or the value of **input_field** if not included. -• If the **-union, -intersection**, or **-symdiff** option was specified, apply that logic to the **input_field** and **mask_field** values at each grid point to determine the output value. +• If the **-union, -intersection**, or **-symdiff** option was specified, apply that logic to the input field and current masking region values at each grid point to determine the output value. • Write the output value for each grid point to the **out_file**. -This three step process enables the Gen-Vx-Mask tool to be run iteratively on its own output to generate complex masking areas. Additionally, the **-union, -intersection**, and **-symdiff** options control the logic for combining the input data value and current mask value at each grid point. For example, one could define a complex masking region by selecting grid points with an elevation greater than 1000 meters within a specified geographic region by doing the following: +.. note:: -• Run the Gen-Vx-Mask tool to apply data masking by thresholding a field of topography greater than 1000 meters. + While multiple **-type** mask types can be requested in a single run, all requested masking types must use the same **mask_file** setting. -• Rerun the Gen-Vx-Mask tool passing in the output of the first call and applying polyline masking to define the geographic area of interest. - - Use the **-intersection** option to only select grid points whose value is non-zero in both the input field and the current mask. +An example of defining the northwest hemisphere of the earth, as defined by latitudes >= 0 and longitudes < 0, in a single run is shown below: + +.. code-block:: none + + gen_vx_mask G004 G004 northwest_hemisphere.nc \ + -type lat,lon -thresh ge0,lt0 \ + -intersection -name nw_hemisphere -An example of the gen_vx_mask calling sequence is shown below: + +The Gen-Vx-Mask tool to be run iteratively on its own output using different **mask_file** settings to generate complex masking areas. The **-union, -intersection**, and **-symdiff** options control the logic for combining the input field and current mask values at each grid point. For example, one could define a complex masking region by selecting grid points with an elevation greater than 1000 meters within a Contiguous United States geographic region by doing the following: + +• Run Gen-Vx-Mask to apply data masking by thresholding a field of topography greater than 1000 meters. + +• Run Gen-Vx-Mask a second time on the output from the first call and applying polyline masking to define the geographic area of interest. Use the **-intersection** option to only select grid points whose value is non-zero in both the input field and the current mask. + +An example of this Gen-Vx-Mask calling sequence is shown below: .. code-block:: none - gen_vx_mask sample_fcst.grb \ - CONUS.poly CONUS_poly.nc + gen_vx_mask fcst.grib fcst.grib TOPO_mask.nc \ + -type data \ + -mask_field 'name="TOPO"; level="L0";' \ + -thresh '>1000' + + gen_vx_mask TOPO_mask.nc CONUS.poly TOPO_CONUS_mask.nc \ + -type poly \ + -intersection -name TOPO_CONUS_mask + -In this example, the Gen-Vx-Mask tool will read the ASCII Lat/Lon file named **CONUS.poly** and apply the default polyline masking method to the domain on which the data in the file **sample_fcst.grib** resides. It will create a NetCDF file containing a bitmap for the domain with a value of 1 for all grid points inside the CONUS polyline and a value of 0 for all grid points outside. It will write an output NetCDF file named **CONUS_poly.nc**. +Here, Gen-Vx-Mask uses the **data** masking type to read topography data (**TOPO**) from a GRIB file and thresholds the values **>1000** to define a topography mask. The second run of Gen-Vx-Mask uses the **poly** masking type to read the ASCII Lat/Lon file named **CONUS.poly** and select all grid points within that region to define a polyline mask. When reading its own output, Gen-Vx-Mask automatically reads the topography mask as the **input_field** and applies the **intersection** logic to combine it with the polyline mask, selecting grid points where both conditions are true. The resulting complex mask is written to the output NetCDF file named **TOPO_CONUS_mask.nc**. Feature-Relative Methods ======================== From 4ff2c26e69a67d93c6c08d4ad560c24be586750e Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 6 Nov 2024 19:39:27 +0000 Subject: [PATCH 12/14] Per #2966, update logic to fix using data masking twice, add a unit test to demonstrate, and update the mask_type attribute to include the magic string for the gridded data used for data masking. --- internal/test_unit/xml/unit_gen_vx_mask.xml | 21 +++++++++++++++++++++ src/tools/other/gen_vx_mask/gen_vx_mask.cc | 17 ++++++++++++----- src/tools/other/gen_vx_mask/gen_vx_mask.h | 4 +++- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/internal/test_unit/xml/unit_gen_vx_mask.xml b/internal/test_unit/xml/unit_gen_vx_mask.xml index a3dd56a2c0..5128485162 100644 --- a/internal/test_unit/xml/unit_gen_vx_mask.xml +++ b/internal/test_unit/xml/unit_gen_vx_mask.xml @@ -493,6 +493,27 @@ + + + + + + &MET_BIN;/gen_vx_mask + \ + &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F012.grib2 \ + &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F012.grib2 \ + &OUTPUT_DIR;/gen_vx_mask/DATA_DATA_LAT_LON_mask.nc \ + -type data,data,lat,lon \ + -mask_field 'name="LAND"; level="L0";' \ + -mask_field 'name="TMP"; level="L0";' \ + -thresh eq1,lt273,gt0,lt0 \ + -intersection -v 5 + + + &OUTPUT_DIR;/gen_vx_mask/DATA_DATA_LAT_LON_mask.nc + + + diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.cc b/src/tools/other/gen_vx_mask/gen_vx_mask.cc index 29df66bb01..f22d57fea1 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.cc +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.cc @@ -85,10 +85,13 @@ int met_main(int argc, char *argv[]) { // Process each -type setting for(int i=0; i0) mask_type_desc_cs << " " << setlogic_to_abbr(set_logic) << " "; mask_type_desc_cs << masktype_to_string(mask_type); + if(mask_type == MaskType::Data) mask_type_desc_cs << "(" << data_desc_cs << ")"; if(thresh.get_type() != thresh_na) mask_type_desc_cs << thresh.get_str(); - // Process the mask file - static DataPlane dp_cur; - process_mask_file(dp_cur); - // Combine with prior masks if(dp_mask.nxy() == 0) dp_mask = dp_cur; else dp_mask = combine(dp_mask, dp_cur, set_logic); @@ -507,6 +511,9 @@ static void get_data_plane(const ConcatString &file_name, // Store the units string if no threhsold was specified if(thresh.get_type() == thresh_na) units_cs = vi_ptr->units(); + // Store the description of the data + data_desc_cs = vi_ptr->magic_str(); + // Clean up if(vi_ptr) { delete vi_ptr; vi_ptr = (VarInfo *) nullptr; } diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.h b/src/tools/other/gen_vx_mask/gen_vx_mask.h index 193a9dbdf6..be559c27dc 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.h +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.h @@ -103,6 +103,7 @@ static bool have_input_data = true; static ConcatString input_field_str; static StringArray mask_field_opts; static ConcatString mask_field_str; +static ConcatString data_desc_cs; static SetLogic set_logic = SetLogic::None; static bool complement = false; static ThreshArray thresh_opts; @@ -122,7 +123,8 @@ static std::vector shape_recs; static MaskPoly poly_mask; // Grid on which the data field resides -static Grid grid, grid_mask; +static Grid grid; +static Grid grid_mask; // Configuration object for reading config strings static MetConfig global_config; From 0177be501f624def2e158caeb13f090ef8940210 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 6 Nov 2024 22:36:09 +0000 Subject: [PATCH 13/14] Per #2966, adjust the logic slightly to revert to existing behavior where we only write the timing information of the input data to the gen_vx_mask output when no threshold was applied. This should reduce the number of diffs flagged by PR #3008 --- src/tools/other/gen_vx_mask/gen_vx_mask.cc | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.cc b/src/tools/other/gen_vx_mask/gen_vx_mask.cc index f22d57fea1..b74133ea1e 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.cc +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.cc @@ -131,14 +131,18 @@ int met_main(int argc, char *argv[]) { // Combine the input data with the current binary mask static DataPlane dp_out; - if(have_input_data && - (mask_type == MaskType::Poly || - mask_type == MaskType::Poly_XY || - mask_type == MaskType::Shape || - mask_type == MaskType::Box || - mask_type == MaskType::Grid || - thresh.get_type() != thresh_na)) { - dp_out = combine(dp_input, dp_mask, set_logic); + if(mask_type == MaskType::Poly || + mask_type == MaskType::Poly_XY || + mask_type == MaskType::Shape || + mask_type == MaskType::Box || + mask_type == MaskType::Grid || + thresh.get_type() != thresh_na) { + + // Combination logic based on presence of input data + SetLogic logic = (have_input_data ? + set_logic : SetLogic::None); + + dp_out = combine(dp_input, dp_mask, logic); } // Otherwise, pass through the distance or raw values else { From f016f5d741dbc931e2f9c8314c18102c24d2c00e Mon Sep 17 00:00:00 2001 From: MET Tools Test Account Date: Fri, 22 Nov 2024 18:17:26 +0000 Subject: [PATCH 14/14] Per #2966, update details about the -union, -intersection, and -symdiff options in the usage statement and documentation as recommended by @CPKalb. --- docs/Users_Guide/masking.rst | 21 ++++++++++++++------- src/tools/other/gen_vx_mask/gen_vx_mask.cc | 7 ++++--- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/Users_Guide/masking.rst b/docs/Users_Guide/masking.rst index 41c12c89b2..9d75704820 100644 --- a/docs/Users_Guide/masking.rst +++ b/docs/Users_Guide/masking.rst @@ -61,7 +61,11 @@ Required Arguments for gen_vx_mask 3. The **out_file** is the output NetCDF mask file to be written. -4. The **-type string** is a comma-separated list of masking types to be applied. The application will print an error message and exit if "-type string" is not specified at least once on the command line. Use multiple times for multiple mask types. See the description of supported types below. +4. The **-type string** is a comma-separated list of masking types to be applied. The application will print an error message and exit if "-type string" is not specified at least once on the command line. Use multiple times for multiple mask types. See a list of supported masking types described below. + +.. note:: + + While multiple **-type** mask types can be requested in a single run, all requested masking types must use the same **mask_file** setting. Optional Arguments for gen_vx_mask ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -72,7 +76,7 @@ Optional Arguments for gen_vx_mask 7. The **-complement** option can be used to compute the complement of the area defined by "mask_file". -8. The **-union | -intersection | -symdiff** options specify how to combine the "input_field" with the current mask. +8. The **-union | -intersection | -symdiff** options specify how to combine multiple binary masks. Applies to masks read from the "input_field" and those generated during the current run. 9. The **-thresh string** option is a comma-separated list of thresholds to be applied. Use multiple times for multiple mask types. @@ -136,6 +140,9 @@ The Gen-Vx-Mask tool supports the following types of masking region definition s The polyline, polyline XY, box, circle, and track masking methods all read an ASCII file containing Lat/Lon locations. Those files must contain a string, which defines the name of the masking region, followed by a series of whitespace-separated latitude (degrees north) and longitude (degree east) values. +Logic for gen_vx_mask +^^^^^^^^^^^^^^^^^^^^^ + The Gen-Vx-Mask tool performs three main steps, described below. 1. Determine the input grid definition. @@ -148,7 +155,7 @@ The Gen-Vx-Mask tool performs three main steps, described below. • If the **input_grid** is the output from a previous run of Gen-Vx-Mask, automatically initialize each input field value with the previously-generated mask value. -2. Determine the current masking region. +2. Process each of the requested masking regions. • For each **-type** mask type option requested, process the **mask_file** setting. @@ -160,6 +167,8 @@ The Gen-Vx-Mask tool performs three main steps, described below. • Apply logic to combine the newly generated masking region with those defined by previous **-type** mask type options to create a **mask_field**. + • By default, compute the **-union** of multiple masks, unless **-intersection** or **-symdiff** were specified to override this default. + 3. Apply logic to combine the input field and current masking region and write the **out_file**. • By default, the output value at each grid point is set to the value of current masking region if included in the mask, or the value of **input_field** if not included. @@ -168,10 +177,8 @@ The Gen-Vx-Mask tool performs three main steps, described below. • Write the output value for each grid point to the **out_file**. -.. note:: - - While multiple **-type** mask types can be requested in a single run, all requested masking types must use the same **mask_file** setting. - +Examples for gen_vx_mask +^^^^^^^^^^^^^^^^^^^^^^^^ An example of defining the northwest hemisphere of the earth, as defined by latitudes >= 0 and longitudes < 0, in a single run is shown below: diff --git a/src/tools/other/gen_vx_mask/gen_vx_mask.cc b/src/tools/other/gen_vx_mask/gen_vx_mask.cc index b74133ea1e..e2d2580436 100644 --- a/src/tools/other/gen_vx_mask/gen_vx_mask.cc +++ b/src/tools/other/gen_vx_mask/gen_vx_mask.cc @@ -1746,9 +1746,10 @@ __attribute__((noreturn)) static void usage() { << "\t\t\"-complement\" computes the complement of the current " << "mask (optional).\n" - << "\t\t\"-union | -intersection | -symdiff\" specify how " - << "to combine the \"input_field\" with the current mask " - << "(optional).\n" + << "\t\t\"-union | -intersection | -symdiff\" specify how to combine " + << "multiple binary masks (optional).\n" + << "\t\t Applies to masks read from the \"input_field\" and those " + << "generated during the current run.\n" << "\t\t\"-thresh string\" is a comma-separated list of thresholds " << "to be applied (optional).\n"