From 536a7a0420e21a044b63fb1fb164800b1fb24dd8 Mon Sep 17 00:00:00 2001 From: akrherz Date: Wed, 29 Jan 2025 08:35:59 -0600 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=8F=EF=B8=8F=20Fix=20label=20typo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pylib/iemweb/autoplot/scripts/p85.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylib/iemweb/autoplot/scripts/p85.py b/pylib/iemweb/autoplot/scripts/p85.py index 6f36bfcd8..b10c69faa 100644 --- a/pylib/iemweb/autoplot/scripts/p85.py +++ b/pylib/iemweb/autoplot/scripts/p85.py @@ -179,7 +179,7 @@ def plotter(ctx: dict): if ctx["scale"] == "100": ax.set_xlim(0, 100) ax.set_xticks([0, 5, 25, 50, 75, 95, 100]) - ax.set_xlabel(f"Frequency [%s] (Hour Timezone: {tzname})") + ax.set_xlabel(f"Frequency [%] (Hour Timezone: {tzname})") ax.set_ylim(-0.5, 23.5) return fig, df From 11dddb7bb047c39f2b3b490fe44f74a1bcae98f9 Mon Sep 17 00:00:00 2001 From: akrherz Date: Wed, 29 Jan 2025 10:10:20 -0600 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=9A=A7=20Whack-a-mole=20some=20CoCoRa?= =?UTF-8?q?HS=20issues=20found?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- htdocs/cocorahs/current.phtml | 176 ++++++++---------- htdocs/cocorahs/obs.phtml | 17 +- htdocs/request/download.phtml | 2 +- htdocs/sites/networks.php | 4 +- include/imagemaps.php | 21 ++- pylib/iemweb/autoplot/scripts200/p207.py | 128 +++++++------ .../cocorahs/cocorahs_data_ingest.py | 5 +- .../ingestors/cocorahs/cocorahs_stations.py | 7 +- 8 files changed, 179 insertions(+), 181 deletions(-) diff --git a/htdocs/cocorahs/current.phtml b/htdocs/cocorahs/current.phtml index b98e55138..90f64ddc7 100644 --- a/htdocs/cocorahs/current.phtml +++ b/htdocs/cocorahs/current.phtml @@ -1,59 +1,38 @@ -'; -$mesosite = iemdb("mesosite"); -$rs = pg_query( - $mesosite, - "SELECT id, name from networks where id ~* '_COCORAHS' ORDER by name ASC"); -for ($i = 0; $row = pg_fetch_assoc($rs); $i++) { - $sel = ''; - if ($network == $row["id"]) { - $sel = " selected='SELECTED'"; - } - $nselect .= sprintf( - "\n", - $row["id"], - $sel, - $row["name"] - ); +function ffmt($val){ + if (is_null($val)) return "M"; + if ($val > 0 and $val < 0.005) return "T"; + return sprintf("%.2f", $val); } -$nselect .= ""; + +$nselect = selectNetworkType("COCORAHS", $network); $t->title = "CoCoRaHS Last Observation"; -$t->headextra = << - - -EOF; $t->current_network = "CoCoRaHS"; -$nt = new NetworkTable($network); - $dbconn = iemdb("coop"); $year = date("Y"); $sql = <<= $2 GROUP by iemid) select t.*, m.pmonth, m.count, @@ -62,38 +41,45 @@ from today t JOIN month m on (t.iemid = m.iemid) ORDER by t.id asc EOM; $day1 = new DateTime(); -; +$days_this_month = intval($day1->format("d")); $rs = pg_prepare($dbconn, "CS", $sql); - $rs = pg_execute($dbconn, "CS", array($network, $day1->format("Y-m") .'-01')); - $monthDict = Array(); - $db = Array(); - for( $i=0; $row = pg_fetch_array($rs); $i++){ - $site = $row["id"]; - $db[$site] = Array( - 'pmonth' => $row["pmonth"], - 'count' => $row["count"], - 'snow'=> "", 'snowd'=>"", 'ratio'=>"", 'pday'=>"", 'pmonth'=>"", 'pmiss' => ""); - $db[$site]['ts'] = strtotime($row["local_valid"]); - $db[$site]['sid'] = $site; - $db[$site]['name'] = $nt->table[$site]["name"]; - $db[$site]['county'] = $nt->table[$site]["county"]; - $db[$site]['pday'] = $row["precip"]; - $db[$site]['snow'] = ($row["snow"] >= 0) ? $row["snow"] : " "; - $db[$site]['snowd'] = ($row["snowd"] >= 0) ? $row["snowd"] : " "; - $db[$site]["ratio"] = -1; - if ($db[$site]["snow"] > 0.0001 && $db[$site]["pday"] > 0.0001) - { - $db[$site]["ratio"] = intval( $db[$site]["snow"] / $db[$site]["pday"] ); - } +$rs = pg_execute($dbconn, "CS", array($network, $day1->format("Y-m") . '-01')); +$db = array(); +for ($i = 0; $row = pg_fetch_array($rs); $i++) { + $site = $row["id"]; + $db[$site] = array( + 'sid' => $site, + 'ts' => strtotime($row["local_valid"]), + 'name' => $row["name"], + 'county' => $row["county"], + 'pmonth' => $row["pmonth"], + 'count' => $row["count"], + 'snow' => ($row["snow"] >= 0) ? $row["snow"] : " ", + 'snowd' => ($row["snowd"] >= 0) ? $row["snowd"] : " ", + 'ratio' => -1, + 'pday' => $row["precip"], + 'pmonth' => $row["pmonth"], + 'pmiss' => $days_this_month - $row["count"], + ); + if ($db[$site]["snow"] > 0.0001 && $db[$site]["pday"] > 0.0001) { + $db[$site]["ratio"] = intval($db[$site]["snow"] / $db[$site]["pday"]); + } } $db = aSortBySecondIndex($db, $sortcol, "desc"); -$cols = Array("ts" => "Valid", "county" => "County", - "sid" => "Site ID", "name" => "Station Name", - "tmpf" => "Ob Temperature", "max_tmpf" => "24 hour High", - "min_tmpf" => "24 hour Low", "snow" => "24 hour Snowfall", - "snowd" => "Snowfall Depth", "pday" => "24 hour rainfall", - "pmonth" => "Precipitation for Month"); +$cols = array( + "ts" => "Valid", + "county" => "County", + "sid" => "Site ID", + "name" => "Station Name", + "tmpf" => "Ob Temperature", + "max_tmpf" => "24 hour High", + "min_tmpf" => "24 hour Low", + "snow" => "24 hour Snowfall", + "snowd" => "Snowfall Depth", + "pday" => "24 hour rainfall", + "pmonth" => "Precipitation for Month" +); $baseurl2 = "current.phtml?sortcol="; $content = <<View by State: {$nselect}
- + @@ -127,50 +113,44 @@ Option 1: View by State: {$nselect} EOM; - $oddrow = true; - $now = time(); - foreach($db as $site => $value){ - $tdiff = $now - $value["ts"]; - if ( intval( date("Y", $value["ts"]) ) < 2003 ) continue; - $oddrow = ! $oddrow; - - $content .= " - - +$oddrow = true; +$now = time(); +foreach ($db as $site => $value) { + $tdiff = $now - $value["ts"]; + $obslink = sprintf("obs.phtml?station=%s&network=%s", $site, $network); + $content .= << + + + + EOM; - $content .= "'; - } else { - $content .= ">". date("h:i A", $value["ts"]) .""; - } + $content .= ""; - if ($value["pday"] == 0.0001) $value["pday"] = "T"; - if ($value["pmonth"] == 0.0001) $value["pmonth"] = "T"; - if ($value["pday"] < 0) $value["pday"] = "M"; - if ($value["snow"] == 0.0001) $value["snow"] = "T"; - if ($value["snowd"] == 0.0001) $value["snowd"] = "T"; - if ($value["snow"] < 0) $value["snow"] = "M"; + if ($value["pday"] == 0.0001) $value["pday"] = "T"; + if ($value["pmonth"] == 0.0001) $value["pmonth"] = "T"; + if ($value["pday"] < 0) $value["pday"] = "M"; + if ($value["snow"] == 0.0001) $value["snow"] = "T"; + if ($value["snowd"] == 0.0001) $value["snowd"] = "T"; + if ($value["snow"] < 0) $value["snow"] = "M"; - $content .= ""; - $content .= ""; + $content .= ""; - $content .= ""; - if ($value["ratio"] > 0) $content .= ""; + $content .= ""; + if ($value["ratio"] > 0) $content .= ""; else $content .= ""; - $content .= ""; - $content .= ""; - } + $content .= ""; + $content .= ""; +} $content .= <<
Add: SiteID:
$site{$value["name"]}{$value["county"]} + +$site{$value["name"]}{$value["county"]} (24*3600) || date("Ymd") != date("Ymd", $value["ts"]) ){ - $content .= 'bgcolor="red">'. date("d M Y h:i A", $value["ts"]) .'" . date("h:i A", $value["ts"]) . "". $value["pday"] ."". $value["pmonth"]; - if ($value["pmiss"] > 0) $content .= " (". $value["pmiss"] .")"; + $content .= "" . $value["pday"] . "" . ffmt($value["pmonth"]); + if ($value["pmiss"] > 0) $content .= " (" . $value["pmiss"] . ")"; $content .= "". $value["snow"] ."". $value["ratio"] ."" . $value["snow"] . "" . $value["ratio"] . "". $value["snowd"] ."
" . $value["snowd"] . "
-
+ EOM; $t->content = $content; $t->render('sortables.phtml'); diff --git a/htdocs/cocorahs/obs.phtml b/htdocs/cocorahs/obs.phtml index 8a962c175..8f9191b91 100644 --- a/htdocs/cocorahs/obs.phtml +++ b/htdocs/cocorahs/obs.phtml @@ -5,7 +5,6 @@ require_once "../../include/database.inc.php"; $access = iemdb("coop"); require_once "../../include/myview.php"; require_once "../../include/forms.php"; -require_once "../../include/network.php"; $t = new MyView(); $t->title = "CoCoRaHS Observations"; @@ -23,21 +22,19 @@ if ( $sortdir = "DESC"; } -$nt = new NetworkTable($network); - $ys = yearSelect(2007, $year, "year"); if (strlen($station) > 0) { $rs = pg_prepare( $access, "SELECTOR", - "SELECT s.*, t.id as station from cocorahs_$year s JOIN stations t + "SELECT s.*, t.id as station, t.name, t.county from cocorahs_$year s JOIN stations t ON (t.iemid = s.iemid) WHERE t.id = $1 ORDER by $sortvar $sortdir"); $rs = pg_execute($access, "SELECTOR", array($station)); } else { $rs = pg_prepare($access, "SELECTOR", - "SELECT s.*, t.id as station + "SELECT s.*, t.id as station, t.name, t.county from cocorahs_$year s JOIN stations t ON (t.iemid = s.iemid) WHERE day = $1 and network = $2 ORDER by $sortvar $sortdir"); @@ -70,16 +67,12 @@ for ($i = 0; $row = pg_fetch_assoc($rs); $i++) { if (strlen($station) > 0) { $table .= $nwsli; } else { - $table .= "" . $nwsli . ""; + $table .= "{$nwsli}"; } - $table .= "" . $nt->table[$nwsli]["name"] . "" . $nt->table[$nwsli]["county"] . ""; + $table .= "" . $row["name"] . "" . $row["county"] . ""; - if (strlen($date) > 0) { - $table .= $row["day"]; - } else { - $table .= "" . $row["day"] . ""; - } + $table .= "" . $row["day"] . ""; $table .= "" . $rain . "" . $snow . "" . $snowd . "\n"; } diff --git a/htdocs/request/download.phtml b/htdocs/request/download.phtml index 1678cbca1..52ed8afd8 100644 --- a/htdocs/request/download.phtml +++ b/htdocs/request/download.phtml @@ -16,7 +16,7 @@ $network = isset($_GET['network']) ? xssafe($_GET['network']): "IA_ASOS"; $station = ""; $t->title = "Download ASOS/AWOS/METAR Data"; -$netselect = selectAzosNetwork($network); +$netselect = selectNetworkType("ASOS", $network); $y1select = yearSelect2(1928, date("Y"), "year1"); $y2select = yearSelect2(1928, date("Y"), "year2"); diff --git a/htdocs/sites/networks.php b/htdocs/sites/networks.php index 16bccac98..5fe3cc837 100644 --- a/htdocs/sites/networks.php +++ b/htdocs/sites/networks.php @@ -41,11 +41,13 @@ function pretty_date($val, $fmt = "M d, Y") } if ($network == '_ALL_') { + // Too much memory at the moment $rs = pg_query( $pgconn, "SELECT id, name, elevation, archive_begin, archive_end, network, " . "ST_x(geom) as lon, ST_y(geom) as lat, null as attributes, state, " . - "synop, country from stations WHERE online = 'y' ORDER by name" + "synop, country from stations WHERE online = 't' and ". + "network !~* '_COCORAHS' ORDER by name" ); $cities = array(); for ($i = 0; $row = pg_fetch_array($rs); $i++) { diff --git a/include/imagemaps.php b/include/imagemaps.php index 4a1d12833..5bdccb36f 100644 --- a/include/imagemaps.php +++ b/include/imagemaps.php @@ -58,21 +58,32 @@ function ugcStateSelect($state, $selected) } -function selectAzosNetwork($network) +/** + * Select a network type + * @param nettype the network type + * @param selected the selected network id + * @return the select box + */ +function selectNetworkType($nettype, $selected) { - $network = strtoupper($network); - include_once dirname(__FILE__) . "/database.inc.php"; + $selected = strtoupper($selected); + require_once dirname(__FILE__) . "/database.inc.php"; $dbconn = iemdb('mesosite'); - $rs = pg_exec($dbconn, "SELECT * from networks WHERE id ~* 'ASOS' ORDER by name ASC"); + $rs = pg_prepare( + $dbconn, + "SELECT_NETWORK_BY_TYPE", + "SELECT * from networks WHERE id ~* $1 ORDER by name ASC"); + $rs = pg_execute($dbconn, "SELECT_NETWORK_BY_TYPE", array($nettype)); $s = "\n"; + pg_close($dbconn); return $s; } diff --git a/pylib/iemweb/autoplot/scripts200/p207.py b/pylib/iemweb/autoplot/scripts200/p207.py index 757e7e0f1..7bc16f715 100644 --- a/pylib/iemweb/autoplot/scripts200/p207.py +++ b/pylib/iemweb/autoplot/scripts200/p207.py @@ -1,6 +1,6 @@ """ Generates an analysis map of snowfall or freezing rain data -based on NWS Local Storm Reports and NWS COOP Data. This autoplot +based on NWS Local Storm Reports, NWS COOP Data, and CoCoRaHS. This autoplot presents a number of tunables including:
  • The window of hours to look before the specified valid time to @@ -71,8 +71,8 @@ "plot": "Inject zeros and show on plot as red 'Z'", } PDICT5 = { - "yes": "Include any NWS COOP Reports, if possible.", - "no": "Do not include any COOP reports.", + "yes": "Include any reports, if possible.", + "no": "Do not include any reports.", } PDICT6 = {"snow": "Snowfall", "ice": "Freezing Rain / Ice Storm (LSRs Only)"} PDICT7 = { @@ -170,6 +170,13 @@ def get_description(): default="no", name="coop", ), + { + "type": "select", + "options": PDICT5, + "label": "Include CoCoRaHS reports too?", + "default": "yes", + "name": "cocorahs", + }, dict( type="select", options=PDICT7, @@ -196,7 +203,7 @@ def get_description(): return desc -def load_data(ctx, basets, endts): +def load_data(ctx: dict, basets: datetime, endts: datetime): """Generate a dataframe with the data we want to analyze.""" with get_sqlalchemy_conn("postgis") as conn: df: gpd.GeoDataFrame = gpd.read_postgis( @@ -223,67 +230,69 @@ def load_data(ctx, basets, endts): df["nwsli"] = df.index.values df["plotme"] = True df["source"] = "LSR" - if ctx["coop"] == "no" or ctx["v"] == "ice": - return df # More work to do days = [] now = basets while now <= endts: days.append(now.date()) now += timedelta(hours=24) - with get_sqlalchemy_conn("iem") as conn: - df2: gpd.GeoDataFrame = gpd.read_postgis( - text( - """SELECT state, wfo, id as nwsli, - sum(snow) as val, ST_x(geom) as lon, ST_y(geom) as lat, - ST_Transform(geom, 2163) as geo - from summary s JOIN stations t on (s.iemid = t.iemid) - WHERE s.day = ANY(:days) - and t.network ~* 'COOP' and snow >= 0 and - coop_valid >= :basets and coop_valid <= :endts - GROUP by state, wfo, nwsli, lon, lat, geo - ORDER by val DESC - """ - ), - conn, - params={ - "days": days, - "basets": basets, - "endts": endts, - }, - index_col=None, - geom_col="geo", - ) - df2[USEME] = True - df2["plotme"] = True - df2["source"] = "COOP" - with get_sqlalchemy_conn("coop") as conn: - df3: gpd.GeoDataFrame = gpd.read_postgis( - text( - """SELECT state, wfo, id as nwsli, - sum(snow) as val, ST_x(geom) as lon, ST_y(geom) as lat, - ST_Transform(geom, 2163) as geo - from alldata_cocorahs s JOIN stations t on (s.iemid = t.iemid) - WHERE s.day = ANY(:days) - and t.network ~* '_COCORAHS' and snow >= 0 and - obvalid >= :basets and obvalid <= :endts - GROUP by state, wfo, nwsli, lon, lat, geo - ORDER by val DESC - """ - ), - conn, - params={ - "days": days, - "basets": basets, - "endts": endts, - }, - index_col=None, - geom_col="geo", - ) - df3[USEME] = True - df3["plotme"] = True - df3["source"] = "COCORAHS" - return pd.concat([df, df2, df3], ignore_index=True, sort=False) + if ctx["coop"] == "yes" or ctx["v"] == "ice": + with get_sqlalchemy_conn("iem") as conn: + df2: gpd.GeoDataFrame = gpd.read_postgis( + text( + """SELECT state, wfo, id as nwsli, + sum(snow) as val, ST_x(geom) as lon, ST_y(geom) as lat, + ST_Transform(geom, 2163) as geo + from summary s JOIN stations t on (s.iemid = t.iemid) + WHERE s.day = ANY(:days) + and t.network ~* 'COOP' and snow >= 0 and + coop_valid >= :basets and coop_valid <= :endts + GROUP by state, wfo, nwsli, lon, lat, geo + ORDER by val DESC + """ + ), + conn, + params={ + "days": days, + "basets": basets, + "endts": endts, + }, + index_col=None, + geom_col="geo", + ) + df2[USEME] = True + df2["plotme"] = True + df2["source"] = "COOP" + df = pd.concat([df, df2], ignore_index=True, sort=False) + if ctx["cocorahs"] == "yes": + with get_sqlalchemy_conn("coop") as conn: + df3: gpd.GeoDataFrame = gpd.read_postgis( + text( + """SELECT state, wfo, id as nwsli, + sum(snow) as val, ST_x(geom) as lon, ST_y(geom) as lat, + ST_Transform(geom, 2163) as geo + from alldata_cocorahs s JOIN stations t on (s.iemid = t.iemid) + WHERE s.day = ANY(:days) + and t.network ~* '_COCORAHS' and snow >= 0 and + obvalid >= :basets and obvalid <= :endts + GROUP by state, wfo, nwsli, lon, lat, geo + ORDER by val DESC + """ + ), + conn, + params={ + "days": days, + "basets": basets, + "endts": endts, + }, + index_col=None, + geom_col="geo", + ) + df3[USEME] = True + df3["plotme"] = True + df3["source"] = "COCORAHS" + df = pd.concat([df, df3], ignore_index=True, sort=False) + return df def compute_grid_bounds(ctx, csector): @@ -459,6 +468,7 @@ def plotter(ctx: dict): sector = "state" if len(csector) == 2 else csector _t = " & COOP" if ctx["coop"] == "yes" else "" + _t += " & CoCoRaHS" if ctx["cocorahs"] == "yes" else "" title = f"NWS Local Storm Report{_t} Snowfall Total Analysis" if ctx["v"] == "ice": title = "NWS Local Storm Reports of Freezing Rain + Ice" diff --git a/scripts/ingestors/cocorahs/cocorahs_data_ingest.py b/scripts/ingestors/cocorahs/cocorahs_data_ingest.py index dc0a1aadb..805e09457 100644 --- a/scripts/ingestors/cocorahs/cocorahs_data_ingest.py +++ b/scripts/ingestors/cocorahs/cocorahs_data_ingest.py @@ -46,9 +46,6 @@ def main(dt: Optional[date]) -> None: LOG.info("Found %s station defined", len(stations.index)) dbconn, cursor = get_dbconnc("coop") - # https://data.cocorahs.org/export/exportreports.aspx?ReportType=Daily& - # dtf=1&Format=CSV&ReportDateType=timestamp&Date=1/29/2025%2012:00%20AM - # &TimesInGMT=True if dt is not None: # Export for a given date url = ( @@ -62,7 +59,7 @@ def main(dt: Optional[date]) -> None: url = ( "https://data.cocorahs.org/export/exportreports.aspx" "?ReportType=Daily&dtf=1&Format=CSV&ReportDateType=timestamp" - f"&Date={since:%-m/%d/%Y}%20{since:%I:%m}%20{since:%p}&" + f"&Date={since:%-m/%d/%Y%%20%I:%m%%20%p}&" "TimesInGMT=True" ) with StringIO() as sio: diff --git a/scripts/ingestors/cocorahs/cocorahs_stations.py b/scripts/ingestors/cocorahs/cocorahs_stations.py index 8d2412252..f6e4d6421 100644 --- a/scripts/ingestors/cocorahs/cocorahs_stations.py +++ b/scripts/ingestors/cocorahs/cocorahs_stations.py @@ -60,7 +60,12 @@ def main(newerthan: datetime): ) for sid, row in upstream.iterrows(): - if row["Latitude"] == 0 or row["Longitude"] == 0: + if ( + row["Latitude"] == 0 + or row["Longitude"] == 0 + or row["Elevation"] < -900 + or pd.isna(row["StationName"]) + ): continue network = f"{row['State']}_COCORAHS" sname = row["StationName"].strip().replace("'", " ")