Merge pull request #1102 from akrherz/250209-2

Omnibus
akrherz · Feb 10, 2025 · 8c0f975 · 8c0f975
2 parents 8b4ead6 + a7822b4
commit 8c0f975
Show file tree

Hide file tree

Showing 14 changed files with 65 additions and 59 deletions.
diff --git a/htdocs/nws/cf6map.php b/htdocs/nws/cf6map.php
@@ -7,7 +7,7 @@
 force_https();
 $t = new MyView();
 $t->title = "Map of Daily NWS CF6 reports";
-$OL = '9.2.4';
+$OL = '10.3.0';
 $t->headextra = <<<EOF
 <link rel="stylesheet" href="/vendor/openlayers/{$OL}/ol.css" type="text/css">
 <link rel="stylesheet" href="/vendor/jquery-ui/1.12.1/jquery-ui.min.css" />

diff --git a/pylib/iemweb/autoplot/scripts/p62.py b/pylib/iemweb/autoplot/scripts/p62.py
@@ -11,11 +11,10 @@
 import matplotlib.colors as mpcolors
 import numpy as np
 import pandas as pd
-from pyiem.database import get_sqlalchemy_conn
+from pyiem.database import get_sqlalchemy_conn, sql_helper
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure
 from pyiem.plot.colormaps import nwssnow
-from sqlalchemy import text
 
 from iemweb.autoplot import ARG_STATION
 
@@ -62,7 +61,7 @@ def plotter(ctx: dict):
     obs = np.ma.ones((eyear - syear + 1, 183), "f") * -1
     with get_sqlalchemy_conn("coop") as conn:
         df = pd.read_sql(
-            text("""
+            sql_helper("""
             SELECT year, extract(doy from day) as doy, snowd, day,
             case when month < 6 then year - 1 else year end as winter_year
             from alldata WHERE station = :station and

diff --git a/pylib/iemweb/autoplot/scripts/p68.py b/pylib/iemweb/autoplot/scripts/p68.py
@@ -13,10 +13,9 @@
 
 import pandas as pd
 from matplotlib.ticker import MaxNLocator
-from pyiem.database import get_sqlalchemy_conn
+from pyiem.database import get_sqlalchemy_conn, sql_helper
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure, fitbox
-from sqlalchemy import text
 
 
 def get_description():
@@ -45,13 +44,14 @@ def plotter(ctx: dict):
     sqllim = "wfo = :wfo and " if station != "_ALL" else ""
     with get_sqlalchemy_conn("postgis") as conn:
         df = pd.read_sql(
-            text(
-                f"""
+            sql_helper(
+                """
             SELECT distinct extract(year from issue) as year,
             phenomena, significance from warnings WHERE
             {sqllim} phenomena is not null and significance is not null
             and issue > '2005-01-01'
-            """
+            """,
+                sqllim=sqllim,
             ),
             conn,
             params=params,
@@ -71,9 +71,10 @@ def plotter(ctx: dict):
     fig = figure(figsize=(10, 14 if station != "_ALL" else 21), apctx=ctx)
     fitbox(fig, title, 0.1, 0.97, 0.97, 0.99)
     fitbox(fig, subtitle, 0.1, 0.97, 0.95, 0.97)
-    ax = [None, None]
-    ax[0] = fig.add_axes([0.05, 0.75, 0.93, 0.2])
-    ax[1] = fig.add_axes([0.05, 0.03, 0.93, 0.68])
+    ax = [
+        fig.add_axes((0.05, 0.75, 0.93, 0.2)),
+        fig.add_axes((0.05, 0.03, 0.93, 0.68)),
+    ]
 
     ax[0].bar(
         gdf.index.values, gdf["wfo"], width=0.8, fc="b", ec="b", align="center"

diff --git a/pylib/iemweb/autoplot/scripts/p76.py b/pylib/iemweb/autoplot/scripts/p76.py
@@ -22,12 +22,11 @@
 import pandas as pd
 from matplotlib.ticker import MaxNLocator
 from metpy.units import units
-from pyiem.database import get_sqlalchemy_conn
+from pyiem.database import get_sqlalchemy_conn, sql_helper
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure_axes
 from pyiem.util import utc
 from scipy import stats
-from sqlalchemy import text
 
 from iemweb.util import month2months
 
@@ -219,7 +218,7 @@ def get_data(ctx, startyear):
         )
     with get_sqlalchemy_conn("asos") as conn:
         df = pd.read_sql(
-            text(
+            sql_helper(
                 """
             WITH obs as (
                 SELECT valid at time zone :tzname as valid, tmpf, dwpf, relh,

diff --git a/pylib/iemweb/autoplot/scripts/p8.py b/pylib/iemweb/autoplot/scripts/p8.py
@@ -11,10 +11,9 @@
 
 import numpy as np
 import pandas as pd
-from pyiem.database import get_sqlalchemy_conn
+from pyiem.database import get_sqlalchemy_conn, sql_helper
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure_axes
-from sqlalchemy import text
 
 from iemweb.autoplot import ARG_STATION
 
@@ -52,7 +51,7 @@ def plotter(ctx: dict):
 
     with get_sqlalchemy_conn("coop") as conn:
         df = pd.read_sql(
-            text(
+            sql_helper(
                 """
     with months as (
       select year, month, p, avg(p) OVER (PARTITION by month) from (

diff --git a/pylib/iemweb/autoplot/scripts/p85.py b/pylib/iemweb/autoplot/scripts/p85.py
@@ -9,10 +9,9 @@
 from datetime import datetime
 
 import pandas as pd
-from pyiem.database import get_sqlalchemy_conn
+from pyiem.database import get_sqlalchemy_conn, sql_helper
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure
-from sqlalchemy import text
 
 PDICT = {
     "above": "At or Above",
@@ -107,10 +106,11 @@ def plotter(ctx: dict):
     tzname = ctx["_nt"].sts[station]["tzname"]
     with get_sqlalchemy_conn("asos") as conn:
         df = pd.read_sql(
-            text(f"""
+            sql_helper(
+                """
         WITH data as (
             SELECT valid at time zone :tzname  + '10 minutes'::interval as v,
-            {varname}::{CASTS.get(varname, "numeric")} as datum
+            {varname}::{cst} as datum
             from alldata where station = :station and {varname} is not null
             and extract(month from valid) = :month and report_type = 3)
 
@@ -125,7 +125,10 @@ def plotter(ctx: dict):
         sum(case when datum >= :thres THEN 1 ELSE 0 END) as above,
         count(*) from data
         GROUP by hour ORDER by hour ASC
-        """),
+        """,
+                varname=varname,
+                cst=CASTS.get(varname, "numeric"),
+            ),
             conn,
             params={
                 "station": station,

diff --git a/pylib/iemweb/autoplot/scripts/p9.py b/pylib/iemweb/autoplot/scripts/p9.py
@@ -8,10 +8,9 @@
 
 import matplotlib.dates as mdates
 import pandas as pd
-from pyiem.database import get_sqlalchemy_conn
+from pyiem.database import get_sqlalchemy_conn, sql_helper
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure_axes
-from sqlalchemy import text
 
 from iemweb.autoplot import ARG_STATION
 
@@ -107,11 +106,13 @@ def plotter(ctx: dict):
         title = f"base={ceiling}"
     with get_sqlalchemy_conn("coop") as conn:
         df = pd.read_sql(
-            text(
-                f"""SELECT year, sday, {gfunc} as {glabel} from alldata WHERE
+            sql_helper(
+                """SELECT year, sday, {gfunc} as {glabel} from alldata WHERE
             station = :sid and year > 1892 and sday != '0229'
             and sday >= :sday and sday <= :eday ORDER by day ASC
-            """
+            """,
+                gfunc=gfunc,
+                glabel=glabel,
             ),
             conn,
             params={

diff --git a/pylib/iemweb/autoplot/scripts/p99.py b/pylib/iemweb/autoplot/scripts/p99.py
@@ -10,10 +10,9 @@
 
 import matplotlib.dates as mdates
 import pandas as pd
-from pyiem.database import get_sqlalchemy_conn
+from pyiem.database import get_sqlalchemy_conn, sql_helper
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure
-from sqlalchemy import text
 
 from iemweb.autoplot import ARG_STATION
 
@@ -50,7 +49,7 @@ def plotter(ctx: dict):
     clstation = ctx["_nt"].sts[station]["ncei91"]
     with get_sqlalchemy_conn("coop") as conn:
         df = pd.read_sql(
-            text("""
+            sql_helper("""
             WITH days as (
                 select generate_series(:sts, :ets,
                     '1 day'::interval)::date as day,

diff --git a/pylib/iemweb/autoplot/scripts100/p101.py b/pylib/iemweb/autoplot/scripts100/p101.py
@@ -11,11 +11,10 @@
 
 import numpy as np
 import pandas as pd
-from pyiem.database import get_sqlalchemy_conn
+from pyiem.database import get_sqlalchemy_conn, sql_helper
 from pyiem.exceptions import NoDataFound
 from pyiem.nws import vtec
 from pyiem.plot import figure_axes
-from sqlalchemy import text
 
 
 def get_description():
@@ -71,12 +70,15 @@ def plotter(ctx: dict):
     rows = []
     with get_sqlalchemy_conn("postgis") as conn:
         res = conn.execute(
-            text(f"""
+            sql_helper(
+                """
             select phenomena, significance, min(issue), count(*) from warnings
             where ugc is not null and issue > :sts
             and issue < :ets {wfo_limiter}
             GROUP by phenomena, significance ORDER by count DESC
-        """),
+        """,
+                wfo_limiter=wfo_limiter,
+            ),
             params,
         )
         if res.rowcount == 0:

diff --git a/pylib/iemweb/autoplot/scripts100/p102.py b/pylib/iemweb/autoplot/scripts100/p102.py
@@ -12,11 +12,10 @@
 
 import numpy as np
 import pandas as pd
-from pyiem.database import get_sqlalchemy_conn
+from pyiem.database import get_sqlalchemy_conn, sql_helper
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure_axes
 from pyiem.reference import lsr_events
-from sqlalchemy import text
 
 MARKERS = ["8", ">", "<", "v", "o", "h", "*"]
 
@@ -86,16 +85,20 @@ def plotter(ctx: dict):
         else:
             typetext_limiter = " and typetext = ANY(:tt)"
             params["tt"] = ltype
+    params["sts"] = date(syear, 1, 1)
+    params["ets"] = date(eyear + 1, 1, 1)
     with get_sqlalchemy_conn("postgis") as conn:
         df = pd.read_sql(
-            text(
-                f"""
+            sql_helper(
+                """
             select extract(year from valid)::int as yr, upper(source) as src,
             count(*) from lsrs
-            where valid > '{syear}-01-01' and
-            valid < '{eyear + 1}-01-01' {wfo_limiter} {typetext_limiter}
+            where valid > :sts and
+            valid < :ets {wfo_limiter} {typetext_limiter}
             GROUP by yr, src
-        """
+        """,
+                wfo_limiter=wfo_limiter,
+                typetext_limiter=typetext_limiter,
             ),
             conn,
             params=params,

diff --git a/pylib/iemweb/autoplot/scripts100/p103.py b/pylib/iemweb/autoplot/scripts100/p103.py
@@ -9,10 +9,9 @@
 
 import numpy as np
 import pandas as pd
-from pyiem.database import get_sqlalchemy_conn
+from pyiem.database import get_sqlalchemy_conn, sql_helper
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure
-from sqlalchemy import text
 
 from iemweb.autoplot import ARG_STATION
 
@@ -47,7 +46,8 @@ def plotter(ctx: dict):
     )
     with get_sqlalchemy_conn("coop") as conn:
         df = pd.read_sql(
-            text(f"""
+            sql_helper(
+                """
         WITH obs as (
             SELECT day, month, high, low, {year} as season
             from alldata WHERE station = :station),
@@ -69,7 +69,9 @@ def plotter(ctx: dict):
         level, 'fall' as typ from lows WHERE rank = 1) UNION
         (SELECT season as year, day, extract(doy from day) as doy,
         level, 'spring' as typ from highs WHERE rank = 1)
-        """),
+        """,
+                year=year,
+            ),
             conn,
             params={"station": station},
         )
@@ -82,9 +84,9 @@ def plotter(ctx: dict):
     fig = figure(title=title, apctx=ctx)
     x = 0.1
     ax = [
-        fig.add_axes([x, 0.7, 0.88, 0.2]),
-        fig.add_axes([x, 0.4, 0.88, 0.2]),
-        fig.add_axes([x, 0.1, 0.88, 0.2]),
+        fig.add_axes((x, 0.7, 0.88, 0.2)),
+        fig.add_axes((x, 0.4, 0.88, 0.2)),
+        fig.add_axes((x, 0.1, 0.88, 0.2)),
     ]
     dyear = df2.groupby(["year"]).count()
     ax[0].bar(

diff --git a/pylib/iemweb/autoplot/scripts100/p105.py b/pylib/iemweb/autoplot/scripts100/p105.py
@@ -10,10 +10,9 @@
 
 import numpy as np
 import pandas as pd
-from pyiem.database import get_sqlalchemy_conn
+from pyiem.database import get_sqlalchemy_conn, sql_helper
 from pyiem.exceptions import NoDataFound
 from pyiem.plot import figure
-from sqlalchemy import text
 
 from iemweb.autoplot import ARG_STATION
 
@@ -60,7 +59,7 @@ def plotter(ctx: dict):
     use_trace = ctx["trace"] == "yes"
     with get_sqlalchemy_conn("coop") as conn:
         df = pd.read_sql(
-            text("""
+            sql_helper("""
         with data as (
         select sday, day, precip from alldata
         where station = :station),

diff --git a/scripts/ingestors/asos_1minute/parse_ncei_asos1minute.py b/scripts/ingestors/asos_1minute/parse_ncei_asos1minute.py
@@ -21,9 +21,8 @@
 import click
 import httpx
 import pandas as pd
-from pyiem.database import get_dbconn, get_sqlalchemy_conn
+from pyiem.database import get_dbconn, get_sqlalchemy_conn, sql_helper
 from pyiem.util import exponential_backoff, logger, set_property, utc
-from sqlalchemy import text
 from tqdm import tqdm
 
 LOG = logger()
@@ -354,11 +353,12 @@ def init_dataframes(
 
 def merge_archive_end(df, dt):
     """Figure out our archive end times."""
+    table = f"t{dt:%Y%m}_1minute"
     with get_sqlalchemy_conn("asos1min") as conn:
         df2 = pd.read_sql(
-            text(
-                f"SELECT station, max(valid) from t{dt:%Y%m}_1minute "
-                "GROUP by station"
+            sql_helper(
+                "SELECT station, max(valid) from {table} GROUP by station",
+                table=table,
             ),
             conn,
             index_col="station",

diff --git a/scripts/ingestors/madis/to_iemaccess.py b/scripts/ingestors/madis/to_iemaccess.py
@@ -74,7 +74,7 @@ def provider2network(provider, name):
         if network.endswith("DOT"):
             if len(network) == 5:
                 return f"{network[:2]}_RWIS"
-            if tokens[-2] in ["UTAH", "WA", "NV", "MT"]:
+            if tokens[-2] == "UTAH" or len(tokens[-2]) == 2:
                 return f"{tokens[-2][:2]}_RWIS"
             LOG.warning("How to convert %s into a network?", repr(tokens))
             return None
@@ -233,7 +233,6 @@ def main(offset: int):
             subprocess.call(["sh", "SYNC_STATIONS.sh"])
             os.chdir("../ingestors/madis")
             LOG.info("...done with sync.")
-        del iem
     icursor.close()
     pgconn.commit()
     pgconn.close()