diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index cd85d70..c76bd53 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -88,6 +88,8 @@ jobs: Copy-Item README.md dist/grib2pf/ Copy-Item ACKNOWLEDGMENTS.md dist/grib2pf/ Copy-Item products.txt dist/grib2pf/_internal + Copy-Item hrrr_wrfsfcf00_products.csv dist/grib2pf/_internal + Copy-Item hrrr_wrfsfcf01_products.csv dist/grib2pf/_internal Copy-Item examples dist/grib2pf/examples -Recurse Copy-Item palettes dist/grib2pf/_internal/palettes -Recurse Copy-Item presets dist/grib2pf/_internal/presets -Recurse diff --git a/aws.py b/aws.py index 91f2556..8ac43e9 100755 --- a/aws.py +++ b/aws.py @@ -51,9 +51,73 @@ def get_url(self, expires = 60): ExpiresIn = expires, ) +class AWSHRRRHandler: + def __init__(self, product, bucketName = "noaa-hrrr-bdp-pds", config = None): + if config is None: + config = Config(signature_version = UNSIGNED) + + self.product = product + self.bucketName = bucketName + self.client = boto3.client("s3", config=config) + self.mostRecentKey = None + + def update_key(self): + now = time.gmtime(time.time() - 3600) + args = { + "Bucket": self.bucketName, + "Prefix": time.strftime("hrrr.%Y%m%d/" + self.product["location"] + "/", now) + } + if self.mostRecentKey is not None: + args["StartAfter"] = self.mostRecentKey + + pager = self.client.get_paginator("list_objects_v2") + pages = pager.paginate(**args) + + + mostRecent = None + for page in pages: + if 'Contents' not in page: + continue + + for obj in page["Contents"]: + path = obj["Key"].split("/") + if len(path) != 3: + continue + if mostRecent is not None and \ + obj["LastModified"] <= mostRecent["LastModified"]: + continue + parts = path[2].split(".") + if len(parts) != 4 or \ + parts[3] != "grib2" or \ + parts[2] != self.product["fileType"]: + continue + mostRecent = obj + + if mostRecent is not None: + self.mostRecentKey = mostRecent["Key"] + + return mostRecent is not None + + + def get_url(self, idx = False, expires = 60): + key = self.mostRecentKey + if idx: + key += ".idx" + + return self.client.generate_presigned_url( + 'get_object', + Params = { + 'Bucket': self.bucketName, + 'Key': key, + }, + ExpiresIn = expires, + ) + if __name__ == "__main__": def test(): - handler = AWSHandler("CONUS/MergedBaseReflectivity_00.50/") + handler = AWSHRRRHandler({"location": "conus", "fileType": "wrfsfcf00"}) + + #AWSHandler("CONUS/MergedBaseReflectivity_00.50/") handler.update_key() print(handler.get_url()) while not handler.update_key(): diff --git a/grib2pf-ui.py b/grib2pf-ui.py index 08a3628..959ddf5 100755 --- a/grib2pf-ui.py +++ b/grib2pf-ui.py @@ -15,13 +15,120 @@ import os import random import subprocess +import csv location = os.path.split(__file__)[0] MIME_TYPE = "application/x.grib2pf-placefile" -class ProductsDialog(QDialog): +class HRRRProductsDialog(QDialog): + def __init__(self): + QDialog.__init__(self) + + self.selectedProduct = "" + + self.products = { + "conus": [], + "alaska": [], + } + with open(os.path.join(location, "hrrr_wrfsfcf01_products.csv")) as file: + reader = csv.reader(file) + for row in reader: + self.products["conus"].append(list(row)) + self.products["alaska"].append(list(row)) + + self.mainLayout = QVBoxLayout(self) + self.bottomLayout = QHBoxLayout() + + self.model = QStandardItemModel() + self.proxyModel = QSortFilterProxyModel() + + self.proxyModel.setSourceModel(self.model) + self.proxyModel.setFilterCaseSensitivity(Qt.CaseInsensitive) + self.proxyModel.setFilterKeyColumn(1) + self.proxyModel.setSortCaseSensitivity(Qt.CaseInsensitive) + + self.infoBox = QLabel() + self.locationComboBox = QComboBox() + self.mainView = QTreeView() + self.searchBar = QLineEdit() + self.selectButton = QPushButton("Select") + self.cancelButton = QPushButton("Cancel") + + self.locationComboBox.currentTextChanged.connect(self.location_selected) + self.selectButton.clicked.connect(self.select_pressed) + self.cancelButton.clicked.connect(self.reject) + self.searchBar.textChanged.connect(self.update_search) + + self.infoBox.setTextFormat(Qt.RichText) + self.infoBox.setOpenExternalLinks(True) + self.infoBox.setText(""" + + + + + +
HRRR Tablehttps://www.nco.ncep.noaa.gov/pmb/products/hrrr/hrrr.t00z.wrfsfcf00.grib2.shtml
+ """) + + self.mainView.setModel(self.proxyModel) + self.mainView.setSortingEnabled(True) + self.mainView.sortByColumn(0, Qt.AscendingOrder) + + self.mainLayout.addWidget(self.infoBox) + self.mainLayout.addWidget(self.locationComboBox) + self.mainLayout.addWidget(self.mainView) + + self.bottomLayout.addWidget(self.searchBar) + self.bottomLayout.addWidget(self.selectButton) + self.bottomLayout.addWidget(self.cancelButton) + + self.mainLayout.addLayout(self.bottomLayout) + + for loc in self.products.keys(): + self.locationComboBox.addItem(loc) + self.locationComboBox.setCurrentText("CONUS") + + self.setMinimumWidth(600) + self.setMinimumHeight(600) + self.setSizeGripEnabled(True) + + def update_search(self, *args): + self.proxyModel.setFilterWildcard(self.searchBar.text()) + + def select_pressed(self, *args): + indexes = self.mainView.selectedIndexes() + if len(indexes) == 0: + self.reject() + return + + index = self.proxyModel.mapToSource(indexes[0]) + + self.selectedProduct = { + "productId": self.model.item(index.row(), 1).text(), + "location": self.locationComboBox.currentText(), + "fileType": "wrfsfcf01", + } + self.accept() + + def location_selected(self, *args): + self.model.clear() + data = self.products.get(self.locationComboBox.currentText(), {}) + + root = self.model.invisibleRootItem() + + for prod, name in data: + name = QStandardItem(name) + prod = QStandardItem(prod) + name.setEditable(False) + prod.setEditable(False) + root.appendRow([name, prod]) + + self.model.setHorizontalHeaderLabels(["Product Name", "Product ID"]) + self.mainView.resizeColumnToContents(0) + +class MRMSProductsDialog(QDialog): def __init__(self): QDialog.__init__(self) @@ -127,28 +234,44 @@ def __init__(self): self.mainLayout = QHBoxLayout(self) self.mainLayout.setContentsMargins(0, 0, 0, 0) - self.display = QLineEdit() - self.dialog = ProductsDialog() + self.display = QLineEdit() + self.mrmsDialog = MRMSProductsDialog() + self.hrrrDialog = HRRRProductsDialog() self.dialogButton = QToolButton() self.dialogButton.setText("...") self.dialogButton.clicked.connect(self.run_dialog) + self.display.setReadOnly(True) + self.mainLayout.addWidget(self.display) self.mainLayout.addWidget(self.dialogButton) + self.set_data_type("mrms") self.set_product("") + def set_data_type(self, dataType): + self.dataType = dataType + + def dialog(self): + if self.dataType == "mrms": + return self.mrmsDialog + elif self.dataType == "hrrr": + return self.hrrrDialog + else: + print("ProductSelect dialog called without setting data type") + return + def set_product(self, text): - self.dialog.selectedProduct = text - self.display.setText(text) + self.dialog().selectedProduct = text + self.display.setText(str(text)) def get_product(self): - return self.display.text() + return self.dialog().selectedProduct def run_dialog(self, *args): - self.dialog.exec() - self.set_product(self.dialog.selectedProduct) + self.dialog().exec() + self.set_product(self.dialog().selectedProduct) class FileInput(QWidget): @@ -230,13 +353,9 @@ class PlacefileEditor(QWidget): MAIN_TYPES = [ ["Basic", "basic"], ["MRMS Typed Reflectivity", "MRMSTypedReflectivity"], + ["HRRR", "HRRR"], ] - def _make_enabled_callback(self, widget, enabler): - def callback(*args, **kwargs): - widget.setEnabled(enabler.isChecked()) - return callback - def __init__(self, *args, **kwargs): QWidget.__init__(self, *args, **kwargs) @@ -366,18 +485,25 @@ def __init__(self, *args, **kwargs): TYPED_PREC_ONLY = {"rainPalette", "snowPalette", "hailPalette", "typeProduct", "reflProduct"} NOT_TYPED_PREC_ONLY = {"product", "palette", "url"} + NOT_HRRR = {"gzipped"} def change_enabled_callback(self, *args): - aws = self.dataWidgets["aws"].isChecked() + hrrr = self.dataWidgets["mainType"].currentData() == "HRRR" typedPrec = self.dataWidgets["mainType"].currentData() == "MRMSTypedReflectivity" + aws = self.dataWidgets["aws"].isChecked() or hrrr or typedPrec + + self.dataWidgets["product"].set_data_type("hrrr" if hrrr else "mrms") for name, widget in self.dataWidgets.items(): s = (name in self.enableWidgets and not self.enableWidgets[name].isChecked()) or \ (name in self.AWS_ONLY and not aws) or \ (name in self.NOT_AWS_ONLY and aws) or \ (name in self.TYPED_PREC_ONLY and not typedPrec) or \ - (name in self.NOT_TYPED_PREC_ONLY and typedPrec) + (name in self.NOT_TYPED_PREC_ONLY and typedPrec) or \ + (name in self.NOT_HRRR and hrrr) widget.setEnabled(not s) + + def set_settings(self, settings): for name, widget in self.dataWidgets.items(): enabled = name in settings @@ -407,12 +533,7 @@ def get_settings(self): settings = {} awsState = self.dataWidgets["aws"].isChecked() for name, widget in self.dataWidgets.items(): - if name in self.enableWidgets and \ - not self.enableWidgets[name].isChecked(): - continue - elif awsState and name in self.NOT_AWS_ONLY: - continue - elif not awsState and name in self.AWS_ONLY: + if not widget.isEnabled(): continue if isinstance(widget, QLineEdit): settings[name] = widget.text() diff --git a/grib2pf.py b/grib2pf.py index 8d987b1..274aa40 100755 --- a/grib2pf.py +++ b/grib2pf.py @@ -10,7 +10,7 @@ from multiprocessing import Process import sys -from aws import AWSHandler +from aws import AWSHandler, AWSHRRRHandler from grib2pflib import Grib2PfLib, Settings, ColorTable, MRMSTypedReflSettings location = os.path.split(__file__)[0] @@ -244,6 +244,128 @@ def _log(self, *args, **kwargs): t = time.strftime(TIME_FMT).format(format(round((time.time() % 1) * 1000), "0>3")) print(t, f"[{self.title}]", *args, **kwargs) +class HRRRPlaceFiles: + def __init__(self, hrrrs): + self.hrrrs = hrrrs + self.proc = None + + self.timeout = 3600 + self.logName = "HRRR " + hrrrs[0]["product"]["fileType"] + self.products = {} + + for i, hrrr in enumerate(hrrrs): + self.timeout = min(hrrr.get("timeout", 30), self.timeout) + self.products[hrrr["product"]["productId"]] = i + + self.aws = AWSHRRRHandler(self.hrrrs[0]["product"]) + self.verbose = True + + def _get_offsets(self, indexURL): + offsets = [-1] * len(self.hrrrs) + res = requests.get(indexURL, timeout = self.timeout) + for line in res.text.splitlines(): + _, offset, date, ID = line.split(":", 3) + if ID in self.products: + offsets[self.products[ID]] = int(offset) + + for i in range(len(offsets)): + if offsets[i] == -1: + self._log("Could not find a product") + offsets[i] = 0 + + return offsets + + def _generate(self, url, indexURL): + self._log(f"Generating images") + + offsets = self._get_offsets(indexURL) + messages = [] + for hrrr, offset in zip(self.hrrrs, offsets): + messages.append({ + "imageFile": hrrr["imageFile"], + "palette": hrrr.get("palette", None), + "imageWidth": hrrr.get("imageWidth", 1920), + "imageHeight": hrrr.get("imageHeight", 1080), + "title": hrrr.get("title", "HRRR Data"), + "mode": hrrr.get("mode", "Nearest_Data"), + "minimum": hrrr.get("minimum", -998), + "offset": offset, + }) + + settings = Settings( + url = url, + gzipped = False, + timeout = self.timeout, + logName = self.logName, + verbose = True, + messages = messages + ) + + lib = Grib2PfLib() + err, lonL, lonR, latT, latB = lib.generate_image(settings) + if err: + self._log("Error generating image") + sys.exit(err) + + latT = round(latT, 3) + latB = round(latB, 3) + lonL = round(lonL - 360, 3) + lonR = round(lonR - 360, 3) + + for hrrr in self.hrrrs: + self._log(f"Generating placefile {hrrr['placeFile']}", title = + hrrr.get("title", "HRRR Data")) + + with open(hrrr["placeFile"], "w") as file: + file.write(PLACEFILE_TEMPLATE.format( + title = hrrr.get("title", "HRRR Data"), + refresh = hrrr.get("refresh", 15), + imageURL = hrrr.get("imageURL", hrrr["imageFile"]), + latT = latT, + latB = latB, + lonL = lonL, + lonR = lonR, + threshold = hrrr.get("threshold", 0), + )) + self._log("Finished generating") + sys.exit(0) + + def generate(self): + if not self.aws.update_key(): + return + + if self.proc is not None and self.proc.is_alive(): + self._log("Killing old process. Likely failed to update.") + self.proc.kill() + self.proc.join() + self.proc.close() + self.proc = None + elif self.proc is not None: + print("clossing process") + self.proc.close() + self.proc = None + + url = self.aws.get_url(False) + indexURL = self.aws.get_url(True) + + aws = self.aws + self.aws = None + + self.proc = Process(target = self._generate, args = (url, indexURL), + daemon = True) + self.proc.start() + + self.aws = aws + + def _log(self, *args, **kwargs): + if "title" in kwargs: + logName = kwargs.pop("title") + else: + logName = self.logName + + if self.verbose: + t = time.strftime(TIME_FMT).format(format(round((time.time() % 1) * 1000), "0>3")) + print(t, f"[{logName}]", *args, **kwargs) async def run_setting(settings): mainType = settings.get("mainType", "basic") @@ -311,13 +433,33 @@ async def run_setting(settings): last = time.time() placefile.generate() +async def run_hrrrs(hrrrs): + placefile = HRRRPlaceFiles(hrrrs) + while True: + placefile.generate() + await asyncio.sleep(hrrrs[0].get("pullPeriod", 10)) + async def run_settings(settings): if isinstance(settings, dict): await run_setting(settings) elif isinstance(settings, list): async with asyncio.TaskGroup() as tg: + hrrrs = {} for setting in settings: - tg.create_task(run_setting(setting)) + if setting.get("mainType", "") == "HRRR": + location = setting["product"]["location"] + fileType = setting["product"]["fileType"] + + hrrrs.setdefault(location, {}) + hrrrs[location].setdefault(fileType, []) + + hrrrs[location][fileType].append(setting) + else: + tg.create_task(run_setting(setting)) + if len(hrrrs) > 0: + for location in hrrrs.values(): + for fileType in location.values(): + tg.create_task(run_hrrrs(fileType)) def main(): import argparse diff --git a/hrrr_wrfsfcf00_products.csv b/hrrr_wrfsfcf00_products.csv new file mode 100644 index 0000000..acefa66 --- /dev/null +++ b/hrrr_wrfsfcf00_products.csv @@ -0,0 +1,170 @@ +REFC:entire atmosphere:anl:,"Composite reflectivity [dB]" +RETOP:cloud top:anl:,"Echo Top [m]" +var discipline=0 center=7 local_table=1 parmcat=16 parm=201:entire atmosphere:anl:,"Echo Top [m]" +VIL:entire atmosphere:anl:,"Vertically-Integrated Liquid Water [kg/m^2]" +VIS:surface:anl:,"Visibility [m]" +REFD:1000 m above ground:anl:,"Reflectivity [dB]" +REFD:4000 m above ground:anl:,"Reflectivity [dB]" +REFD:263 K level:anl:,"Reflectivity [dB]" +GUST:surface:anl:,"Wind Speed (Gust) [m/s]" +UGRD:250 mb:anl:,"U-Component of Wind [m/s]" +VGRD:250 mb:anl:,"V-Component of Wind [m/s]" +UGRD:300 mb:anl:,"U-Component of Wind [m/s]" +VGRD:300 mb:anl:,"V-Component of Wind [m/s]" +HGT:500 mb:anl:,"Geopotential Height [gpm]" +TMP:500 mb:anl:,"Temperature [K]" +DPT:500 mb:anl:,"Dew Point Temperature [K]" +UGRD:500 mb:anl:,"U-Component of Wind [m/s]" +VGRD:500 mb:anl:,"V-Component of Wind [m/s]" +HGT:700 mb:anl:,"Geopotential Height [gpm]" +TMP:700 mb:anl:,"Temperature [K]" +DPT:700 mb:anl:,"Dew Point Temperature [K]" +DZDT:700 mb:anl:,"Vertical Velocity (Geometric) [m/s]" +UGRD:700 mb:anl:,"U-Component of Wind [m/s]" +VGRD:700 mb:anl:,"V-Component of Wind [m/s]" +HGT:850 mb:anl:,"Geopotential Height [gpm]" +TMP:850 mb:anl:,"Temperature [K]" +DPT:850 mb:anl:,"Dew Point Temperature [K]" +UGRD:850 mb:anl:,"U-Component of Wind [m/s]" +VGRD:850 mb:anl:,"V-Component of Wind [m/s]" +TMP:925 mb:anl:,"Temperature [K]" +DPT:925 mb:anl:,"Dew Point Temperature [K]" +UGRD:925 mb:anl:,"U-Component of Wind [m/s]" +VGRD:925 mb:anl:,"V-Component of Wind [m/s]" +TMP:1000 mb:anl:,"Temperature [K]" +DPT:1000 mb:anl:,"Dew Point Temperature [K]" +UGRD:1000 mb:anl:,"U-Component of Wind [m/s]" +VGRD:1000 mb:anl:,"V-Component of Wind [m/s]" +MAXUVV:100-1000 mb above ground:0-0 day max fcst:,"Hourly Maximum of Upward Vertical Velocity in the lowest 400hPa [m/s]" +MAXDVV:100-1000 mb above ground:0-0 day max fcst:,"Hourly Maximum of Downward Vertical Velocity in the lowest 400hPa [m/s]" +DZDT:0.5-0.8 sigma layer:0-0 day ave fcst:,"Vertical Velocity (Geometric) [m/s]" +MSLMA:mean sea level:anl:,"(MAPS System Reduction) [Pa]" +HGT:1000 mb:anl:,"Geopotential Height [gpm]" +MAXREF:1000 m above ground:0-0 day max fcst:,"Hourly Maximum of Simulated Reflectivity at 1 km AGL [dB]" +REFD:263 K level:0-0 day max fcst:,"Reflectivity [dB]" +MXUPHL:5000-2000 m above ground:0-0 day max fcst:,"Hourly Maximum of Updraft Helicity over Layer 2km to 5 km AGL [m^2/s^2]" +MNUPHL:5000-2000 m above ground:0-0 day min fcst:,"Hourly Minimum of Updraft Helicity [m^2/s^2]" +MXUPHL:2000-0 m above ground:0-0 day max fcst:,"Hourly Maximum of Updraft Helicity over Layer 2km to 5 km AGL [m^2/s^2]" +MNUPHL:2000-0 m above ground:0-0 day min fcst:,"Hourly Minimum of Updraft Helicity [m^2/s^2]" +MXUPHL:3000-0 m above ground:0-0 day max fcst:,"Hourly Maximum of Updraft Helicity over Layer 2km to 5 km AGL [m^2/s^2]" +MNUPHL:3000-0 m above ground:0-0 day min fcst:,"Hourly Minimum of Updraft Helicity [m^2/s^2]" +RELV:2000-0 m above ground:0-0 day max fcst:,"Relative Vorticity [1/s]" +RELV:1000-0 m above ground:0-0 day max fcst:,"Relative Vorticity [1/s]" +HAIL:entire atmosphere:0-0 day max fcst:,"Hail [m]" +HAIL:0.1 sigma level:0-0 day max fcst:,"Hail [m]" +HAIL:surface:0-0 day max fcst:,"Hail [m]" +TCOLG:entire atmosphere (considered as a single layer):0-0 day max fcst:,"Total Column Integrate Graupel [kg/m^2]" +LTNGSD:1 m above ground:anl:,"Lightning Potential Index (LPI) (see Note) [J/kg]" +LTNGSD:2 m above ground:anl:,"Lightning Potential Index (LPI) (see Note) [J/kg]" +LTNG:entire atmosphere:anl:,"Lightning [non-dim]" +UGRD:80 m above ground:anl:,"U-Component of Wind [m/s]" +VGRD:80 m above ground:anl:,"V-Component of Wind [m/s]" +PRES:surface:anl:,"Pressure [Pa]" +HGT:surface:anl:,"Geopotential Height [gpm]" +TMP:surface:anl:,"Temperature [K]" +ASNOW:surface:0-0 day acc fcst:,"Total Snowfall [m]" +MSTAV:0 m underground:anl:,"Moisture Availability [%]" +CNWAT:surface:anl:,"Plant Canopy Surface Water [kg/m^2]" +WEASD:surface:anl:,"Water Equivalent of Accumulated Snow Depth [kg/m^2]" +SNOWC:surface:anl:,"Snow Cover [%]" +SNOD:surface:anl:,"Snow Depth [m]" +TMP:2 m above ground:anl:,"Temperature [K]" +POT:2 m above ground:anl:,"Potential Temperature [K]" +SPFH:2 m above ground:anl:,"Specific Humidity [kg/kg]" +DPT:2 m above ground:anl:,"Dew Point Temperature [K]" +RH:2 m above ground:anl:,"Relative Humidity [%]" +MASSDEN:8 m above ground:anl:,"Mass Density (Concentration) [kg/m^3]" +UGRD:10 m above ground:anl:,"U-Component of Wind [m/s]" +VGRD:10 m above ground:anl:,"V-Component of Wind [m/s]" +WIND:10 m above ground:0-0 day max fcst:,"Wind Speed [m/s]" +MAXUW:10 m above ground:0-0 day max fcst:,"Component of Hourly Maximum 10m Wind Speed [m/s]" +MAXVW:10 m above ground:0-0 day max fcst:,"Component of Hourly Maximum 10m Wind Speed [m/s]" +CPOFP:surface:anl:,"Percent frozen precipitation [%]" +PRATE:surface:anl:,"Precipitation Rate [kg/m^2/s]" +APCP:surface:0-0 day acc fcst:,"Total Precipitation [kg/m^2]" +WEASD:surface:0-0 day acc fcst:,"Water Equivalent of Accumulated Snow Depth [kg/m^2]" +FROZR:surface:0-0 day acc fcst:,"Frozen Rain [kg/m^2]" +FRZR:surface:0-0 day acc fcst:,"Freezing Rain [kg/m^2]" +SSRUN:surface:0-0 day acc fcst:,"Storm Surface Runoff [kg/m^2]" +BGRUN:surface:0-0 day acc fcst:,"Baseflow-Groundwater Runoff [kg/m^2]" +CSNOW:surface:anl:,"Categorical Snow [-]" +CICEP:surface:anl:,"Categorical Ice Pellets [-]" +CFRZR:surface:anl:,"Categorical Freezing Rain [-]" +CRAIN:surface:anl:,"Categorical Rain [-]" +SFCR:surface:anl:,"Surface Roughness [m]" +FRICV:surface:anl:,"Frictional Velocity [m/s]" +SHTFL:surface:anl:,"Sensible Heat Net Flux [W/m^2]" +LHTFL:surface:anl:,"Latent Heat Net Flux [W/m^2]" +VEG:surface:anl:,"Vegetation [%]" +VEGMIN:surface:anl:,"Vegetation [%]" +VEGMAX:surface:anl:,"Vegetation [%]" +LAI:surface:anl:,"Leaf Area Index [Numeric]" +GFLUX:surface:anl:,"Ground Heat Flux [W/m^2]" +VGTYP:surface:anl:,"Vegetation Type [Integer(0- 13)]" +LFTX:500-1000 mb:anl:,"Surface Lifted Index [K]" +CAPE:surface:anl:,"Convective Available Potential Energy [J/kg]" +CIN:surface:anl:,"Convective Inhibition [J/kg]" +PWAT:entire atmosphere (considered as a single layer):anl:,"Precipitable Water [kg/m^2]" +AOTK:entire atmosphere (considered as a single layer):anl:,"Aerosol Optical Thickness [Numeric]" +COLMD:entire atmosphere (considered as a single layer):anl:,"Column-Integrated Mass Density [kg/m^2]" +TCOLW:entire atmosphere:anl:,"Column-Integrated Mass Density [kg/m^2]" +TCOLI:entire atmosphere:anl:,"Column-Integrated Mass Density [kg/m^2]" +TCDC:boundary layer cloud layer:anl:,"Total Cloud Cover [%]" +LCDC:low cloud layer:anl:,"Low Cloud Cover [%]" +MCDC:middle cloud layer:anl:,"Medium Cloud Cover [%]" +HCDC:high cloud layer:anl:,"High Cloud Cover [%]" +TCDC:entire atmosphere:anl:,"Total Cloud Cover [%]" +HGT:cloud ceiling:anl:,"Geopotential Height [gpm]" +HGT:cloud base:anl:,"Geopotential Height [gpm]" +PRES:cloud base:anl:,"Pressure [Pa]" +PRES:cloud top:anl:,"Pressure [Pa]" +HGT:cloud top:anl:,"Geopotential Height [gpm]" +ULWRF:top of atmosphere:anl:,"Upward Long-Wave Rad. Flux [W/m^2]" +DSWRF:surface:anl:,"Downward Short-Wave Radiation Flux [W/m^2]" +DLWRF:surface:anl:,"Downward Long-Wave Rad. Flux [W/m^2]" +USWRF:surface:anl:,"Upward Short-Wave Radiation Flux [W/m^2]" +ULWRF:surface:anl:,"Upward Long-Wave Rad. Flux [W/m^2]" +CFNSF:surface:anl:,"Cloud Forcing Net Solar Flux [W/m^2]" +VBDSF:surface:anl:,"Visible Beam Downward Solar Flux [W/m^2]" +VDDSF:surface:anl:,"Visible Diffuse Downward Solar Flux [W/m^2]" +USWRF:top of atmosphere:anl:,"Upward Short-Wave Radiation Flux [W/m^2]" +HLCY:3000-0 m above ground:anl:,"Storm Relative Helicity [m^2/s^2]" +HLCY:1000-0 m above ground:anl:,"Storm Relative Helicity [m^2/s^2]" +USTM:0-6000 m above ground:anl:,"U-Component Storm Motion [m/s]" +VSTM:0-6000 m above ground:anl:,"V-Component Storm Motion [m/s]" +VUCSH:0-1000 m above ground:anl:,"Vertical U-Component Shear [1/s]" +VVCSH:0-1000 m above ground:anl:,"Vertical V-Component Shear [1/s]" +VUCSH:0-6000 m above ground:anl:,"Vertical U-Component Shear [1/s]" +VVCSH:0-6000 m above ground:anl:,"Vertical V-Component Shear [1/s]" +HGT:0C isotherm:anl:,"Geopotential Height [gpm]" +RH:0C isotherm:anl:,"Relative Humidity [%]" +PRES:0C isotherm:anl:,"Pressure [Pa]" +HGT:highest tropospheric freezing level:anl:,"Geopotential Height [gpm]" +RH:highest tropospheric freezing level:anl:,"Relative Humidity [%]" +PRES:highest tropospheric freezing level:anl:,"Pressure [Pa]" +HGT:263 K level:anl:,"Geopotential Height [gpm]" +HGT:253 K level:anl:,"Geopotential Height [gpm]" +4LFTX:180-0 mb above ground:anl:,"Best (4 layer) Lifted Index [K]" +CAPE:180-0 mb above ground:anl:,"Convective Available Potential Energy [J/kg]" +CIN:180-0 mb above ground:anl:,"Convective Inhibition [J/kg]" +HPBL:surface:anl:,"Planetary Boundary Layer Height [m]" +HGT:level of adiabatic condensation from sfc:anl:,"Geopotential Height [gpm]" +CAPE:90-0 mb above ground:anl:,"Convective Available Potential Energy [J/kg]" +CIN:90-0 mb above ground:anl:,"Convective Inhibition [J/kg]" +CAPE:255-0 mb above ground:anl:,"Convective Available Potential Energy [J/kg]" +CIN:255-0 mb above ground:anl:,"Convective Inhibition [J/kg]" +HGT:equilibrium level:anl:,"Geopotential Height [gpm]" +PLPL:255-0 mb above ground:anl:,"Pressure of level from which parcel was lifted [Pa]" +CAPE:0-3000 m above ground:anl:,"Convective Available Potential Energy [J/kg]" +HGT:level of free convection:anl:,"Geopotential Height [gpm]" +EFHL:surface:anl:,"Geopotential Height [gpm]" +CANGLE:0-500 m above ground:anl:,"Geopotential Height [gpm]" +LAYTH:261 K level - 256 K level:anl:,"Layer Thickness [m]" +ESP:0-3000 m above ground:anl:,"Layer Thickness [m]" +RHPW:entire atmosphere:anl:,"Relative Humidity with Respect to Precipitable Water [%]" +LAND:surface:anl:,"Land Cover (0=sea, 1=land) [Proportion]" +ICEC:surface:anl:,"Ice Cover [Proportion]" +SBT123:top of atmosphere:anl:,"Simulated Brightness Temperature for GOES 12, Channel 3 [K]" +SBT124:top of atmosphere:anl:,"Simulated Brightness Temperature for GOES 12, Channel 4 [K]" +SBT113:top of atmosphere:anl:,"Simulated Brightness Temperature for GOES 11, Channel 3 [K]" +SBT114:top of atmosphere:anl:,"Simulated Brightness Temperature for GOES 11, Channel 4 [K]" diff --git a/hrrr_wrfsfcf01_products.csv b/hrrr_wrfsfcf01_products.csv new file mode 100644 index 0000000..8c42c27 --- /dev/null +++ b/hrrr_wrfsfcf01_products.csv @@ -0,0 +1,170 @@ +REFC:entire atmosphere:1 hour fcst:,"Composite reflectivity [dB]" +RETOP:cloud top:1 hour fcst:,"Echo Top [m]" +var discipline=0 center=7 local_table=1 parmcat=16 parm=201:entire atmosphere:1 hour fcst:,"Echo Top [m]" +VIL:entire atmosphere:1 hour fcst:,"Vertically-Integrated Liquid Water [kg/m^2]" +VIS:surface:1 hour fcst:,"Visibility [m]" +REFD:1000 m above ground:1 hour fcst:,"Reflectivity [dB]" +REFD:4000 m above ground:1 hour fcst:,"Reflectivity [dB]" +REFD:263 K level:1 hour fcst:,"Reflectivity [dB]" +GUST:surface:1 hour fcst:,"Wind Speed (Gust) [m/s]" +UGRD:250 mb:1 hour fcst:,"U-Component of Wind [m/s]" +VGRD:250 mb:1 hour fcst:,"V-Component of Wind [m/s]" +UGRD:300 mb:1 hour fcst:,"U-Component of Wind [m/s]" +VGRD:300 mb:1 hour fcst:,"V-Component of Wind [m/s]" +HGT:500 mb:1 hour fcst:,"Geopotential Height [gpm]" +TMP:500 mb:1 hour fcst:,"Temperature [K]" +DPT:500 mb:1 hour fcst:,"Dew Point Temperature [K]" +UGRD:500 mb:1 hour fcst:,"U-Component of Wind [m/s]" +VGRD:500 mb:1 hour fcst:,"V-Component of Wind [m/s]" +HGT:700 mb:1 hour fcst:,"Geopotential Height [gpm]" +TMP:700 mb:1 hour fcst:,"Temperature [K]" +DPT:700 mb:1 hour fcst:,"Dew Point Temperature [K]" +DZDT:700 mb:1 hour fcst:,"Vertical Velocity (Geometric) [m/s]" +UGRD:700 mb:1 hour fcst:,"U-Component of Wind [m/s]" +VGRD:700 mb:1 hour fcst:,"V-Component of Wind [m/s]" +HGT:850 mb:1 hour fcst:,"Geopotential Height [gpm]" +TMP:850 mb:1 hour fcst:,"Temperature [K]" +DPT:850 mb:1 hour fcst:,"Dew Point Temperature [K]" +UGRD:850 mb:1 hour fcst:,"U-Component of Wind [m/s]" +VGRD:850 mb:1 hour fcst:,"V-Component of Wind [m/s]" +TMP:925 mb:1 hour fcst:,"Temperature [K]" +DPT:925 mb:1 hour fcst:,"Dew Point Temperature [K]" +UGRD:925 mb:1 hour fcst:,"U-Component of Wind [m/s]" +VGRD:925 mb:1 hour fcst:,"V-Component of Wind [m/s]" +TMP:1000 mb:1 hour fcst:,"Temperature [K]" +DPT:1000 mb:1 hour fcst:,"Dew Point Temperature [K]" +UGRD:1000 mb:1 hour fcst:,"U-Component of Wind [m/s]" +VGRD:1000 mb:1 hour fcst:,"V-Component of Wind [m/s]" +MAXUVV:100-1000 mb above ground:0-1 hour max fcst:,"Hourly Maximum of Upward Vertical Velocity in the lowest 400hPa [m/s]" +MAXDVV:100-1000 mb above ground:0-1 hour max fcst:,"Hourly Maximum of Downward Vertical Velocity in the lowest 400hPa [m/s]" +DZDT:0.5-0.8 sigma layer:0-1 hour ave fcst:,"Vertical Velocity (Geometric) [m/s]" +MSLMA:mean sea level:1 hour fcst:,"(MAPS System Reduction) [Pa]" +HGT:1000 mb:1 hour fcst:,"Geopotential Height [gpm]" +MAXREF:1000 m above ground:0-1 hour max fcst:,"Hourly Maximum of Simulated Reflectivity at 1 km AGL [dB]" +REFD:263 K level:0-1 hour max fcst:,"Reflectivity [dB]" +MXUPHL:5000-2000 m above ground:0-1 hour max fcst:,"Hourly Maximum of Updraft Helicity over Layer 2km to 5 km AGL [m^2/s^2]" +MNUPHL:5000-2000 m above ground:0-1 hour min fcst:,"Hourly Minimum of Updraft Helicity [m^2/s^2]" +MXUPHL:2000-0 m above ground:0-1 hour max fcst:,"Hourly Maximum of Updraft Helicity over Layer 2km to 5 km AGL [m^2/s^2]" +MNUPHL:2000-0 m above ground:0-1 hour min fcst:,"Hourly Minimum of Updraft Helicity [m^2/s^2]" +MXUPHL:3000-0 m above ground:0-1 hour max fcst:,"Hourly Maximum of Updraft Helicity over Layer 2km to 5 km AGL [m^2/s^2]" +MNUPHL:3000-0 m above ground:0-1 hour min fcst:,"Hourly Minimum of Updraft Helicity [m^2/s^2]" +RELV:2000-0 m above ground:0-1 hour max fcst:,"Relative Vorticity [1/s]" +RELV:1000-0 m above ground:0-1 hour max fcst:,"Relative Vorticity [1/s]" +HAIL:entire atmosphere:0-1 hour max fcst:,"Hail [m]" +HAIL:0.1 sigma level:0-1 hour max fcst:,"Hail [m]" +HAIL:surface:0-1 hour max fcst:,"Hail [m]" +TCOLG:entire atmosphere (considered as a single layer):0-1 hour max fcst:,"Total Column Integrate Graupel [kg/m^2]" +LTNGSD:1 m above ground:1 hour fcst:,"Lightning Potential Index (LPI) (see Note) [J/kg]" +LTNGSD:2 m above ground:1 hour fcst:,"Lightning Potential Index (LPI) (see Note) [J/kg]" +LTNG:entire atmosphere:1 hour fcst:,"Lightning [non-dim]" +UGRD:80 m above ground:1 hour fcst:,"U-Component of Wind [m/s]" +VGRD:80 m above ground:1 hour fcst:,"V-Component of Wind [m/s]" +PRES:surface:1 hour fcst:,"Pressure [Pa]" +HGT:surface:1 hour fcst:,"Geopotential Height [gpm]" +TMP:surface:1 hour fcst:,"Temperature [K]" +ASNOW:surface:0-1 hour acc fcst:,"Total Snowfall [m]" +MSTAV:0 m underground:1 hour fcst:,"Moisture Availability [%]" +CNWAT:surface:1 hour fcst:,"Plant Canopy Surface Water [kg/m^2]" +WEASD:surface:1 hour fcst:,"Water Equivalent of Accumulated Snow Depth [kg/m^2]" +SNOWC:surface:1 hour fcst:,"Snow Cover [%]" +SNOD:surface:1 hour fcst:,"Snow Depth [m]" +TMP:2 m above ground:1 hour fcst:,"Temperature [K]" +POT:2 m above ground:1 hour fcst:,"Potential Temperature [K]" +SPFH:2 m above ground:1 hour fcst:,"Specific Humidity [kg/kg]" +DPT:2 m above ground:1 hour fcst:,"Dew Point Temperature [K]" +RH:2 m above ground:1 hour fcst:,"Relative Humidity [%]" +MASSDEN:8 m above ground:1 hour fcst:,"Mass Density (Concentration) [kg/m^3]" +UGRD:10 m above ground:1 hour fcst:,"U-Component of Wind [m/s]" +VGRD:10 m above ground:1 hour fcst:,"V-Component of Wind [m/s]" +WIND:10 m above ground:0-1 hour max fcst:,"Wind Speed [m/s]" +MAXUW:10 m above ground:0-1 hour max fcst:,"Component of Hourly Maximum 10m Wind Speed [m/s]" +MAXVW:10 m above ground:0-1 hour max fcst:,"Component of Hourly Maximum 10m Wind Speed [m/s]" +CPOFP:surface:1 hour fcst:,"Percent frozen precipitation [%]" +PRATE:surface:1 hour fcst:,"Precipitation Rate [kg/m^2/s]" +APCP:surface:0-1 hour acc fcst:,"Total Precipitation [kg/m^2]" +WEASD:surface:0-1 hour acc fcst:,"Water Equivalent of Accumulated Snow Depth [kg/m^2]" +FROZR:surface:0-1 hour acc fcst:,"Frozen Rain [kg/m^2]" +FRZR:surface:0-1 hour acc fcst:,"Freezing Rain [kg/m^2]" +SSRUN:surface:0-1 hour acc fcst:,"Storm Surface Runoff [kg/m^2]" +BGRUN:surface:0-1 hour acc fcst:,"Baseflow-Groundwater Runoff [kg/m^2]" +CSNOW:surface:1 hour fcst:,"Categorical Snow [-]" +CICEP:surface:1 hour fcst:,"Categorical Ice Pellets [-]" +CFRZR:surface:1 hour fcst:,"Categorical Freezing Rain [-]" +CRAIN:surface:1 hour fcst:,"Categorical Rain [-]" +SFCR:surface:1 hour fcst:,"Surface Roughness [m]" +FRICV:surface:1 hour fcst:,"Frictional Velocity [m/s]" +SHTFL:surface:1 hour fcst:,"Sensible Heat Net Flux [W/m^2]" +LHTFL:surface:1 hour fcst:,"Latent Heat Net Flux [W/m^2]" +VEG:surface:1 hour fcst:,"Vegetation [%]" +VEGMIN:surface:1 hour fcst:,"Vegetation [%]" +VEGMAX:surface:1 hour fcst:,"Vegetation [%]" +LAI:surface:1 hour fcst:,"Leaf Area Index [Numeric]" +GFLUX:surface:1 hour fcst:,"Ground Heat Flux [W/m^2]" +VGTYP:surface:1 hour fcst:,"Vegetation Type [Integer(0- 13)]" +LFTX:500-1000 mb:1 hour fcst:,"Surface Lifted Index [K]" +CAPE:surface:1 hour fcst:,"Convective Available Potential Energy [J/kg]" +CIN:surface:1 hour fcst:,"Convective Inhibition [J/kg]" +PWAT:entire atmosphere (considered as a single layer):1 hour fcst:,"Precipitable Water [kg/m^2]" +AOTK:entire atmosphere (considered as a single layer):1 hour fcst:,"Aerosol Optical Thickness [Numeric]" +COLMD:entire atmosphere (considered as a single layer):1 hour fcst:,"Column-Integrated Mass Density [kg/m^2]" +TCOLW:entire atmosphere:1 hour fcst:,"Column-Integrated Mass Density [kg/m^2]" +TCOLI:entire atmosphere:1 hour fcst:,"Column-Integrated Mass Density [kg/m^2]" +TCDC:boundary layer cloud layer:1 hour fcst:,"Total Cloud Cover [%]" +LCDC:low cloud layer:1 hour fcst:,"Low Cloud Cover [%]" +MCDC:middle cloud layer:1 hour fcst:,"Medium Cloud Cover [%]" +HCDC:high cloud layer:1 hour fcst:,"High Cloud Cover [%]" +TCDC:entire atmosphere:1 hour fcst:,"Total Cloud Cover [%]" +HGT:cloud ceiling:1 hour fcst:,"Geopotential Height [gpm]" +HGT:cloud base:1 hour fcst:,"Geopotential Height [gpm]" +PRES:cloud base:1 hour fcst:,"Pressure [Pa]" +PRES:cloud top:1 hour fcst:,"Pressure [Pa]" +HGT:cloud top:1 hour fcst:,"Geopotential Height [gpm]" +ULWRF:top of atmosphere:1 hour fcst:,"Upward Long-Wave Rad. Flux [W/m^2]" +DSWRF:surface:1 hour fcst:,"Downward Short-Wave Radiation Flux [W/m^2]" +DLWRF:surface:1 hour fcst:,"Downward Long-Wave Rad. Flux [W/m^2]" +USWRF:surface:1 hour fcst:,"Upward Short-Wave Radiation Flux [W/m^2]" +ULWRF:surface:1 hour fcst:,"Upward Long-Wave Rad. Flux [W/m^2]" +CFNSF:surface:1 hour fcst:,"Cloud Forcing Net Solar Flux [W/m^2]" +VBDSF:surface:1 hour fcst:,"Visible Beam Downward Solar Flux [W/m^2]" +VDDSF:surface:1 hour fcst:,"Visible Diffuse Downward Solar Flux [W/m^2]" +USWRF:top of atmosphere:1 hour fcst:,"Upward Short-Wave Radiation Flux [W/m^2]" +HLCY:3000-0 m above ground:1 hour fcst:,"Storm Relative Helicity [m^2/s^2]" +HLCY:1000-0 m above ground:1 hour fcst:,"Storm Relative Helicity [m^2/s^2]" +USTM:0-6000 m above ground:1 hour fcst:,"U-Component Storm Motion [m/s]" +VSTM:0-6000 m above ground:1 hour fcst:,"V-Component Storm Motion [m/s]" +VUCSH:0-1000 m above ground:1 hour fcst:,"Vertical U-Component Shear [1/s]" +VVCSH:0-1000 m above ground:1 hour fcst:,"Vertical V-Component Shear [1/s]" +VUCSH:0-6000 m above ground:1 hour fcst:,"Vertical U-Component Shear [1/s]" +VVCSH:0-6000 m above ground:1 hour fcst:,"Vertical V-Component Shear [1/s]" +HGT:0C isotherm:1 hour fcst:,"Geopotential Height [gpm]" +RH:0C isotherm:1 hour fcst:,"Relative Humidity [%]" +PRES:0C isotherm:1 hour fcst:,"Pressure [Pa]" +HGT:highest tropospheric freezing level:1 hour fcst:,"Geopotential Height [gpm]" +RH:highest tropospheric freezing level:1 hour fcst:,"Relative Humidity [%]" +PRES:highest tropospheric freezing level:1 hour fcst:,"Pressure [Pa]" +HGT:263 K level:1 hour fcst:,"Geopotential Height [gpm]" +HGT:253 K level:1 hour fcst:,"Geopotential Height [gpm]" +4LFTX:180-0 mb above ground:1 hour fcst:,"Best (4 layer) Lifted Index [K]" +CAPE:180-0 mb above ground:1 hour fcst:,"Convective Available Potential Energy [J/kg]" +CIN:180-0 mb above ground:1 hour fcst:,"Convective Inhibition [J/kg]" +HPBL:surface:1 hour fcst:,"Planetary Boundary Layer Height [m]" +HGT:level of adiabatic condensation from sfc:1 hour fcst:,"Geopotential Height [gpm]" +CAPE:90-0 mb above ground:1 hour fcst:,"Convective Available Potential Energy [J/kg]" +CIN:90-0 mb above ground:1 hour fcst:,"Convective Inhibition [J/kg]" +CAPE:255-0 mb above ground:1 hour fcst:,"Convective Available Potential Energy [J/kg]" +CIN:255-0 mb above ground:1 hour fcst:,"Convective Inhibition [J/kg]" +HGT:equilibrium level:1 hour fcst:,"Geopotential Height [gpm]" +PLPL:255-0 mb above ground:1 hour fcst:,"Pressure of level from which parcel was lifted [Pa]" +CAPE:0-3000 m above ground:1 hour fcst:,"Convective Available Potential Energy [J/kg]" +HGT:level of free convection:1 hour fcst:,"Geopotential Height [gpm]" +EFHL:surface:1 hour fcst:,"Geopotential Height [gpm]" +CANGLE:0-500 m above ground:1 hour fcst:,"Geopotential Height [gpm]" +LAYTH:261 K level - 256 K level:1 hour fcst:,"Layer Thickness [m]" +ESP:0-3000 m above ground:1 hour fcst:,"Layer Thickness [m]" +RHPW:entire atmosphere:1 hour fcst:,"Relative Humidity with Respect to Precipitable Water [%]" +LAND:surface:1 hour fcst:,"Land Cover (0=sea, 1=land) [Proportion]" +ICEC:surface:1 hour fcst:,"Ice Cover [Proportion]" +SBT123:top of atmosphere:1 hour fcst:,"Simulated Brightness Temperature for GOES 12, Channel 3 [K]" +SBT124:top of atmosphere:1 hour fcst:,"Simulated Brightness Temperature for GOES 12, Channel 4 [K]" +SBT113:top of atmosphere:1 hour fcst:,"Simulated Brightness Temperature for GOES 11, Channel 3 [K]" +SBT114:top of atmosphere:1 hour fcst:,"Simulated Brightness Temperature for GOES 11, Channel 4 [K]" diff --git a/source/grib2pf.c b/source/grib2pf.c index 6c79114..d234e00 100644 --- a/source/grib2pf.c +++ b/source/grib2pf.c @@ -91,6 +91,8 @@ size_t chunk_from_server(void *contents, size_t size, size_t nmemb, void *userp) DownloadingData* data = userp; size_t inputSize = size * nmemb; + LogSettings log = {.verbose = true, .logName = "test" }; + if (inputSize == 0) { fprintf(stderr, "Got empty response from the server\n"); return 0; @@ -100,7 +102,6 @@ size_t chunk_from_server(void *contents, size_t size, size_t nmemb, void *userp) fprintf(stderr, "Got more data after finished inflating\n"); } - if (data->gzipped) { data->strm.next_in = contents; data->strm.avail_in = inputSize; @@ -134,20 +135,27 @@ size_t chunk_from_server(void *contents, size_t size, size_t nmemb, void *userp) } } } else { - while (data->out.size - data->out.current < size) { - uint8_t* ptr = realloc(data->out.data, data->out.size + CHUNCK_SIZE); + while (data->out.size - data->out.current < inputSize) { + size_t newSize; + if (data->out.size == 0) { + newSize = CHUNCK_SIZE; + } else { + newSize = data->out.size * 2; + } + uint8_t* ptr = realloc(data->out.data, newSize); if (ptr == NULL) { fprintf(stderr, "Could not allocate buffer\n"); return CURL_WRITEFUNC_ERROR; } data->out.data = ptr; - data->out.size += CHUNCK_SIZE; + data->out.size = newSize; } memcpy(data->out.data + data->out.current, contents, inputSize); data->out.current += inputSize; } + return inputSize; } @@ -574,6 +582,7 @@ int generate_image(const Settings* settings, OutputCoords* output) { ImageData imData = generate_image_data(message, data.gribStart, data.totalSize, settings->verbose); + logS.logName = message->title; if (imData.error) { continue; } @@ -628,6 +637,7 @@ int generate_image(const Settings* settings, OutputCoords* output) { png_image_free(&image); free(imageBuffer); } + logS.logName = settings->logName; free(data.data);