Skip to content

Commit

Permalink
Add validation for all files being downloaded
Browse files Browse the repository at this point in the history
  • Loading branch information
jacobbieker committed Oct 24, 2023
1 parent a38aa40 commit 9b25ab7
Showing 1 changed file with 7 additions and 1 deletion.
8 changes: 7 additions & 1 deletion nwp/assets/cams/cams.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import datetime as dt

import os
import cdsapi
import dagster
from dagster import AssetObservation
Expand Down Expand Up @@ -231,3 +231,9 @@ def fetch_cams_forecast_for_day(context: dagster.OpExecutionContext, config: CAM
)
)

# Validate that all files were downloaded
for var in multi_variables + single_variables:
fname: str = f'{config.raw_dir}/{date.strftime("%Y%m%d")}{it[:2]}_{var}.grib'
if not os.path.isfile(fname):
raise FileNotFoundError(f"File {fname} was not downloaded.")

0 comments on commit 9b25ab7

Please sign in to comment.