diff --git a/dag/climate.yml b/dag/climate.yml index 84dd5b430731..4436974e2b25 100644 --- a/dag/climate.yml +++ b/dag/climate.yml @@ -32,7 +32,7 @@ steps: # Copernicus Climate Change Service - Surface temperature. # data://meadow/climate/2023-12-20/surface_temperature: - - snapshot://climate/2024-11-05/surface_temperature.zip + - snapshot://climate/2024-12-05/surface_temperature.zip - snapshot://countries/2023-12-27/world_bank.zip data://garden/climate/2023-12-20/surface_temperature: - data://meadow/climate/2023-12-20/surface_temperature diff --git a/etl/steps/data/meadow/climate/2023-12-20/surface_temperature.py b/etl/steps/data/meadow/climate/2023-12-20/surface_temperature.py index 6e0fa1f3be60..92aa10be5435 100644 --- a/etl/steps/data/meadow/climate/2023-12-20/surface_temperature.py +++ b/etl/steps/data/meadow/climate/2023-12-20/surface_temperature.py @@ -137,11 +137,12 @@ def run(dest_dir: str) -> None: f"It wasn't possible to extract temperature data for {len(small_countries)} small countries as they are too small for the resolution of the Copernicus data." ) # Define the start and end dates - da["date"] = pd.to_datetime(da["date"].astype(str), format="%Y%m%d") + da["valid_time"] = pd.to_datetime(da["valid_time"].astype(str), format="ISO8601") + print(da["valid_time"]) # Now you can access the 'dt' accessor - start_time = da["date"].min().dt.date.astype(str).item() - end_time = da["date"].max().dt.date.astype(str).item() + start_time = da["valid_time"].min().dt.date.astype(str).item() + end_time = da["valid_time"].max().dt.date.astype(str).item() # Generate a date range from start_time to end_time with monthly frequency month_middles = pd.date_range(start=start_time, end=end_time, freq="MS") + pd.offsets.Day(14) diff --git a/snapshots/climate/2024-11-19/total_precipitation.py b/snapshots/climate/2024-11-19/total_precipitation.py index 261639c7b029..05c119573ead 100644 --- a/snapshots/climate/2024-11-19/total_precipitation.py +++ b/snapshots/climate/2024-11-19/total_precipitation.py @@ -1,4 +1,4 @@ -"""Script to create a snapshot of the monthly averaged surface temperature data from 1950 to present from the Copernicus Climate Change Service. +"""Script to create a snapshot of the precipitation data from 1950 to present from the Copernicus Climate Change Service. The script assumes that the data is available on the CDS API. Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS diff --git a/snapshots/climate/2024-12-05/surface_temperature.py b/snapshots/climate/2024-12-05/surface_temperature.py new file mode 100644 index 000000000000..bd1f036f9d5e --- /dev/null +++ b/snapshots/climate/2024-12-05/surface_temperature.py @@ -0,0 +1,56 @@ +"""Script to create a snapshot of the monthly averaged surface temperature data from 1950 to present from the Copernicus Climate Change Service. + +The script assumes that the data is available on the CDS API. +Instructions on how to access the API on a Mac are here: https://confluence.ecmwf.int/display/CKB/How+to+install+and+use+CDS+API+on+macOS + +More information on how to access the data is here: hhttps://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels-monthly-means?tab=overview + +The data is downloaded as a NetCDF file. Tutorials for using the Copernicus API are here and work with the NETCDF format are here: https://ecmwf-projects.github.io/copernicus-training-c3s/cds-tutorial.html +""" + + +import tempfile +from pathlib import Path + +# CDS API +import cdsapi +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +def main(upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"climate/{SNAPSHOT_VERSION}/surface_temperature.zip") + snap + # Save data as a compressed temporary file. + with tempfile.TemporaryDirectory() as temp_dir: + output_file = Path(temp_dir) / "era5_monthly_t2m_eur.nc" + + client = cdsapi.Client() + + dataset = "reanalysis-era5-single-levels-monthly-means" + request = { + "product_type": ["monthly_averaged_reanalysis"], + "variable": ["2m_temperature"], + "year": [str(year) for year in range(1940, 2025)], + "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], + "time": "00:00", + "area": [90, -180, -90, 180], + "data_format": "netcdf", + "download_format": "zip", + } + + client.retrieve(dataset, request, output_file) + + # Upload snapshot. + snap.create_snapshot(filename=output_file, upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/climate/2024-12-05/surface_temperature.zip.dvc b/snapshots/climate/2024-12-05/surface_temperature.zip.dvc new file mode 100644 index 000000000000..4e136913e454 --- /dev/null +++ b/snapshots/climate/2024-12-05/surface_temperature.zip.dvc @@ -0,0 +1,26 @@ +meta: + origin: + title_snapshot: ERA5 Monthly Averaged Data on Single Levels from 1940 to Present - Monthly Averages of 2m Surface Temperature + title: ERA5 monthly averaged data on single levels from 1940 to present + description: |- + ERA5 is the latest climate reanalysis produced by ECMWF, providing hourly data on many atmospheric, land-surface and sea-state parameters together with estimates of uncertainty. + + ERA5 data are available in the Climate Data Store on regular latitude-longitude grids at 0.25° x 0.25° resolution, with atmospheric parameters on 37 pressure levels. + + ERA5 is available from 1940 and continues to be extended forward in time, with daily updates being made available 5 days behind real time + + Initial release data, i.e., data no more than three months behind real time, are called ERA5T. + producer: Contains modified Copernicus Climate Change Service information + version_producer: 2 + citation_full: |- + Hersbach, H., Bell, B., Berrisford, P., Biavati, G., Horányi, A., Muñoz Sabater, J., Nicolas, J., Peubey, C., Radu, R., Rozum, I., Schepers, D., Simmons, A., Soci, C., Dee, D., Thépaut, J-N. (2023): ERA5 monthly averaged data on single levels from 1940 to present. Copernicus Climate Change Service (C3S) Climate Data Store (CDS), DOI: 10.24381/cds.f17050d7 (Accessed on 19-Nov-2024) + url_main: https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels-monthly-means?tab=overview + date_accessed: 2024-12-05 + date_published: 2019-12-04 + license: + name: Copernicus License + url: https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels-monthly-means?tab=overview +outs: + - md5: 63aa98ba0272c654be00eaa4779c253d + size: 1396486999 + path: surface_temperature.zip