-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #123 from US-GHG-Center/staging
Release to Production
- Loading branch information
Showing
58 changed files
with
50,694 additions
and
4,313 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,221 @@ | ||
| { | ||
| "cells": [ | ||
| { | ||
| "cell_type": "raw", | ||
| "id": "0190df52-e0dc-4c6e-9a94-c290c7b86ef9", | ||
| "metadata": { | ||
| "vscode": { | ||
| "languageId": "raw" | ||
| } | ||
| }, | ||
| "source": [ | ||
| "---\n", | ||
| "title: GRA²PES Greenhouse Gas and Air Quality Species\n", | ||
| "description: Monthly, 0.036 degree resolution emissions of fossil fuel carbon dioxide (ffCO₂), carbon monoxide (CO), nitrogen oxide (NOₓ), sulfur dioxide (SO₂), and particulate matter (PM₂.₅) emissions for the year 2021 over the Contiguous United States from the Greenhouse gas And Air Pollutants Emissions System (GRA²PES).\n", | ||
| "author: Siddharth Chaudhary, Paridhi Parajuli\n", | ||
| "date: 30 August 2024\n", | ||
| "execute:\n", | ||
| " freeze: true\n", | ||
| "---" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "id": "6477da5d-92e7-4d79-b49c-2c29a11ec1a1", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "This script was used to transform the GRA2PES dataset to Cloud Optimized GeoTIFF (COG) format for display in the Greenhouse Gas (GHG) Center." | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "ca65368b-5280-4685-948e-d92fb20c9318", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "import xarray as xr\n", | ||
| "import os\n", | ||
| "import glob\n", | ||
| "from datetime import datetime\n", | ||
| "import boto3\n", | ||
| "import s3fs\n", | ||
| "import tempfile\n", | ||
| "import numpy as np\n", | ||
| "\n", | ||
| "import rasterio\n", | ||
| "from rasterio.enums import Resampling\n", | ||
| "from rio_cogeo.cogeo import cog_translate\n", | ||
| "from rio_cogeo.profiles import cog_profiles\n" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "7d8ad9e2-e18b-4ca8-b68e-fadbe14d112a", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "config = {\n", | ||
| " \"data_acquisition_method\": \"s3\",\n", | ||
| " \"raw_data_bucket\" : \"gsfc-ghg-store\",\n", | ||
| " \"raw_data_prefix\": \"GRA2PES/monthly_subset_regrid/2021\", \n", | ||
| " \"cog_data_bucket\": \"ghgc-data-store-develop\",\n", | ||
| " \"cog_data_prefix\": \"transformed_cogs/GRAAPES\",\n", | ||
| " \"date_fmt\" :\"%Y%m\",\n", | ||
| " \"transformation\": {}\n", | ||
| "}" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "b7a394ef-d598-4769-aeb5-d22447b2d5f0", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "session = boto3.session.Session()\n", | ||
| "s3_client = session.client(\"s3\")\n", | ||
| "\n", | ||
| "raw_data_bucket = config[\"raw_data_bucket\"]\n", | ||
| "raw_data_prefix= config[\"raw_data_prefix\"]\n", | ||
| "\n", | ||
| "cog_data_bucket = config['cog_data_bucket']\n", | ||
| "cog_data_prefix= config[\"cog_data_prefix\"]\n", | ||
| "\n", | ||
| "date_fmt=config['date_fmt']\n", | ||
| "\n", | ||
| "fs = s3fs.S3FileSystem()" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "04d90bf6-cd92-466e-9b23-23b927810de6", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "def get_all_s3_keys(bucket, model_name, ext):\n", | ||
| " \"\"\"Get a list of all keys in an S3 bucket.\"\"\"\n", | ||
| " keys = []\n", | ||
| "\n", | ||
| " kwargs = {\"Bucket\": bucket, \"Prefix\": f\"{model_name}/\"}\n", | ||
| " while True:\n", | ||
| " resp = s3_client.list_objects_v2(**kwargs)\n", | ||
| " for obj in resp[\"Contents\"]:\n", | ||
| " if obj[\"Key\"].endswith(ext) and \"historical\" not in obj[\"Key\"]:\n", | ||
| " keys.append(obj[\"Key\"])\n", | ||
| "\n", | ||
| " try:\n", | ||
| " kwargs[\"ContinuationToken\"] = resp[\"NextContinuationToken\"]\n", | ||
| " except KeyError:\n", | ||
| " break\n", | ||
| "\n", | ||
| " return keys\n", | ||
| "\n", | ||
| "keys = get_all_s3_keys(raw_data_bucket, raw_data_prefix, \".nc4\")\n", | ||
| "\n", | ||
| "def download_s3_objects(bucket, keys, download_dir):\n", | ||
| " \"\"\"Download all S3 objects listed in keys to the specified local directory.\"\"\"\n", | ||
| " if not os.path.exists(download_dir):\n", | ||
| " os.makedirs(download_dir)\n", | ||
| "\n", | ||
| " for key in keys:\n", | ||
| " local_filename = os.path.join(download_dir, os.path.basename(key))\n", | ||
| " try:\n", | ||
| " s3_client.download_file(bucket, key, local_filename)\n", | ||
| " print(f\"Downloaded {key} to {local_filename}\")\n", | ||
| " except (NoCredentialsError, PartialCredentialsError) as e:\n", | ||
| " print(f\"Credentials error: {e}\")\n", | ||
| " except Exception as e:\n", | ||
| " print(f\"Failed to download {key}: {e}\")\n", | ||
| "\n", | ||
| "download_s3_objects(raw_data_bucket, keys, \"data\")\n", | ||
| "\n" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "b74c98b4-7059-4f47-b6b2-ae9be7bc62dc", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "def extract_date_from_key(key):\n", | ||
| " # Split the key to isolate the part that contains the date\n", | ||
| " parts = key.split('_')\n", | ||
| " for part in parts:\n", | ||
| " # Check if the part is numeric and has the length of 6 (YYYYMM format)\n", | ||
| " if part.isdigit() and len(part) == 6:\n", | ||
| " return part\n", | ||
| " return None" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "id": "7c273250-65f3-4e28-905c-d27ef4e373e3", | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "COG_PROFILE = {\"driver\": \"COG\", \"compress\": \"DEFLATE\"}\n", | ||
| "OVERVIEW_LEVELS = 4 \n", | ||
| "OVERVIEW_RESAMPLING = 'average'\n", | ||
| "\n", | ||
| "for key in glob.glob(\"data/*.nc4\"):\n", | ||
| " xds= xr.open_dataset(key)\n", | ||
| " xds = xds.assign_coords(lon=(((xds.lon + 180) % 360) - 180)).sortby(\"lon\")\n", | ||
| " \n", | ||
| " for var in [\"PM25-PRI\",\"CO2\",\"CO\",\"NOX\",\"SOX\"]:\n", | ||
| " yearmonth = extract_date_from_key(key)\n", | ||
| " filename = f\"output/GRA2PESv1.0_total_{(\"-\").join(var.split('_'))}_{yearmonth}.tif\"\n", | ||
| " data = getattr(xds,var)\n", | ||
| " data.rio.set_spatial_dims(\"lon\", \"lat\", inplace=True)\n", | ||
| " data.rio.write_crs(\"epsg:4326\", inplace=True)\n", | ||
| " \n", | ||
| " # Create a temporary file to hold the COG\n", | ||
| " with tempfile.NamedTemporaryFile(suffix='.tif', delete=False) as temp_file:\n", | ||
| " data.rio.to_raster(f\"temp_{yearmonth}_{var}.tif\", **COG_PROFILE, nodata=-9999)\n", | ||
| " # Create COG with overviews and nodata value\n", | ||
| " cog_translate(\n", | ||
| " f\"temp_{yearmonth}_{var}.tif\",\n", | ||
| " temp_file.name,\n", | ||
| " cog_profiles.get(\"deflate\"),\n", | ||
| " overview_level=OVERVIEW_LEVELS,\n", | ||
| " overview_resampling=OVERVIEW_RESAMPLING,\n", | ||
| " nodata=-9999\n", | ||
| " )\n", | ||
| " \n", | ||
| " # Move the temporary file to the desired local path\n", | ||
| " os.rename(temp_file.name, filename)\n", | ||
| " \n", | ||
| " if os.path.exists(f\"temp_{yearmonth}_{var}.tif\"):\n", | ||
| " os.remove(f\"temp_{yearmonth}_{var}.tif\")\n", | ||
| " del data\n", | ||
| " print(f\"Done for: {filename}\")\n", | ||
| " " | ||
| ] | ||
| } | ||
| ], | ||
| "metadata": { | ||
| "kernelspec": { | ||
| "display_name": "Python 3 (ipykernel)", | ||
| "language": "python", | ||
| "name": "python3" | ||
| }, | ||
| "language_info": { | ||
| "codemirror_mode": { | ||
| "name": "ipython", | ||
| "version": 3 | ||
| }, | ||
| "file_extension": ".py", | ||
| "mimetype": "text/x-python", | ||
| "name": "python", | ||
| "nbconvert_exporter": "python", | ||
| "pygments_lexer": "ipython3", | ||
| "version": "3.12.3" | ||
| } | ||
| }, | ||
| "nbformat": 4, | ||
| "nbformat_minor": 5 | ||
| } |
Oops, something went wrong.