-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #153 from US-GHG-Center/staging
Fix broken links for TM5
- Loading branch information
Showing
5 changed files
with
152 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| { | ||
| "cells": [ | ||
| { | ||
| "cell_type": "raw", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "---\n", | ||
| "title: TM5-4DVar Isotopic CH₄ Inverse Fluxes\n", | ||
| "description: Documentation of data transformation\n", | ||
| "author: Vishal Gaur\n", | ||
| "date: August 31, 2023\n", | ||
| "execute:\n", | ||
| " freeze: true\n", | ||
| "---" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "This script was used to transform the TM5-4DVar Isotopic CH₄ Inverse Fluxes dataset from netCDF to Cloud Optimized GeoTIFF (COG) format for display in the Greenhouse Gas (GHG) Center." | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "import os\n", | ||
| "import xarray\n", | ||
| "import re\n", | ||
| "import pandas as pd\n", | ||
| "import json\n", | ||
| "import tempfile\n", | ||
| "import boto3\n", | ||
| "from datetime import datetime" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "session = boto3.session.Session()\n", | ||
| "s3_client = session.client(\"s3\")\n", | ||
| "bucket_name = (\n", | ||
| " \"ghgc-data-store-dev\" # S3 bucket where the COGs are stored after transformation\n", | ||
| ")\n", | ||
| "FOLDER_NAME = \"tm5-ch4-inverse-flux\"\n", | ||
| "\n", | ||
| "files_processed = pd.DataFrame(\n", | ||
| " columns=[\"file_name\", \"COGs_created\"]\n", | ||
| ") # A dataframe to keep track of the files that we have transformed into COGs\n", | ||
| "\n", | ||
| "# Reading the raw netCDF files from local machine\n", | ||
| "for name in os.listdir(FOLDER_NAME):\n", | ||
| " xds = xarray.open_dataset(f\"{FOLDER_NAME}/{name}\", engine=\"netcdf4\")\n", | ||
| " xds = xds.rename({\"latitude\": \"lat\", \"longitude\": \"lon\"})\n", | ||
| " xds = xds.assign_coords(lon=(((xds.lon + 180) % 360) - 180)).sortby(\"lon\")\n", | ||
| " variable = [var for var in xds.data_vars if \"global\" not in var]\n", | ||
| "\n", | ||
| " for time_increment in range(0, len(xds.months)):\n", | ||
| " filename = name.split(\"/ \")[-1]\n", | ||
| " filename_elements = re.split(\"[_ .]\", filename)\n", | ||
| " start_time = datetime(int(filename_elements[-2]), time_increment + 1, 1)\n", | ||
| " for var in variable:\n", | ||
| " data = getattr(xds.isel(months=time_increment), var)\n", | ||
| " data = data.isel(lat=slice(None, None, -1))\n", | ||
| " data.rio.set_spatial_dims(\"lon\", \"lat\", inplace=True)\n", | ||
| " data.rio.write_crs(\"epsg:4326\", inplace=True)\n", | ||
| "\n", | ||
| " # # insert date of generated COG into filename\n", | ||
| " filename_elements.pop()\n", | ||
| " filename_elements[-1] = start_time.strftime(\"%Y%m\")\n", | ||
| " filename_elements.insert(2, var)\n", | ||
| " cog_filename = \"_\".join(filename_elements)\n", | ||
| " # # add extension\n", | ||
| " cog_filename = f\"{cog_filename}.tif\"\n", | ||
| "\n", | ||
| " with tempfile.NamedTemporaryFile() as temp_file:\n", | ||
| " data.rio.to_raster(\n", | ||
| " temp_file.name,\n", | ||
| " driver=\"COG\",\n", | ||
| " )\n", | ||
| " s3_client.upload_file(\n", | ||
| " Filename=temp_file.name,\n", | ||
| " Bucket=bucket_name,\n", | ||
| " Key=f\"{FOLDER_NAME}/{cog_filename}\",\n", | ||
| " )\n", | ||
| "\n", | ||
| " files_processed = files_processed._append(\n", | ||
| " {\"file_name\": name, \"COGs_created\": cog_filename},\n", | ||
| " ignore_index=True,\n", | ||
| " )\n", | ||
| "\n", | ||
| " print(f\"Generated and saved COG: {cog_filename}\")\n", | ||
| "\n", | ||
| "# Generate the json file with the metadata that is present in the netCDF files.\n", | ||
| "with tempfile.NamedTemporaryFile(mode=\"w+\") as fp:\n", | ||
| " json.dump(xds.attrs, fp)\n", | ||
| " json.dump({\"data_dimensions\": dict(xds.dims)}, fp)\n", | ||
| " json.dump({\"data_variables\": list(xds.data_vars)}, fp)\n", | ||
| " fp.flush()\n", | ||
| "\n", | ||
| " s3_client.upload_file(\n", | ||
| " Filename=fp.name,\n", | ||
| " Bucket=bucket_name,\n", | ||
| " Key=f\"{FOLDER_NAME}/metadata.json\",\n", | ||
| " )\n", | ||
| "\n", | ||
| "# creating the csv file with the names of files transformed.\n", | ||
| "files_processed.to_csv(\n", | ||
| " f\"s3://{bucket_name}/{FOLDER_NAME}/files_converted.csv\",\n", | ||
| ")\n", | ||
| "print(\"Done generating COGs\")\n" | ||
| ] | ||
| } | ||
| ], | ||
| "metadata": { | ||
| "language_info": { | ||
| "name": "python" | ||
| }, | ||
| "orig_nbformat": 4 | ||
| }, | ||
| "nbformat": 4, | ||
| "nbformat_minor": 2 | ||
| } |
9 changes: 9 additions & 0 deletions
9
data_workflow/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.qmd
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| --- | ||
| title: TM5-4DVar Isotopic CH₄ Inverse Fluxes | ||
| description: "Global, monthly 1 degree resolution methane emission estimates from microbial, fossil and pyrogenic sources derived using inverse modeling, version 1" | ||
| --- | ||
| <object data="./reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf" type="application/pdf" width="1024px" height="1000px"> | ||
| <embed src="./reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf"> | ||
| <p>This browser does not support PDFs. Please download the PDF to view it: <a href="./reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf">Download PDF</a>.</p> | ||
| </embed> | ||
| </object> |
Binary file added
BIN
+194 KB
...tion_reports/reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf
Binary file not shown.
9 changes: 9 additions & 0 deletions
9
...verification_reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.qmd
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| --- | ||
| title: TM5-4DVar Isotopic CH₄ Inverse Fluxes | ||
| description: "Global, monthly 1 degree resolution methane emission estimates from microbial, fossil and pyrogenic sources derived using inverse modeling, version 1" | ||
| --- | ||
| <object data="./reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf" type="application/pdf" width="1024px" height="1000px"> | ||
| <embed src="./reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf"> | ||
| <p>This browser does not support PDFs. Please download the PDF to view it: <a href="./reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf">Download PDF</a>.</p> | ||
| </embed> | ||
| </object> |