Update the content for upcoming release (#141)

* ceated python script to fetch json files and generate statistics

* rename all TM54Dvar to CT-CH4

* tm5 reports updated

* gosat verification report updated

* update ct-ch4 notebook

* added draft content for goes

---------

.gitignore

@@ -3,3 +3,5 @@
 .env
 /.quarto/
 _site/
+.yaml
+.json

_quarto.yml

@@ -55,7 +55,7 @@ website:
             contents:
               - user_data_notebooks/oco2-mip-co2budget-yeargrid-v1_User_Notebook.ipynb
               - user_data_notebooks/odiac-ffco2-monthgrid-v2023_User_Notebook.ipynb
-              - user_data_notebooks/tm54dvar-ch4flux-monthgrid-v1_User_Notebook.ipynb
+              - user_data_notebooks/ct-ch4-monthgrid-v2023_User_Notebook.ipynb
               - user_data_notebooks/epa-ch4emission-grid-v2express_User_Notebook.ipynb
               - user_data_notebooks/vulcan-ffco2-yeargrid-v4_User_Notebook.ipynb
               - user_data_notebooks/gra2pes-ghg-monthgrid-v1_User_Notebook.ipynb
@@ -65,11 +65,12 @@ website:
               - user_data_notebooks/micasa-carbonflux-daygrid-v1_User_Notebook.ipynb
               - user_data_notebooks/gosat-based-ch4budget-yeargrid-v1_User_Notebook.ipynb
               - user_data_notebooks/oco2-mip-co2budget-yeargrid-v1_User_Notebook.ipynb
-              - user_data_notebooks/tm54dvar-ch4flux-monthgrid-v1_User_Notebook.ipynb
+              - user_data_notebooks/ct-ch4-monthgrid-v2023_User_Notebook.ipynb
               - user_data_notebooks/lpjeosim-wetlandch4-grid-v1_User_Notebook.ipynb
           - section: Large Emissions Events
             contents:
               - user_data_notebooks/emit-ch4plume-v1_User_Notebook.ipynb
+              - user_data_notebooks/goes-ch4plume-v1_User_Notebook.ipynb              
           - section: Greenhouse Gas Concentrations
             contents:
               - user_data_notebooks/noaa-insitu_User_Notebook.ipynb
@@ -87,7 +88,7 @@ website:
             contents:
               - cog_transformation/oco2-mip-co2budget-yeargrid-v1.ipynb
               - cog_transformation/odiac-ffco2-monthgrid-v2023.ipynb
-              - cog_transformation/tm54dvar-ch4flux-monthgrid-v1.ipynb
+              - cog_transformation/ct-ch4-monthgrid-v2023.ipynb
               - cog_transformation/epa-ch4emission-grid-v2express.ipynb   
               - cog_transformation/vulcan-ffco2-yeargrid-v4.ipynb
               - cog_transformation/gra2pes-ghg-monthgrid-v1.ipynb
@@ -96,7 +97,7 @@ website:
               - cog_transformation/eccodarwin-co2flux-monthgrid-v5.ipynb
               - cog_transformation/gosat-based-ch4budget-yeargrid-v1.ipynb
               - cog_transformation/oco2-mip-co2budget-yeargrid-v1.ipynb
-              - cog_transformation/tm54dvar-ch4flux-monthgrid-v1.ipynb
+              - cog_transformation/ct-ch4-monthgrid-v2023.ipynb
           - section: Large Emissions Events
             contents:
               - cog_transformation/emit-ch4plume-v1.ipynb
@@ -118,7 +119,7 @@ website:
             contents:
               - processing_and_verification_reports/oco2-mip-co2budget-yeargrid-v1_Processing and Verification Report.qmd
               - processing_and_verification_reports/odiac-ffco2-monthgrid-v2023_Processing and Verification Report.qmd
-              - processing_and_verification_reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.qmd
+              - processing_and_verification_reports/ct-ch4-monthgrid-v2023_Processing and Verification Report.qmd
               - processing_and_verification_reports/epa-ch4emission-grid-v2express_Processing and Verification Report.qmd 
               - processing_and_verification_reports/vulcan-ffco2-yeargrid-v4_Processing and Verification Report.qmd  
               - processing_and_verification_reports/gra2pes-ghg-monthgrid-v1_Processing and Verification Report.qmd
@@ -128,11 +129,12 @@ website:
               - processing_and_verification_reports/micasa-carbonflux-daygrid-v1_Processing and Verification Report.qmd
               - processing_and_verification_reports/gosat-based-ch4budget-yeargrid-v1_Processing and Verification Report.qmd
               - processing_and_verification_reports/oco2-mip-co2budget-yeargrid-v1_Processing and Verification Report.qmd
-              - processing_and_verification_reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.qmd
+              - processing_and_verification_reports/ct-ch4-monthgrid-v2023_Processing and Verification Report.qmd
               - processing_and_verification_reports/lpjeosim-wetlandch4-grid-v1_Processing and Verification Report.qmd
           - section: Large Emissions Events
             contents:
               - processing_and_verification_reports/emit-ch4plume-v1_Processing and Verification Report.qmd
+              - processing_and_verification_reports/goes-ch4plume-v1_Processing and Verification Report.qmd
           - section: Greenhouse Gas Concentrations
             contents:
               - processing_and_verification_reports/noaa-gggrn-co2-concentrations_Processing and Verification Report.qmd
@@ -151,7 +153,7 @@ website:
             contents:
               - data_workflow/oco2-mip-co2budget-yeargrid-v1_Data_Flow.qmd
               - data_workflow/odiac-ffco2-monthgrid-v2023_Data_Flow.qmd
-              - data_workflow/tm54dvar-ch4flux-monthgrid-v1_Data_Flow.qmd
+              - data_workflow/ct-ch4-monthgrid-v2023_Data_Flow.qmd
               - data_workflow/epa-ch4emission-grid-v2express_Data_Flow.qmd
               - data_workflow/vulcan-ffco2-yeargrid-v4_Data_Flow.qmd  
               - data_workflow/gra2pes-ghg-monthgrid-v1_Data_Flow.qmd 
@@ -161,11 +163,12 @@ website:
               - data_workflow/micasa-carbonflux-daygrid-v1_Data_Flow.qmd
               - data_workflow/gosat-based-ch4budget-yeargrid-v1_Data_Flow.qmd
               - data_workflow/oco2-mip-co2budget-yeargrid-v1_Data_Flow.qmd
-              - data_workflow/tm54dvar-ch4flux-monthgrid-v1_Data_Flow.qmd
+              - data_workflow/ct-ch4-monthgrid-v2023_Data_Flow.qmd
               - data_workflow/lpjeosim-wetlandch4-grid-v1_Data_Flow.qmd
           - section: Large Emissions Events
             contents:
               - data_workflow/emit-ch4plume-v1_Data_Flow.qmd
+              - data_workflow/goes-ch4plume-v1_Data_Flow.qmd              
           - section: Greenhouse Gas Concentrations
             contents:
               - data_workflow/noaa-gggrn-co2-concentrations_Data_Flow.qmd

cog_transformation/tm54dvar-ch4flux-monthgrid-v1.ipynb → cog_transformation/ct-ch4-monthgrid-v2023.ipynb

@@ -2,10 +2,14 @@
  "cells": [
   {
    "cell_type": "raw",
-   "metadata": {},
+   "metadata": {
+    "vscode": {
+     "languageId": "raw"
+    }
+   },
    "source": [
     "---\n",
-    "title: TM5-4DVar Isotopic CH₄ Inverse Fluxes\n",
+    "title: CarbonTracker-CH₄ Isotopic Methane Inverse Fluxes\n",
     "description: Documentation of data transformation\n",
     "author: Vishal Gaur\n",
     "date: August 31, 2023\n",
@@ -18,7 +22,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "This script was used to transform the TM5-4DVar Isotopic CH₄ Inverse Fluxes dataset from netCDF to Cloud Optimized GeoTIFF (COG) format for display in the Greenhouse Gas (GHG) Center."
+    "This script was used to transform the CarbonTracker-CH₄ Isotopic Methane Inverse Fluxes dataset from netCDF to Cloud Optimized GeoTIFF (COG) format for display in the Greenhouse Gas (GHG) Center."
    ]
   },
   {

data_workflow/ct-ch4-monthgrid-v2023_Data_Flow.qmd

@@ -0,0 +1,3 @@
+# CarbonTracker-CH₄ Isotopic Methane Inverse Fluxes
+
+![Data Flow Diagram Extending From Acquisition/Creation to User Delivery](./media/ct-ch4-monthgrid-v2023_Data_Flow.png)

data_workflow/goes-ch4plume-v1_Data_Flow.qmd

@@ -0,0 +1,4 @@
+# Geostationary Satellite Observations of Extreme and Transient Methane Emissions from Oil and Gas Infrastructure
+
+
+![Data Flow Diagram Extending From Acquisition/Creation to User Delivery](./media/goes-ch4plume-v1_Data_Flow.png)

datatransformationcode.qmd

@@ -17,7 +17,7 @@ Explore, analyze, and make a difference with the US GHG Center.
 ## Gridded Anthropogenic Greenhouse Gas Emissions
 1. [OCO-2 MIP Top-Down CO₂ Budgets](cog_transformation/oco2-mip-co2budget-yeargrid-v1.ipynb)
 2. [ODIAC Fossil Fuel CO₂ Emissions](cog_transformation/odiac-ffco2-monthgrid-v2023.ipynb)
-3. [TM5-4DVar Isotopic CH₄ Inverse Fluxes](cog_transformation/tm54dvar-ch4flux-monthgrid-v1.ipynb)
+3. [CarbonTracker-CH₄ Isotopic Methane Inverse Fluxes](cog_transformation/ct-ch4-monthgrid-v2023.ipynb)
 4. [U.S. Gridded Anthropogenic Methane Emissions Inventory](cog_transformation/epa-ch4emission-grid-v2express.ipynb)
 5. [Vulcan Fossil Fuel CO₂ Emissions](cog_transformation/vulcan-ffco2-yeargrid-v4.ipynb)
 6. [GRA²PES Greenhouse Gas and Air Quality Species](cog_transformation/gra2pes-ghg-monthgrid-v1.ipynb)
@@ -26,7 +26,7 @@ Explore, analyze, and make a difference with the US GHG Center.
 1. [Air-Sea CO₂ Flux, ECCO-Darwin Model v5](cog_transformation/eccodarwin-co2flux-monthgrid-v5.ipynb)
 2. [GOSAT-based Top-down Total and Natural Methane Emissions](cog_transformation/gosat-based-ch4budget-yeargrid-v1.ipynb)
 3. [OCO-2 MIP Top-Down CO₂ Budgets](cog_transformation/oco2-mip-co2budget-yeargrid-v1.ipynb)
-4. [TM5-4DVar Isotopic CH₄ Inverse Fluxes](cog_transformation/tm54dvar-ch4flux-monthgrid-v1.ipynb)
+4. [CarbonTracker-CH₄ Isotopic Methane Inverse Fluxes](cog_transformation/ct-ch4-monthgrid-v2023.ipynb)
 
 
 ## Large Emissions Events

datausage.qmd

@@ -16,8 +16,8 @@ Explore, analyze, and make a difference with the US GHG Center.
    - [Intermediate level notebook to read and visualize](user_data_notebooks/oco2-mip-National-co2budget.ipynb) National CO₂ Budgets using OCO-2 MIP Top-Down CO₂ Budget country total data. This notebook utilizes the country totals available at [https://ceos.org/gst/carbon-dioxide.html](https://ceos.org/gst/carbon-dioxide.html), which compliment the global 1° x 1° gridded CO₂ Budget data featured in the [US GHG Center](https://earth.gov/ghgcenter/data-catalog/oco2-mip-co2budget-yeargrid-v1).
 2. ODIAC Fossil Fuel CO₂ Emissions
    - [Beginner level notebook](user_data_notebooks/odiac-ffco2-monthgrid-v2023_User_Notebook.ipynb) to access, visualize, explore statistics, and create a time series of the ODIAC Fossil Fuel CO₂ Emissions dataset.
-3. TM5-4DVar Isotopic CH₄ Inverse Fluxes
-   - [Beginner level notebook](user_data_notebooks/tm54dvar-ch4flux-monthgrid-v1_User_Notebook.ipynb) to access, visualize, explore statistics, and create a time series of the TM5-4DVar Isotopic CH₄ Inverse Fluxes dataset.
+3. CarbonTracker-CH₄ Isotopic Methane Inverse Fluxes
+   - [Beginner level notebook](user_data_notebooks/ct-ch4-monthgrid-v2023_User_Notebook.ipynb) to access, visualize, explore statistics, and create a time series of the TM5-4DVar Isotopic CH₄ Inverse Fluxes dataset.
 4. U.S. Gridded Anthropogenic Methane Emissions Inventory
    - [Beginner level notebook](user_data_notebooks/epa-ch4emission-grid-v2express_User_Notebook.ipynb) to access, visualize, explore statistics, and create a time series of the U.S. Gridded Anthropogenic Methane Emissions Inventory dataset.
 5. Vulcan Fossil Fuel CO₂ Emissions
@@ -35,14 +35,16 @@ Explore, analyze, and make a difference with the US GHG Center.
 4. OCO-2 MIP Top-Down CO₂ Budgets
    - [Beginner level notebook](user_data_notebooks/oco2-mip-co2budget-yeargrid-v1_User_Notebook.ipynb) to access, visualize, explore statistics, and create a time series of the OCO-2 MIP Top-Down CO₂ Budgets dataset.
    - [Intermediate level notebook to read and visualize](user_data_notebooks/oco2-mip-National-co2budget.ipynb)National CO₂ Budgets using OCO-2 MIP Top-Down CO₂ Budget country total data. This notebook utilizes the country totals available at ceos.org/gst/carbon-dioxide, which compliment the global 1° x 1° gridded CO₂ Budget data featured in the US GHG Center.
-5. TM5-4DVar Isotopic CH₄ Inverse Fluxes
-   - [Beginner level notebook](user_data_notebooks/tm54dvar-ch4flux-monthgrid-v1_User_Notebook.ipynb) to access, visualize, explore statistics, and create a time series of the TM5-4DVar Isotopic CH₄ Inverse Fluxes dataset.
+5. CarbonTracker-CH₄ Isotopic Methane Inverse Fluxes
+   - [Beginner level notebook](user_data_notebooks/ct-ch4-monthgrid-v2023_User_Notebook.ipynb) to access, visualize, explore statistics, and create a time series of the TM5-4DVar Isotopic CH₄ Inverse Fluxes dataset.
 6. Wetland Methane Emissions, LPJ-EOSIM model
    - [Beginner level notebook](user_data_notebooks/lpjeosim-wetlandch4-grid-v1_User_Notebook.ipynb) to access, visualize, explore statistics, and create a time series of the Wetland Methane Emissions, LPJ-EOSIM model dataset.
 
 ## Large Emissions Events
 1. EMIT Methane Point Source Plume Complexes
    - [Beginner level notebook](user_data_notebooks/emit-ch4plume-v1_User_Notebook.ipynb) to access, visualize, explore statistics, and create a time series of the EMIT Methane Point Source Plume Complexes dataset.
+2. Geostationary Satellite Observations of Extreme and Transient Methane Emissions from Oil and Gas Infrastructure
+   - [Beginner level notebook](user_data_notebooks/goes-ch4plume-v1_User_Notebook.ipynb) to access, visualize, explore statistics, and create a time series of the EMIT Methane Point Source Plume Complexes dataset.
 
 ## Greenhouse Gas Concentrations
 1. Atmospheric Carbon Dioxide Concentrations from NOAA Global Monitoring Laboratory

generating_statistics_for_validation/generate_statistics.py

@@ -0,0 +1,58 @@
+import numpy as np
+import boto3
+import pandas as pd
+import json
+import yaml
+
+# Step 1: Connect to the S3 bucket
+s3 = boto3.client('s3')
+bucket_name = 'ghgc-data-store-develop'
+prefix = 'transformed_cogs/tm5-4dvar-update-noaa/'  # optional: specify if files are in a specific folder within the bucket
+
+# Step 2: List all JSON files in the bucket
+response = s3.list_objects_v2(Bucket=bucket_name, Prefix=prefix)
+files = [file['Key'] for file in response.get('Contents', []) if file['Key'].endswith('.json') and 'fossil_emis_2015' in file['Key']]
+print(len(files))
+# Step 3: Load each JSON file into a DataFrame and combine
+dataframes = []
+for file_key in files:
+    obj = s3.get_object(Bucket=bucket_name, Key=file_key)
+    data = json.load(obj['Body'])  # Load JSON content
+    df = pd.json_normalize(data)    # Normalize JSON to flatten nested structures if needed
+    dataframes.append(df)
+# Combine all DataFrames into one
+all_data = pd.concat(dataframes, ignore_index=True)
+for name in all_data.columns[3:]:
+    all_data[name] = all_data[name].astype(np.float32)
+
+# Step 4: Calculate mean, std, min, and max for overall data
+mean_values_netcdf = all_data["mean_value_netcdf"].mean()
+std_values_netcdf = all_data["std_value_netcdf"].std()
+min_values_netcdf = all_data["minimum_value_netcdf"].min()
+max_values_netcdf = all_data["maximum_value_netcdf"].max()
+mean_values_cog = all_data["mean_value_cog"].mean()
+std_values_cog = all_data["std_value_cog"].std()
+min_values_cog = all_data["minimum_value_cog"].min()
+max_values_cog = all_data["maximum_value_cog"].max()
+
+data_to_save = {
+    "netcdf":{
+        "mean":mean_values_netcdf,
+        "std":std_values_netcdf,
+        "min":min_values_netcdf,
+        "max":max_values_netcdf,
+    },
+    "COG": {
+        "mean":mean_values_cog,
+        "std":std_values_cog,
+        "min":min_values_cog,
+        "max":max_values_cog,
+    }
+
+}
+print(data_to_save)
+# # Write the data to a YAML file
+# output_yaml_path = "config.yaml"
+# with open(output_yaml_path, "w") as yaml_file:
+#     yaml.dump(data_to_save, yaml_file, default_flow_style=False)
+

processing_and_verification_reports/ct-ch4-monthgrid-v2023_Processing and Verification Report.qmd

@@ -0,0 +1,9 @@
+---
+title: CarbonTracker-CH₄ Isotopic Methane Inverse Fluxes
+description: "Global, monthly 1 degree resolution methane emission estimates from microbial, fossil and pyrogenic sources derived using inverse modeling, version 2023"
+---
+<object data="./reports/ct-ch4-monthgrid-v2023_Processing and Verification Report.pdf" type="application/pdf" width="1024px" height="1000px">
+    <embed src="./reports/ct-ch4-monthgrid-v2023_Processing and Verification Report.pdf">
+        <p>This browser does not support PDFs. Please download the PDF to view it: <a href="./reports/ct-ch4-monthgrid-v2023_Processing and Verification Report.pdf">Download PDF</a>.</p>
+    </embed>
+</object>

processing_and_verification_reports/goes-ch4plume-v1_Processing and Verification Report.qmd

@@ -0,0 +1,9 @@
+---
+title: Geostationary Satellite Observations of Extreme and Transient Methane Emissions from Oil and Gas Infrastructure
+description: "A sample of methane plumes from point sources observed since 2019 by the U.S. Geostationary Operational Environmental Satellites (GOES) over North and South America"
+---
+<object data="./reports/goes-ch4plume-v1_Processing and Verification Report.pdf" type="application/pdf" width="1024px" height="1000px">
+    <embed src="./reports/goes-ch4plume-v1_Processing and Verification Report.pdf">
+        <p>This browser does not support PDFs. Please download the PDF to view it: <a href="./reports/goes-ch4plume-v1_Processing and Verification Report.pdf">Download PDF</a>.</p>
+    </embed>
+</object>

processing_and_verification_reports/gosat-based-ch4budget-yeargrid-v1_Processing and Verification Report.qmd

@@ -1,6 +1,6 @@
 ---
 title: GOSAT-based Top-down Total and Natural Methane Emissions
-description: "Annual methane emissions gridded globally at 1° resolution for 2019, version"
+description: "Total and wetland yearly methane emissions derived using the GEOS-Chem global chemistry transport model with inclusion of GOSAT data for 2010 to 2022 on a 4 x 5 degree (lat/lon) grid "
 ---
 <object data="./reports/gosat-based-ch4budget-yeargrid-v1_Processing and Verification Report.pdf" type="application/pdf" width="1024px" height="1000px">
     <embed src="./reports/gosat-based-ch4budget-yeargrid-v1_Processing and Verification Report.pdf">

processing_and_verification_reports/gra2pes-ghg-monthgrid-v1_Processing and Verification Report.qmd

@@ -3,7 +3,7 @@ title: GRA²PES Greenhouse Gas and Air Quality Species
 description: "Monthly, 0.036 degree resolution emissions of carbon dioxide (CO₂), carbon monoxide (CO), nitrogen oxide (NOₓ), sulfur dioxide (SO₂), and particulate matter (PM2.5) emissions for the year 2021 over the Contiguous United States from the Greenhouse gas And Air Pollutants Emissions System (GRA²PES)."
 ---
 <object data="./reports/gra2pes-ghg-monthgrid-v1_Processing and Verification Report.pdf" type="application/pdf" width="1024px" height="1000px">
-    <embed src="./reports/tm54dvar-ch4flux-monthgrid-v1_Processing and Verification Report.pdf">
+    <embed src="./reports/gra2pes-ghg-monthgrid-v1_Processing and Verification Report.pdf">
         <p>This browser does not support PDFs. Please download the PDF to view it: <a href="./reports/gra2pes-ghg-monthgrid-v1_Processing and Verification Report.pdf">Download PDF</a>.</p>
     </embed>
 </object>