From 2754b13f3209d44332737eb71fa7513668210942 Mon Sep 17 00:00:00 2001 From: elpamart Date: Tue, 12 Sep 2023 17:01:23 +0200 Subject: [PATCH 1/6] Includes preprocessing for the nutrient assimilation capacity indicator --- .../nutrient_assimilation_capacity/Makefile | 25 ++++++ .../nutrient_assimilation_capacity/README | 65 ++++++++++++++ .../process_data.py | 90 +++++++++++++++++++ 3 files changed, 180 insertions(+) create mode 100644 data/preprocessing/nutrient_assimilation_capacity/Makefile create mode 100644 data/preprocessing/nutrient_assimilation_capacity/README create mode 100644 data/preprocessing/nutrient_assimilation_capacity/process_data.py diff --git a/data/preprocessing/nutrient_assimilation_capacity/Makefile b/data/preprocessing/nutrient_assimilation_capacity/Makefile new file mode 100644 index 000000000..8251ba1ac --- /dev/null +++ b/data/preprocessing/nutrient_assimilation_capacity/Makefile @@ -0,0 +1,25 @@ +# Makefile for downloading, processing, and uploading data +# Variables +DATA_DIR=data/ +checksums_dir=../../../../h3_data_importer/data_checksums +AWS_S3_BUCKET_URL=s3://landgriffon-raw-data + +# Targets +.PHONY: unzip-limiting-nutrient + +all: unzip-limiting-nutrient + +# First you need to download the data manually from https://figshare.com/articles/figure/DRP_NO3_TN_TP_rasters/14527638/1?file=31154728 and save it in nutrient_assimilation_capacity/data +unzip-limiting-nutrient: + unzip -q -u $(DATA_DIR)/hybas_l03_v1c_Cases.zip -d $(DATA_DIR)/ + +# Preprocess the data before ingesting instead of performing these calculations on the database +process-limiting-nutrients: + python process_data.py $(DATA_DIR)/hybas_l03_v1c_Cases + +upload_results: + aws s3 cp $(DATA_DIR)/hybas_l03_v1c_Cases/nutrient_assimilation_capacity.shp ${AWS_S3_BUCKET_URL}/processed/nutrients_assimilation_capacity/ + +write_checksum: + cd $(DATA_DIR)/hybas_l03_v1c_Cases && sha256sum nutrient_assimilation_capacity.shp > $(checksums_dir)/nutrient_assimilation_capacity + diff --git a/data/preprocessing/nutrient_assimilation_capacity/README b/data/preprocessing/nutrient_assimilation_capacity/README new file mode 100644 index 000000000..7d8f6cb5c --- /dev/null +++ b/data/preprocessing/nutrient_assimilation_capacity/README @@ -0,0 +1,65 @@ +# Data Processing Pipeline README + +This repository contains a data processing pipeline implemented using a Makefile and Python script to download, preprocess, upload, and generate checksums for data files. The pipeline is designed to work with geospatial data related to nutrient assimilation capacity. + +## Prerequisites + +Before running the pipeline, ensure you have the following prerequisites in place: + +1. **Data Download**: You need to manually download the data from [here](https://figshare.com/articles/figure/DRP_NO3_TN_TP_rasters/14527638/1?file=31154728) and save it in the `data/` directory. + +2. **Python Dependencies**: The preprocessing script requires Python and the following Python packages: + - `geopandas` + - Other dependencies as specified in your `process_data.py` script. + +3. **AWS Credentials**: To upload results to an AWS S3 bucket, you should have AWS credentials configured on your machine. + +## Usage + +### 1. Download and Unzip Data + +Use the following command to download and unzip the data: + +```bash +make unzip-limiting-nutrient +``` +This command will download the data and place it in the data/ directory. + +### 2. Preprocess Data + +Before ingesting the data into your database, preprocess it using the Python script. Run the following command: + +``` bash +make process-limiting-nutrients +``` +This command will execute the process_data.py script, which performs data preprocessing, including reprojection and calculation of nutrient reduction percentages. + +### 3. Upload Process Data + +To upload the processed data to an AWS S3 bucket, use the following command: + +```bash +make upload_results +``` +Make sure you have AWS credentials configured to access the specified S3 bucket. + +### 4. Generate Checksum + +Generate a SHA-256 checksum for the processed data by running the following command: + +```bash +make write_checksum +``` +This command will calculate the checksum and save it in the data_checksums/ directory. + +## Configuration + +You can configure the pipeline by modifying the variables at the top of the Makefile: + +- `DATA_DIR`: Specify the directory where data files are stored. +- `checksums_dir`: Define the directory where checksum files will be saved. +- `AWS_S3_BUCKET_URL`: Set the AWS S3 bucket URL for uploading results. + +Feel free to adapt this pipeline to suit your specific data processing needs and directory structure. + +`Note`: Make sure you have the necessary permissions and access to the data sources and AWS resources mentioned in this README before running the pipeline. diff --git a/data/preprocessing/nutrient_assimilation_capacity/process_data.py b/data/preprocessing/nutrient_assimilation_capacity/process_data.py new file mode 100644 index 000000000..246f21116 --- /dev/null +++ b/data/preprocessing/nutrient_assimilation_capacity/process_data.py @@ -0,0 +1,90 @@ +""" Reads the limiting nutrients equal area vector file, reporjects the file to EPSG4326 and estimates the percentage of reduction needed to meet a good water quality conditions. + +Usage: +process_data.py + +Arguments: + Folder containing the limiting nutrients shapefile +""" +import os +import logging +from pathlib import Path +import argparse + +import geopandas as gpd + +logging.basicConfig(level=logging.INFO) +log = logging.getLogger("preprocessing_limiting_nutrients_file") + +def check_and_reproject_to_4326(gdf): + """ + Checks if a GeoDataFrame is in CRS 4326 (WGS84) and reprojects it if not. + + Parameters: + - gdf: GeoDataFrame to check and reproject if needed. + + Returns: + - Reprojected GeoDataFrame (if reprojected) or the original GeoDataFrame (if already in 4326). + """ + if gdf.crs is None or gdf.crs.to_epsg() != 4326: + log.info("Reprojecting GeoDataFrame to EPSG:4326 (WGS84)...") + try: + # Reproject to EPSG:4326 + gdf = gdf.to_crs(epsg=4326) + log.info("Reprojection successful.") + except: + log.error("Reprojection failed with error") + else: + log.info("GeoDataFrame is already in EPSG:4326 (WGS84).") + + return gdf + +# Define the function to calculate perc_reduction +def calculate_perc_reduction(row): + if row['Cases_v2_1'] == 4 and row['TP_con_V2_']: + return ((row['TP_con_V2_'] - 0.046) / row['TP_con_V2_']) * 100 + elif row['Cases_v2_1'] == 2 and row['TN_con_V2_']: + return ((row['TN_con_V2_'] - 0.7) / row['TN_con_V2_']) * 100 + else: + return 0 + +def process_folder(folder): + vec_extensions = "gdb gpkg shp json geojson".split() + path = Path(folder) + vectors = [] + for ext in vec_extensions: + vectors.extend(path.glob(f"*.{ext}")) + if not vectors: + log.error(f"No vectors with extension {vec_extensions} found in {folder}") + return + if len(vectors) == 1: #folder just contains one vector file + # Read the shapefile + gdf = gpd.read_file(vectors[0]) + # Check and reproject to EPSG:4326 + gdf = check_and_reproject_to_4326(gdf) + # Calculate perc_reduction and add it as a new column + gdf['perc_reduc'] = gdf.apply(calculate_perc_reduction, axis=1) + # Save the processed data to a new shapefile + gdf = gdf[['Cases_v2_1', 'perc_reduc', 'geometry']] + output_file = os.path.join(folder, 'nutrient_assimilation_capacity.shp') + log.info(f"Saving preprocessed file to {output_file}") + gdf.to_file(output_file) + else: + mssg = ( + f"Found more than one vector file in {folder}." + f" For now we only support folders with just one vector file." + ) + logging.error(mssg) + return + +def main(): + # Parse command-line arguments + parser = argparse.ArgumentParser(description="Process limiting nutrients vector files.") + parser.add_argument("folder", type=str, help="Path to the folder containing vector files") + args = parser.parse_args() + + # Process the specified folder + process_folder(args.folder) + +if __name__ == "__main__": + main() From 5059b4df721c9c159fab8c9f67a3b64c1ebcf399 Mon Sep 17 00:00:00 2001 From: elpamart Date: Wed, 13 Sep 2023 08:32:47 +0200 Subject: [PATCH 2/6] fix issues in preprocessing and updates data ingestion to include the nutrient assimilation ccapacity indicator --- data/h3_data_importer/Makefile | 35 +++++++++++++--- ...id_limiting_nutrients_global_metadata.json | 42 +++++++++++++++++++ ...nutrient_assimilation_global_metadata.json | 42 +++++++++++++++++++ .../nutrient_assimilation_capacity/Makefile | 9 +++- .../process_data.py | 18 ++++---- 5 files changed, 130 insertions(+), 16 deletions(-) create mode 100644 data/h3_data_importer/contextual_layers_metadata/h3_grid_limiting_nutrients_global_metadata.json create mode 100644 data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_assimilation_global_metadata.json diff --git a/data/h3_data_importer/Makefile b/data/h3_data_importer/Makefile index 16aeb16f2..7c9ad0767 100644 --- a/data/h3_data_importer/Makefile +++ b/data/h3_data_importer/Makefile @@ -20,7 +20,8 @@ WORKDIR_HDI=data/contextual/hdi WORKDIR_DEFORESTATION=data/hansen_loss WORKDIR_GHG=data/forest_ghg WORKDIR_WOODPULP=data/woodpulp -WORKING_NATURAL_CROP_CONVERSION=data/natural_crop_conversion +WORKDIR_NATURAL_CROP_CONVERSION=data/natural_crop_conversion +WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY=data/nutrient_assimilation_capacity CHECKSUMS_PATH=data_checksums @@ -52,6 +53,7 @@ indicators: make convert-forestGHG make convert-satDeforestation make convert-naturalCropConversion + make convert-nutrientAssimilationCapacity contextual-layers: convert-hdi-contextual convert-blue-water-contextual @@ -224,14 +226,14 @@ convert-forestGHG: download-forestGHG python raster_folder_to_h3_table.py $(WORKDIR_GHG) h3_grid_ghg_global indicator GHG_LUC_T 2021 --h3-res=6 --thread-count=$(PARALLELIZATION_FACTOR) download-naturalCropConversion: - mkdir -p $(WORKING_NATURAL_CROP_CONVERSION) - aws s3 sync $(AWS_S3_BUCKET_URL)/processed/natural_crop_conversion $(WORKING_NATURAL_CROP_CONVERSION) - cd $(WORKING_NATURAL_CROP_CONVERSION) && sha256sum --check ../../$(CHECKSUMS_PATH)/natural_crop_conversion + mkdir -p $(WORKDIR_NATURAL_CROP_CONVERSION) + aws s3 sync $(AWS_S3_BUCKET_URL)/processed/natural_crop_conversion $(WORKDIR_NATURAL_CROP_CONVERSION) + cd $(WORKDIR_NATURAL_CROP_CONVERSION) && sha256sum --check ../../$(CHECKSUMS_PATH)/natural_crop_conversion convert-naturalCropConversion: download-naturalCropConversion @echo "Converting natural crop conversion data... " - python raster_folder_to_h3_table.py $(WORKING_NATURAL_CROP_CONVERSION) h3_grid_natural_crop_conversion_global indicator NECR 2022 --h3-res=6 --thread-count=$(PARALLELIZATION_FACTOR) + python raster_folder_to_h3_table.py $(WORKDIR_NATURAL_CROP_CONVERSION) h3_grid_natural_crop_conversion_global indicator NECR 2022 --h3-res=6 --thread-count=$(PARALLELIZATION_FACTOR) @@ -308,7 +310,7 @@ convert-woodpulp: download-woodpulp python raster_folder_to_h3_table.py $(WORKDIR_WOODPULP)/ha h3_grid_woodpulp_ha harvest_area gfw_plantations 2021 --h3-res=6 --thread-count=$(PARALLELIZATION_FACTOR) python raster_folder_to_h3_table.py $(WORKDIR_WOODPULP)/prod h3_grid_woodpulp_prod production gfw_plantations 2021 --h3-res=6 --thread-count=$(PARALLELIZATION_FACTOR) ################### -# Contextual data # +# Aqueduct data # ################### # Aqueduct Global water risk contextual data @@ -322,6 +324,27 @@ extract-aqueduct: download-aqueduct convert-aqueduct: extract-aqueduct python vector_folder_to_h3_table.py $(WORKDIR_AQUEDUCT)/Aqueduct40_waterrisk_download_Y2023M07D05/GDB h3_grid_aqueduct_global bws_cat aqueduct "Environmental datasets" 2022 --indicator=UWUSR_T --h3-res=6 --layer=baseline_annual +######################################### +# Nutrient assimilation capacity data # +######################################### + +download-nutrient-assimilation-capacity: + mkdir -p $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY) + aws s3 sync $(AWS_S3_BUCKET_URL)/processed/nutrients_assimilation_capacity/ $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY) + +extract-nutrient-assimilation-capacity:download-nutrient-assimilation-capacity + unzip -q -u $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY)/nutrient_assimilation_capacity.zip -d $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY)/ + cd $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY) && sha256sum --check ../../$(CHECKSUMS_PATH)/nutrient_assimilation_capacity + +convert-nutrientAssimilationCapacity: extract-nutrient-assimilation-capacity + @echo "Converting nutrient assimilation capacity data... " + python vector_folder_to_h3_table.py $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY) h3_grid_nutrient_assimilation_global perc_reduc nutrient_assimilation_capacity "Environmental datasets" 2023 --indicator=NAC --h3-res=6 + @echo "Including contextual layer... " + python vector_folder_to_h3_table.py $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY) h3_grid_limiting_nutrients_global Cases_v2_1 limiting_nutrient "Environmental datasets" 2023 + +################### +# Contextual data # +################### download-hdi-contextual: mkdir -p $(WORKDIR_HDI) diff --git a/data/h3_data_importer/contextual_layers_metadata/h3_grid_limiting_nutrients_global_metadata.json b/data/h3_data_importer/contextual_layers_metadata/h3_grid_limiting_nutrients_global_metadata.json new file mode 100644 index 000000000..a8f393785 --- /dev/null +++ b/data/h3_data_importer/contextual_layers_metadata/h3_grid_limiting_nutrients_global_metadata.json @@ -0,0 +1,42 @@ +{ + "$schema": "./contextual_metadata_schema.json", + "name": "Modeled limiting nutrients", + "description": "Modeled limiting nutrients from McDowell et al. (2020)", + "aggType": "mode", + "legend": { + "name": "Modeled limiting nutrients", + "id": "", + "unit": null, + "min": 0, + "type": "category", + "items": [ + { + "value": 0, + "label": "No Data", + "color": "#9c9c9c" + }, + { + "value": 1, + "label": "N-limited growth acceptable", + "color": "#d2ff73" + }, + { + "value": 2, + "label": "N-limited growth undesirable", + "color": "#3aa809" + }, + { + "value": 3, + "label": "P-limited growth acceptable", + "color": "#fea87c" + }, + { + "value": 4, + "label": "P-limited growth undersiable", + "color": "#a73800" + } + ] + }, + "source": "McDowell et al. (2020)", + "license": "CC BY 4.0" +} diff --git a/data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_assimilation_global_metadata.json b/data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_assimilation_global_metadata.json new file mode 100644 index 000000000..be1f19238 --- /dev/null +++ b/data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_assimilation_global_metadata.json @@ -0,0 +1,42 @@ +{ + "$schema": "./contextual_metadata_schema.json", + "name": "Nutrient Assimilation Capacity", + "description": "Modeled limiting nutrients from McDowell et al. (2020).", + "aggType": "mode", + "legend": { + "name": "Nutrient Assimilation Capacity", + "id": "", + "unit": "", + "min": 0, + "type": "range", + "items": [ + { + "value": 10, + "label": "10%", + "color": "#f6eff7" + }, + { + "value": 30, + "label": "30%", + "color": "#bdc9e1" + }, + { + "value": 50, + "label": "50%", + "color": "#67a9cf" + }, + { + "value": 60, + "label": "60%", + "color": "#1c9099" + }, + { + "value": 90, + "label": "90%", + "color": "#016c59" + } + ] + }, + "source": "McDowell et al. (2020)", + "license": "CC BY 4.0" +} diff --git a/data/preprocessing/nutrient_assimilation_capacity/Makefile b/data/preprocessing/nutrient_assimilation_capacity/Makefile index 8251ba1ac..6a5947dd4 100644 --- a/data/preprocessing/nutrient_assimilation_capacity/Makefile +++ b/data/preprocessing/nutrient_assimilation_capacity/Makefile @@ -15,10 +15,15 @@ unzip-limiting-nutrient: # Preprocess the data before ingesting instead of performing these calculations on the database process-limiting-nutrients: - python process_data.py $(DATA_DIR)/hybas_l03_v1c_Cases + mkdir -p $(DATA_DIR)/nutrient_assimilation_capacity + python process_data.py $(DATA_DIR)/hybas_l03_v1c_Cases $(DATA_DIR)/nutrient_assimilation_capacity + +# Create a zip archive of the shapefile and related files +zip-shapefile: + cd $(DATA_DIR)/nutrient_assimilation_capacity && zip -r nutrient_assimilation_capacity.zip nutrient_assimilation_capacity.* upload_results: - aws s3 cp $(DATA_DIR)/hybas_l03_v1c_Cases/nutrient_assimilation_capacity.shp ${AWS_S3_BUCKET_URL}/processed/nutrients_assimilation_capacity/ + aws s3 cp $(DATA_DIR)/nutrient_assimilation_capacity/nutrient_assimilation_capacity.zip ${AWS_S3_BUCKET_URL}/processed/nutrients_assimilation_capacity/ write_checksum: cd $(DATA_DIR)/hybas_l03_v1c_Cases && sha256sum nutrient_assimilation_capacity.shp > $(checksums_dir)/nutrient_assimilation_capacity diff --git a/data/preprocessing/nutrient_assimilation_capacity/process_data.py b/data/preprocessing/nutrient_assimilation_capacity/process_data.py index 246f21116..c04cf4246 100644 --- a/data/preprocessing/nutrient_assimilation_capacity/process_data.py +++ b/data/preprocessing/nutrient_assimilation_capacity/process_data.py @@ -48,14 +48,15 @@ def calculate_perc_reduction(row): else: return 0 -def process_folder(folder): +def process_folder(input_folder, output_folder): vec_extensions = "gdb gpkg shp json geojson".split() - path = Path(folder) + input_path = Path(input_folder) + output_path = Path(output_folder) vectors = [] for ext in vec_extensions: - vectors.extend(path.glob(f"*.{ext}")) + vectors.extend(input_path.glob(f"*.{ext}")) if not vectors: - log.error(f"No vectors with extension {vec_extensions} found in {folder}") + log.error(f"No vectors with extension {vec_extensions} found in {input_folder}") return if len(vectors) == 1: #folder just contains one vector file # Read the shapefile @@ -66,12 +67,12 @@ def process_folder(folder): gdf['perc_reduc'] = gdf.apply(calculate_perc_reduction, axis=1) # Save the processed data to a new shapefile gdf = gdf[['Cases_v2_1', 'perc_reduc', 'geometry']] - output_file = os.path.join(folder, 'nutrient_assimilation_capacity.shp') + output_file = output_path / 'nutrient_assimilation_capacity.shp' log.info(f"Saving preprocessed file to {output_file}") gdf.to_file(output_file) else: mssg = ( - f"Found more than one vector file in {folder}." + f"Found more than one vector file in {input_folder}." f" For now we only support folders with just one vector file." ) logging.error(mssg) @@ -80,11 +81,12 @@ def process_folder(folder): def main(): # Parse command-line arguments parser = argparse.ArgumentParser(description="Process limiting nutrients vector files.") - parser.add_argument("folder", type=str, help="Path to the folder containing vector files") + parser.add_argument("input_folder", type=str, help="Path to the input folder containing vector files") + parser.add_argument("output_folder", type=str, help="Path to the output folder to save processed data") args = parser.parse_args() # Process the specified folder - process_folder(args.folder) + process_folder(args.input_folder, args.output_folder) if __name__ == "__main__": main() From 21dac910261fdc9746cf0f7b0660c08dbcf827f9 Mon Sep 17 00:00:00 2001 From: elpamart Date: Wed, 13 Sep 2023 18:05:49 +0200 Subject: [PATCH 3/6] rename indicator to nutreint load reduction --- data/base_data_importer/data/2.indicator.csv | 2 +- data/h3_data_importer/Makefile | 26 +++++++++---------- ...3_grid_nutrient_load_global_metadata.json} | 4 +-- .../Makefile | 12 ++++----- .../README | 2 +- .../process_data.py | 2 +- 6 files changed, 24 insertions(+), 24 deletions(-) rename data/h3_data_importer/contextual_layers_metadata/{h3_grid_nutrient_assimilation_global_metadata.json => h3_grid_nutrient_load_global_metadata.json} (91%) rename data/preprocessing/{nutrient_assimilation_capacity => nutrient_load_reduction}/Makefile (58%) rename data/preprocessing/{nutrient_assimilation_capacity => nutrient_load_reduction}/README (96%) rename data/preprocessing/{nutrient_assimilation_capacity => nutrient_load_reduction}/process_data.py (97%) diff --git a/data/base_data_importer/data/2.indicator.csv b/data/base_data_importer/data/2.indicator.csv index ac7bc68c8..fba1e700c 100644 --- a/data/base_data_importer/data/2.indicator.csv +++ b/data/base_data_importer/data/2.indicator.csv @@ -10,5 +10,5 @@ "776f101e-b574-4e36-8410-81e573afab83","Satelligence deforestation","Satelligence deforestation","SAT_DF","Forest loss occurring near material production areas. Deforestation is either calculated using Satelligence models trained to identify transitions from forest to non-forest states in satellite imagery, or using global tree cover loss data from Global Forest Watch.","6970f9b8-eba0-4fee-b6ee-2723ce6604d4","{""name"": ""Satelligence deforestation"",""short name"": ""Satelligence deforestation"",""name code"": ""SAT_DF"",""indicator type"": ""landscape-level"",""units"":""ha/yr"",""description"": ""Forest loss occurring near material production areas. Deforestation is either calculated using Satelligence models trained to identify transitions from forest to non-forest states in satellite imagery, or using global tree cover loss data from Global Forest Watch."",""license"": ""CC BY 4.0"",""geographic coverage"": ""Global coverage (excluding Antarctica and other Arctic islands)."",""citation"": [""Satelligence, 2022. Forest Change Bulletins. https://satelligence.com/technology"",""Hansen, M. C., P. V. Potapov, R. Moore, M. Hancher, S. A. Turubanova, A. Tyukavina, D. Thau, et al. 2013. “High-Resolution Global Maps of 21st-Century Forest Cover Change.” Science 342 (6160): 850–53. https://doi.org/10.1126/science.1244693. ""],""source"":[""Satelligence 2022"",""Hansen 2001-2021""],""frequency of updates"": ""Annual"",""date of content"": ""2001-2021"",""resolution"": ""30x30 meters""}" "169c0638-9cf5-4048-b448-2652dc56ee6a","Satelligence deforestation risk","Satelligence deforestation risk","SAT_DF_R","Forest loss occurring near material production areas. Deforestation is either calculated using Satelligence models trained to identify transitions from forest to non-forest states in satellite imagery, or using global tree cover loss data from Global Forest Watch.","6970f9b8-eba0-4fee-b6ee-2723ce6604d4","{""name"": ""Satelligence deforestation risk"",""short name"": ""Satelligence deforestation risk"",""name code"": ""SAT_DF_R"",""indicator type"": ""landscape-level"",""units"":""ha/yr"",""description"": ""Forest loss occurring near material production areas. Deforestation is either calculated using Satelligence models trained to identify transitions from forest to non-forest states in satellite imagery, or using global tree cover loss data from Global Forest Watch."",""license"": ""CC BY 4.0"",""geographic coverage"": ""Global coverage (excluding Antarctica and other Arctic islands)."",""citation"": [""Satelligence, 2022. Forest Change Bulletins. https://satelligence.com/technology"",""Hansen, M. C., P. V. Potapov, R. Moore, M. Hancher, S. A. Turubanova, A. Tyukavina, D. Thau, et al. 2013. “High-Resolution Global Maps of 21st-Century Forest Cover Change.” Science 342 (6160): 850–53. https://doi.org/10.1126/science.1244693. ""],""source"":[""Satelligence 2022"",""Hansen 2001-2021""],""frequency of updates"": ""Annual"",""date of content"": ""2001-2021"",""resolution"": ""30x30 meters""}" "d5f945c9-8636-45a2-a7c9-67a1dc8e687a","Water quality","Water quality","WQ","The Water Quality indicator estimates the annual average water volume required to assimilate the nutrient load. It focuses on the grey water footprint of a commodity, which indicates the volume of freshwater required to absorb the pollutants present, based on prevailing ambient water quality standards. In the context of crop production, the grey water footprint serves as an indicator of the volume of freshwater pollution. By quantifying water quality, this indicator provides insights into the amount of water necessary to assimilate the nutrients that ultimately reach either ground- or surface water.","9c0da38a-6371-4c79-879b-218fc39c4700","{""name"": ""Water quality"",""short name"": ""Water quality"",""name code"": ""WQ"",""indicator type"": ""farm-level"",""units"": ""Mm3/yr"",""description"": ""The Water Quality indicator estimates the annual average water volume required to assimilate the nutrient load. It focuses on the grey water footprint of a commodity, which indicates the volume of freshwater required to absorb the pollutants present, based on prevailing ambient water quality standards. In the context of crop production, the grey water footprint serves as an indicator of the volume of freshwater pollution. By quantifying water quality, this indicator provides insights into the amount of water necessary to assimilate the nutrients that ultimately reach either ground- or surface water."",""license"": ""CC BY 4.0"",""geographic coverage"": ""Global coverage."",""citation"": [""Mekonnen, M.M. & Hoekstra, A.Y. (2011) The green, blue and grey water footprint of crops and derived crop products, Hydrology and Earth System Sciences, 15(5): 1577-1600."", ""Mekonnen, M.M. & Hoekstra, A.Y. (2012) A global assessment of the water footprint of farm animal products, Ecosystems, 15(3): 401–415.""],""source"": [ ""Mekonnen 2011"", ""Mekonnen 2012""],""frequency of updates"": """",""date of content"": ""1996-2005"",""resolution"": """"}" -"a39394be-ad57-41bc-9c2c-be0949ec6193","Nutrient assimilation capacity","Nutrient assimilation capacity","NAC","The Nutrient Assimilation Capacity indicator aims to assess and quantify the maximum amount of nutrient load that will attain the desired instream nutrient concentration.","9c0da38a-6371-4c79-879b-218fc39c4700","{""name"": ""Nutrient assimilation capacity"",""short name"": ""Nutrient assimilation capacity"",""name code"": ""NAC"",""indicator type"": ""farm-level"",""units"": ""Mm3/yr"",""description"": ""The Nutrient Assimilation Capacity indicator aims to assess and quantify the maximum amount of nutrient load that will attain the desired instream nutrient concentration."",""license"": ""CC BY 4.0"",""geographic coverage"": ""Global coverage."",""citation"": [ ""McDowell, R. W., A. Noble, P. Pletnyakov, B. E. Haggard and L. M. Mosley, 2020. Global Mapping of Freshwater Nutrient Enrichment and Periphyton Growth Potential. Scientific Reports.https://doi.org/10.1038/s41598-020-60279-w"", ""Benjamin S. Halpern, Melanie Frazier, Juliette Verstaen, Paul-Eric Rayner, Gage Clawson, Julia L. Blanchard, Richard S. Cottrell, Halley E. Froehlich, Jessica A. Gephart, Nis S. Jacobsen, Caitlin D. Kuempel, Peter B. McIntyre, Marc Metian, Daniel Moran, Kirsty L. Nash, Johannes Többen, David R. Williams. (2021) The environmental footprint of global food production.Scientific Reports.https://doi.org/10.1038/s41893-022-00965-x"" ,""Mekonnen, M.M. & Hoekstra, A.Y. (2011) The green, blue and grey water footprint of crops and derived crop products, Hydrology and Earth System Sciences, 15(5): 1577-1600."",""Mekonnen, M.M. & Hoekstra, A.Y. (2012) A global assessment of the water footprint of farm animal products, Ecosystems, 15(3): 401–415.""],""source"": [ ""McDowell et al 2020"", ""Mekonnen 2011"", ""Mekonnen 2012""],""frequency of updates"": """",""date of content"": """",""resolution"": """"}" +"a39394be-ad57-41bc-9c2c-be0949ec6193","Nutrient load reduction","Nutrient load reduction","NLR","The Nutrient Load Reduction indicator aims to assess and quantify the maximum amount of nutrient load that will attain the desired instream nutrient concentration.","9c0da38a-6371-4c79-879b-218fc39c4700","{""name"": ""Nutrient load reduction"",""short name"": ""Nutrient load reduction"",""name code"": ""NAC"",""indicator type"": ""farm-level"",""units"": ""Mm3/yr"",""description"": ""The Nutrient Load Reduction indicator aims to assess and quantify the maximum amount of nutrient load that will attain the desired instream nutrient concentration."",""license"": ""CC BY 4.0"",""geographic coverage"": ""Global coverage."",""citation"": [ ""McDowell, R. W., A. Noble, P. Pletnyakov, B. E. Haggard and L. M. Mosley, 2020. Global Mapping of Freshwater Nutrient Enrichment and Periphyton Growth Potential. Scientific Reports.https://doi.org/10.1038/s41598-020-60279-w"", ""Benjamin S. Halpern, Melanie Frazier, Juliette Verstaen, Paul-Eric Rayner, Gage Clawson, Julia L. Blanchard, Richard S. Cottrell, Halley E. Froehlich, Jessica A. Gephart, Nis S. Jacobsen, Caitlin D. Kuempel, Peter B. McIntyre, Marc Metian, Daniel Moran, Kirsty L. Nash, Johannes Többen, David R. Williams. (2021) The environmental footprint of global food production.Scientific Reports.https://doi.org/10.1038/s41893-022-00965-x"" ,""Mekonnen, M.M. & Hoekstra, A.Y. (2011) The green, blue and grey water footprint of crops and derived crop products, Hydrology and Earth System Sciences, 15(5): 1577-1600."",""Mekonnen, M.M. & Hoekstra, A.Y. (2012) A global assessment of the water footprint of farm animal products, Ecosystems, 15(3): 401–415.""],""source"": [ ""McDowell et al 2020"", ""Mekonnen 2011"", ""Mekonnen 2012""],""frequency of updates"": """",""date of content"": """",""resolution"": """"}" "5c133ba4-da24-46db-9c6c-ece7520f01b0","Natural ecosystems conversion risk","Natural ecosystems conversion risk","NECR","The Natural Ecosystems Conversion Risk indicator quantifies the conversion of natural ecosystems occurring near commodity production areas and allocates the risk to the quantity of commodity sourced. IT uses a statistical land use change method to calculate deforestation within a buffered region around the sourcing location. Remote sensing imagery and machine learning algorithms are used to identify forest loss, and the annual rate of deforestation is weighted by commodity production to account for areas where the commodity is grown. The indicator provides insight into the risk of sourcing contributing to deforestation, distinguishing between natural forest loss and loss from plantations. Human land use is factored in by combining crop data, pasture and plantation areas.","6970f9b8-eba0-4fee-b6ee-2723ce6604d4","{""name"": ""Natural ecosystems conversion risk"",""short name"": ""Natural ecosystems conversion risk"",""name code"": ""NECR"",""indicator type"": ""landscape-level"",""units"": ""ha/yr"",""description"": ""The Natural Ecosystems Conversion Risk indicator quantifies the conversion of natural ecosystems occurring near commodity production areas and allocates the risk to the quantity of commodity sourced. It uses a statistical land use change method to calculate deforestation within a buffered region around the sourcing location. Remote sensing imagery and machine learning algorithms are used to identify forest loss, and the annual rate of deforestation is weighted by commodity production to account for areas where the commodity is grown. The indicator provides insight into the risk of sourcing contributing to deforestation, distinguishing between natural forest loss and loss from plantations. Human land use is factored in by combining crop data, pasture and plantation areas."",""license"": ""CC BY 4.0"",""geographic coverage"": ""Global coverage."",""citation"": [ ""Karra, Kontgis, et al. “Global land use/land cover with Sentinel-2 and deep learning.” IGARSS 2021-2021 IEEE International Geoscience and Remote Sensing Symposium. IEEE, 2021."", ""Elise Mazur, Michelle Sims, Elizabeth Goldman, Martina Schneider, Marco Daldoss Pirri, Craig R. Beatty, Fred Stolle. 2023. Natural Lands Map v1.0""],""source"": [ ""Esri/Sentinel-2 10m land use/land cover"", ""SBTN/WRI/Systemiq/WWF""],""frequency of updates"": ""Annual"",""date of content"": ""2023"",""resolution"": ""10x10 kilometers""}" diff --git a/data/h3_data_importer/Makefile b/data/h3_data_importer/Makefile index 7c9ad0767..f3976570e 100644 --- a/data/h3_data_importer/Makefile +++ b/data/h3_data_importer/Makefile @@ -21,7 +21,7 @@ WORKDIR_DEFORESTATION=data/hansen_loss WORKDIR_GHG=data/forest_ghg WORKDIR_WOODPULP=data/woodpulp WORKDIR_NATURAL_CROP_CONVERSION=data/natural_crop_conversion -WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY=data/nutrient_assimilation_capacity +WORKDIR_NUTRIENT_LOAD_REDUCTION=data/nutrient_load_reduction CHECKSUMS_PATH=data_checksums @@ -53,7 +53,7 @@ indicators: make convert-forestGHG make convert-satDeforestation make convert-naturalCropConversion - make convert-nutrientAssimilationCapacity + make convert-nutrientLoadReduction contextual-layers: convert-hdi-contextual convert-blue-water-contextual @@ -325,22 +325,22 @@ convert-aqueduct: extract-aqueduct python vector_folder_to_h3_table.py $(WORKDIR_AQUEDUCT)/Aqueduct40_waterrisk_download_Y2023M07D05/GDB h3_grid_aqueduct_global bws_cat aqueduct "Environmental datasets" 2022 --indicator=UWUSR_T --h3-res=6 --layer=baseline_annual ######################################### -# Nutrient assimilation capacity data # +# Nutrient load reduction data # ######################################### -download-nutrient-assimilation-capacity: - mkdir -p $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY) - aws s3 sync $(AWS_S3_BUCKET_URL)/processed/nutrients_assimilation_capacity/ $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY) +download-nutrient-load-reduction: + mkdir -p $(WORKDIR_NUTRIENT_LOAD_REDUCTION) + aws s3 sync $(AWS_S3_BUCKET_URL)/processed/nutrients_load_reduction/ $(WORKDIR_NUTRIENT_LOAD_REDUCTION) -extract-nutrient-assimilation-capacity:download-nutrient-assimilation-capacity - unzip -q -u $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY)/nutrient_assimilation_capacity.zip -d $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY)/ - cd $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY) && sha256sum --check ../../$(CHECKSUMS_PATH)/nutrient_assimilation_capacity +extract-nutrient-load-reduction:download-nutrient-load-reduction + unzip -q -u $(WORKDIR_NUTRIENT_LOAD_REDUCTION)/nutrient_load_reduction.zip -d $(WORKDIR_NUTRIENT_LOAD_REDUCTION)/ + cd $(WORKDIR_NUTRIENT_LOAD_REDUCTION) && sha256sum --check ../../$(CHECKSUMS_PATH)/nutrient_load_reduction -convert-nutrientAssimilationCapacity: extract-nutrient-assimilation-capacity - @echo "Converting nutrient assimilation capacity data... " - python vector_folder_to_h3_table.py $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY) h3_grid_nutrient_assimilation_global perc_reduc nutrient_assimilation_capacity "Environmental datasets" 2023 --indicator=NAC --h3-res=6 +convert-nutrientLoadReduction: extract-nutrient-load-reduction + @echo "Converting nutrient load reduction data... " + python vector_folder_to_h3_table.py $(WORKDIR_NUTRIENT_LOAD_REDUCTION) h3_grid_nutrient_load_global perc_reduc nutrient_load_reduction "Environmental datasets" 2023 --indicator=NLR --h3-res=6 @echo "Including contextual layer... " - python vector_folder_to_h3_table.py $(WORKDIR_NUTRIENT_ASSIMILATION_CAPACITY) h3_grid_limiting_nutrients_global Cases_v2_1 limiting_nutrient "Environmental datasets" 2023 + python vector_folder_to_h3_table.py $(WORKDIR_NUTRIENT_LOAD_REDUCTION) h3_grid_limiting_nutrients_global Cases_v2_1 limiting_nutrient "Environmental datasets" 2023 ################### # Contextual data # diff --git a/data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_assimilation_global_metadata.json b/data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_load_global_metadata.json similarity index 91% rename from data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_assimilation_global_metadata.json rename to data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_load_global_metadata.json index be1f19238..b3d9812a2 100644 --- a/data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_assimilation_global_metadata.json +++ b/data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_load_global_metadata.json @@ -1,10 +1,10 @@ { "$schema": "./contextual_metadata_schema.json", - "name": "Nutrient Assimilation Capacity", + "name": "Nutrient Load Reduction", "description": "Modeled limiting nutrients from McDowell et al. (2020).", "aggType": "mode", "legend": { - "name": "Nutrient Assimilation Capacity", + "name": "Nutrient Load Reduction", "id": "", "unit": "", "min": 0, diff --git a/data/preprocessing/nutrient_assimilation_capacity/Makefile b/data/preprocessing/nutrient_load_reduction/Makefile similarity index 58% rename from data/preprocessing/nutrient_assimilation_capacity/Makefile rename to data/preprocessing/nutrient_load_reduction/Makefile index 6a5947dd4..ea3eb2429 100644 --- a/data/preprocessing/nutrient_assimilation_capacity/Makefile +++ b/data/preprocessing/nutrient_load_reduction/Makefile @@ -9,22 +9,22 @@ AWS_S3_BUCKET_URL=s3://landgriffon-raw-data all: unzip-limiting-nutrient -# First you need to download the data manually from https://figshare.com/articles/figure/DRP_NO3_TN_TP_rasters/14527638/1?file=31154728 and save it in nutrient_assimilation_capacity/data +# First you need to download the data manually from https://figshare.com/articles/figure/DRP_NO3_TN_TP_rasters/14527638/1?file=31154728 and save it in nutrient_load_reduction/data unzip-limiting-nutrient: unzip -q -u $(DATA_DIR)/hybas_l03_v1c_Cases.zip -d $(DATA_DIR)/ # Preprocess the data before ingesting instead of performing these calculations on the database process-limiting-nutrients: - mkdir -p $(DATA_DIR)/nutrient_assimilation_capacity - python process_data.py $(DATA_DIR)/hybas_l03_v1c_Cases $(DATA_DIR)/nutrient_assimilation_capacity + mkdir -p $(DATA_DIR)/nutrient_load_reduction + python process_data.py $(DATA_DIR)/hybas_l03_v1c_Cases $(DATA_DIR)/nutrient_load_reduction # Create a zip archive of the shapefile and related files zip-shapefile: - cd $(DATA_DIR)/nutrient_assimilation_capacity && zip -r nutrient_assimilation_capacity.zip nutrient_assimilation_capacity.* + cd $(DATA_DIR)/nutrient_load_reduction && zip -r nutrient_load_reduction.zip nutrient_load_reduction.* upload_results: - aws s3 cp $(DATA_DIR)/nutrient_assimilation_capacity/nutrient_assimilation_capacity.zip ${AWS_S3_BUCKET_URL}/processed/nutrients_assimilation_capacity/ + aws s3 cp $(DATA_DIR)/nutrient_load_reduction/nutrient_load_reduction.zip ${AWS_S3_BUCKET_URL}/processed/nutrients_load_reduction/ write_checksum: - cd $(DATA_DIR)/hybas_l03_v1c_Cases && sha256sum nutrient_assimilation_capacity.shp > $(checksums_dir)/nutrient_assimilation_capacity + cd $(DATA_DIR)/nutrient_load_reduction && sha256sum nutrient_load_reduction.shp > $(checksums_dir)/nutrient_load_reduction diff --git a/data/preprocessing/nutrient_assimilation_capacity/README b/data/preprocessing/nutrient_load_reduction/README similarity index 96% rename from data/preprocessing/nutrient_assimilation_capacity/README rename to data/preprocessing/nutrient_load_reduction/README index 7d8f6cb5c..d7af21041 100644 --- a/data/preprocessing/nutrient_assimilation_capacity/README +++ b/data/preprocessing/nutrient_load_reduction/README @@ -1,6 +1,6 @@ # Data Processing Pipeline README -This repository contains a data processing pipeline implemented using a Makefile and Python script to download, preprocess, upload, and generate checksums for data files. The pipeline is designed to work with geospatial data related to nutrient assimilation capacity. +This repository contains a data processing pipeline implemented using a Makefile and Python script to download, preprocess, upload, and generate checksums for data files. The pipeline is designed to work with geospatial data related to nutrient load reduction. ## Prerequisites diff --git a/data/preprocessing/nutrient_assimilation_capacity/process_data.py b/data/preprocessing/nutrient_load_reduction/process_data.py similarity index 97% rename from data/preprocessing/nutrient_assimilation_capacity/process_data.py rename to data/preprocessing/nutrient_load_reduction/process_data.py index c04cf4246..9c9f41805 100644 --- a/data/preprocessing/nutrient_assimilation_capacity/process_data.py +++ b/data/preprocessing/nutrient_load_reduction/process_data.py @@ -67,7 +67,7 @@ def process_folder(input_folder, output_folder): gdf['perc_reduc'] = gdf.apply(calculate_perc_reduction, axis=1) # Save the processed data to a new shapefile gdf = gdf[['Cases_v2_1', 'perc_reduc', 'geometry']] - output_file = output_path / 'nutrient_assimilation_capacity.shp' + output_file = output_path / 'nutrient_load_reduction.shp' log.info(f"Saving preprocessed file to {output_file}") gdf.to_file(output_file) else: From e5891c3077a49146102e8904e671b5473a7f9c4d Mon Sep 17 00:00:00 2001 From: elpamart Date: Thu, 14 Sep 2023 09:47:33 +0200 Subject: [PATCH 4/6] refine calculation definition and extend metadata description --- .../h3_grid_limiting_nutrients_global_metadata.json | 2 +- .../h3_grid_nutrient_load_global_metadata.json | 2 +- data/preprocessing/nutrient_load_reduction/process_data.py | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/data/h3_data_importer/contextual_layers_metadata/h3_grid_limiting_nutrients_global_metadata.json b/data/h3_data_importer/contextual_layers_metadata/h3_grid_limiting_nutrients_global_metadata.json index a8f393785..5d36c0660 100644 --- a/data/h3_data_importer/contextual_layers_metadata/h3_grid_limiting_nutrients_global_metadata.json +++ b/data/h3_data_importer/contextual_layers_metadata/h3_grid_limiting_nutrients_global_metadata.json @@ -1,7 +1,7 @@ { "$schema": "./contextual_metadata_schema.json", "name": "Modeled limiting nutrients", - "description": "Modeled limiting nutrients from McDowell et al. (2020)", + "description": "Geographical distribution and estimated area occupied by four different types of catchments across the globe based on predicted median nutrient concentrations. These catchments are classified as either 'acceptable periphyton growth and N-limitation' (catchment type 1), 'undesirable periphyton growth and N-limitation' (catchment type 2), 'acceptable periphyton growth and P-limitation' (catchment type 3) or 'undesirable periphyton growth and P-limitation' (catchment type 4) ", "aggType": "mode", "legend": { "name": "Modeled limiting nutrients", diff --git a/data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_load_global_metadata.json b/data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_load_global_metadata.json index b3d9812a2..1530c605d 100644 --- a/data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_load_global_metadata.json +++ b/data/h3_data_importer/contextual_layers_metadata/h3_grid_nutrient_load_global_metadata.json @@ -1,7 +1,7 @@ { "$schema": "./contextual_metadata_schema.json", "name": "Nutrient Load Reduction", - "description": "Modeled limiting nutrients from McDowell et al. (2020).", + "description": "Geospatial representation indicating the extent to which basin-wide nutrient loads need to be reduced to achieve the desired state of nature in various regions. This layer illustrateS the required proportional nutrient load reduction for each specific watershed or basin to no longer be considered polluted and to attain the desired instream nutrient concentration.", "aggType": "mode", "legend": { "name": "Nutrient Load Reduction", diff --git a/data/preprocessing/nutrient_load_reduction/process_data.py b/data/preprocessing/nutrient_load_reduction/process_data.py index 9c9f41805..6d6479809 100644 --- a/data/preprocessing/nutrient_load_reduction/process_data.py +++ b/data/preprocessing/nutrient_load_reduction/process_data.py @@ -39,7 +39,10 @@ def check_and_reproject_to_4326(gdf): return gdf -# Define the function to calculate perc_reduction +# Calculation of the required Load Reduction. +# This equation is applied to only the basin-specific limiting nutrient as identified by McDowell et al. (2020) +# The global concentration thresholds values for Total N (0.70 mg-N/L) and Total P (0.046 mg-P/L) represent acceptable levels of algal growth. +# More information can be found on the LandGriffon v2.0 methodology def calculate_perc_reduction(row): if row['Cases_v2_1'] == 4 and row['TP_con_V2_']: return ((row['TP_con_V2_'] - 0.046) / row['TP_con_V2_']) * 100 From 153e0d5ef4d728da2cc6f7ee6c0cefff1d3ba58e Mon Sep 17 00:00:00 2001 From: elpamart Date: Fri, 15 Sep 2023 18:15:13 +0200 Subject: [PATCH 5/6] remove unused lib in preprocessing and make some small changes in Makefile --- data/preprocessing/nutrient_load_reduction/Makefile | 4 ++-- data/preprocessing/nutrient_load_reduction/process_data.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/data/preprocessing/nutrient_load_reduction/Makefile b/data/preprocessing/nutrient_load_reduction/Makefile index ea3eb2429..9c69efba2 100644 --- a/data/preprocessing/nutrient_load_reduction/Makefile +++ b/data/preprocessing/nutrient_load_reduction/Makefile @@ -5,9 +5,9 @@ checksums_dir=../../../../h3_data_importer/data_checksums AWS_S3_BUCKET_URL=s3://landgriffon-raw-data # Targets -.PHONY: unzip-limiting-nutrient +.PHONY: unzip-limiting-nutrient process-limiting-nutrients zip-shapefile upload_results write_checksum: -all: unzip-limiting-nutrient +all: unzip-limiting-nutrient process-limiting-nutrients zip-shapefile upload_results write_checksum: # First you need to download the data manually from https://figshare.com/articles/figure/DRP_NO3_TN_TP_rasters/14527638/1?file=31154728 and save it in nutrient_load_reduction/data unzip-limiting-nutrient: diff --git a/data/preprocessing/nutrient_load_reduction/process_data.py b/data/preprocessing/nutrient_load_reduction/process_data.py index 6d6479809..e77e9dd27 100644 --- a/data/preprocessing/nutrient_load_reduction/process_data.py +++ b/data/preprocessing/nutrient_load_reduction/process_data.py @@ -1,12 +1,12 @@ """ Reads the limiting nutrients equal area vector file, reporjects the file to EPSG4326 and estimates the percentage of reduction needed to meet a good water quality conditions. Usage: -process_data.py +process_data.py Arguments: - Folder containing the limiting nutrients shapefile + Input folder containing the limiting nutrients shapefile + Output folder to export the required percentage reduction """ -import os import logging from pathlib import Path import argparse From a1aabedad0e86ce5e799b2fec83106760e1a4c8b Mon Sep 17 00:00:00 2001 From: elpamart Date: Tue, 19 Sep 2023 19:08:54 +0200 Subject: [PATCH 6/6] include description and checksum --- .../data_checksums/nutrient_load_reduction | 1 + .../nutrient_load_reduction/process_data.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 data/h3_data_importer/data_checksums/nutrient_load_reduction diff --git a/data/h3_data_importer/data_checksums/nutrient_load_reduction b/data/h3_data_importer/data_checksums/nutrient_load_reduction new file mode 100644 index 000000000..387fca2da --- /dev/null +++ b/data/h3_data_importer/data_checksums/nutrient_load_reduction @@ -0,0 +1 @@ +cd63dfeebf87434f7d0a79b04ca228f618c987bdb70a1760eca165cf6cf48a4e nutrient_load_reduction.shp diff --git a/data/preprocessing/nutrient_load_reduction/process_data.py b/data/preprocessing/nutrient_load_reduction/process_data.py index e77e9dd27..dff73a602 100644 --- a/data/preprocessing/nutrient_load_reduction/process_data.py +++ b/data/preprocessing/nutrient_load_reduction/process_data.py @@ -39,11 +39,15 @@ def check_and_reproject_to_4326(gdf): return gdf -# Calculation of the required Load Reduction. -# This equation is applied to only the basin-specific limiting nutrient as identified by McDowell et al. (2020) -# The global concentration thresholds values for Total N (0.70 mg-N/L) and Total P (0.046 mg-P/L) represent acceptable levels of algal growth. -# More information can be found on the LandGriffon v2.0 methodology def calculate_perc_reduction(row): + """ + Calculation of the required Load Reduction. + + This equation is applied to only the basin-specific limiting nutrient as identified by McDowell et al. (2020) + The global concentration thresholds values for Total N (0.70 mg-N/L) and Total P (0.046 mg-P/L) represent acceptable levels of algal growth. + + More information can be found on the LandGriffon v2.0 methodology + """ if row['Cases_v2_1'] == 4 and row['TP_con_V2_']: return ((row['TP_con_V2_'] - 0.046) / row['TP_con_V2_']) * 100 elif row['Cases_v2_1'] == 2 and row['TN_con_V2_']: