Adds 5 extra earthstat datasets

data/h3_data_importer/Makefile

@@ -23,7 +23,7 @@ WORKDIR_GHG_FARM=data/ghg_farm
 WORKDIR_LIVESTOCK_PROCESSED=data/livestock_processed
 WORKDIR_LIVESTOCK_GHG=data/livestock_ghg
 WORKDIR_DEFAULT_COMMODITY=data/default_commodity
-WORKDIR_GRASSNES=data/grassnes
+WORKDIR_EARTHSTAT=data/earthstat
 
 
 CHECKSUMS_PATH=data_checksums
@@ -39,8 +39,9 @@ all:
 		false; \
 	fi
 	@echo "AWS S3 bucket access confirmed, proceeding with import..."
-	make clean
-	make crop indicators
+#	make clean
+#	make crop indicators
+	make convert-earthstat
 	make delete-h3-tables
 
 crop: convert-mapspam-crop-production convert-mapspam-crop-harvest convert-glw3-livestock convert-livestock-processed convert-default-commodity convert-grassnes
@@ -70,20 +71,18 @@ convert-mapspam-crop-harvest: download-mapspam-crop-harvest
 	python raster_folder_to_h3_table.py $(WORKDIR_MAPSPAM_HA) h3_grid_spam2010v2r0_global_ha harvest_area spam 2010 --h3-res=6 --thread-count=$(PARALLELIZATION_FACTOR)
 
 # ---------------------------------------------
-# Grassnes (earthstat) - production and harvest
+# EARTHSTAT - production and harvest
 # ---------------------------------------------
 
-download-grassnes:
-	mkdir -p $(WORKDIR_GRASSNES)
-	aws s3 sync $(AWS_S3_BUCKET_URL)/processed/grassnes/ $(WORKDIR_GRASSNES)
-	cd $(WORKDIR_GRASSNES) && sha256sum --check ../../$(CHECKSUMS_PATH)/grassnes
-
-convert-grassnes: download-grassnes
-	mkdir -p $(WORKDIR_GRASSNES)/production $(WORKDIR_GRASSNES)/harvest
-	mv $(WORKDIR_GRASSNES)/earthstat_global_prod_grassnes_t.tif $(WORKDIR_GRASSNES)/production
-	mv $(WORKDIR_GRASSNES)/earthstat_global_harvest_grassnes_ha.tif $(WORKDIR_GRASSNES)/harvest
-	python raster_folder_to_h3_table.py $(WORKDIR_GRASSNES)/production h3_grid_grassnes_global_prod production earthstat 2000 --h3-res=6 --thread-count=$(PARALLELIZATION_FACTOR)
-	python raster_folder_to_h3_table.py $(WORKDIR_GRASSNES)/harvest h3_grid_grassnes_global_ha harvest_area earthstat 2000 --h3-res=6 --thread-count=$(PARALLELIZATION_FACTOR)
+download-earthstat:
+	mkdir -p $(WORKDIR_EARTHSTAT)/production $(WORKDIR_EARTHSTAT)/harvest
+	aws s3 sync $(AWS_S3_BUCKET_URL)/processed/earthstat/ $(WORKDIR_EARTHSTAT)
+	cd $(WORKDIR_EARTHSTAT) && sha256sum --check ../../$(CHECKSUMS_PATH)/earthstat
+
+
+convert-earthstat: download-earthstat
+	python raster_folder_to_h3_table.py $(WORKDIR_EARTHSTAT)/production h3_grid_earthstat_global_prod production earthstat 2000 --h3-res=6 --thread-count=$(PARALLELIZATION_FACTOR)
+	python raster_folder_to_h3_table.py $(WORKDIR_EARTHSTAT)/harvest h3_grid_earthstat_global_ha harvest_area earthstat 2000 --h3-res=6 --thread-count=$(PARALLELIZATION_FACTOR)
 
 
 

data/h3_data_importer/data_checksums/earthstat

@@ -0,0 +1,10 @@
+e9bc4b5da0a989a2dee26bb5f77cab64dd72103c33328d0e6b6497ffc162ba4d  harvest/earthstat_global_alfalfa_harvest_ha.tif
+984c66e6326b9b66594d04314485adcc622255a13bd71512847b4f1b1e06f61d  harvest/earthstat_global_clover_harvest_ha.tif
+705abfab0294c2bfca4d2a5e333bf0b7237339b52b52880f7cfb78a21c7c2aa3  harvest/earthstat_global_grassnes_harvest_ha.tif
+1b360b651c7f376daa750266ccec0df67306f0692727b3738c35f959bd055b9f  harvest/earthstat_global_maizefor_harvest_ha.tif
+5c4c5bd48c14f51030ad5cd19aafbe0dddb154d8dc612b85cae5b75e0c69af06  harvest/earthstat_global_triticale_harvest_ha.tif
+a73ae9e7af6af3ba06176f65b7af96e3d98f71d4c7f3d04f15ba0c8641d5bf5e  production/earthstat_global_alfalfa_production_t.tif
+74b2c08865d430eef071beae49c00e5a4f4d6f4a63b0fd8e228dfe09873fd2b3  production/earthstat_global_clover_production_t.tif
+1bee00b30181661f8c8a5de5dec9a28983fff3ee973ff8465a46376ba7195f38  production/earthstat_global_grassnes_production_t.tif
+7b794071b81f59a939ed156042817ca83863f16fda434577048a075d997489e1  production/earthstat_global_maizefor_production_t.tif
+bca39c9f4ef90f65a8416e663181dcaa488ca5007b9b3a3e3587c15f80fc4793  production/earthstat_global_triticale_production_t.tif

data/preprocessing/earthstat/Makefile

@@ -0,0 +1,71 @@
+checksums_dir=../../../h3_data_importer/data_checksums
+data_dir=data
+resampling_resolution="0.083333"
+
+AWS_S3_BUCKET_URL=s3://landgriffon-raw-data
+GRASSNES_URL=https://s3.us-east-2.amazonaws.com/earthstatdata/HarvestedAreaYield175Crops_Indvidual_Geotiff/grassnes_HarvAreaYield_Geotiff.zip
+CLOVER_URL=https://s3.us-east-2.amazonaws.com/earthstatdata/HarvestedAreaYield175Crops_Indvidual_Geotiff/clover_HarvAreaYield_Geotiff.zip
+MAIZEFOR_URL=https://s3.us-east-2.amazonaws.com/earthstatdata/HarvestedAreaYield175Crops_Indvidual_Geotiff/maizefor_HarvAreaYield_Geotiff.zip
+ALFA_URL=https://s3.us-east-2.amazonaws.com/earthstatdata/HarvestedAreaYield175Crops_Indvidual_Geotiff/alfalfa_HarvAreaYield_Geotiff.zip
+TRITICALE_URL=https://s3.us-east-2.amazonaws.com/earthstatdata/HarvestedAreaYield175Crops_Indvidual_Geotiff/triticale_HarvAreaYield_Geotiff.zip
+
+include ../../../.env
+
+export AWS_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY
+
+.PHONY: all download_grassnes extract_grassnes_rasters resample_grassnes upload_results write_checksum
+
+all: clean download extract resample upload_results write_checksum
+
+download:
+	wget $(GRASSNES_URL) -O data/grassnes_HarvAreaYield_Geotiff.zip
+	wget $(CLOVER_URL) -O data/clover_HarvAreaYield_Geotiff.zip
+	wget $(MAIZEFOR_URL) -O data/maizefor_HarvAreaYield_Geotiff.zip
+	wget $(ALFA_URL) -O data/alfalfa_HarvAreaYield_Geotiff.zip
+	wget $(TRITICALE_URL) -O data/triticale_HarvAreaYield_Geotiff.zip
+
+extract:
+	unzip -oj data/grassnes_HarvAreaYield_Geotiff.zip \
+		grassnes_HarvAreaYield_Geotiff/grassnes_Production.tif \
+		grassnes_HarvAreaYield_Geotiff/grassnes_HarvestedAreaHectares.tif \
+		-d data/
+	rm data/grassnes_HarvAreaYield_Geotiff.zip
+
+	unzip -oj data/clover_HarvAreaYield_Geotiff.zip \
+		clover_HarvAreaYield_Geotiff/clover_Production.tif \
+		clover_HarvAreaYield_Geotiff/clover_HarvestedAreaHectares.tif \
+		-d data/
+	rm data/clover_HarvAreaYield_Geotiff.zip
+
+	unzip -oj data/maizefor_HarvAreaYield_Geotiff.zip \
+		maizefor_HarvAreaYield_Geotiff/maizefor_Production.tif \
+		maizefor_HarvAreaYield_Geotiff/maizefor_HarvestedAreaHectares.tif \
+		-d data/
+	rm data/maizefor_HarvAreaYield_Geotiff.zip
+
+	unzip -oj data/alfalfa_HarvAreaYield_Geotiff.zip \
+		alfalfa_HarvAreaYield_Geotiff/alfalfa_Production.tif \
+		alfalfa_HarvAreaYield_Geotiff/alfalfa_HarvestedAreaHectares.tif \
+		-d data/
+	rm data/alfalfa_HarvAreaYield_Geotiff.zip
+
+	unzip -oj data/triticale_HarvAreaYield_Geotiff.zip \
+		triticale_HarvAreaYield_Geotiff/triticale_Production.tif \
+		triticale_HarvAreaYield_Geotiff/triticale_HarvestedAreaHectares.tif \
+		-d data/
+	rm data/triticale_HarvAreaYield_Geotiff.zip
+
+resample:
+	mkdir -p $(data_dir)/harvest $(data_dir)/production
+	bash resample.sh
+
+upload_results:
+	aws s3 sync $(data_dir) ${AWS_S3_BUCKET_URL}/processed/earthstat/
+
+write_checksum:
+	cd $(data_dir) && sha256sum harvest/* > $(checksums_dir)/earthstat
+	cd $(data_dir) && sha256sum production/* >> $(checksums_dir)/earthstat
+
+clean:
+	rm -rf $(data_dir)/*

data/preprocessing/earthstat/resample.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+
+data_dir="data"
+resampling_resolution="0.083333"
+
+for file in $data_dir/*.tif; do
+  filename=$(basename "$file")
+  crop_type="${filename%%_*}"
+  if [[ $file == *"HarvestedAreaHectares"* ]]; then
+   harv_or_prod="harvest"
+   unit="ha"
+  elif [[ $file == *"Production"* ]]; then
+   harv_or_prod="production"
+   unit="t"
+  fi
+  outfile="earthstat_global_${crop_type}_${harv_or_prod}_${unit}.tif"
+  echo "Resampling $file to $outfile"
+  rio warp \
+   $file \
+   $data_dir/$harv_or_prod/"$outfile" \
+   --resampling sum \
+   --res $resampling_resolution \
+   --src-nodata nan \
+   --dst-nodata -1 \
+   --co compress=deflate \
+   --co predictor=3 \
+   --overwrite
+  rm "$file"
+done