📊 Forest loss by driver (#2964)

* adding meadow step with raster processing

* garden step

* wip

* fix dag

* test

* remove assigning origins

* adding earth engine script WIP

* exporting to GDrive

* updating instructions

* separate script for reproject raster

* adding more description of steps and creating separate script for reprojecting

* including poetry lock

* wip

* snapshots

* adding in click options

* adding subprocess to submit jobs

* adding snapshot step

* adding tree cover loss by driver, over time

* upload

* gardening

* bare bones metadata

* updating poetry.lock

* converting m2 to ha

* fix folder

* improve comments

* adding # type: ignore as per Mojmir's suggestion

* converting code to driver

* adding poetry lock

* getting snapshots working

* updating poetry.lock

* reading in specific columns

* updating snapshot in case it's corrupted

* fixing vehicles

* cleaning up the countries

* combining alaska + hawaii into USA, canaries into Spain etc

* some metadata

* metadata

* deleting repeated comment

* trying to add significant figures

* adding regions

* excluding another coutnry

* consistent snapshot instructions

dag/forests.yml

@@ -7,3 +7,12 @@ steps:
     - data://garden/regions/2023-01-01/regions
   data://grapher/forests/2024-05-08/ifl:
     - data://garden/forests/2024-05-08/ifl
+
+  # Dominant Driver of Forest Loss
+  data://meadow/forests/2024-07-10/tree_cover_loss_by_driver:
+    - snapshot://forests/2024-07-10/tree_cover_loss_by_driver.csv
+  data://garden/forests/2024-07-10/tree_cover_loss_by_driver:
+    - data://meadow/forests/2024-07-10/tree_cover_loss_by_driver
+    - data://garden/regions/2023-01-01/regions
+  data://grapher/forests/2024-07-10/tree_cover_loss_by_driver:
+    - data://garden/forests/2024-07-10/tree_cover_loss_by_driver

etl/steps/data/garden/forests/2024-07-10/tree_cover_loss_by_driver.countries.json

@@ -0,0 +1,205 @@
+{
+  "Afghanistan": "Afghanistan",
+  "Albania": "Albania",
+  "Algeria": "Algeria",
+  "Andorra": "Andorra",
+  "Angola": "Angola",
+  "Argentina": "Argentina",
+  "Armenia": "Armenia",
+  "Australia": "Australia",
+  "Austria": "Austria",
+  "Azerbaijan": "Azerbaijan",
+  "Bahamas, The": "Bahamas",
+  "Bangladesh": "Bangladesh",
+  "Barbados": "Barbados",
+  "Belarus": "Belarus",
+  "Belgium": "Belgium",
+  "Belize": "Belize",
+  "Benin": "Benin",
+  "Bhutan": "Bhutan",
+  "Bolivia": "Bolivia",
+  "Botswana": "Botswana",
+  "Brazil": "Brazil",
+  "Bulgaria": "Bulgaria",
+  "Burkina Faso": "Burkina Faso",
+  "Burundi": "Burundi",
+  "Cambodia": "Cambodia",
+  "Cameroon": "Cameroon",
+  "Canada": "Canada",
+  "Chile": "Chile",
+  "China": "China",
+  "Colombia": "Colombia",
+  "Comoros": "Comoros",
+  "Costa Rica": "Costa Rica",
+  "Croatia": "Croatia",
+  "Cuba": "Cuba",
+  "Cyprus": "Cyprus",
+  "Denmark": "Denmark",
+  "Dominica": "Dominica",
+  "Dominican Republic": "Dominican Republic",
+  "Ecuador": "Ecuador",
+  "El Salvador": "El Salvador",
+  "Equatorial Guinea": "Equatorial Guinea",
+  "Eritrea": "Eritrea",
+  "Estonia": "Estonia",
+  "Ethiopia": "Ethiopia",
+  "Fiji": "Fiji",
+  "Finland": "Finland",
+  "France": "France",
+  "Gabon": "Gabon",
+  "Gambia, The": "Gambia",
+  "Georgia": "Georgia",
+  "Germany": "Germany",
+  "Ghana": "Ghana",
+  "Greece": "Greece",
+  "Grenada": "Grenada",
+  "Guatemala": "Guatemala",
+  "Guinea": "Guinea",
+  "Guinea-Bissau": "Guinea-Bissau",
+  "Guyana": "Guyana",
+  "Haiti": "Haiti",
+  "Honduras": "Honduras",
+  "Hungary": "Hungary",
+  "India": "India",
+  "Indonesia": "Indonesia",
+  "Iraq": "Iraq",
+  "Ireland": "Ireland",
+  "Israel": "Israel",
+  "Italy": "Italy",
+  "Jamaica": "Jamaica",
+  "Japan": "Japan",
+  "Jordan": "Jordan",
+  "Kazakhstan": "Kazakhstan",
+  "Kenya": "Kenya",
+  "Kosovo": "Kosovo",
+  "Latvia": "Latvia",
+  "Lebanon": "Lebanon",
+  "Lesotho": "Lesotho",
+  "Liberia": "Liberia",
+  "Libya": "Libya",
+  "Liechtenstein": "Liechtenstein",
+  "Lithuania": "Lithuania",
+  "Luxembourg": "Luxembourg",
+  "Madagascar": "Madagascar",
+  "Malawi": "Malawi",
+  "Malaysia": "Malaysia",
+  "Maldives": "Maldives",
+  "Mali": "Mali",
+  "Mauritania": "Mauritania",
+  "Mauritius": "Mauritius",
+  "Mexico": "Mexico",
+  "Moldova": "Moldova",
+  "Mongolia": "Mongolia",
+  "Montenegro": "Montenegro",
+  "Morocco": "Morocco",
+  "Mozambique": "Mozambique",
+  "Namibia": "Namibia",
+  "Nepal": "Nepal",
+  "Netherlands": "Netherlands",
+  "New Zealand": "New Zealand",
+  "Nicaragua": "Nicaragua",
+  "Niger": "Niger",
+  "Nigeria": "Nigeria",
+  "Norway": "Norway",
+  "Pakistan": "Pakistan",
+  "Palau": "Palau",
+  "Panama": "Panama",
+  "Papua New Guinea": "Papua New Guinea",
+  "Paraguay": "Paraguay",
+  "Peru": "Peru",
+  "Philippines": "Philippines",
+  "Poland": "Poland",
+  "Portugal": "Portugal",
+  "Romania": "Romania",
+  "Rwanda": "Rwanda",
+  "Saint Lucia": "Saint Lucia",
+  "San Marino": "San Marino",
+  "Saudi Arabia": "Saudi Arabia",
+  "Senegal": "Senegal",
+  "Serbia": "Serbia",
+  "Sierra Leone": "Sierra Leone",
+  "Singapore": "Singapore",
+  "Slovenia": "Slovenia",
+  "Somalia": "Somalia",
+  "South Africa": "South Africa",
+  "South Sudan": "South Sudan",
+  "Spain": "Spain",
+  "Sri Lanka": "Sri Lanka",
+  "Sudan": "Sudan",
+  "Suriname": "Suriname",
+  "Sweden": "Sweden",
+  "Switzerland": "Switzerland",
+  "Tajikistan": "Tajikistan",
+  "Tanzania": "Tanzania",
+  "Thailand": "Thailand",
+  "Timor-Leste": "East Timor",
+  "Togo": "Togo",
+  "Tunisia": "Tunisia",
+  "Turkey": "Turkey",
+  "Turkmenistan": "Turkmenistan",
+  "Uganda": "Uganda",
+  "Ukraine": "Ukraine",
+  "United Kingdom": "United Kingdom",
+  "Uruguay": "Uruguay",
+  "Uzbekistan": "Uzbekistan",
+  "Vanuatu": "Vanuatu",
+  "Vietnam": "Vietnam",
+  "Zambia": "Zambia",
+  "Zimbabwe": "Zimbabwe",
+  "Aruba": "Aruba",
+  "Bosnia & Herzegovina": "Bosnia and Herzegovina",
+  "Brunei": "Brunei",
+  "Burma": "Myanmar",
+  "Cote d'Ivoire": "Cote d'Ivoire",
+  "Curacao": "Curacao",
+  "Czechia": "Czechia",
+  "Egypt": "Egypt",
+  "French Guiana": "French Guiana",
+  "Gaza Strip": "Gaza Strip",
+  "Guadeloupe": "Guadeloupe",
+  "Hong Kong": "Hong Kong",
+  "Iran": "Iran",
+  "Isle of Man": "Isle of Man",
+  "Kyrgyzstan": "Kyrgyzstan",
+  "Laos": "Laos",
+  "Macau": "Macao",
+  "Macedonia": "North Macedonia",
+  "Martinique": "Martinique",
+  "Mayotte": "Mayotte",
+  "Montserrat": "Montserrat",
+  "New Caledonia": "New Caledonia",
+  "Puerto Rico": "Puerto Rico",
+  "Reunion": "Reunion",
+  "Russia": "Russia",
+  "Slovakia": "Slovakia",
+  "Swaziland": "Eswatini",
+  "Syria": "Syria",
+  "Taiwan": "Taiwan",
+  "Trinidad & Tobago": "Trinidad and Tobago",
+  "United States": "United States",
+  "Venezuela": "Venezuela",
+  "Yemen": "Yemen",
+  "Akrotiri": "Akrotiri and Dhekelia",
+  "Antigua & Barbuda": "Antigua and Barbuda",
+  "British Virgin Is": "British Virgin Islands",
+  "Cayman Is": "Cayman Islands",
+  "Central African Rep": "Central African Republic",
+  "Dem Rep of the Congo": "Democratic Republic of Congo",
+  "Fed States of Micronesia": "Micronesia (country)",
+  "Korea, North": "North Korea",
+  "Korea, South": "South Korea",
+  "Rep of the Congo": "Congo",
+  "Sint Maarten": "Sint Maarten (Dutch part)",
+  "Solomon Is": "Solomon Islands",
+  "St Kitts & Nevis": "Saint Kitts and Nevis",
+  "St Martin": "Saint Martin (French part)",
+  "St Vincent & the Grenadines": "Saint Vincent and the Grenadines",
+  "Turks & Caicos Is": "Turks and Caicos Islands",
+  "US Virgin Is": "United States Virgin Islands",
+  "West Bank": "Palestine",
+  "Portugal (Madeira Is)": "Portugal",
+  "Spain (Africa)": "Spain",
+  "Spain (Canary Is)": "Spain",
+  "United States (Alaska)": "United States",
+  "United States (Hawaii)": "United States"
+}

etl/steps/data/garden/forests/2024-07-10/tree_cover_loss_by_driver.excluded_countries.json

@@ -0,0 +1,7 @@
+[
+  "Abyei Area",
+  "Invernada Area",
+  "Navassa I",
+  "No Man's Land",
+  "Akrotiri and Dhekelia"
+]

etl/steps/data/garden/forests/2024-07-10/tree_cover_loss_by_driver.meta.yml

@@ -0,0 +1,52 @@
+# NOTE: To learn more about the fields, hover over their names.
+definitions:
+  common:
+    presentation:
+      topic_tags:
+        - Forests & Deforestation
+      grapher_config:
+        note: |-
+          The method behind this data have changed over time, with a major methodological change occurring in 2015. Be cautious comparing old and new data, especially data from before and after 2015.
+    unit: hectares
+    short_unit: ha
+    processing_level: major
+    description_processing: >-
+      This indicator is calculated by Our World in Data based on data from the Global Forest Watch. Time-series data of tree cover loss in areas of forest where tree canopy cover was greater than or equal to 30% in 2000 was extracted and cross-referenced with data on the dominant driver of deforestation in each area, based on [Curtis et al 2018](https://science.sciencemag.org/content/361/6407/1108). Values may be slightly different to those shown on Global Forest Watch due to differences in the country boundaries used.
+    display:
+      roundingMode: significantFigures
+      numSignificantFigures: 3
+
+# Learn more about the available fields:
+# http://docs.owid.io/projects/etl/architecture/metadata/reference/
+dataset:
+  update_period_days: 0
+
+
+tables:
+  tree_cover_loss_by_driver:
+    variables:
+      commodity_driven_deforestation:
+        title: Tree cover loss driven by commodity-driven deforestation
+        description_short: Tree cover loss from commodity-driven deforestation is typically large-scale deforestation linked primarily to commercial agricultural expansion.  This type of tree cover loss usually represents permanent deforestation.
+        display:
+          name: Commodity-driven deforestation
+      forestry:
+        title: Tree cover loss driven by forestry
+        description_short: Tree cover loss from forestry is typically temporary loss from plantation and natural forest harvesting, with some deforestation of primary forests. This type of tree cover loss usually represents temporary loss.
+        display:
+          name: Forestry
+      shifting_agriculture:
+        title: Tree cover loss driven by shifting agriculture
+        description_short: Tree cover loss from shifting agriculture is typically temporary loss or permanent deforestation due to small- and medium-scale agriculture. This type of tree cover loss usually represents temporary loss.
+        display:
+          name: Shifting agriculture
+      urbanization:
+        title: Tree cover loss driven by urbanization
+        description_short: Tree cover loss from urbanization is typically deforestation for expansion of urban centers. This type of tree cover loss usually represents permanent deforestation.
+        display:
+          name: Urbanization
+      wildfire:
+        title: Tree cover loss driven by wildfire
+        description_short: Tree cover loss from wildfire is typically temporary loss, it does not include fire clearing for agriculture. This type of tree cover loss usually represents temporary loss.
+        display:
+          name: Wildfire

etl/steps/data/garden/forests/2024-07-10/tree_cover_loss_by_driver.py

@@ -0,0 +1,63 @@
+"""Load a meadow dataset and create a garden dataset."""
+
+from owid.catalog import Table
+
+from etl.data_helpers import geo
+from etl.helpers import PathFinder, create_dataset
+
+# Get paths and naming conventions for current step.
+paths = PathFinder(__file__)
+REGIONS = ["Asia", "Europe", "Africa", "Oceania", "North America", "South America"]
+
+
+def run(dest_dir: str) -> None:
+    #
+    # Load inputs.
+    #
+    # Load meadow dataset.
+    ds_meadow = paths.load_dataset("tree_cover_loss_by_driver")
+    # Load regions dataset.
+    ds_regions = paths.load_dataset("regions")
+    # Read table from meadow dataset.
+    tb = ds_meadow["tree_cover_loss_by_driver"].reset_index()
+
+    #
+    # Process data.
+    #
+    tb = geo.harmonize_countries(
+        df=tb, countries_file=paths.country_mapping_path, excluded_countries_file=paths.excluded_countries_path
+    )
+    # Some regions are broken down into smaller regions in the dataset, so we need to aggregate them here e.g. Alaska and Hawaii are recorded separately in the dataset, but the geo.harmonize_countries function renames them as United States
+    tb = tb.groupby(["country", "year", "category"]).sum().reset_index()
+    tb["year"] = tb["year"].astype(int) + 2000
+    # Convert m2 to ha
+    tb["area"] = tb["area"].astype(float).div(10000)
+    tb = convert_codes_to_drivers(tb)
+
+    tb = tb.pivot(index=["country", "year"], columns="category", values="area").reset_index()
+    tb = geo.add_regions_to_table(tb, ds_regions=ds_regions, regions=REGIONS, min_num_values_per_year=1)
+    tb = tb.format(["country", "year"])
+
+    #
+    # Save outputs.
+    #
+    # Create a new garden dataset with the same metadata as the meadow dataset.
+    ds_garden = create_dataset(
+        dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata
+    )
+
+    # Save changes in the new garden dataset.
+    ds_garden.save()
+
+
+def convert_codes_to_drivers(tb: Table) -> Table:
+    """ """
+    code_dict = {
+        1: "commodity_driven_deforestation",
+        2: "shifting_agriculture",
+        3: "forestry",
+        4: "wildfire",
+        5: "urbanization",
+    }
+    tb["category"] = tb["category"].replace(code_dict)
+    return tb

etl/steps/data/grapher/forests/2024-07-10/tree_cover_loss_by_driver.py

@@ -0,0 +1,32 @@
+"""Load a garden dataset and create a grapher dataset."""
+
+from etl.helpers import PathFinder, create_dataset
+
+# Get paths and naming conventions for current step.
+paths = PathFinder(__file__)
+
+
+def run(dest_dir: str) -> None:
+    #
+    # Load inputs.
+    #
+    # Load garden dataset.
+    ds_garden = paths.load_dataset("tree_cover_loss_by_driver")
+
+    # Read table from garden dataset.
+    tb = ds_garden["tree_cover_loss_by_driver"]
+
+    #
+    # Process data.
+    #
+
+    #
+    # Save outputs.
+    #
+    # Create a new grapher dataset with the same metadata as the garden dataset.
+    ds_grapher = create_dataset(
+        dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_garden.metadata
+    )
+
+    # Save changes in the new grapher dataset.
+    ds_grapher.save()