📊 Update natural disasters data

dag/main.yml

@@ -178,28 +178,6 @@ steps:
   data://grapher/un/2022-11-29/undp_hdr:
     - data://garden/un/2022-11-29/undp_hdr
 
-  #
-  # EM-DAT Natural disasters (2023).
-  #
-  data://meadow/emdat/2023-09-20/natural_disasters:
-    - snapshot://emdat/2023-09-20/natural_disasters.xlsx
-  # The following dataset has a table for yearly data and another for decadal data.
-  data://garden/emdat/2023-09-20/natural_disasters:
-    - data://meadow/emdat/2023-09-20/natural_disasters
-    - data://garden/demography/2023-03-31/population
-    - data://garden/wb/2023-04-30/income_groups
-    - data://garden/regions/2023-01-01/regions
-    - data://garden/worldbank_wdi/2023-05-29/wdi
-  # The following dataset has all (yearly and decadal) variables together.
-  data://grapher/emdat/2023-09-20/natural_disasters:
-    - data://garden/emdat/2023-09-20/natural_disasters
-  # The following dataset has only global data, and entity corresponds to the type of disaster.
-  data://grapher/emdat/2023-09-20/natural_disasters_global_by_type:
-    - data://garden/emdat/2023-09-20/natural_disasters
-  # Natural disasters explorer.
-  data://explorers/emdat/2023-09-20/natural_disasters:
-    - data://garden/emdat/2023-09-20/natural_disasters
-
   # Country profiles - overview
   data://garden/country_profile/2022/overview:
     - backport://backport/owid/latest/dataset_5599_ihme__global_burden_of_disease__deaths_and_dalys__institute_for_health_metrics_and_evaluation__2022_04
@@ -721,6 +699,58 @@ steps:
   data://grapher/wb/2024-03-11/income_groups:
     - data://garden/wb/2024-03-11/income_groups
 
+  #
+  # EM-DAT Natural disasters.
+  #
+  data://meadow/emdat/2024-04-11/natural_disasters:
+    - snapshot://emdat/2024-04-11/natural_disasters.xlsx
+
+  # The following dataset has a table for yearly data and another for decadal data.
+  data://garden/emdat/2024-04-11/natural_disasters:
+    - data://meadow/emdat/2024-04-11/natural_disasters
+    - data://garden/demography/2023-03-31/population
+    - data://garden/wb/2024-03-11/income_groups
+    - data://garden/regions/2023-01-01/regions
+    - data://garden/worldbank_wdi/2023-05-29/wdi
+
+  # The following dataset has all (yearly and decadal) variables together.
+  data://grapher/emdat/2024-04-11/natural_disasters:
+    - data://garden/emdat/2024-04-11/natural_disasters
+
+  # The following dataset has only global data, and entity corresponds to the type of disaster.
+  data://grapher/emdat/2024-04-11/natural_disasters_global_by_type:
+    - data://garden/emdat/2024-04-11/natural_disasters
+
+  # Natural disasters explorer.
+  data://explorers/emdat/latest/natural_disasters:
+    - data://garden/emdat/2024-04-11/natural_disasters
+
+  ######################################################################################################################
+  # Older versions that should be archived once they are not used by any other steps.
+  #
+  # EM-DAT Natural disasters (2023).
+  #
+  data://meadow/emdat/2023-09-20/natural_disasters:
+    - snapshot://emdat/2023-09-20/natural_disasters.xlsx
+  # The following dataset has a table for yearly data and another for decadal data.
+  data://garden/emdat/2023-09-20/natural_disasters:
+    - data://meadow/emdat/2023-09-20/natural_disasters
+    - data://garden/demography/2023-03-31/population
+    - data://garden/wb/2023-04-30/income_groups
+    - data://garden/regions/2023-01-01/regions
+    - data://garden/worldbank_wdi/2023-05-29/wdi
+  # The following dataset has all (yearly and decadal) variables together.
+  data://grapher/emdat/2023-09-20/natural_disasters:
+    - data://garden/emdat/2023-09-20/natural_disasters
+  # The following dataset has only global data, and entity corresponds to the type of disaster.
+  data://grapher/emdat/2023-09-20/natural_disasters_global_by_type:
+    - data://garden/emdat/2023-09-20/natural_disasters
+  # Natural disasters explorer.
+  data://explorers/emdat/2023-09-20/natural_disasters:
+    - data://garden/emdat/2023-09-20/natural_disasters
+
+  ######################################################################################################################
+
 include:
   - dag/open_numbers.yml
   - dag/faostat.yml

etl/steps/data/explorers/emdat/latest/natural_disasters.py

@@ -44,7 +44,7 @@ def create_wide_tables(table: Table) -> Table:
     """
     # Adapt disaster type names to match those in the old explorer files.
     table = table.reset_index()
-    table["type"] = table["type"].replace(DISASTER_TYPE_RENAMING)
+    table["type"] = table.astype({"type": str})["type"].replace(DISASTER_TYPE_RENAMING)
 
     # Create wide table.
     table_wide = table.pivot(index=["country", "year"], columns="type", join_column_levels_with="_")
@@ -56,7 +56,8 @@ def create_wide_tables(table: Table) -> Table:
             .replace("total_dead", "deaths")
             .replace("total_damages_per_gdp", "total_damages_pct_gdp")
             for column in table_wide.columns
-        }
+        },
+        errors="raise",
     )
 
     # Remove unnecessary columns.
@@ -77,23 +78,32 @@ def create_wide_tables(table: Table) -> Table:
     ]
 
     # Set an appropriate index and sort conveniently.
-    table_wide = table_wide.set_index(["country", "year"], verify_integrity=True).sort_index()
+    table_wide = table_wide.format()
 
     return table_wide
 
 
 def run(dest_dir: str) -> None:
+    #
+    # Load inputs.
+    #
     # Load the latest dataset from garden.
     ds_garden = paths.load_dataset("natural_disasters")
 
-    # Load tables with yearly and decadal data.
+    # Read tables with yearly and decadal data.
     tb_yearly = ds_garden["natural_disasters_yearly"]
     tb_decadal = ds_garden["natural_disasters_decadal"]
 
+    #
+    # Process data.
+    #
     # Create wide tables adapted to the old format in explorers.
     tb_yearly_wide = create_wide_tables(table=tb_yearly)
     tb_decadal_wide = create_wide_tables(table=tb_decadal)
 
+    #
+    # Save outputs.
+    #
     # Initialize a new grapher dataset and add dataset metadata.
     ds_grapher = create_dataset(
         dest_dir,

etl/steps/data/garden/emdat/2024-04-11/natural_disasters.countries.json

@@ -11,7 +11,7 @@
   "Australia": "Australia",
   "Austria": "Austria",
   "Azerbaijan": "Azerbaijan",
-  "Bahamas (the)": "Bahamas",
+  "Bahamas": "Bahamas",
   "Bangladesh": "Bangladesh",
   "Barbados": "Barbados",
   "Belarus": "Belarus",
@@ -24,6 +24,7 @@
   "Bosnia and Herzegovina": "Bosnia and Herzegovina",
   "Botswana": "Botswana",
   "Brazil": "Brazil",
+  "British Virgin Islands": "British Virgin Islands",
   "Brunei Darussalam": "Brunei",
   "Bulgaria": "Bulgaria",
   "Burkina Faso": "Burkina Faso",
@@ -32,26 +33,28 @@
   "Cambodia": "Cambodia",
   "Cameroon": "Cameroon",
   "Canada": "Canada",
-  "Cayman Islands (the)": "Cayman Islands",
+  "Cayman Islands": "Cayman Islands",
   "Central African Republic": "Central African Republic",
   "Chad": "Chad",
   "Chile": "Chile",
   "China": "China",
   "Colombia": "Colombia",
-  "Comoros (the)": "Comoros",
-  "Congo (the Democratic Republic of the)": "Democratic Republic of Congo",
-  "Congo (the)": "Congo",
-  "Cook Islands (the)": "Cook Islands",
+  "Comoros": "Comoros",
+  "Congo": "Congo",
+  "Cook Islands": "Cook Islands",
   "Costa Rica": "Costa Rica",
   "Croatia": "Croatia",
   "Cuba": "Cuba",
   "Cyprus": "Cyprus",
+  "Czechia": "Czechia",
   "Czechoslovakia": "Czechoslovakia",
   "C\u00f4te d\u2019Ivoire": "Cote d'Ivoire",
+  "Democratic People's Republic of Korea": "North Korea",
+  "Democratic Republic of the Congo": "Democratic Republic of Congo",
   "Denmark": "Denmark",
   "Djibouti": "Djibouti",
   "Dominica": "Dominica",
-  "Dominican Republic (the)": "Dominican Republic",
+  "Dominican Republic": "Dominican Republic",
   "Ecuador": "Ecuador",
   "Egypt": "Egypt",
   "El Salvador": "El Salvador",
@@ -65,8 +68,9 @@
   "French Guiana": "French Guiana",
   "French Polynesia": "French Polynesia",
   "Gabon": "Gabon",
-  "Gambia (the)": "Gambia",
+  "Gambia": "Gambia",
   "Georgia": "Georgia",
+  "German Democratic Republic": "East Germany",
   "Germany": "Germany",
   "Ghana": "Ghana",
   "Greece": "Greece",
@@ -79,7 +83,6 @@
   "Guyana": "Guyana",
   "Haiti": "Haiti",
   "Honduras": "Honduras",
-  "Hong Kong": "Hong Kong",
   "Hungary": "Hungary",
   "Iceland": "Iceland",
   "India": "India",
@@ -96,24 +99,23 @@
   "Kazakhstan": "Kazakhstan",
   "Kenya": "Kenya",
   "Kiribati": "Kiribati",
-  "Korea (the Democratic People's Republic of)": "North Korea",
   "Kuwait": "Kuwait",
   "Kyrgyzstan": "Kyrgyzstan",
-  "Lao People's Democratic Republic (the)": "Laos",
+  "Lao People's Democratic Republic": "Laos",
   "Latvia": "Latvia",
   "Lebanon": "Lebanon",
   "Lesotho": "Lesotho",
   "Liberia": "Liberia",
   "Libya": "Libya",
   "Lithuania": "Lithuania",
   "Luxembourg": "Luxembourg",
-  "Macao": "Macao",
   "Madagascar": "Madagascar",
   "Malawi": "Malawi",
   "Malaysia": "Malaysia",
   "Maldives": "Maldives",
   "Mali": "Mali",
   "Malta": "Malta",
+  "Marshall Islands": "Marshall Islands",
   "Martinique": "Martinique",
   "Mauritania": "Mauritania",
   "Mauritius": "Mauritius",
@@ -131,29 +133,32 @@
   "New Caledonia": "New Caledonia",
   "New Zealand": "New Zealand",
   "Nicaragua": "Nicaragua",
-  "Niger (the)": "Niger",
+  "Niger": "Niger",
   "Nigeria": "Nigeria",
   "Niue": "Niue",
+  "North Macedonia": "North Macedonia",
+  "Northern Mariana Islands": "Northern Mariana Islands",
   "Norway": "Norway",
   "Oman": "Oman",
   "Pakistan": "Pakistan",
   "Palau": "Palau",
-  "Palestine, State of": "Palestine",
   "Panama": "Panama",
   "Papua New Guinea": "Papua New Guinea",
   "Paraguay": "Paraguay",
   "Peru": "Peru",
-  "Philippines (the)": "Philippines",
+  "Philippines": "Philippines",
   "Poland": "Poland",
   "Portugal": "Portugal",
   "Puerto Rico": "Puerto Rico",
   "Qatar": "Qatar",
+  "Republic of Korea": "South Korea",
+  "Republic of Moldova": "Moldova",
   "Romania": "Romania",
-  "Russian Federation (the)": "Russia",
+  "Russian Federation": "Russia",
   "Rwanda": "Rwanda",
   "R\u00e9union": "Reunion",
   "Saint Barth\u00e9lemy": "Saint Barthelemy",
-  "Saint Helena, Ascension and Tristan da Cunha": "Saint Helena",
+  "Saint Helena": "Saint Helena",
   "Saint Kitts and Nevis": "Saint Kitts and Nevis",
   "Saint Lucia": "Saint Lucia",
   "Saint Martin (French Part)": "Saint Martin (French part)",
@@ -172,57 +177,52 @@
   "Somalia": "Somalia",
   "South Africa": "South Africa",
   "South Sudan": "South Sudan",
+  "Soviet Union": "USSR",
   "Spain": "Spain",
   "Sri Lanka": "Sri Lanka",
-  "Sudan (the)": "Sudan",
+  "State of Palestine": "Palestine",
+  "Sudan": "Sudan",
   "Suriname": "Suriname",
   "Sweden": "Sweden",
   "Switzerland": "Switzerland",
   "Syrian Arab Republic": "Syria",
   "Taiwan (Province of China)": "Taiwan",
   "Tajikistan": "Tajikistan",
-  "Tanzania, United Republic of": "Tanzania",
   "Thailand": "Thailand",
   "Timor-Leste": "East Timor",
   "Togo": "Togo",
   "Tokelau": "Tokelau",
   "Tonga": "Tonga",
   "Trinidad and Tobago": "Trinidad and Tobago",
   "Tunisia": "Tunisia",
-  "Turkey": "Turkey",
   "Turkmenistan": "Turkmenistan",
-  "Turks and Caicos Islands (the)": "Turks and Caicos Islands",
+  "Turks and Caicos Islands": "Turks and Caicos Islands",
   "Tuvalu": "Tuvalu",
   "Uganda": "Uganda",
   "Ukraine": "Ukraine",
+  "United Arab Emirates": "United Arab Emirates",
+  "United Kingdom of Great Britain and Northern Ireland": "United Kingdom",
+  "United Republic of Tanzania": "Tanzania",
+  "United States Virgin Islands": "United States Virgin Islands",
+  "United States of America": "United States",
   "Uruguay": "Uruguay",
   "Uzbekistan": "Uzbekistan",
   "Vanuatu": "Vanuatu",
   "Venezuela (Bolivarian Republic of)": "Venezuela",
   "Viet Nam": "Vietnam",
-  "Wallis and Futuna": "Wallis and Futuna",
+  "Wallis and Futuna Islands": "Wallis and Futuna",
   "Yemen": "Yemen",
+  "Yemen Arab Republic": "Yemen Arab Republic",
+  "People's Democratic Republic of Yemen": "Yemen People's Republic",
   "Yugoslavia": "Yugoslavia",
   "Zambia": "Zambia",
   "Zimbabwe": "Zimbabwe",
   "Azores Islands": "Portugal",
-  "Canary Is": "Spain",
-  "Czech Republic (the)": "Czechia",
-  "Germany Dem Rep": "East Germany",
-  "Germany Fed Rep": "West Germany",
-  "Korea (the Republic of)": "South Korea",
-  "Macedonia (the former Yugoslav Republic of)": "North Macedonia",
-  "Marshall Islands (the)": "Marshall Islands",
-  "Moldova (the Republic of)": "Moldova",
-  "Netherlands (the)": "Netherlands",
-  "Northern Mariana Islands (the)": "Northern Mariana Islands",
+  "Canary Islands": "Spain",
+  "China, Hong Kong Special Administrative Region": "Hong Kong",
+  "China, Macao Special Administrative Region": "Macao",
+  "Germany Federal Republic": "West Germany",
+  "Netherlands (Kingdom of the)": "Netherlands",
   "Serbia Montenegro": "Serbia and Montenegro",
-  "Soviet Union": "USSR",
-  "United Arab Emirates (the)": "United Arab Emirates",
-  "United Kingdom of Great Britain and Northern Ireland (the)": "United Kingdom",
-  "United States of America (the)": "United States",
-  "Virgin Island (British)": "British Virgin Islands",
-  "Virgin Island (U.S.)": "United States Virgin Islands",
-  "Yemen Arab Rep": "North Yemen",
-  "Yemen P Dem Rep": "South Yemen"
+  "T\u00fcrkiye": "Turkey"
 }