📊 update life expectancy dependencies (#3675)

* 📊 update life expectancy dependencies * wip: un_wpp_lt * 🐛 ci/cd broken hmd grapher * fix dimension error in jinja * typo * wip * wip * wip * wip * wip * change version * re-structure * minor fix * wip * exclude countries * wip * wip * wip * wip * wip lt * man outsurvival probability to woman * archive * gini_le * success message in anomalist * bug in index access * wip * wip * wip * upgrade dependency * archive datasets
owid · Dec 3, 2024 · f860f1d · f860f1d
1 parent f9d9a42
commit f860f1d
Show file tree

Hide file tree

Showing 31 changed files with 2,406 additions and 84 deletions.
diff --git a/apps/wizard/app_pages/anomalist/app.py b/apps/wizard/app_pages/anomalist/app.py
@@ -819,6 +819,7 @@ def _score_table(df: pd.DataFrame) -> pd.DataFrame:
         # Show controls only if needed
         if len(items) > items_per_page:
             pagination.show_controls(mode="bar")
-
+else:
+    st.success("Ha! We did not find any no anomalies in the selected datasets! What were the odds of that?")
 # Reset state
 set_states({"anomalist_datasets_submitted": False})
diff --git a/dag/archive/demography.yml b/dag/archive/demography.yml
@@ -70,3 +70,51 @@ steps:
     - data://garden/hmd/2023-09-19/hmd
   data://grapher/demography/2023-09-27/survivor_percentiles:
     - data://garden/demography/2023-09-27/survivor_percentiles
+
+  # Phi-gender life expectancy inequality
+  data://garden/demography/2023-10-03/phi_gender_le:
+    - data://garden/demography/2023-10-03/life_tables
+  data://grapher/demography/2023-10-03/phi_gender_le:
+    - data://garden/demography/2023-10-03/phi_gender_le
+
+  # Broken limits of Life Expectancy
+  data://garden/demography/2023-10-20/broken_limits_le:
+    - data://garden/demography/2023-10-03/life_tables
+    - data://garden/hmd/2023-09-19/hmd
+  data://grapher/demography/2023-10-20/broken_limits_le:
+    - data://garden/demography/2023-10-20/broken_limits_le
+
+  # Gini Life Expectancy Inequality
+  data://garden/demography/2023-10-04/gini_le:
+    - data://garden/demography/2023-10-03/life_tables
+  data://grapher/demography/2023-10-04/gini_le:
+    - data://garden/demography/2023-10-04/gini_le
+
+  # HMD
+  data://meadow/hmd/2023-09-19/hmd:
+    - snapshot://hmd/2023-09-18/hmd.zip
+  data://garden/hmd/2023-09-19/hmd:
+    - data://meadow/hmd/2023-09-19/hmd
+  data://grapher/hmd/2023-09-19/hmd:
+    - data://garden/hmd/2023-09-19/hmd
+  # UN WPP Life Tables
+  data://meadow/un/2023-10-02/un_wpp_lt:
+    - snapshot://un/2023-10-02/un_wpp_lt_all.zip
+    - snapshot://un/2023-10-02/un_wpp_lt_f.zip
+    - snapshot://un/2023-10-02/un_wpp_lt_m.zip
+  data://garden/un/2023-10-02/un_wpp_lt:
+    - data://meadow/un/2023-10-02/un_wpp_lt
+  # UN WPP + HMD Life Tables
+  data://garden/demography/2023-10-03/life_tables:
+    - data://garden/hmd/2023-09-19/hmd
+    - data://garden/un/2023-10-02/un_wpp_lt
+  data://grapher/demography/2023-10-04/life_tables:
+    - data://garden/demography/2023-10-03/life_tables
+  # OMM: Life Expectancy
+  data://garden/demography/2023-10-09/life_expectancy:
+    - data://garden/demography/2023-10-03/life_tables
+    - data://garden/demography/2023-10-10/zijdeman_et_al_2015
+    - data://garden/demography/2023-10-10/riley_2005
+    - data://garden/un/2022-07-11/un_wpp
+  data://grapher/demography/2023-10-10/life_expectancy:
+    - data://garden/demography/2023-10-09/life_expectancy
diff --git a/dag/covid.yml b/dag/covid.yml
@@ -27,7 +27,7 @@ steps:
     - data://garden/regions/2023-01-01/regions
     # Demography
     - data://garden/demography/2024-07-15/population
-    - data://garden/demography/2023-10-09/life_expectancy
+    - data://garden/demography/2024-12-03/life_expectancy
     - data://garden/un/2024-07-12/un_wpp
     # Econ
     - data://garden/wb/2024-10-07/world_bank_pip
@@ -325,4 +325,3 @@ steps:
     - grapher://grapher/covid/latest/covax
     - grapher://grapher/covid/latest/infections_model
     - grapher://grapher/covid/latest/xm_who
-
diff --git a/dag/demography.yml b/dag/demography.yml
@@ -111,17 +111,32 @@ steps:
   data://grapher/demography/2023-07-03/world_population_comparison:
     - data://garden/demography/2023-06-27/world_population_comparison
 
+  # Maddison working paper (2022)
+  data://meadow/ggdc/2024-01-19/maddison_federico_paper:
+    - snapshot://ggdc/2024-01-19/maddison_federico_paper.xlsx
+  data://garden/ggdc/2024-01-19/maddison_federico_paper:
+    - data://meadow/ggdc/2024-01-19/maddison_federico_paper
+
+  # UN WPP experiments
+  data://garden/un/2024-03-14/un_wpp_most:
+    - data://garden/un/2022-07-11/un_wpp
+  data://grapher/un/2024-03-14/un_wpp_most:
+    - data://garden/un/2024-03-14/un_wpp_most
   ########################################################################
   # Life expectancy                                                      #
   ########################################################################
 
-  # HMD
-  data://meadow/hmd/2023-09-19/hmd:
-    - snapshot://hmd/2023-09-18/hmd.zip
-  data://garden/hmd/2023-09-19/hmd:
-    - data://meadow/hmd/2023-09-19/hmd
-  data://grapher/hmd/2023-09-19/hmd:
-    - data://garden/hmd/2023-09-19/hmd
+  # Zijdeman et al
+  data://meadow/demography/2023-10-10/zijdeman_et_al_2015:
+    - snapshot://demography/2023-10-10/zijdeman_et_al_2015.xlsx
+  data://garden/demography/2023-10-10/zijdeman_et_al_2015:
+    - data://meadow/demography/2023-10-10/zijdeman_et_al_2015
+
+  # Riley
+  data://meadow/demography/2023-10-10/riley_2005:
+    - snapshot://demography/2023-10-10/riley_2005.pdf
+  data://garden/demography/2023-10-10/riley_2005:
+    - data://meadow/demography/2023-10-10/riley_2005
 
   # Human Mortality Database
   data://meadow/hmd/2024-12-01/hmd:
@@ -131,53 +146,54 @@ steps:
   data://grapher/hmd/2024-12-01/hmd:
     - data://garden/hmd/2024-12-01/hmd
 
-  # Gini Life Expectancy Inequality
-  data://garden/demography/2023-10-04/gini_le:
-    - data://garden/demography/2023-10-03/life_tables
-  data://grapher/demography/2023-10-04/gini_le:
-    - data://garden/demography/2023-10-04/gini_le
-
-  # Phi-gender life expectancy inequality
-  data://garden/demography/2023-10-03/phi_gender_le:
-    - data://garden/demography/2023-10-03/life_tables
-  data://grapher/demography/2023-10-03/phi_gender_le:
-    - data://garden/demography/2023-10-03/phi_gender_le
-
   # UN WPP Life Tables
-  data://meadow/un/2023-10-02/un_wpp_lt:
-    - snapshot://un/2023-10-02/un_wpp_lt_all.zip
-    - snapshot://un/2023-10-02/un_wpp_lt_f.zip
-    - snapshot://un/2023-10-02/un_wpp_lt_m.zip
-  data://garden/un/2023-10-02/un_wpp_lt:
-    - data://meadow/un/2023-10-02/un_wpp_lt
+  data://meadow/un/2024-12-02/un_wpp_lt:
+    - snapshot://un/2024-12-02/un_wpp_lt_m.csv
+    - snapshot://un/2024-12-02/un_wpp_lt_all.csv
+    - snapshot://un/2024-12-02/un_wpp_lt_f.csv
+  data://garden/un/2024-12-02/un_wpp_lt:
+    - data://meadow/un/2024-12-02/un_wpp_lt
 
-  # UN WPP + HMD Life Tables
-  data://garden/demography/2023-10-03/life_tables:
-    - data://garden/hmd/2023-09-19/hmd
-    - data://garden/un/2023-10-02/un_wpp_lt
-  data://grapher/demography/2023-10-04/life_tables:
-    - data://garden/demography/2023-10-03/life_tables
-
-  # Zijdeman et al
-  data://meadow/demography/2023-10-10/zijdeman_et_al_2015:
-    - snapshot://demography/2023-10-10/zijdeman_et_al_2015.xlsx
-  data://garden/demography/2023-10-10/zijdeman_et_al_2015:
-    - data://meadow/demography/2023-10-10/zijdeman_et_al_2015
+  # Survivorship ages (HMD-derived)
+  data://garden/demography/2024-12-02/survivor_percentiles:
+    - data://garden/hmd/2024-12-01/hmd
+  data://grapher/demography/2024-12-02/survivor_percentiles:
+    - data://garden/demography/2024-12-02/survivor_percentiles
 
-  # Riley
-  data://meadow/demography/2023-10-10/riley_2005:
-    - snapshot://demography/2023-10-10/riley_2005.pdf
-  data://garden/demography/2023-10-10/riley_2005:
-    - data://meadow/demography/2023-10-10/riley_2005
+  # UN WPP + HMD Life Tables
+  data://garden/demography/2024-12-03/life_tables:
+    - data://garden/hmd/2024-12-01/hmd
+    - data://garden/un/2024-12-02/un_wpp_lt
+  data://grapher/demography/2024-12-03/life_tables:
+    - data://garden/demography/2024-12-03/life_tables
 
   # OMM: Life Expectancy
-  data://garden/demography/2023-10-09/life_expectancy:
-    - data://garden/demography/2023-10-03/life_tables
-    - data://garden/demography/2023-10-10/zijdeman_et_al_2015
+  data://garden/demography/2024-12-03/life_expectancy:
     - data://garden/demography/2023-10-10/riley_2005
-    - data://garden/un/2022-07-11/un_wpp
-  data://grapher/demography/2023-10-10/life_expectancy:
-    - data://garden/demography/2023-10-09/life_expectancy
+    - data://garden/demography/2023-10-10/zijdeman_et_al_2015
+    - data://garden/demography/2024-12-03/life_tables
+    - data://garden/un/2024-07-12/un_wpp
+  data://grapher/demography/2024-12-03/life_expectancy:
+    - data://garden/demography/2024-12-03/life_expectancy
+
+  # Broken limits of Life Expectancy
+  data://garden/demography/2024-12-03/broken_limits_le:
+    - data://garden/hmd/2024-12-01/hmd
+    - data://garden/demography/2024-12-03/life_tables
+  data://grapher/demography/2024-12-03/broken_limits_le:
+    - data://garden/demography/2024-12-03/broken_limits_le
+
+  # Phi-gender life expectancy inequality
+  data://garden/demography/2024-12-03/phi_gender_le:
+    - data://garden/demography/2024-12-03/life_tables
+  data://grapher/demography/2024-12-03/phi_gender_le:
+    - data://garden/demography/2024-12-03/phi_gender_le
+
+  # Gini Life Expectancy Inequality
+  data://garden/demography/2024-12-03/gini_le:
+    - data://garden/demography/2024-12-03/life_tables
+  data://grapher/demography/2024-12-03/gini_le:
+    - data://garden/demography/2024-12-03/gini_le
 
   # Life Expectancy OECD
   data://meadow/oecd/2023-10-11/life_expectancy_birth:
@@ -187,13 +203,6 @@ steps:
   data://grapher/oecd/2023-10-11/life_expectancy_birth:
     - data://garden/oecd/2023-10-11/life_expectancy_birth
 
-  # Broken limits of Life Expectancy
-  data://garden/demography/2023-10-20/broken_limits_le:
-    - data://garden/demography/2023-10-03/life_tables
-    - data://garden/hmd/2023-09-19/hmd
-  data://grapher/demography/2023-10-20/broken_limits_le:
-    - data://garden/demography/2023-10-20/broken_limits_le
-
   # Contribution to sex gap in Life Expectancy
   data://meadow/demography/2023-11-08/le_sex_gap_age_contribution:
     - snapshot://demography/2023-11-08/le_sex_gap_age_contribution.zip
@@ -210,18 +219,6 @@ steps:
   data://grapher/demography/2023-11-08/modal_age_death:
     - data://garden/demography/2023-11-08/modal_age_death
 
-  # Maddison working paper (2022)
-  data://meadow/ggdc/2024-01-19/maddison_federico_paper:
-    - snapshot://ggdc/2024-01-19/maddison_federico_paper.xlsx
-  data://garden/ggdc/2024-01-19/maddison_federico_paper:
-    - data://meadow/ggdc/2024-01-19/maddison_federico_paper
-
-  # UN WPP experiments
-  data://garden/un/2024-03-14/un_wpp_most:
-    - data://garden/un/2022-07-11/un_wpp
-  data://grapher/un/2024-03-14/un_wpp_most:
-    - data://garden/un/2024-03-14/un_wpp_most
-
   ########################################################################
   # Fertility                                                            #
   ########################################################################
@@ -247,9 +244,3 @@ steps:
     - data://meadow/demography/2024-11-26/multiple_births
   data://grapher/demography/2024-11-26/multiple_births:
     - data://garden/demography/2024-11-26/multiple_births
-
-  # Survivorship ages (HMD-derived)
-  data://garden/demography/2024-12-02/survivor_percentiles:
-    - data://garden/hmd/2024-12-01/hmd
-  data://grapher/demography/2024-12-02/survivor_percentiles:
-    - data://garden/demography/2024-12-02/survivor_percentiles
diff --git a/etl/steps/data/garden/demography/2024-12-03/broken_limits_le.meta.yml b/etl/steps/data/garden/demography/2024-12-03/broken_limits_le.meta.yml
@@ -0,0 +1,51 @@
+# NOTE: To learn more about the fields, hover over their names.
+definitions:
+  common:
+    presentation:
+      topic_tags:
+        - Life Expectancy
+
+
+# Learn more about the available fields:
+# http://docs.owid.io/projects/etl/architecture/metadata/reference/dataset/
+dataset:
+  title: "Life Expectancy: Broken limits"
+  update_period_days: 365
+
+
+# Learn more about the available fields:
+# http://docs.owid.io/projects/etl/architecture/metadata/reference/tables/
+tables:
+  broken_limits_le:
+    variables:
+      life_expectancy:
+        title: &le_name Maximum life expectancy
+        unit: years
+        description_short: |-
+          <%- if (sex == 'female') -%>
+          Maximum life expectancy recorded in a given year (among females).
+          <%- elif (sex == 'male') -%>
+          Maximum life expectancy recorded in a given year (among males).
+          <%- elif (sex == 'all') -%>
+          Maximum life expectancy recorded in a given year.
+          <%- endif -%>
+        description_key:
+          - Period life expectancy is a metric that summarizes death rates across all age groups in one particular year. For a given year, it represents the average lifespan for a hypothetical group of people, if they experienced the same age-specific death rates throughout their lives as the age-specific death rates seen in that particular year.
+          - Records are only shown for countries in the Human Mortality Database. Prior to 1950, we use HMD (2023) data. From 1950 onwards, we use UN WPP (2022) data.
+        display:
+          name: *le_name
+        presentation:
+          title_public: *le_name
+          title_variant: ""
+          attribution_short: HMD; UN WPP
+          topic_tags:
+            - Life Expectancy
+          grapher_config:
+            hasMapTab: true
+
+      country_with_max_le:
+        title: Country with yearly maximum life expectancy
+        unit: ""
+        description_short: |-
+          Name of the country with the yearly maximum life expectancy registered<%- if (sex == 'female') %> among females<% elif (sex == 'male') %> among males<% endif -%>.
+        description_processing: This indicator is meant to be used as an auxiliary indicator.
diff --git a/etl/steps/data/garden/demography/2024-12-03/broken_limits_le.py b/etl/steps/data/garden/demography/2024-12-03/broken_limits_le.py
@@ -0,0 +1,76 @@
+"""Load a meadow dataset and create a garden dataset.
+
+We only consider data from countries that are present in HMD. And, additionally, we only consider entries for these countries since the year they first appear in the HMD dataset (even if for that period we use UN WPP data, i.e. post-1950)
+"""
+
+from owid.catalog import Table
+
+from etl.helpers import PathFinder, create_dataset
+
+# Get paths and naming conventions for current step.
+paths = PathFinder(__file__)
+# Year to start tracking. Note that in the first years, few countries have data. Hence, we start in a later year, where more countries have data.
+YEAR_FIRST = 1840
+
+
+def run(dest_dir: str) -> None:
+    #
+    # Load inputs.
+    #
+    # Load meadow dataset.
+    ds_meadow = paths.load_dataset("life_tables")
+    ds_hmd = paths.load_dataset("hmd")
+
+    # Read table from meadow dataset.
+    tb = ds_meadow.read("life_tables", reset_index=False)
+    tb_hmd = ds_hmd.read("life_tables")
+
+    #
+    # Process data.
+    #
+    # Filter relevant dimensions
+    tb = tb.loc[(slice(None), slice(None), slice(None), "0", "period"), ["life_expectancy"]].reset_index()
+
+    # Keep relevant columns and rows
+    tb = tb.drop(columns=["type", "age"]).dropna()
+
+    # Rename column
+    tb = tb.rename(columns={"location": "country"})
+
+    # Get country-sex and first year of LE reported in HMD
+    tb_hmd = get_first_year_of_country_in_hmd(tb_hmd)
+
+    # Only preserve countries coming from HDM
+    tb = tb.merge(tb_hmd, on=["country", "sex"], suffixes=("", "_min"))
+    tb = tb[tb["year"] >= tb["year_min"]].drop(columns=["year_min"])
+
+    # Get max for each year
+    tb = tb.loc[tb.groupby(["year", "sex"], observed=True)["life_expectancy"].idxmax()]
+
+    # Organise columns
+    tb["country_with_max_le"] = tb["country"]
+    tb["country"] = tb["country"] + " " + tb["year"].astype("string")
+
+    # First year
+    tb = tb[tb["year"] >= YEAR_FIRST]
+
+    # Set index
+    tb = tb.format(["country", "year", "sex"], short_name="broken_limits_le")
+
+    #
+    # Save outputs.
+    #
+    # Create a new garden dataset with the same metadata as the meadow dataset.
+    ds_garden = create_dataset(
+        dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata
+    )
+
+    # Save changes in the new garden dataset.
+    ds_garden.save()
+
+
+def get_first_year_of_country_in_hmd(tb_hmd: Table) -> Table:
+    tb_hmd = tb_hmd.loc[(tb_hmd["type"] == "period") & (tb_hmd["age"] == "0")]
+    tb_hmd = tb_hmd.loc[:, ["country", "year", "sex", "life_expectancy"]].dropna()
+    tb_hmd = tb_hmd.groupby(["country", "sex"], observed=True, as_index=False)["year"].min()
+    return tb_hmd