📊 VDEM: new release v14

apps/wizard/pages/charts/variable_config.py

@@ -218,7 +218,7 @@ def ask_and_get_variable_mapping(search_form) -> "VariableConfig":
             # Show only first 100 variables to map (otherwise app crashes)
             if len(suggestions) > LIMIT_VARS_TO_MAP:
                 st.warning(
-                    f"Too many variables to map! Showing only the first {LIMIT_VARS_TO_MAP}. If you want to map more variables, do it in batches. That is, first map this batch and approve the generated chart revisions in admin. Once you are done, run this app again. Make sure you have approved the previously generated revisions!"
+                    f"Too many variables to map ({len(suggestions)})! Showing only the first {LIMIT_VARS_TO_MAP}. If you want to map more variables, do it in batches. That is, first map this batch and approve the generated chart revisions in admin. Once you are done, run this app again. Make sure you have approved the previously generated revisions!"
                 )
                 suggestions = suggestions[:LIMIT_VARS_TO_MAP]
 
@@ -313,6 +313,7 @@ def ask_and_get_variable_mapping(search_form) -> "VariableConfig":
 def show_explore_df(df_data, variable_old, variable_new, variable_id_to_display, element_check) -> None:
     if element_check:  # type: ignore
         with st.container(border=True):
+            # plot_comparison_two_variables(df_data, variable_old, variable_new, variable_id_to_display)  # type: ignore
             try:
                 plot_comparison_two_variables(df_data, variable_old, variable_new, variable_id_to_display)  # type: ignore
             except Exception:
@@ -363,9 +364,15 @@ def build_df_comparison_two_variables_cached(df, variable_old, variable_new, var
     df_variables.loc[:, "value"] = df_variables.value.astype(float)
     # Reshape dataframe
     df_variables = df_variables.pivot(index=["entityName", "year"], columns="variableId", values="value").reset_index()
-    df_variables["Relative difference (abs, %)"] = (
-        (100 * (df_variables[variable_old] - df_variables[variable_new]) / df_variables[variable_old]).round(2)
+    mask = df_variables[variable_old] == 0
+    df_variables.loc[~mask, "Relative difference (abs, %)"] = (
+        (
+            100
+            * (df_variables.loc[~mask, variable_old] - df_variables.loc[~mask, variable_new])
+            / df_variables.loc[~mask, variable_old]
+        ).round(2)
     ).abs()
+    df_variables.loc[mask, "Relative difference (abs, %)"] = float("inf")
     df_variables = df_variables.rename(columns=var_id_to_display).sort_values(
         "Relative difference (abs, %)", ascending=False
     )
@@ -386,13 +393,15 @@ def plot_comparison_two_variables(df, variable_old, variable_new, var_id_to_disp
     # st.write(countries)
     # if countries:
     #     df_variables = df_variables[df_variables["entityName"].isin(countries)]
-    score = round(100 - df_variables["Relative difference (abs, %)"].mean(), 1)
+    relative_diff = df_variables["Relative difference (abs, %)"]
+    relative_diff.loc[relative_diff == float("inf")] = float("nan")
+    score = round(100 - relative_diff.mean(), 1)
     if score == 100:
-        score = round(100 - df_variables["Relative difference (abs, %)"].mean(), 2)
+        score = round(100 - relative_diff.mean(), 2)
         if score == 100:
-            score = round(100 - df_variables["Relative difference (abs, %)"].mean(), 3)
+            score = round(100 - relative_diff.mean(), 3)
             if score == 100:
-                score = round(100 - df_variables["Relative difference (abs, %)"].mean(), 4)
+                score = round(100 - relative_diff.mean(), 4)
     num_nan_score = df_variables["Relative difference (abs, %)"].isna().sum()
 
     nrows_0 = df_variables.shape[0]

dag/democracy.yml

@@ -8,3 +8,13 @@ steps:
     - data://garden/demography/2023-03-31/population
   data://grapher/democracy/2024-03-07/bmr:
     - data://garden/democracy/2024-03-07/bmr
+
+  # Varieties of Democracy (2024)
+  data://meadow/democracy/2024-03-07/vdem:
+    - snapshot://democracy/2024-03-18/vdem.zip
+  data://garden/democracy/2024-03-07/vdem:
+    - data://meadow/democracy/2024-03-07/vdem
+    - data://garden/regions/2023-01-01/regions
+    - data://garden/demography/2023-03-31/population
+  data://grapher/democracy/2024-03-07/vdem:
+    - data://garden/democracy/2024-03-07/vdem

etl/steps/data/garden/democracy/2024-03-07/bmr.py

@@ -41,6 +41,7 @@
 import yaml
 from owid.catalog import Dataset, Variable
 from owid.catalog.tables import Table, concat
+from shared import add_population_in_dummies, expand_observations, from_wide_to_long
 
 from etl.data_helpers import geo
 from etl.helpers import PathFinder, create_dataset
@@ -443,7 +444,28 @@ def make_tables_population_counters(tb: Table, ds_regions: Dataset, ds_populatio
     tb_ = make_table_with_dummies(tb_, ds_regions)
 
     # Add population in dummies (population value replaces 1, 0 otherwise)
-    tb_ = add_population_in_dummies(tb_, ds_population)
+    tb_ = add_population_in_dummies(
+        tb_,
+        ds_population,
+        expected_countries_without_population=[
+            "Pakistan (former)",
+            "Korea (former)",
+            "Duchy of Parma and Piacenza",
+            "Orange Free State",
+            "Federal Republic of Central America",
+            "Grand Duchy of Tuscany",
+            "Democratic Republic of Vietnam",
+            "Kingdom of Saxony",
+            "Duchy of Modena and Reggio",
+            "Kingdom of the Two Sicilies",
+            "Kingdom of Sardinia",
+            "Great Colombia",
+            "Grand Duchy of Baden",
+            "Kingdom of Wurttemberg",
+            "Republic of Vietnam",
+            "Kingdom of Bavaria",
+        ],
+    )
 
     # Get region aggregates
     tb_ = geo.add_regions_to_table(
@@ -569,42 +591,6 @@ def make_table_with_dummies(
     return tb_
 
 
-def from_wide_to_long(tb: Table) -> Table:
-    """Format a particular shape of table from wide to long format.
-
-    The expected input table format is:
-
-    | year | country | indicator_a_1 | indicator_a_2 | indicator_b_1 | indicator_b_2 |
-    |------|---------|---------------|---------------|---------------|---------------|
-    | 2000 |   USA   |       1       |       2       |       3       |       4       |
-    | 2000 |   CAN   |       5       |       6       |       7       |       8       |
-
-    The generated output is:
-
-    | year | country |  category  | indicator_a | indicator_b |
-    |------|---------|------------|-------------|-------------|
-    | 2000 | USA     | category_1 |      1      |       3     |
-    | 2000 | USA     | category_2 |      2      |       4     |
-    """
-    # Melt the DataFrame to long format
-    tb = tb.melt(id_vars=["year", "country"], var_name="indicator_type", value_name="value")
-
-    # Extract indicator names and types
-    tb["indicator"] = tb["indicator_type"].apply(lambda x: "_".join(x.split("_")[:-1]))
-    tb["category"] = tb["indicator_type"].apply(lambda x: x.split("_")[-1])
-
-    # Drop the original 'indicator_type' column as it's no longer needed
-    tb.drop("indicator_type", axis=1, inplace=True)
-
-    # Pivot the table to get 'indicator_a' and 'indicator_b' as separate columns
-    tb = tb.pivot(index=["year", "country", "category"], columns="indicator", values="value").reset_index()
-
-    # Rename the columns to match your requirements
-    tb.columns.name = None  # Remove the hierarchy
-
-    return tb
-
-
 def expand_observations_without_leading_to_duplicates(tb: Table, ds_regions: Dataset) -> Table:
     """Expand observations (accounting for overlaps between former and current countries).
 
@@ -636,68 +622,6 @@ def expand_observations_without_leading_to_duplicates(tb: Table, ds_regions: Dat
     return tb
 
 
-def expand_observations(tb: Table) -> Table:
-    """Expand to have a row per (year, country)."""
-    # Add missing years for each triplet ("warcode", "campcode", "ccode")
-
-    # List of countries
-    regions = set(tb["country"])
-
-    # List of possible years
-    years = np.arange(tb["year"].min(), tb["year"].max() + 1)
-
-    # New index
-    new_idx = pd.MultiIndex.from_product([years, regions], names=["year", "country"])
-
-    # Reset index
-    tb = tb.set_index(["year", "country"]).reindex(new_idx).reset_index()
-
-    # Type of `year`
-    tb["year"] = tb["year"].astype("int")
-    return tb
-
-
-def add_population_in_dummies(tb: Table, ds_population: Dataset):
-    # Add population column
-    tb = geo.add_population_to_table(
-        tb,
-        ds_population,
-        interpolate_missing_population=True,
-        expected_countries_without_population=[
-            "Pakistan (former)",
-            "Korea (former)",
-            "Duchy of Parma and Piacenza",
-            "Orange Free State",
-            "Federal Republic of Central America",
-            "Grand Duchy of Tuscany",
-            "Democratic Republic of Vietnam",
-            "Kingdom of Saxony",
-            "Duchy of Modena and Reggio",
-            "Kingdom of the Two Sicilies",
-            "Kingdom of Sardinia",
-            "Great Colombia",
-            "Grand Duchy of Baden",
-            "Kingdom of Wurttemberg",
-            "Republic of Vietnam",
-            "Kingdom of Bavaria",
-        ],
-    )
-    tb = cast(Table, tb.dropna(subset="population"))
-    # Add metadata (origins combined indicator+population)
-    cols = [col for col in tb.columns if col not in ["year", "country", "population"]]
-    meta = {col: tb[col].metadata for col in cols} | {"population": tb["population"].metadata}
-    ## Encode population in indicators: Population if 1, 0 otherwise
-    tb[cols] = tb[cols].multiply(tb["population"], axis=0)
-    tb = tb.drop(columns="population")
-    ## Add metadata back (combine origins from population)
-    for col in cols:
-        metadata = meta[col]
-        metadata.origins += meta["population"].origins
-        tb[col].metadata = meta[col]
-
-    return tb
-
-
 def _get_countries_to_ignore_population(ds_regions: Dataset) -> Set[str]:
     """List of countries to ignore when working with population.
 

etl/steps/data/garden/democracy/2024-03-07/shared.py

@@ -0,0 +1,118 @@
+from typing import Callable, List, Optional, cast
+
+import numpy as np
+import pandas as pd
+from owid.catalog import Dataset, Table
+
+from etl.data_helpers import geo
+
+
+def from_wide_to_long(
+    tb: Table,
+    indicator_name_callback: Optional[Callable] = None,
+    indicator_category_callback: Optional[Callable] = None,
+    column_dimension_name: str = "category",
+) -> Table:
+    """Format a particular shape of table from wide to long format.
+
+    tb: Table with wide format.
+    indicator_name_callback: Function to extract the indicator name from the column name.
+    indicator_category_callback: Function to extract the indicator category from the column name.
+
+    If no `indicator_name_callback` and `indicator_category_callback` are provided, it proceed expects the following input:
+
+    | year | country | indicator_a_1 | indicator_a_2 | indicator_b_1 | indicator_b_2 |
+    |------|---------|---------------|---------------|---------------|---------------|
+    | 2000 |   USA   |       1       |       2       |       3       |       4       |
+    | 2000 |   CAN   |       5       |       6       |       7       |       8       |
+
+    and then generates the output:
+
+    | year | country |  category  | indicator_a | indicator_b |
+    |------|---------|------------|-------------|-------------|
+    | 2000 | USA     | category_1 |      1      |       3     |
+    | 2000 | USA     | category_2 |      2      |       4     |
+    """
+    # Melt the DataFrame to long format
+    tb = tb.melt(id_vars=["year", "country"], var_name="indicator_type", value_name="value")
+
+    # Get callables
+    if indicator_name_callback is None:
+
+        def default_indicator_name(x):
+            return "_".join(x.split("_")[:-1])
+
+        indicator_name_callback = default_indicator_name
+
+    if indicator_category_callback is None:
+
+        def default_indicator_category(x):
+            return x.split("_")[-1]
+
+        indicator_category_callback = default_indicator_category
+
+    # Extract indicator names and types
+    tb["indicator"] = tb["indicator_type"].apply(indicator_name_callback)
+    tb[column_dimension_name] = tb["indicator_type"].apply(indicator_category_callback)
+
+    # Drop the original 'indicator_type' column as it's no longer needed
+    tb.drop("indicator_type", axis=1, inplace=True)
+
+    # Pivot the table to get 'indicator_a' and 'indicator_b' as separate columns
+    tb = tb.pivot(index=["year", "country", column_dimension_name], columns="indicator", values="value").reset_index()
+
+    # Rename the columns to match your requirements
+    tb.columns.name = None  # Remove the hierarchy
+
+    return tb
+
+
+def expand_observations(tb: Table) -> Table:
+    """Expand to have a row per (year, country)."""
+    # Add missing years for each triplet ("warcode", "campcode", "ccode")
+
+    # List of countries
+    regions = set(tb["country"])
+
+    # List of possible years
+    years = np.arange(tb["year"].min(), tb["year"].max() + 1)
+
+    # New index
+    new_idx = pd.MultiIndex.from_product([years, regions], names=["year", "country"])
+
+    # Reset index
+    tb = tb.set_index(["year", "country"]).reindex(new_idx).reset_index()
+
+    # Type of `year`
+    tb["year"] = tb["year"].astype("int")
+    return tb
+
+
+def add_population_in_dummies(
+    tb: Table,
+    ds_population: Dataset,
+    expected_countries_without_population: Optional[List[str]] = None,
+    drop_population: bool = True,
+):
+    # Add population column
+    tb = geo.add_population_to_table(
+        tb,
+        ds_population,
+        interpolate_missing_population=True,
+        expected_countries_without_population=expected_countries_without_population,
+    )
+    tb = cast(Table, tb.dropna(subset="population"))
+    # Add metadata (origins combined indicator+population)
+    cols = [col for col in tb.columns if col not in ["year", "country", "population"]]
+    meta = {col: tb[col].metadata for col in cols} | {"population": tb["population"].metadata}
+    ## Encode population in indicators: Population if 1, 0 otherwise
+    tb[cols] = tb[cols].multiply(tb["population"], axis=0)
+    if drop_population:
+        tb = tb.drop(columns="population")
+    ## Add metadata back (combine origins from population)
+    for col in cols:
+        metadata = meta[col]
+        metadata.origins += meta["population"].origins
+        tb[col].metadata = meta[col]
+
+    return tb