✨ improve grapher chart component

apps/pr/cli.py

@@ -373,7 +373,7 @@ def bake_branch_name(repo, pr_title, no_llm, remote_branches):
     category = pr_title.category
 
     # Get input title (without emoji, scope, etc.)
-    title = _extract_relevant_title_for_branch_name(pr_title.title, not no_llm)
+    title = _extract_relevant_title_for_branch_name(pr_title.title, category, not no_llm)
 
     # Bake complete PR branch name
     # name = f"{user}-{category}-{title}"
@@ -383,13 +383,14 @@ def bake_branch_name(repo, pr_title, no_llm, remote_branches):
     # if name in remote_branches:
     #     log.info("Generating a hash for this branch name to prevent name collisions.")
     #     name = f"{name}-{user}"
-    if name in remote_branches:
+    local_branches = [branch.name for branch in repo.branches]
+    if (name in remote_branches) or (name in local_branches):
         log.info("Generating a hash for this branch name to prevent name collisions.")
         name = f"{name}-{generate_short_hash()}"
     return name
 
 
-def _extract_relevant_title_for_branch_name(text_in: str, use_llm) -> str:
+def _extract_relevant_title_for_branch_name(text_in: str, category: str, use_llm) -> str:
     """
     Process the input string by:
     1. Removing all symbols, keeping only letters and numbers.
@@ -411,8 +412,13 @@ def _extract_relevant_title_for_branch_name(text_in: str, use_llm) -> str:
 
     # Split into tokens/words
     tokens = cleaned_text.split()
+
+    # Clean if there is word included in category
+    tokens = [t for t in tokens if t.lower() != category]
+
     # Keep only the first 3 tokens
     tokens = tokens[:3]
+
     # Combine tokens with '-'
     name = "-".join(tokens).lower()
 

dag/demography.yml

@@ -1,7 +1,7 @@
 steps:
-  ####################################
-  # Population #######################
-  ####################################
+  ########################################################################
+  # Population                                                           #
+  ########################################################################
 
   # Population OMM #############################
   # 2022-12-08: HYDE 3.2 + Gapminder + UN WPP 2022
@@ -111,9 +111,9 @@ steps:
   data://grapher/demography/2023-07-03/world_population_comparison:
     - data://garden/demography/2023-06-27/world_population_comparison
 
-  ####################################
-  # Life expectancy ##################
-  ####################################
+  ########################################################################
+  # Life expectancy                                                      #
+  ########################################################################
 
   # HMD
   data://meadow/hmd/2023-09-19/hmd:
@@ -220,53 +220,9 @@ steps:
   data://grapher/un/2024-03-14/un_wpp_most:
     - data://garden/un/2024-03-14/un_wpp_most
 
-  # Migration UN DESA
-  data://meadow/un/2024-07-16/migrant_stock:
-    - snapshot://un/2024-07-16/migrant_stock_dest_origin.xlsx
-    - snapshot://un/2024-07-16/migrant_stock_origin.xlsx
-    - snapshot://un/2024-07-16/migrant_stock_dest.xlsx
-    - snapshot://un/2024-07-16/migrant_stock_age_sex.xlsx
-  data://garden/un/2024-07-16/migrant_stock:
-    - data://meadow/un/2024-07-16/migrant_stock
-  data://grapher/un/2024-07-16/migrant_stock:
-    - data://garden/un/2024-07-16/migrant_stock
-
-  # Migration UN DESA (flows)
-  data://grapher/migration/2024-11-20/migrant_stock_flows:
-    - data://garden/un/2024-07-16/migrant_stock
-
-  # Internal displacement monitoring centre
-  data://meadow/idmc/2024-08-02/internal_displacement:
-    - snapshot://idmc/2024-08-02/internal_displacement.xlsx
-    - data://garden/demography/2024-07-15/population
-  data://grapher/idmc/2024-08-02/internal_displacement:
-    - data://meadow/idmc/2024-08-02/internal_displacement
-
-  # UNHCR refugee data
-  data://meadow/un/2024-07-25/refugee_data:
-    - snapshot://un/2024-07-25/refugee_data.zip
-  data://garden/un/2024-07-25/refugee_data:
-    - data://meadow/un/2024-07-25/refugee_data
-    - data://garden/demography/2024-07-15/population
-    - data://garden/un/2024-07-25/resettlement
-  data://grapher/un/2024-07-25/refugee_data:
-    - data://garden/un/2024-07-25/refugee_data
-
-  # UNHCR resettlement data
-  data://meadow/un/2024-07-25/resettlement:
-    - snapshot://un/2024-07-25/resettlement.zip
-  data://garden/un/2024-07-25/resettlement:
-    - data://meadow/un/2024-07-25/resettlement
-    - data://garden/demography/2024-07-15/population
-
-  # Child migration (UNICEF)
-  data://meadow/unicef/2024-07-30/child_migration:
-    - snapshot://unicef/2024-07-30/child_migration.csv
-  data://garden/unicef/2024-07-30/child_migration:
-    - data://meadow/unicef/2024-07-30/child_migration
-    - data://garden/demography/2024-07-15/population
-  data://grapher/unicef/2024-07-30/child_migration:
-    - data://garden/unicef/2024-07-30/child_migration
+  ########################################################################
+  # Fertility                                                            #
+  ########################################################################
 
   # Mothers by decadal age-group
   data://garden/un/2024-10-01/births_by_age:
@@ -281,3 +237,11 @@ steps:
     - data://meadow/hmd/2024-11-19/hfd
   data://grapher/hmd/2024-11-19/hfd:
     - data://garden/hmd/2024-11-19/hfd
+
+  # Multiple births
+  data://meadow/demography/2024-11-26/multiple_births:
+    - snapshot://demography/2024-11-26/multiple_births.7z
+  data://garden/demography/2024-11-26/multiple_births:
+    - data://meadow/demography/2024-11-26/multiple_births
+  data://grapher/demography/2024-11-26/multiple_births:
+    - data://garden/demography/2024-11-26/multiple_births

dag/main.yml

@@ -100,7 +100,6 @@ steps:
   data://grapher/homicide/2024-10-30/unodc:
     - data://garden/homicide/2024-10-30/unodc
 
-
   #
   # UN
   data://grapher/un/2022-07-11/un_wpp:
@@ -321,8 +320,6 @@ steps:
   data://grapher/terrorism/2023-07-20/global_terrorism_database:
     - data://garden/terrorism/2023-07-20/global_terrorism_database
 
-
-
   # Colonial Dates Dataset (COLDAT)
   data://meadow/harvard/2023-09-18/colonial_dates_dataset:
     - snapshot://harvard/2023-09-18/colonial_dates_dataset.csv
@@ -355,7 +352,6 @@ steps:
   data://garden/gapminder/2023-09-21/under_five_mortality:
     - data://meadow/gapminder/2023-09-21/under_five_mortality
 
-
   # Older vintage UN IGME (with longer time-series)
   data://meadow/un/2018/igme:
     - snapshot://un/2018/igme.csv
@@ -763,7 +759,7 @@ steps:
   data-private://grapher/language/2024-07-17/ethnologue:
     - data-private://garden/language/2024-07-17/ethnologue
 
-# Child Mortality Estimates - UN IGME
+  # Child Mortality Estimates - UN IGME
   data://meadow/un/2024-09-11/igme:
     - snapshot://un/2024-09-11/igme.zip
   data://garden/un/2024-09-11/igme:
@@ -774,17 +770,15 @@ steps:
   data://grapher/un/2024-09-11/igme:
     - data://garden/un/2024-09-11/igme
 
-
-# Long-run child mortality, Gapminder + UN IGME
+  # Long-run child mortality, Gapminder + UN IGME
   data://garden/un/2024-09-16/long_run_child_mortality:
     - data://garden/un/2024-09-11/igme
     - data://garden/gapminder/2023-09-18/under_five_mortality
     - data://garden/gapminder/2023-09-21/under_five_mortality
   data://grapher/un/2024-09-16/long_run_child_mortality:
     - data://garden/un/2024-09-16/long_run_child_mortality
 
-
-# UN SDG (2024)
+  # UN SDG (2024)
   data://meadow/un/2024-08-27/un_sdg:
     - snapshot://un/2024-08-27/un_sdg.feather
   data://garden/un/2024-08-27/un_sdg:
@@ -794,7 +788,7 @@ steps:
   data://grapher/un/2024-08-27/un_sdg:
     - data://garden/un/2024-08-27/un_sdg
 
-# OECD Official Development Assistance (ODA)
+  # OECD Official Development Assistance (ODA)
   data://meadow/oecd/2024-08-21/official_development_assistance:
     - snapshot://oecd/2024-08-21/official_development_assistance_dac1.zip
     - snapshot://oecd/2024-08-21/official_development_assistance_dac2a.zip
@@ -806,23 +800,23 @@ steps:
   data://grapher/oecd/2024-08-21/official_development_assistance:
     - data://garden/oecd/2024-08-21/official_development_assistance
 
-# Oil Spills
+  # Oil Spills
   data://meadow/itopf/2024-10-16/oil_spills:
     - snapshot://itopf/2024-10-16/oil_spills.pdf
   data://garden/itopf/2024-10-16/oil_spills:
     - data://meadow/itopf/2024-10-16/oil_spills
   data://grapher/itopf/2024-10-16/oil_spills:
     - data://garden/itopf/2024-10-16/oil_spills
 
-# UN SD census data
+  # UN SD census data
   data://meadow/un/2024-10-21/census_dates:
     - snapshot://un/2024-10-21/census_dates.csv
   data://garden/un/2024-10-21/census_dates:
     - data://meadow/un/2024-10-21/census_dates
   data://grapher/un/2024-10-21/census_dates:
     - data://garden/un/2024-10-21/census_dates
 
-#  World Peace Foundation - Famines
+  #  World Peace Foundation - Famines
   data://meadow/wpf/2024-10-03/famines:
     - snapshot://wpf/2024-10-03/famines.xlsx
   data://garden/wpf/2024-10-03/famines:
@@ -861,7 +855,7 @@ steps:
   data-private://grapher/owid/latest/ig_countries:
     - data-private://garden/owid/latest/ig_countries
 
-# Migration between regions, based on UN DESA flows
+  # Migration between regions, based on UN DESA flows
   data://garden/migration/2024-11-18/migration_between_regions:
     - data://garden/un/2024-07-16/migrant_stock
     - data://garden/regions/2023-01-01/regions
@@ -913,3 +907,4 @@ include:
   - dag/chartbook.yml
   - dag/minerals.yml
   - dag/tourism.yml
+  - dag/migration.yml

dag/migration.yml

@@ -0,0 +1,44 @@
+steps:
+  # Migration UN DESA
+  data://meadow/un/2024-07-16/migrant_stock:
+    - snapshot://un/2024-07-16/migrant_stock_dest_origin.xlsx
+    - snapshot://un/2024-07-16/migrant_stock_origin.xlsx
+    - snapshot://un/2024-07-16/migrant_stock_dest.xlsx
+    - snapshot://un/2024-07-16/migrant_stock_age_sex.xlsx
+  data://garden/un/2024-07-16/migrant_stock:
+    - data://meadow/un/2024-07-16/migrant_stock
+  data://grapher/un/2024-07-16/migrant_stock:
+    - data://garden/un/2024-07-16/migrant_stock
+
+  # Internal displacement monitoring centre
+  data://meadow/idmc/2024-08-02/internal_displacement:
+    - snapshot://idmc/2024-08-02/internal_displacement.xlsx
+    - data://garden/demography/2024-07-15/population
+  data://grapher/idmc/2024-08-02/internal_displacement:
+    - data://meadow/idmc/2024-08-02/internal_displacement
+
+  # UNHCR refugee data
+  data://meadow/un/2024-07-25/refugee_data:
+    - snapshot://un/2024-07-25/refugee_data.zip
+  data://garden/un/2024-07-25/refugee_data:
+    - data://meadow/un/2024-07-25/refugee_data
+    - data://garden/demography/2024-07-15/population
+    - data://garden/un/2024-07-25/resettlement
+  data://grapher/un/2024-07-25/refugee_data:
+    - data://garden/un/2024-07-25/refugee_data
+
+  # UNHCR resettlement data
+  data://meadow/un/2024-07-25/resettlement:
+    - snapshot://un/2024-07-25/resettlement.zip
+  data://garden/un/2024-07-25/resettlement:
+    - data://meadow/un/2024-07-25/resettlement
+    - data://garden/demography/2024-07-15/population
+
+  # Child migration (UNICEF)
+  data://meadow/unicef/2024-07-30/child_migration:
+    - snapshot://unicef/2024-07-30/child_migration.csv
+  data://garden/unicef/2024-07-30/child_migration:
+    - data://meadow/unicef/2024-07-30/child_migration
+    - data://garden/demography/2024-07-15/population
+  data://grapher/unicef/2024-07-30/child_migration:
+    - data://garden/unicef/2024-07-30/child_migration

docs/guides/etl-tips.md

@@ -136,7 +136,6 @@ You can adapt the function to your needs, and perform different kinds of expansi
     )
     ```
 
-
 ## Deprecate code
 Our codebase has lots of code. Some of it may no longer be maintained or used. To avoid confusion, it is a good practice to slowly deprecate code. This can be done by adding a deprecation warning to the code, and then removing it after a certain period of time:
 
@@ -149,3 +148,23 @@ from deprecated import deprecated
 Make sure to point users to an alternative function or method that they can use instead.
 
 Please deprecate function with care, and make sure to check if the function is widely used, and communicate the deprecation to the team.
+
+## Add entity annotations to your dataset
+Just add the field `display.entityAnnotationsMap` to the desired indicator.
+
+```yaml
+display:
+    entityAnnotationsMap: |-
+        Spain: Some annotation
+        France: Another annotation
+```
+
+!!! note "Space is limited"
+
+    The space for annotations in charts is limited. Please be mindful and keep the annotations short and to the point. 2-4 words is usually enough, ideally 2.
+
+
+<figure markdown="span">
+    <img src="../../assets/annotations-chart.png" alt="OWID chart with annotations" style="width:80%;">
+    <figcaption>Example chart with entity annotations. Note that the space for annotations.</figcaption>
+</figure>

docs/guides/pull-requests.md

@@ -19,7 +19,7 @@ Once you've created a PR, the automation user [@owidbot](https://github.com/owid
 - data-diff: Changes introduced in the data compared to PRODUCTION.
 
 <figure markdown="span">
-    <img src="../../assets/pr-1.png" alt="Chart Upgrader" style="width:50%;">
+    <img src="../../assets/pr-1.png" alt="Chart Upgrader" style="width:80%;">
     <figcaption>PR, and [comment by @owidbot](https://github.com/owid/etl/pull/3563#issuecomment-2485397175), as of 19th November 2024</figcaption>
 </figure>
 

etl/steps/data/garden/demography/2024-11-26/multiple_births.countries.json

@@ -0,0 +1,27 @@
+{
+    "Australia": "Australia",
+    "Austria": "Austria",
+    "Canada": "Canada",
+    "Chile": "Chile",
+    "Czech Republic": "Czechia",
+    "Denmark": "Denmark",
+    "Finland": "Finland",
+    "France": "France",
+    "Germany": "Germany",
+    "Greece": "Greece",
+    "Iceland": "Iceland",
+    "Italy": "Italy",
+    "Japan": "Japan",
+    "Lithuania": "Lithuania",
+    "Netherlands": "Netherlands",
+    "New Zealand": "New Zealand",
+    "Norway": "Norway",
+    "Republic of Korea": "South Korea",
+    "Spain": "Spain",
+    "Sweden": "Sweden",
+    "Switzerland": "Switzerland",
+    "United States": "United States",
+    "Uruguay": "Uruguay",
+    "UK-England and Wales": "England and Wales",
+    "UK-Scotland": "Scotland"
+}

etl/steps/data/garden/demography/2024-11-26/multiple_births.excluded_countries.json

@@ -0,0 +1,2 @@
+[
+]