Merge branch 'master' into data-mean-paternal-age

dag/archive/poverty_inequality.yml

@@ -43,3 +43,12 @@ steps:
     - data://meadow/ophi/2023-07-05/multidimensional_poverty_index
   data://grapher/ophi/2023-07-05/multidimensional_poverty_index:
     - data://garden/ophi/2023-07-05/multidimensional_poverty_index
+
+  # Poverty projections from the World Bank
+  data://meadow/wb/2024-06-26/poverty_projections:
+    - snapshot://wb/2024-06-26/poverty_projections_number_global.csv
+    - snapshot://wb/2024-06-26/poverty_projections_share_regions.csv
+  data://garden/wb/2024-06-26/poverty_projections:
+    - data://meadow/wb/2024-06-26/poverty_projections
+  data://grapher/wb/2024-06-26/poverty_projections:
+    - data://garden/wb/2024-06-26/poverty_projections

dag/health.yml

@@ -980,3 +980,26 @@ steps:
     - data-private://garden/antibiotics/2024-12-04/microbe_total_pathogens
   data-private://grapher/antibiotics/2024-12-04/microbe_total_pathogens_amr:
     - data-private://garden/antibiotics/2024-12-04/microbe_total_pathogens_amr
+
+  # MICROBE - neonatal deaths by pathogen and amr resistance
+  data-private://meadow/antibiotics/2024-12-05/microbe_neonatal_total_amr:
+    - snapshot-private://antibiotics/2024-12-05/microbe_neonatal_total_amr.csv
+  data-private://garden/antibiotics/2024-12-05/microbe_neonatal_total_amr:
+    - data-private://meadow/antibiotics/2024-12-05/microbe_neonatal_total_amr
+    - data-private://garden/antibiotics/2024-11-20/microbe
+  data-private://grapher/antibiotics/2024-12-05/microbe_neonatal_total_amr:
+    - data-private://garden/antibiotics/2024-12-05/microbe_neonatal_total_amr
+    # MICROBE - total deaths by syndrome
+  data-private://meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome:
+    - snapshot-private://antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.csv
+  data-private://garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome:
+    - data-private://meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome
+
+  # MICROBE - total deaths by syndrome and amr resistance
+  data-private://meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr:
+    - snapshot-private://antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.csv
+  data-private://garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr:
+    - data-private://meadow/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr
+    - data-private://garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome
+  data-private://grapher/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr:
+    - data-private://garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr

dag/poverty_inequality.yml

@@ -114,19 +114,18 @@ steps:
   data://grapher/oecd/2024-04-30/affordable_housing_database:
     - data://garden/oecd/2024-04-30/affordable_housing_database
 
-  # Poverty projections from the World Bank
-  data://meadow/wb/2024-06-26/poverty_projections:
-    - snapshot://wb/2024-06-26/poverty_projections_number_global.csv
-    - snapshot://wb/2024-06-26/poverty_projections_share_regions.csv
-  data://garden/wb/2024-06-26/poverty_projections:
-    - data://meadow/wb/2024-06-26/poverty_projections
-  data://grapher/wb/2024-06-26/poverty_projections:
-    - data://garden/wb/2024-06-26/poverty_projections
-
   # Institute of Global Homelessness - Better Data Project
   data://meadow/igh/2024-07-05/better_data_homelessness:
     - snapshot://igh/2024-07-05/better_data_homelessness.xlsx
   data://garden/igh/2024-07-05/better_data_homelessness:
     - data://meadow/igh/2024-07-05/better_data_homelessness
   data://grapher/igh/2024-07-05/better_data_homelessness:
     - data://garden/igh/2024-07-05/better_data_homelessness
+
+  # Poverty projections from the Poverty, Prosperity and Planet Report 2024
+  data://meadow/wb/2024-12-03/poverty_projections:
+    - snapshot://wb/2024-12-03/reproducibility_package_poverty_prosperity_planet.zip
+  data://garden/wb/2024-12-03/poverty_projections:
+    - data://meadow/wb/2024-12-03/poverty_projections
+  data://grapher/wb/2024-12-03/poverty_projections:
+    - data://garden/wb/2024-12-03/poverty_projections

etl/grapher_helpers.py

@@ -212,13 +212,19 @@ def _cached_jinja_template(text: str) -> jinja2.environment.Template:
     return jinja_env.from_string(text)
 
 
-def _expand_jinja_text(text: str, dim_dict: Dict[str, str]) -> str:
+def _expand_jinja_text(text: str, dim_dict: Dict[str, str]) -> Union[str, bool]:
     if not _uses_jinja(text):
         return text
 
     try:
         # NOTE: we're stripping the result to avoid trailing newlines
-        return _cached_jinja_template(text).render(dim_dict).strip()
+        out = _cached_jinja_template(text).render(dim_dict).strip()
+        # Convert strings to booleans. Getting boolean directly from Jinja is not possible
+        if out in ("false", "False", "FALSE"):
+            return False
+        elif out in ("true", "True", "TRUE"):
+            return True
+        return out
     except jinja2.exceptions.TemplateSyntaxError as e:
         new_message = f"{e.message}\n\nDimensions:\n{dim_dict}\n\nTemplate:\n{text}\n"
         raise e.__class__(new_message, e.lineno, e.name, e.filename) from e

etl/steps/data/garden/antibiotics/2024-12-05/microbe_neonatal_total_amr.countries.json

@@ -0,0 +1,3 @@
+{
+  "Global": "World"
+}

etl/steps/data/garden/antibiotics/2024-12-05/microbe_neonatal_total_amr.meta.yml

@@ -0,0 +1,42 @@
+# NOTE: To learn more about the fields, hover over their names.
+definitions:
+  common:
+    presentation:
+      topic_tags:
+        - Antibiotics
+
+# Learn more about the available fields:
+# http://docs.owid.io/projects/etl/architecture/metadata/reference/
+dataset:
+  update_period_days: 365
+
+tables:
+  microbe_neonatal_total_amr:
+    variables:
+      amr_attributable_deaths:
+        title: Neonatal deaths from infections attributed to AMR, by pathogen
+        unit: deaths
+        description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from infections that are attributed to antimicrobial resistance.
+        presentation:
+          title_public: Neonatal deaths from infections attributed to AMR, by pathogen
+        display:
+          roundingMode: significantFigures
+          numSignificantFigures: 3
+      non_amr_attributable_deaths:
+        title: Neonatal global deaths from infections not attributed to AMR, by pathogen
+        unit: deaths
+        description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from infections that are not attributed to antimicrobial resistance.
+        presentation:
+          title_public: Neonatal global deaths from infections not attributed to AMR, by pathogen
+        display:
+          roundingMode: significantFigures
+          numSignificantFigures: 3
+      total_deaths:
+        title: Neonatal global deaths from infections
+        unit: deaths
+        description_short: Estimated number of [neonates](#dod:neonatal) – newborns under 28 days of age –  who die each year from infections.
+        presentation:
+          title_public: Neonatal global deaths from infections
+        display:
+          roundingMode: significantFigures
+          numSignificantFigures: 3

etl/steps/data/garden/antibiotics/2024-12-05/microbe_neonatal_total_amr.py

@@ -0,0 +1,45 @@
+"""Load a meadow dataset and create a garden dataset."""
+
+from etl.data_helpers import geo
+from etl.helpers import PathFinder, create_dataset
+
+# Get paths and naming conventions for current step.
+paths = PathFinder(__file__)
+
+
+def run(dest_dir: str) -> None:
+    #
+    # Load inputs.
+    #
+    # Load meadow dataset.
+    ds_meadow = paths.load_dataset("microbe_neonatal_total_amr")
+    ds_total = paths.load_dataset("microbe")
+
+    # Read table from meadow dataset.
+    tb = (
+        ds_meadow.read("microbe_neonatal_total_amr")
+        .drop(columns=["upper", "lower"])
+        .rename(columns={"value": "amr_attributable_deaths"})
+    )
+    tb_total = ds_total.read("microbe").drop(columns=["upper", "lower"]).rename(columns={"value": "total_deaths"})
+
+    #
+    # Process data.
+    #
+    tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path)
+
+    tb = tb.merge(tb_total, on=["country", "year", "pathogen", "pathogen_type"], how="inner")
+    tb["non_amr_attributable_deaths"] = tb["total_deaths"] - tb["amr_attributable_deaths"]
+    tb = tb.drop(columns=["country", "pathogen_type"]).rename(columns={"pathogen": "country"})
+    tb = tb.format(["country", "year"])
+
+    #
+    # Save outputs.
+    #
+    # Create a new garden dataset with the same metadata as the meadow dataset.
+    ds_garden = create_dataset(
+        dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata
+    )
+
+    # Save changes in the new garden dataset.
+    ds_garden.save()

etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.countries.json

@@ -0,0 +1,3 @@
+{
+  "Global": "World"
+}

etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.meta.yml

@@ -0,0 +1,45 @@
+# NOTE: To learn more about the fields, hover over their names.
+definitions:
+  common:
+    presentation:
+      topic_tags:
+        - Antibiotics
+
+# Learn more about the available fields:
+# http://docs.owid.io/projects/etl/architecture/metadata/reference/
+dataset:
+  update_period_days: 365
+
+tables:
+  microbe_total_deaths_by_syndrome:
+    variables:
+      value:
+        title: Total deaths from << infectious_syndrome.lower() >>
+        unit: deaths
+        description_short: Estimated number of deaths from << infectious_syndrome.lower() >>.
+        presentation:
+          title_public: Total deaths from << infectious_syndrome.lower() >>
+        display:
+          roundingMode: significantFigures
+          numSignificantFigures: 3
+          name: << pathogen >>
+      lower:
+        title: Lower bound of deaths from << infectious_syndrome.lower() >>
+        unit: deaths
+        description_short: Estimated number of deaths from << infectious_syndrome.lower() >>.
+        presentation:
+          title_public: Lower bound of deaths from << infectious_syndrome.lower() >>
+        display:
+          roundingMode: significantFigures
+          numSignificantFigures: 3
+          name: << pathogen >>
+      upper:
+        title: Upper bound of deaths from << infectious_syndrome.lower() >>
+        unit: deaths
+        description_short: Estimated number of deaths from << infectious_syndrome.lower() >>.
+        presentation:
+          title_public: Upper bound of deaths from << infectious_syndrome.lower() >>
+        display:
+          roundingMode: significantFigures
+          numSignificantFigures: 3
+          name: << pathogen >>

etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome.py

@@ -0,0 +1,35 @@
+"""Load a meadow dataset and create a garden dataset."""
+
+from etl.data_helpers import geo
+from etl.helpers import PathFinder, create_dataset
+
+# Get paths and naming conventions for current step.
+paths = PathFinder(__file__)
+
+
+def run(dest_dir: str) -> None:
+    #
+    # Load inputs.
+    #
+    # Load meadow dataset.
+    ds_meadow = paths.load_dataset("microbe_total_deaths_by_syndrome")
+
+    # Read table from meadow dataset.
+    tb = ds_meadow.read("microbe_total_deaths_by_syndrome")
+
+    #
+    # Process data.
+    #
+    tb = geo.harmonize_countries(df=tb, countries_file=paths.country_mapping_path)
+    tb = tb.format(["country", "year", "infectious_syndrome"])
+
+    #
+    # Save outputs.
+    #
+    # Create a new garden dataset with the same metadata as the meadow dataset.
+    ds_garden = create_dataset(
+        dest_dir, tables=[tb], check_variables_metadata=True, default_metadata=ds_meadow.metadata
+    )
+
+    # Save changes in the new garden dataset.
+    ds_garden.save()

etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.countries.json

@@ -0,0 +1,3 @@
+{
+  "Global": "World"
+}

etl/steps/data/garden/antibiotics/2024-12-05/microbe_total_deaths_by_syndrome_amr.meta.yml

@@ -0,0 +1,42 @@
+# NOTE: To learn more about the fields, hover over their names.
+definitions:
+  common:
+    presentation:
+      topic_tags:
+        - Antibiotics
+
+# Learn more about the available fields:
+# http://docs.owid.io/projects/etl/architecture/metadata/reference/
+dataset:
+  update_period_days: 365
+
+tables:
+  microbe_total_deaths_by_syndrome_amr:
+    variables:
+      amr_attributable_deaths:
+        title: Total deaths from infections attributed to AMR, by syndrome
+        unit: deaths
+        description_short: Estimated number of deaths each year from infections that are attributed to antimicrobial resistance.
+        presentation:
+          title_public: Total deaths from infections attributed to AMR, by syndrome
+        display:
+          roundingMode: significantFigures
+          numSignificantFigures: 3
+      non_amr_attributable_deaths:
+        title: Total deaths from infections not attributed to AMR, by syndrome
+        unit: deaths
+        description_short: Estimated number of deaths each year from infections that are not attributed to antimicrobial resistance.
+        presentation:
+          title_public: Total deaths from infections not attributed to AMR, by syndrome
+        display:
+          roundingMode: significantFigures
+          numSignificantFigures: 3
+      total_deaths:
+        title: Total deaths from infections, by syndrome
+        unit: deaths
+        description_short: Estimated number deaths each year from infections.
+        presentation:
+          title_public: Total deaths from infections, by syndrome
+        display:
+          roundingMode: significantFigures
+          numSignificantFigures: 3