Merge branch 'master' into covid-mdims-b5

.git-blame-ignore-revs

@@ -9,3 +9,6 @@ ab772adce9a987639cb1cde15b9a62a349568582
 
 # switching to ruff
 5647751506f0685c6704aebc31a59632904d0741
+
+# update ruff
+38d649354920a5bd549a4e1ac02185882d13aa60

apps/chart_animation/cli.py

@@ -1,6 +1,5 @@
-"""Create a GIF or video for a given chart URL.
+"""Create a GIF or video for a given chart URL."""
 
-"""
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse

apps/metadata_migrate/cli.py

@@ -209,9 +209,9 @@ def cli(
 
         # empty fields to be filled
         vars[col]["presentation"] = {}
-        vars[col]["presentation"][
-            "title_public"
-        ] = "TBD - Indicator title to be shown in data pages, that overrides the indicator's title."
+        vars[col]["presentation"]["title_public"] = (
+            "TBD - Indicator title to be shown in data pages, that overrides the indicator's title."
+        )
         vars[col]["presentation"]["title_variant"] = "TBD - Indicator's title variant"
         vars[col]["presentation"]["attribution_short"] = "TBD - Indicator's attribution (shorter version)"
         vars[col]["presentation"]["faqs"] = [

apps/metagpt/cli.py

@@ -1,4 +1,5 @@
 """Client module."""
+
 import os
 from typing import Any, Dict, Literal, cast
 

apps/metagpt/prompts.py

@@ -2,6 +2,7 @@
 
 Contains instructions to correctly query chat GPT for the different use cases (snapshot, garden, grapher, etc.). This includes details on what each metadata field means.
 """
+
 import json
 from typing import Any, Dict, List
 

apps/metagpt/utils.py

@@ -1,4 +1,5 @@
 """Auxiliary utils for metagpt app."""
+
 from pathlib import Path
 from typing import Dict, List
 

apps/utils/gpt.py

@@ -1,4 +1,5 @@
 """Auxiliary classes, functions and variables."""
+
 from dataclasses import asdict, dataclass, field
 from typing import Any, Dict, List, Optional, Tuple
 

apps/wizard/app.py

@@ -4,6 +4,7 @@
 
 NOTE: This only works with >1.35 (nightly) version of Streamlit.
 """
+
 import streamlit as st
 
 from apps.wizard.config import WIZARD_CONFIG

apps/wizard/app_pages/anomalist/utils.py

@@ -1,4 +1,5 @@
 """Utils for chart revision tool."""
+
 import time
 from enum import Enum
 from typing import Dict, List, Optional, Tuple

apps/wizard/app_pages/chart_diff/chart_diff.py

@@ -15,7 +15,7 @@
 from etl import grapher_model as gm
 from etl.config import OWID_ENV
 from etl.db import read_sql
-from etl.git_helpers import get_changed_files
+from etl.git_helpers import get_changed_files, log_time
 
 log = get_logger()
 
@@ -540,6 +540,7 @@ def get_diffs_summary_df(self, cache: bool = False, **kwargs) -> pd.DataFrame:
         return pd.DataFrame(summary)
 
 
+@log_time
 def _modified_data_metadata_on_staging(
     source_session: Session, target_session: Session, chart_ids: Optional[List[int]] = None
 ) -> pd.DataFrame:
@@ -644,6 +645,7 @@ def _modified_data_metadata_on_staging(
     return diff
 
 
+@log_time
 def _modified_chart_configs_on_staging(
     source_session: Session, target_session: Session, chart_ids: Optional[List[int]] = None
 ) -> pd.DataFrame:

apps/wizard/app_pages/chart_diff/chart_diff_show.py

@@ -3,6 +3,7 @@
 
 If you want to learn more about it, start from its `show` method.
 """
+
 import difflib
 import json
 import os

apps/wizard/app_pages/dashboard.py

@@ -1,6 +1,5 @@
-"""Create a dashboard with the main information about ETL steps, and the possibility to update them.
+"""Create a dashboard with the main information about ETL steps, and the possibility to update them."""
 
-"""
 import subprocess
 from datetime import datetime
 

apps/wizard/app_pages/dataset_explorer.py

@@ -3,6 +3,7 @@
 - [ ] See its dependencies
 - [ ] Preview its metadata
 """
+
 import tempfile
 from typing import Any, Dict, List, cast
 

apps/wizard/app_pages/expert/app.py

@@ -3,6 +3,7 @@
 references:
 - https://docs.streamlit.io/knowledge-base/tutorials/build-conversational-apps#build-a-chatgpt-like-app
 """
+
 from typing import Any, Dict, cast
 
 import streamlit as st

apps/wizard/app_pages/expert/prompts.py

@@ -1,4 +1,5 @@
-""""Get prompts for GPT-interaction."""
+""" "Get prompts for GPT-interaction."""
+
 import glob
 from typing import List
 

apps/wizard/app_pages/explorer_edit.py

@@ -1,4 +1,5 @@
 """Helper tool to create map brackets for all indicators in an indicator-based explorer."""
+
 from io import StringIO
 
 import streamlit as st

apps/wizard/app_pages/fasttrack/app.py

@@ -2,6 +2,7 @@
 
 TODO: Fully decouple front-end from backend (see fasttrack.utils for back-end, i.e. should not have streamlit calls).
 """
+
 from pathlib import Path
 
 import pandas as pd

apps/wizard/app_pages/fasttrack/fast_import.py

@@ -1,4 +1,5 @@
 """Definition of FasttrackImport object (mainly backend)."""
+
 import datetime as dt
 import difflib
 import html

apps/wizard/app_pages/fasttrack/load.py

@@ -2,6 +2,7 @@
 
 Relies on Streamlit to print messages.
 """
+
 import concurrent.futures
 import datetime as dt
 import json

apps/wizard/app_pages/fasttrack/process.py

@@ -2,6 +2,7 @@
 
 Relies on Streamlit.
 """
+
 import json
 from typing import Dict, List, Tuple
 

apps/wizard/app_pages/fasttrack/utils.py

@@ -1,4 +1,5 @@
 """Utils."""
+
 import os
 from typing import Any, Dict, Optional
 

apps/wizard/app_pages/harmonizer.py

@@ -1,4 +1,5 @@
 """Harmonize entities."""
+
 import json
 from operator import itemgetter
 from pathlib import Path

apps/wizard/app_pages/indicator_upgrade/app.py

@@ -22,6 +22,7 @@
     - Set to True: When the user clicks on "Finish (3/3)" in the third form.
     - Set to False:
 """
+
 import streamlit as st
 from structlog import get_logger
 

apps/wizard/app_pages/indicator_upgrade/charts_update.py

@@ -1,4 +1,5 @@
 """Handle submission of chart updates."""
+
 from http.client import RemoteDisconnected
 from typing import Dict, List
 from urllib.error import URLError

apps/wizard/app_pages/indicator_upgrade/dataset_selection.py

@@ -1,4 +1,5 @@
 """Search configuration."""
+
 from typing import Any, Dict
 
 import pandas as pd

apps/wizard/app_pages/indicator_upgrade/explore_mode.py

@@ -2,6 +2,7 @@
 
 This is currently shown in the indicator upgrader, but might be moved to chart-diff in the future.
 """
+
 from dataclasses import dataclass, field
 from typing import Dict, Optional, Tuple, cast
 

apps/wizard/app_pages/indicator_upgrade/utils.py

@@ -1,4 +1,5 @@
 """Utils for chart revision tool."""
+
 from typing import Dict, Tuple, cast
 
 import pandas as pd

apps/wizard/app_pages/map_brackets.py

@@ -1,6 +1,4 @@
-"""Helper tool to create map brackets for all indicators in an indicator-based explorer.
-
-"""
+"""Helper tool to create map brackets for all indicators in an indicator-based explorer."""
 
 import json
 from pathlib import Path

apps/wizard/app_pages/metagpt.py

@@ -2,6 +2,7 @@
 
 Note that it still relies on apps/metagpt.
 """
+
 import os
 import tempfile
 from pathlib import Path

apps/wizard/app_pages/metaplay.py

@@ -2,6 +2,7 @@
 
 streamlit run app.py
 """
+
 import webbrowser
 from pathlib import Path
 

apps/wizard/app_pages/news.py

@@ -1,4 +1,5 @@
 """Display news from ETL."""
+
 from datetime import datetime
 
 import streamlit as st

apps/wizard/app_pages/owidle.py

@@ -1,4 +1,5 @@
 """Game owidle."""
+
 import datetime as dt
 import math
 from itertools import product

apps/wizard/cli.py

@@ -4,6 +4,7 @@
 
 python cli.py
 """
+
 import logging
 import sys
 from typing import Iterable

apps/wizard/etl_steps/express.py

@@ -1,4 +1,5 @@
 """Garden phase."""
+
 from datetime import date, datetime, timedelta
 from pathlib import Path
 from typing import Any, Dict, List

apps/wizard/etl_steps/garden.py

@@ -1,4 +1,5 @@
 """Garden phase."""
+
 import os
 from pathlib import Path
 from typing import List, cast
@@ -452,9 +453,9 @@ def export_metadata() -> None:
                     f"Use the generated notebook `{notebook_path.relative_to(BASE_DIR)}` to examine the dataset output interactively."
                 )
                 # B/ Generate metadata
-                st.session_state[
-                    "garden.dataset_path"
-                ] = f"data/garden/{form.namespace}/{form.version}/{form.short_name}"
+                st.session_state["garden.dataset_path"] = (
+                    f"data/garden/{form.namespace}/{form.version}/{form.short_name}"
+                )
                 st.markdown("#### Generate metadata")
                 st.markdown(f"Generate metadata file `{form.short_name}.meta.yml` from your dataset with:")
                 st.button(

apps/wizard/etl_steps/grapher.py

@@ -1,4 +1,5 @@
 """Grapher phase."""
+
 from pathlib import Path
 from typing import cast
 

apps/wizard/etl_steps/meadow.py

@@ -1,4 +1,5 @@
 """Meadow phase."""
+
 import os
 from pathlib import Path
 from typing import cast

apps/wizard/etl_steps/snapshot.py

@@ -1,4 +1,5 @@
 """Snapshot phase."""
+
 import os
 import subprocess
 import traceback

apps/wizard/home.py

@@ -1,4 +1,5 @@
 """Home page of wizard."""
+
 from copy import deepcopy
 from typing import Any, Dict, List, Optional
 

apps/wizard/scripts/news.py

@@ -1,4 +1,5 @@
 """Methods to update Wizard DB to have the necessary PR data."""
+
 from typing import Tuple
 
 import requests

apps/wizard/utils/__init__.py

@@ -9,6 +9,7 @@
 
     Also, can imagine apps/wizard/ being renamed to just wizard/, and stuff other than wizard should be either (i) deleted or (ii) migrated elsewhere in etl/.
 """
+
 import argparse
 import ast
 import datetime as dt

apps/wizard/utils/chart_config.py

@@ -1,4 +1,5 @@
 """Tools to generate chart configs."""
+
 from copy import deepcopy
 from typing import Any, Dict, List, Optional
 

apps/wizard/utils/db.py

@@ -5,6 +5,7 @@
 Some of the tools here rely on a local temporary sqlite database. This database is a custom and temporary database used to store data in a server. Not intended for production use.
 
 """
+
 import datetime as dt
 import hashlib
 import os

apps/wizard/utils/io.py

@@ -2,6 +2,7 @@
 
 Together with utils.db and utils.cached, it might need some rethinking on where it goes.
 """
+
 from pathlib import Path
 from typing import Dict, List, Optional
 

dag/archive/demography.yml

@@ -118,3 +118,11 @@ steps:
     - data://garden/un/2022-07-11/un_wpp
   data://grapher/demography/2023-10-10/life_expectancy:
     - data://garden/demography/2023-10-09/life_expectancy
+
+  ## Population by education group (Wittgenstein centre)
+  data://meadow/demography/2023-04-07/population_education_wittgenstein:
+    - snapshot://demography/2023-04-07/population_education_wittgenstein.csv
+  data://garden/demography/2023-04-07/population_education_wittgenstein:
+    - data://meadow/demography/2023-04-07/population_education_wittgenstein
+  data://grapher/demography/2023-04-07/population_education_wittgenstein:
+    - data://garden/demography/2023-04-07/population_education_wittgenstein

dag/archive/education.yml

@@ -9,4 +9,14 @@ steps:
     - data://garden/education/2018-04-18/literacy_rates
 
   data://grapher/wb/2023-07-10/education:
-    - data://garden/wb/2023-07-10/education
+    - data://garden/wb/2023-07-10/education
+
+  # Wittgenstein Center Population and Human Capital Projections - Educational Attainment
+  data://meadow/education/2023-12-15/wittgenstein_center:
+    - snapshot://education/2023-12-15/wittgenstein_center_data.csv
+    - snapshot://education/2023-12-15/wittgenstein_center_dictionary.csv
+  data://garden/education/2023-12-15/wittgenstein_center:
+    - data://meadow/education/2023-12-15/wittgenstein_center
+    - data://garden/education/2023-08-14/oecd_education
+  data://grapher/education/2023-12-15/wittgenstein_center:
+    - data://garden/education/2023-12-15/wittgenstein_center