📊 improve etl-explorer tooling (#3220)

* 📊 update war explorer to path-based * bump streamlit * update icons, titles * wip: explorer editor * wip * wip * object to manipulate explorer config * wip * wip * wip * change module name * improve err message, remove unused code * wip * wip * wip * wip * comment explorer tool for now * explorer tools * add beta tool to migrate id->paths in explorers * format * ci/cd * log warning * change to new explorer creation tool * typo in function name
owid · Sep 5, 2024 · 1e26124 · 1e26124
1 parent 98605b7
commit 1e26124
Show file tree

Hide file tree

Showing 22 changed files with 788 additions and 64 deletions.
diff --git a/apps/wizard/app_pages/chart_diff/app.py b/apps/wizard/app_pages/chart_diff/app.py
@@ -385,7 +385,7 @@ def show_chart_diffs(chart_diffs, pagination_key, source_session: Session, targe
 ########################################
 def main():
     st.title(
-        "Chart ⚡ **:gray[Diff]**",
+        ":material/difference: Chart Diff",
         help=f"""
 **Chart diff** is a living page that compares all ongoing charts between [`production`](http://owid.cloud) and your [`{OWID_ENV.name}`]({OWID_ENV.admin_site}) environment.
 

diff --git a/apps/wizard/app_pages/dashboard.py b/apps/wizard/app_pages/dashboard.py
@@ -123,7 +123,7 @@
 ########################################
 # TITLE and DESCRIPTION
 ########################################
-st.title("ETL Dashboard 📋 :grey[Control panel for ETL steps]")
+st.title(":material/tv_gen: ETL Dashboard **:grey[Control panel for ETL steps]**")
 st.markdown(
     """\
 Explore all active ETL steps, and, if you are working on your local machine, perform some actions.

diff --git a/apps/wizard/app_pages/datainsight_robot.py b/apps/wizard/app_pages/datainsight_robot.py
@@ -13,7 +13,7 @@
     page_title="Data insight robot",
     page_icon="🪄",
 )
-st.title("💡 Data insighter")
+st.title(":material/lightbulb: Data insighter")
 st.markdown("Generate data insights from a chart view.")
 
 

diff --git a/apps/wizard/app_pages/dataset_explorer.py b/apps/wizard/app_pages/dataset_explorer.py
@@ -25,7 +25,7 @@
 )
 st.session_state.export_metadata = st.session_state.get("export_metadata", False)
 
-st.title("🕵️ Dataset Explorer")
+st.title(":material/search: Dataset Explorer")
 
 COLORS = {
     "snapshot": "#FC9090",

diff --git a/apps/wizard/app_pages/explorer_edit.py b/apps/wizard/app_pages/explorer_edit.py
@@ -0,0 +1,64 @@
+"""Helper tool to create map brackets for all indicators in an indicator-based explorer."""
+from io import StringIO
+
+import streamlit as st
+
+from etl.explorer import Explorer
+
+# PAGE CONFIG
+st.set_page_config(
+    page_title="Wizard: Explorer editor",
+    page_icon="🪄",
+)
+st.title(":material/explore: Explorer Editor")
+
+
+with st.container(border=True):
+    st.subheader("IDs to Paths")
+    st.markdown("Migrate all references to indicator IDs for their corresponding indicator paths.")
+
+    uploaded_file = st.file_uploader(
+        label="Upload Explorer config file",
+        type=["csv", "tsv"],
+    )
+
+    if uploaded_file:
+        stringio = StringIO(uploaded_file.getvalue().decode("utf-8"))
+        string_data = stringio.read()
+
+        if uploaded_file.name.endswith("csv"):
+            sep = ","
+        else:
+            sep = "\t"
+
+        explorer = Explorer.from_raw_string(string_data, sep=sep)
+
+        # explorer.convert_ids_to_etl_paths()
+
+        # with st.popover("config"):
+        #     st.dataframe(pd.DataFrame(explorer.config).T.rename(columns={0: "value"}))
+
+        # with st.popover("graphers"):
+        #     st.dataframe(explorer.df_graphers)
+
+        # with st.popover("columns"):
+        #     st.dataframe(explorer.df_columns)
+
+        # Update
+        explorer.convert_ids_to_etl_paths()
+
+        # Downloads
+        st.download_button(
+            "Download `graphers` (CSV)",
+            explorer._df_graphers_output.to_csv(sep=",", index=False),  # type: ignore
+            file_name="graphers.csv",
+        )
+        if not explorer.df_columns.empty:
+            st.download_button(
+                "Download `columns` (CSV)",
+                explorer._df_columns_output.to_csv(sep=",", index=False),  # type: ignore
+                file_name="columns.csv",
+            )
+
+        filename = f"modified-{uploaded_file.name.replace('.csv', '.tsv')}"
+        st.download_button("Download new config (TSV)", explorer.content, file_name=filename)
diff --git a/apps/wizard/app_pages/fasttrack/app.py b/apps/wizard/app_pages/fasttrack/app.py
@@ -63,7 +63,7 @@ def reset_states() -> None:
 # MAIN ###################################################
 ##########################################################
 # TITLE & description
-st.title("Import data via  **:gray[Fast-Track]**")
+st.title(":material/fast_forward: Fast-Track import")
 st.markdown(
     """
             Fast-track is a tool for importing datasets from Google Sheets. The idea is to keep all data and metadata there, and use this interface to import or update the data in grapher database where it can be used to create charts. Fast-track also commits your work to [ETL repository](https://github.com/owid/etl) where you can further process your data with Python.

diff --git a/apps/wizard/app_pages/harmonizer.py b/apps/wizard/app_pages/harmonizer.py
@@ -97,7 +97,7 @@ def sort_indicators(indicators: List[str]) -> List[str]:
 
 
 # Page config
-st.title("🎶 Entity Harmonizer")
+st.title(":material/music_note: Entity Harmonizer")
 
 # Set states
 st.session_state["show_all"] = st.session_state.get("show_all", False)

diff --git a/apps/wizard/app_pages/indicator_upgrade/app.py b/apps/wizard/app_pages/indicator_upgrade/app.py
@@ -52,7 +52,7 @@
     """,
     },
 )
-st.title("Indicator 🧬 **:gray[Upgrader]**")
+st.title(":material/upgrade: Indicator Upgrader")
 st.markdown("Update indicators to their new versions.")  # Get datasets (might take some time)
 
 # Get all datasets

diff --git a/apps/wizard/app_pages/map_brackets.py b/apps/wizard/app_pages/map_brackets.py
@@ -890,7 +890,7 @@ def _create_maximum_instances_message(mb: MapBracketer) -> str:
     page_icon="🪄",
     # initial_sidebar_state="collapsed",
 )
-st.title("🗺️ Map bracketer")
+st.title(":material/map: Map bracketer")
 with st.popover("ℹ️ Learn about it"):
     st.markdown(
         "This tool will find optimal map brackets for a specific variable, and let you manually edit it in a way that is consistent with our guidelines."

diff --git a/apps/wizard/app_pages/news.py b/apps/wizard/app_pages/news.py
@@ -8,7 +8,7 @@
 
 # Page config
 st.set_page_config(page_title="Wizard: News", page_icon="🪄")
-st.title("News 🗞️")
+st.title(":material/newspaper: News")
 st.markdown("Find news and updates from the [etl project](https://github.com/owid/etl).")
 
 # Only run this on live!

diff --git a/apps/wizard/config/config.yml b/apps/wizard/config/config.yml
@@ -15,7 +15,7 @@
 # - description (str): Description of the app. This will be shown in the card of the app in the home page of Wizard.
 # - maintenaner (str): Slack handle of the person responsible for the app.
 # - entrypoint (str): Path to the main file of the app. This file should be the streamlit python script.
-# - icon (str): Emoji or material icon to show in the sidebar menu.
+# - icon (str): Emoji or material icon to show in the sidebar menu. Use from https://fonts.google.com/icons?icon.set=Material+Symbols&icon.style=Rounded
 # - image_url (str): Link to the image you want to show on the card in the home page of Wizard.
 # - disable (bool): Whether the app is disabled in certain settings. To detect if the app is running in a remote setting, it checks the value of the environment variable `ENV`.
 #     bool: Whether the app is disabled in all settings.
@@ -86,44 +86,49 @@ etl:
       image_url: "https://upload.wikimedia.org/wikipedia/commons/b/be/Alonso_%28Renault%29_qualifying_at_USGP_2005.jpg"
 
 sections:
-
   - title: "Data tools"
     description: |-
       Updates, data edits, etc.
     apps:
-    - title: "Indicator Upgrader"
-      alias: indicator-upgrade
-      entrypoint: app_pages/indicator_upgrade/app.py
-      description: Migrate indicators in charts
-      maintainer: "@lucas"
-      icon: ":material/upgrade:"
-      image_url: "https://superheroetc.wordpress.com/wp-content/uploads/2017/05/bulbasaur-line.jpg"
-      disable:
-        production: True
-    - title: "Chart Diff"
-      alias: chart-diff
-      entrypoint: app_pages/chart_diff/app.py
-      description: Compare charts in staging and prod
-      maintainer: "@mojmir"
-      icon: ":material/difference:"
-      image_url: "https://static.wikia.nocookie.net/dragonball/images/6/60/FusionDanceFinaleGotenTrunksBuuSaga.png"
-      disable:
-        production: True
-    - title: "Harmonizer"
-      alias: harmonizer
-      description: "Harmonize a column of a table"
-      maintainer: "@lucas"
-      entrypoint: app_pages/harmonizer.py
-      icon: ":material/music_note:"
-      image_url: "https://upload.wikimedia.org/wikipedia/commons/thumb/c/c1/C_triad.svg/2560px-C_triad.svg.png"
-    - title: "Map Bracketer"
-      alias: map_brackets
-      entrypoint: app_pages/map_brackets.py
-      description: Create optimal map brackets
-      maintainer: "@pablo"
-      icon: ":material/map:"
-      image_url: "https://upload.wikimedia.org/wikipedia/en/8/8c/Human_Language_Families_Map_%28Wikipedia_Colors_.PNG"
-
+      - title: "Indicator Upgrader"
+        alias: indicator-upgrade
+        entrypoint: app_pages/indicator_upgrade/app.py
+        description: Migrate indicators in charts
+        maintainer: "@lucas"
+        icon: ":material/upgrade:"
+        image_url: "https://superheroetc.wordpress.com/wp-content/uploads/2017/05/bulbasaur-line.jpg"
+        disable:
+          production: True
+      - title: "Chart Diff"
+        alias: chart-diff
+        entrypoint: app_pages/chart_diff/app.py
+        description: Compare charts in staging and prod
+        maintainer: "@mojmir"
+        icon: ":material/difference:"
+        image_url: "https://static.wikia.nocookie.net/dragonball/images/6/60/FusionDanceFinaleGotenTrunksBuuSaga.png"
+        disable:
+          production: True
+      - title: "Harmonizer"
+        alias: harmonizer
+        description: "Harmonize a column of a table"
+        maintainer: "@lucas"
+        entrypoint: app_pages/harmonizer.py
+        icon: ":material/music_note:"
+        image_url: "https://upload.wikimedia.org/wikipedia/commons/thumb/c/c1/C_triad.svg/2560px-C_triad.svg.png"
+      - title: "Map Bracketer"
+        alias: map_brackets
+        entrypoint: app_pages/map_brackets.py
+        description: Create optimal map brackets
+        maintainer: "@pablo"
+        icon: ":material/map:"
+        image_url: "https://upload.wikimedia.org/wikipedia/en/8/8c/Human_Language_Families_Map_%28Wikipedia_Colors_.PNG"
+      - title: "Explorer editor"
+        alias: explorer_editor
+        entrypoint: app_pages/explorer_edit.py
+        description: Edit explorer config
+        maintainer: "@lucas"
+        icon: ":material/explore:"
+        image_url: "https://upload.wikimedia.org/wikipedia/en/1/18/Dora_the_Explorer_2004_album_cover.jpg"
 
   - title: "Monitoring"
     description: |-
@@ -199,7 +204,6 @@ sections:
         disable:
           "production": True
           "staging": True
-
 # legacy:
 #   description: |-
 #     The pages below are legacy now. They are kept in case the new pages are not working properly.

diff --git a/apps/wizard/etl_steps/express.py b/apps/wizard/etl_steps/express.py
@@ -301,7 +301,7 @@ def export_metadata() -> None:
 # MAIN ##################################################
 #########################################################
 # TITLE
-st.title("Create step 🐆 **:gray[Express]**")
+st.title(":material/bolt: Express **:gray[Create steps]**")
 
 st.info("Use this step to create Meadow, Garden and Grapher step for a _single dataset_!")
 

diff --git a/apps/wizard/etl_steps/garden.py b/apps/wizard/etl_steps/garden.py
@@ -211,7 +211,7 @@ def export_metadata() -> None:
 # MAIN ##################################################
 #########################################################
 # TITLE
-st.title("Create step  **:gray[Garden]**")
+st.title(":material/deceased: Garden **:gray[Create step]**")
 
 # SIDEBAR
 with st.sidebar:

diff --git a/apps/wizard/etl_steps/grapher.py b/apps/wizard/etl_steps/grapher.py
@@ -97,7 +97,7 @@ def update_state() -> None:
 # MAIN ##################################################
 #########################################################
 # TITLE
-st.title("Create step  **:gray[Grapher]**")
+st.title(":material/database: Grapher  **:gray[Create step]**")
 
 # SIDEBAR
 with st.sidebar:

diff --git a/apps/wizard/etl_steps/meadow.py b/apps/wizard/etl_steps/meadow.py
@@ -127,9 +127,9 @@ def update_state() -> None:
 
 # TITLE
 if st.session_state["meadow.edit_dataset"]:
-    st.title("Edit step  **:gray[Meadow]**")
+    st.title(":material/nature: Meadow  **:gray[Edit step]**")
 else:
-    st.title("Create step  **:gray[Meadow]**")
+    st.title(":material/nature: Meadow  **:gray[Create step]**")
 
 # SIDEBAR
 with st.sidebar:

diff --git a/apps/wizard/etl_steps/snapshot.py b/apps/wizard/etl_steps/snapshot.py
@@ -707,7 +707,7 @@ def run_snap_step() -> None:
 #########################################################
 
 # TITLE
-st.title("Create step **:gray[Snapshot]**")
+st.title(":material/photo_camera: Snapshot **:gray[Create step]**")
 
 # SIDEBAR
 with st.sidebar: