Upgrades visualizer simplification

buildstock_query/tools/upgrades_visualizer/upgrades_visualizer.py

@@ -35,8 +35,6 @@
 # ])
 transforms = [MultiplexerTransform()]
 
-# yaml_path = "/Users/radhikar/Documents/eulpda/EULP-data-analysis/notebooks/EUSS-project-file-example.yml"
-yaml_path = "/Users/radhikar/Documents/largee/resstock/project_national/fact_sheets_category_1.yml"
 opt_sat_path = "/Users/radhikar/Downloads/options_saturations.csv"
 default_end_use = "fuel_use_electricity_total_m_btu"
 
@@ -56,12 +54,40 @@ def filter_cols(all_columns, prefixes=[], suffixes=[]):
     return cols
 
 
-def _get_app(yaml_path: str, opt_sat_path: str, db_name: str = 'euss-tests',
+def get_int_set(input_str):
+    """
+        Convert "1,2,3-6,8,9" to [1, 2, 3, 4, 5, 6, 8, 9]
+    """
+    if not input_str:
+        return set()
+
+    pattern = r'^(\d+(-\d+)?,)*(\d+(-\d+)?)$'
+    if not re.match(pattern, input_str):
+        raise ValueError(f"{input_str} is not a valid pattern for list")
+
+    result = set()
+    segments = input_str.split(',')
+    for segment in segments:
+        if '-' in segment:
+            start, end = map(int, segment.split('-'))
+            result |= set(range(start, end + 1))
+        else:
+            result.add(int(segment))
+
+    return result
+
+
+def _get_app(opt_sat_path: str, db_name: str = 'euss-tests',
              table_name: str = 'res_test_03_2018_10k_20220607',
              workgroup: str = 'largeee',
-             buildstock_type: str = 'resstock'):
-    viz_data = VizData(yaml_path=yaml_path, opt_sat_path=opt_sat_path, db_name=db_name,
-                       run=table_name, workgroup=workgroup, buildstock_type=buildstock_type)
+             buildstock_type: str = 'resstock',
+             include_monthly: bool = True,
+             upgrades_selection_str: str = ''):
+    viz_data = VizData(opt_sat_path=opt_sat_path, db_name=db_name,
+                       run=table_name, workgroup=workgroup, buildstock_type=buildstock_type,
+                       include_monthly=include_monthly,
+                       upgrades_selection=get_int_set(upgrades_selection_str)
+                       )
     return get_app(viz_data)
 
 
@@ -93,7 +119,8 @@ def get_buildings(upgrade):
         return upgrade2res[int(upgrade)]['building_id'].to_list()
 
     def get_plot(end_use, value_type='mean', savings_type='', change_type='',
-                 sync_upgrade=None, filter_bldg=None, group_cols=None, report_upgrade=None):
+                 sync_upgrade=None, filter_bldg=None, group_cols=None, report_upgrade=None,
+                 resolution='annual'):
         filter_bldg = filter_bldg or []
         group_cols = group_cols or []
         sync_upgrade = sync_upgrade or 0
@@ -115,7 +142,7 @@ def get_plot(end_use, value_type='mean', savings_type='', change_type='',
         dbc.Row([dbc.Col(html.H1("Upgrades Visualizer"), width='auto'), dbc.Col(html.Sup("beta"))]),
         # Add a row for annual, vs monthly vs seasonal plot radio buttons
         dbc.Row([dbc.Col(dbc.Label("Resolution: "), width='auto'),
-                 dbc.Col(dcc.RadioItems(["annual", "monthly"], "annual",
+                 dbc.Col(dcc.RadioItems(["annual", "monthly"] if viz_data.include_monthly else ["annual"], "annual",
                                         inline=True, id="radio_resolution"))]),
 
         dbc.Row([dbc.Col(dbc.Label("Visualization Type: "), width='auto'),
@@ -278,7 +305,7 @@ def download_char(n_clicks, bldg_id, bldg_options, bldg_options2, chk_chars):
         bdf = viz_data.upgrade2res[0].filter(pl.col("building_id").is_in(set(bldg_ids))).select(char_cols)
         return dcc.send_bytes(bdf.write_csv, f"chars_{n_clicks}.csv")
 
-    def get_elligible_output_columns(category, fuel):
+    def get_elligible_output_columns(category, fuel, resolution):
         if category == 'energy':
             elligible_cols = viz_data.get_cleaned_up_end_use_cols(resolution, fuel)
         elif category == 'water':
@@ -304,15 +331,6 @@ def get_elligible_output_columns(category, fuel):
             raise ValueError(f"Invalid tab {category}")
         return elligible_cols
 
-    @app.callback(
-        Output('radio_resolution', 'options'),
-        Input('radio_resolution', 'value'),
-    )
-    def update_resolution(res):
-        nonlocal resolution
-        resolution = res
-        return ['annual', 'monthly']
-
     @app.callback(
         Output('dropdown_enduse', "options"),
         Output('dropdown_enduse', "value"),
@@ -322,7 +340,7 @@ def update_resolution(res):
         Input('radio_resolution', 'value')
     )
     def update_enduse_options(view_tab, fuel_type, current_enduse, resolution):
-        elligible_cols = get_elligible_output_columns(view_tab, fuel_type)
+        elligible_cols = get_elligible_output_columns(view_tab, fuel_type, resolution)
         enduse = current_enduse if current_enduse in elligible_cols else elligible_cols[0]
         return sorted(elligible_cols), enduse
 
@@ -772,11 +790,12 @@ def show_char_report(bldg_id, bldg_options, bldg_options2, inp_char: list[str],
         Input('input_building2', 'options'),
         Input('chk-graph', 'value'),
         State("uirevision", "data"),
-        State('report_upgrade', 'value')
+        State('report_upgrade', 'value'),
+        State('radio_resolution', 'value')
     )
     def update_figure(view_tab, grp_by, fuel, enduse, graph_type, savings_type, chng_type,
                       sync_upgrade, selected_bldg, bldg_options, bldg_options2, chk_graph, uirevision,
-                      report_upgrade):
+                      report_upgrade, resolution):
         nonlocal download_csv_df
         if dash.callback_context.triggered_id == 'input_building2' and "Graph" not in chk_graph:
             raise PreventUpdate()
@@ -798,7 +817,8 @@ def update_figure(view_tab, grp_by, fuel, enduse, graph_type, savings_type, chng
             filter_bldg = [int(b) for b in bldg_options]
 
         new_figure, report_df = get_plot(full_name, graph_type, savings_type,
-                                         chng_type, sync_upgrade, filter_bldg, grp_by, report_upgrade)
+                                         chng_type, sync_upgrade, filter_bldg, grp_by, report_upgrade,
+                                         resolution)
 
         uirevision = uirevision or "default"
         new_figure.update_layout(uirevision=uirevision)
@@ -813,8 +833,6 @@ def update_figure(view_tab, grp_by, fuel, enduse, graph_type, savings_type, chng
 def main():
     print("Welcome to Upgrades Visualizer.")
     defaults = load_script_defaults("project_info")
-    yaml_file = inquirer.text(message="Please enter path to the buildstock configuration yml file: ",
-                              default=defaults.get("yaml_file", "")).execute()
     opt_sat_file = inquirer.text(message="Please enter path to the options saturation csv file: ",
                                  default=defaults.get("opt_sat_file", "")).execute()
     workgroup = inquirer.text(message="Please Athena workgroup name: ",
@@ -827,16 +845,25 @@ def main():
                                "separated by comma if baseline and upgrades are in different run] :",
                                default=defaults.get("table_name", "")
                                ).execute()
-    defaults.update({"yaml_file": yaml_file, "opt_sat_file": opt_sat_file, "workgroup": workgroup,
-                     "db_name": db_name, "table_name": table_name})
+    monthly_default = defaults.get("include_monthly", True)
+    include_monthly = inquirer.confirm(f"Do you want to include monthly data ({monthly_default})?",
+                                       default=monthly_default,
+                                       ).execute()
+    upgrades_selection = inquirer.text(message="Please enter upgrade ids separated by comma and dashes (example: 1-3,5,7,8-9) "
+                                       "or leave empty to include all upgrades.",
+                                       default=defaults.get("upgrades_selection", "")).execute()
+    defaults.update({"opt_sat_file": opt_sat_file, "workgroup": workgroup,
+                     "db_name": db_name, "table_name": table_name, "include_monthly": include_monthly,
+                     "upgrades_selection": upgrades_selection})
     save_script_defaults("project_info", defaults)
     if ',' in table_name:
         table_name = table_name.split(',')
-    app = _get_app(yaml_path=yaml_file,
-                   opt_sat_path=opt_sat_file,
+    app = _get_app(opt_sat_path=opt_sat_file,
                    workgroup=workgroup,
                    db_name=db_name,
-                   table_name=table_name)
+                   table_name=table_name,
+                   include_monthly=include_monthly,
+                   upgrades_selection_str=upgrades_selection)
     app.run_server(debug=False, port=8005)
 
 

buildstock_query/tools/upgrades_visualizer/viz_data.py

@@ -3,6 +3,7 @@
 import polars as pl
 from buildstock_query.tools.upgrades_visualizer.plot_utils import PlotParams
 from typing import Union
+import datetime
 
 num2month = {1: "January", 2: "February", 3: "March", 4: "April",
              5: "May", 6: "June", 7: "July", 8: "August",
@@ -12,12 +13,14 @@
 
 class VizData:
     @validate_arguments(config=dict(arbitrary_types_allowed=True, smart_union=True))
-    def __init__(self, yaml_path: str, opt_sat_path: str,
+    def __init__(self, opt_sat_path: str,
                  db_name: str,
                  run: Union[str, tuple[str, str]],
                  workgroup: str = 'largeee',
                  buildstock_type: str = 'resstock',
-                 skip_init: bool = False):
+                 skip_init: bool = False,
+                 include_monthly: bool = True,
+                 upgrades_selection: set = set()):
         if isinstance(run, tuple):
             # Allows for separate baseline and upgrade runs
             # In this case, run[0] is the baseline run and run[1] is the upgrade run
@@ -39,24 +42,30 @@ def __init__(self, yaml_path: str, opt_sat_path: str,
                                         buildstock_type=buildstock_type,
                                         table_name=table,
                                         skip_reports=skip_init)
-        self.yaml_path = yaml_path
         self.opt_sat_path = opt_sat_path
+        self.upgrades_selection = upgrades_selection
+        self.include_monthly = include_monthly
         if not skip_init:
             self.initialize()
 
     def initialize(self):
-        self.ua = self.main_run.get_upgrades_analyzer(yaml_file=self.yaml_path,
-                                                      opt_sat_file=self.opt_sat_path)
+        available_upgrades = self.main_run.get_available_upgrades()
+        available_upgrades = [int(u) for u in available_upgrades]
+        if (unavailable_upgrades := self.upgrades_selection - set(available_upgrades)):
+            raise ValueError(f"Upgrades {unavailable_upgrades} is not available in the run")
+        available_upgrades = self.upgrades_selection
         self.report = pl.from_pandas(self.main_run.report.get_success_report(), include_index=True)
-        self.available_upgrades = list(sorted(set(self.report["upgrade"].unique()) - {0}))
-        self.upgrade2name = {indx+1: f"Upgrade {indx+1}: {upgrade['upgrade_name']}" for indx,
-                             upgrade in enumerate(self.ua.cfg.get('upgrades', []))}
-        self.upgrade2name[0] = "Upgrade 0: Baseline"
-        self.upgrade2shortname = {indx+1: f"Upgrade {indx+1}" for indx,
-                                  upgrade in enumerate(self.ua.cfg.get('upgrades', []))}
+        self.available_upgrades = list(set([int(u) for u in available_upgrades]) - {0})
+        self.upgrade2name = {0: "Upgrade 0: Baseline"}
+        if self.available_upgrades:
+            upgrade_names = self.main_run.get_upgrade_names()
+            self.upgrade2name |= upgrade_names
+
+        self.upgrade2shortname = {indx+1: f"Upgrade {indx+1}" for indx in range(len(self.available_upgrades) + 1)}
         self.chng2bldg = self.get_change2bldgs()
         self.init_annual_results()
-        self.init_monthly_results(self.metadata_df)
+        if self.include_monthly:
+            self.init_monthly_results(self.metadata_df)
         self.all_upgrade_plotting_df = None
 
     def run_obj(self, upgrade: int) -> BuildStockQuery:
@@ -122,10 +131,23 @@ def init_monthly_results(self, metadata_df):
             ts_cols = self._get_ts_enduse_cols(upgrade)
             print(f"Getting monthly results for {upgrade}")
             run_obj = self.run_obj(upgrade)
-            monthly_vals = run_obj.agg.aggregate_timeseries(enduses=ts_cols,
-                                                            group_by=[run_obj.bs_bldgid_column],
-                                                            upgrade_id=upgrade,
-                                                            timestamp_grouping_func='month')
+            monthly_vals_query = run_obj.agg.aggregate_timeseries(get_query_only=True,
+                                                                  enduses=ts_cols,
+                                                                  group_by=[run_obj.bs_bldgid_column],
+                                                                  upgrade_id=upgrade,
+                                                                  timestamp_grouping_func='month',
+                                                                  )
+            month_year = f"{datetime.datetime.now().strftime('%b%Y')}"
+            s3_unload_path = f"s3://resstock-core/athena_unload_results/{month_year}/"
+            if monthly_vals_query in run_obj._query_cache:
+                monthly_vals = run_obj._query_cache[monthly_vals_query].copy()
+            else:
+                pd_cursor = run_obj._conn.cursor(unload=True, s3_staging_dir=s3_unload_path).execute(
+                    monthly_vals_query,
+                    result_reuse_enable=True,
+                    result_reuse_minutes=60 * 24 * 7)
+                monthly_vals = pd_cursor.as_pandas()
+                run_obj._query_cache[monthly_vals_query] = monthly_vals
             run_obj.save_cache()
             monthly_df = pl.from_pandas(monthly_vals, include_index=True)
             monthly_df = monthly_df.with_columns(pl.col('time').dt.month().alias("month"))
@@ -193,7 +215,7 @@ def get_plotting_df(self, upgrade: int,
                 .then(0)
                 .otherwise(pl.col("value"))
                 .alias("value")
-                )
+            )
         return up_df
 
     def get_all_cols(self, resolution: str) -> list[str]: