TS visualizer initial POC

buildstock_query/aggregate_query.py

@@ -94,6 +94,7 @@ def _aggregate_timeseries_light(self,
             new_query = params.copy()
             new_query.enduses = [enduse.name]
             new_query.split_enduses = False
+            new_query.get_query_only = True
             query = self.aggregate_timeseries(params=new_query)
             batch_queries_to_submit.append(query)
 
@@ -109,14 +110,17 @@ def _aggregate_timeseries_light(self,
         result_dfs = self._bsq.get_batch_query_result(batch_id=batch_query_id, combine=False)
         logger.info("Joining the individual enduses result into a single DataFrame")
         group_by = self._bsq._clean_group_by(params.group_by)
-        for res in result_dfs:
-            res.set_index(group_by, inplace=True)
+        if not params.collapse_ts and 'time' not in group_by:
+            group_by.append('time')
+        for i, res in enumerate(result_dfs):
+            if group_by:
+                res.set_index(group_by, inplace=True)
+            if i > 0:
+                res.drop(columns=['sample_count', 'units_count'], inplace=True, errors='ignore')
         self.result_dfs = result_dfs
-        joined_enduses_df = result_dfs[0].drop(columns=['query_id'])
-        for enduse, res in list(zip(params.enduses, result_dfs))[1:]:
-            if not isinstance(enduse, str):
-                enduse = enduse.name
-            joined_enduses_df = joined_enduses_df.join(res[[enduse]])
+        joined_enduses_df = result_dfs[0]
+        for res in result_dfs[1:]:
+            joined_enduses_df = joined_enduses_df.join(res)
 
         logger.info("Joining Completed.")
         return joined_enduses_df.reset_index()

buildstock_query/query_core.py

@@ -156,7 +156,7 @@ def load_cache(self, path: Optional[str] = None):
         pickle_path = pathlib.Path(path) if path else self._get_cache_file_path()
         before_count = len(self._query_cache)
         saved_cache = load_pickle(pickle_path)
-        logger.info(f"{len(saved_cache)} queries cache read from {path}.")
+        logger.info(f"{len(saved_cache)} queries cache read from {pickle_path}.")
         self._query_cache.update(saved_cache)
         self.last_saved_queries = set(saved_cache)
         after_count = len(self._query_cache)
@@ -519,10 +519,12 @@ def get_pandas(future):
             return exe_id, AthenaFutureDf(result_future)
         else:
             if query not in self._query_cache:
-                self._query_cache[query] = self._conn.cursor().execute(query,
-                                                                       result_reuse_enable=self.athena_query_reuse,
-                                                                       result_reuse_minutes=60 * 24 * 7,
-                                                                       ).as_pandas()
+                cursor = self._conn.cursor()
+                self._query_cache[query] = cursor.execute(query,
+                                                          result_reuse_enable=self.athena_query_reuse,
+                                                          result_reuse_minutes=60 * 24 * 7,
+                                                          ).as_pandas()
+                self._log_execution_cost(cursor.query_id)
             return self._query_cache[query].copy()
 
     def print_all_batch_query_status(self) -> None:

buildstock_query/tools/visualizer/__init__.py

@@ -0,0 +1,4 @@
+from .upgrades_visualizer import main as upgrades_visualizer
+from .timeseries_visualizer import main as timeseries_visualizer
+__all__ = ['upgrades_visualizer', 'timeseries_visualizer']
+

buildstock_query/tools/upgrades_visualizer/figure.py → buildstock_query/tools/visualizer/figure.py

@@ -1,5 +1,5 @@
-from buildstock_query.tools.upgrades_visualizer.plot_utils import PlotParams, ValueTypes, human_sort_key, flatten_list
-from buildstock_query.tools.upgrades_visualizer.viz_data import VizData
+from buildstock_query.tools.visualizer.plot_utils import PlotParams, ValueTypes, human_sort_key, flatten_list
+from buildstock_query.tools.visualizer.viz_data import VizData
 import plotly.graph_objects as go
 import polars as pl
 import re