Merge pull request #12 from thisistheplace/feature/multithreading

Performance: added option to move matplotlib figure generation to separate process

.github/workflows/test.yml

@@ -9,6 +9,8 @@ on:
     branches:
       - 'master'
 
+  workflow_dispatch:
+
 jobs:
   test:
     runs-on: ubuntu-22.04

README.md

@@ -12,54 +12,61 @@ This tool was developed as part of a PhD, "Modelling and Design of Local Energy
 1. Install [Python 3.10](https://www.python.org/downloads/) and [Git](https://git-scm.com/).
 
 2. Clone this git repo onto your machine:
-```
-git clone [email protected]:andrewlyden/PyLESA.git
-```
+    ```
+    git clone [email protected]:andrewlyden/PyLESA.git
+    ```
 
 3. Install the `PyLESA` python virtual environment.
 
-If using Linux:
-```
-python3.10 -m venv venv
-source venv/bin/activate
-python -m pip install --upgrade pip
-python -m pip install -r requirements.txt
-```
-
-If using Windows and Powershell:
-```
-python -m venv venv
-.\venv\Scripts\activate.ps1
-python -m pip install --upgrade pip
-python -m pip install -r requirements.txt
-```
+    If using Linux:
+    ```
+    python3.10 -m venv venv
+    source venv/bin/activate
+    python -m pip install --upgrade pip
+    python -m pip install -r requirements.txt
+    ```
+
+    If using Windows and Powershell:
+    ```
+    python -m venv venv
+    .\venv\Scripts\activate.ps1
+    python -m pip install --upgrade pip
+    python -m pip install -r requirements.txt
+    ```
 
 4. Define and gather data on the local energy system to be modelled including resources, demands, supply, storage, grid connection, and control strategy. Define the increments and ranges to be modelled within the required parametric design. Input all this data using one of the template Excel Workbooks from the [inputs](./inputs) folder.
 
 5. Optionally run the demand ([heat_demand.py](./pylesa/demand/heat_demand.py) and [electricity_demand.py](./pylesa/demand/electricity_demand.py)) and resource assessment methods (see PhD thesis for details) to generate hourly profiles depending on available data. Input generated profiles into the Excel Workbook.
 
 6. Using a terminal (e.g. PowerShell) within the clone of the `PyLESA` git repo, run:
 
-If using Linux:
+    If using Linux:
 
-```python
-source venv/bin/activate
-python -m pylesa --help # to display help messages
-python -m pylesa ./inputs/{name of Excel input file}.xlsx my/existing/output/directory --overwrite
-```
+    ```python
+    source venv/bin/activate
+    python -m pylesa --help # to display help messages
+    python -m pylesa ./inputs/{name of Excel input file}.xlsx my/existing/output/directory --overwrite
+    ```
 
-If using Windows and Powershell:
+    If using Windows and Powershell:
 
-```python
-.\venv\Scripts\activate.ps1
-python -m pylesa --help # to display help messages
-python -m pylesa ./inputs/{name of Excel input file}.xlsx my/existing/output/directory --overwrite
-```
+    ```python
+    .\venv\Scripts\activate.ps1
+    python -m pylesa --help # to display help messages
+    python -m pylesa ./inputs/{name of Excel input file}.xlsx my/existing/output/directory --overwrite
+    ```
+
+    Running `python -m pylesa --help` will display the following help message:
+    ![pylesa usage](./img/pylesa_usage.png)
+
+    Note that PyLESA defaults to using 2 compute cores: 1 to run the solver, 1 to generate matplotlib
+    figures and write them to disc. Using the `--singlecore` command-line option will force PyLESA
+    to run on a single core which will increase the overall runtime.
 
-Running `python -m pylesa --help` will display the following help message:
-![pylesa usage](./img/pylesa_usage.png)
+7. After the run is complete, open the outpus folder in your chosen run directory to view the KPI 3D plots and/or operational graphs, as well as .csv outputs (note that an error will be raised if only one simulation combination is run, as 3D plots cannot be processed). There are also raw outputs.pkl files for each simulation combination which contains a vast range of raw outputs.
 
-7. After the run is complete, open the outpus folder in your chosen run directory to view the KPI 3D plots and/or operational graphs, as well as .csv outputs. (Note an error will be raised if only one simulation combination is run, as 3D plots cannot be processed). There are also raw outputs.pkl files for each simulation combination which contains a vast range of raw outputs.
+    Information about the run is written to a `pylesa.log` file located in the output folder. This
+    file contains details of run progress and any warning or error messages that may have occurred.
 
 A video discussing how to run `PyLESA` is available here: https://youtu.be/QsJut9ftCT4
 

pylesa/__init__.py

@@ -1,4 +0,0 @@
-from .constants import DEFAULT_LOGLEVEL
-from .logging import setup_logging
-
-setup_logging(DEFAULT_LOGLEVEL)

pylesa/constants.py

@@ -1,9 +1,9 @@
 import logging
 
-DEFAULT_LOGLEVEL = logging.INFO
+DEFAULT_LOGLEVEL = logging.WARNING
 FILE_LOG_FORMAT = "%(asctime)s: %(levelname)s: %(message)s"
 CONSOLE_LOG_FORMAT = "%(levelname)s: %(message)s"
-LOG_PATH = "pylesa.log"
+LOG_FILENAME = "pylesa.log"
 
 INDIR = "inputs"
 OUTDIR = "outputs"

pylesa/controllers/fixed_order.py

@@ -12,8 +12,7 @@
 from pathlib import Path
 import numpy as np
 import pickle
-
-from progressbar import Bar, ETA, Percentage, ProgressBar, RotatingMarker
+from tqdm import tqdm
 
 from .. import initialise_classes
 from ..io import inputs
@@ -93,13 +92,6 @@ def run_timesteps(self, first_hour, timesteps):
         soc_list = []
         results = []
 
-        # includes a progress bar, purely for aesthetics
-        widgets = ['Running: ' + self.subname, ' ', Percentage(), ' ',
-                   Bar(marker=RotatingMarker(), left='[', right=']'),
-                   ' ', ETA()]
-        pbar = ProgressBar(widgets=widgets, maxval=timesteps)
-        pbar.start()
-
         # can run number of timesteps up to 8760
         # final hour is from first hour plus number of timesteps
         final_hour = first_hour + timesteps
@@ -109,7 +101,11 @@ def run_timesteps(self, first_hour, timesteps):
             raise ValueError(msg)
 
         # run controller for each timestep
-        for timestep in range(first_hour, final_hour):
+        for timestep in tqdm(
+                range(first_hour, final_hour),
+                desc=f"Solving: {self.subname}",
+                leave=False
+            ):
             heat_demand = self.heat_demand[timestep]
             source_temp = self.source_temp[timestep]
             flow_temp = self.flow_temp[timestep]
@@ -161,11 +157,6 @@ def run_timesteps(self, first_hour, timesteps):
             soc_list.append(
                 run['ES']['final_soc'])
 
-            # update for progress bar
-            pbar.update(timestep - first_hour + 1)
-        # stop progress bar
-        pbar.finish()
-
         # write the outputs to a pickle
         file = self.root / OUTDIR / self.subname / 'outputs.pkl'
         with open(file, 'wb') as output_file:

pylesa/controllers/mpc.py

@@ -6,8 +6,8 @@
 from gekko import GEKKO
 from pathlib import Path
 import numpy as np
-from progressbar import Bar, ETA, Percentage, ProgressBar, RotatingMarker
 import pickle
+from tqdm import tqdm
 
 from .. import initialise_classes, tools
 from ..io import inputs
@@ -548,21 +548,18 @@ def moving_horizon(self, pre_calc, first_hour, timesteps):
         horizon = self.horizon
         final_hour = first_hour + timesteps
 
-        # includes a progress bar, purely for aesthetics
-        widgets = ['Running: ' + self.subname, ' ', Percentage(), ' ',
-                   Bar(marker=RotatingMarker(), left='[', right=']'),
-                   ' ', ETA()]
-        pbar = ProgressBar(widgets=widgets, maxval=timesteps)
-        pbar.start()
-
         if final_hour > 8760:
             msg = f'The final timestep is {final_hour} which is beyond the end of the year (8760)'
             LOG.error(msg)
             raise ValueError(msg)
 
         results = []
         next_results = []
-        for hour in range(first_hour, final_hour - 1):
+        for hour in tqdm(
+                range(first_hour, final_hour - 1),
+                desc=f"Solving: {self.subname}",
+                leave=False
+            ):
             final_horizon_hour = hour + horizon
             if final_horizon_hour > 8759:
                 final_horizon_hour = 8759
@@ -669,11 +666,6 @@ def moving_horizon(self, pre_calc, first_hour, timesteps):
                     results.append(r['results'])
                     next_results.append(r['next_results'])
 
-            # update for progress bar
-            pbar.update(hour - first_hour + 1)
-        # stop progress bar
-        pbar.finish()
-
         # write the outputs to a pickle
         file = self.root / OUTDIR / self.subname / 'outputs.pkl'
         with open(file, 'wb') as output_file:

pylesa/io/outputs.py

@@ -7,6 +7,8 @@
 # import register_matplotlib_converters
 import matplotlib
 import matplotlib.pyplot as plt
+import time
+from tqdm import tqdm
 from typing import Dict
 
 from . import inputs
@@ -54,6 +56,7 @@
 
 
 def run_plots(root: str | Path, subname: str):
+    then = time.time()
     root = Path(root).resolve()
     myInputs = inputs.Inputs(root, subname)
     # controller inputs
@@ -82,20 +85,26 @@ def run_plots(root: str | Path, subname: str):
         myPlots.grid()
         myPlots.RES_bar()
 
+    LOG.info(f"Written output files for: {subname}. Time taken: {int(round(time.time() - then,0))} seconds")
 
 def run_KPIs(root: str | Path):
     root = Path(root).resolve()
 
+    jobs = []
+
     my3DPlots = ThreeDPlots(root)
-    my3DPlots.KPIs_to_csv()
-    my3DPlots.plot_opex()
-    my3DPlots.plot_RES()
-    my3DPlots.plot_heat_from_RES()
-    my3DPlots.plot_HP_size_ratio()
-    my3DPlots.plot_HP_utilisation()
-    my3DPlots.plot_capital_cost()
-    my3DPlots.plot_LCOH()
-    my3DPlots.plot_COH()
+    jobs.append(my3DPlots.KPIs_to_csv)
+    jobs.append(my3DPlots.plot_opex)
+    jobs.append(my3DPlots.plot_RES)
+    jobs.append(my3DPlots.plot_heat_from_RES)
+    jobs.append(my3DPlots.plot_HP_size_ratio)
+    jobs.append(my3DPlots.plot_HP_utilisation)
+    jobs.append(my3DPlots.plot_capital_cost)
+    jobs.append(my3DPlots.plot_LCOH)
+    jobs.append(my3DPlots.plot_COH)
+
+    for job in tqdm(jobs, desc=f"Writing KPIs"):
+        job()
 
 
 class Plot(object):

pylesa/io/read_excel.py

@@ -22,7 +22,7 @@ def read_inputs(xlsxpath: str | Path, root: Path) -> None:
         root: path to directory to store intermediary inputs and outputs
     """
     xlsxpath = valid_fpath(xlsxpath)
-    LOG.info(f'Reading MS Excel file: {xlsxpath.name}')
+    LOG.info(f'Reading MS Excel file: {xlsxpath}')
     root = Path(root).resolve()
     myInput = XlsxInput(xlsxpath, root)
 

pylesa/logging.py

@@ -1,7 +1,8 @@
 import enum
 import logging
+from pathlib import Path
 
-from .constants import LOG_PATH, CONSOLE_LOG_FORMAT, FILE_LOG_FORMAT
+from .constants import LOG_FILENAME, CONSOLE_LOG_FORMAT, FILE_LOG_FORMAT
 
 
 class CustomFormatter(logging.Formatter):
@@ -26,21 +27,23 @@ def format(self, record):
         return formatter.format(record)
 
 
-def setup_logging(level: enum.Enum):
+def setup_logging(logdir: str | Path, level: enum.Enum):
     """Sets up logging handlers, removes any existing handlers
 
     A FileHandler and a StreamHandler (console) are created.
 
     Args:
+        logdir: path to directory to write log file
         level: Enum defining the logging level
     """
     root = logging.getLogger()
-    root.setLevel(level)
+    root.setLevel(min(logging.INFO, level))
     root.handlers.clear()
     handlers = []
 
     # Log to a file on disk
-    file_handler = logging.FileHandler(LOG_PATH, "w+")
+    # Level always set at least to INFO
+    file_handler = logging.FileHandler(Path(logdir).resolve() / LOG_FILENAME, "w+")
     file_handler.setFormatter(logging.Formatter(FILE_LOG_FORMAT))
     handlers.append(file_handler)