Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Performance: added option to move matplotlib figure generation to separate process #12

Merged
merged 28 commits into from
Jul 11, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
d95a536
Added threading but bad idea since matplotlib isn't threadsafe
thisistheplace May 31, 2024
e44f630
Moved plotting to separate process
thisistheplace May 31, 2024
086fd35
Removed tqdm output reporting
thisistheplace May 31, 2024
321165e
Improved progress reporting in solvers
thisistheplace May 31, 2024
1214402
Removed unused threading module
thisistheplace May 31, 2024
d488542
Updated requirements
thisistheplace Jun 2, 2024
35019f8
Added commandline option for single core run
thisistheplace Jun 2, 2024
e1362d8
Added threading but bad idea since matplotlib isn't threadsafe
thisistheplace May 31, 2024
6f3e9ac
Moved plotting to separate process
thisistheplace May 31, 2024
44370d6
Removed tqdm output reporting
thisistheplace May 31, 2024
c0615f4
Improved progress reporting in solvers
thisistheplace May 31, 2024
e81fd6e
Removed unused threading module
thisistheplace May 31, 2024
1e3b079
Updated requirements
thisistheplace Jun 2, 2024
938ef9a
Added commandline option for single core run
thisistheplace Jun 2, 2024
6be3f1d
Merged in test_heatpump updates
thisistheplace Jun 3, 2024
80d8276
Started on process tests
thisistheplace Jun 4, 2024
3a00f11
Updated pylesa regression test to use tmpdir
thisistheplace Jun 4, 2024
95c480e
Added logging to multiprocessing
thisistheplace Jun 5, 2024
2fa5be3
Minor logging updates
thisistheplace Jun 10, 2024
a8ee1a2
Improved logging messages and mp logging handlers
thisistheplace Jun 10, 2024
ecb4120
Ran black
thisistheplace Jun 10, 2024
a57c24d
Updated requirements file
thisistheplace Jun 10, 2024
425802a
Added workflow_dispatch?
thisistheplace Jun 10, 2024
f7281e8
Updated readme
thisistheplace Jun 10, 2024
1aac858
Fixed error in test fixture
thisistheplace Jun 10, 2024
17f6b63
Added additional process tests
thisistheplace Jun 10, 2024
c6ef00d
Added comment to pylesa regression test
thisistheplace Jun 10, 2024
ba39fc6
Merged master into branch
thisistheplace Jun 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Moved plotting to separate process
  • Loading branch information
thisistheplace committed Jun 3, 2024
commit 6f3e9acca5b720baf3e238a98e58b1c2d49a5dcd
5 changes: 1 addition & 4 deletions pylesa/controllers/fixed_order.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,10 +101,7 @@ def run_timesteps(self, first_hour, timesteps):
raise ValueError(msg)

# run controller for each timestep
for timestep in tqdm(
range(first_hour, final_hour),
desc=f"Solving: {self.subname}"
):
for timestep in range(first_hour, final_hour):
heat_demand = self.heat_demand[timestep]
source_temp = self.source_temp[timestep]
flow_temp = self.flow_temp[timestep]
Expand Down
46 changes: 27 additions & 19 deletions pylesa/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@
from pathlib import Path
import shutil
import time
from tqdm import tqdm

from . import parametric_analysis
from .controllers import fixed_order
from .controllers import mpc
from .io import inputs, outputs, read_excel
from .io.paths import valid_dir, valid_fpath
from .threads import run_pool
from .process import OutputProcess

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -86,24 +87,31 @@ def main(xlsxpath: str, outdir: str, overwrite: bool = False):
timesteps = controller_info['total_timesteps']
first_hour = controller_info['first_hour']

# Run controller for all combinations
jobs = {}
for i in range(num_combos):
# combo to be run
subname = combinations[i]
jobs[subname] = [controller, subname, outdir, first_hour, timesteps]
# run_solver(controller, subname, outdir, first_hour, timesteps)
run_pool(run_solver, jobs, "Running solver")

# Run outputs for all combinations
jobs = {}
for i in range(num_combos):
# combo to be run
subname = combinations[i]
jobs[subname] = [outdir, subname]
# outputs.run_plots(outdir, subname)
LOG.info('Running output analysis...')
run_pool(outputs.run_plots, jobs, "Writing output")
# Create separate process for writing output
p = OutputProcess()

try:
# Start output process
# Process waits to write output until a job is submitted
p.start(outputs.run_plots)

# Run controller for all combinations
jobs = {}
for i in tqdm(range(num_combos), desc="Jobs"):
# combo to be run
subname = combinations[i]
jobs[subname] = [controller, subname, outdir, first_hour, timesteps]
run_solver(controller, subname, outdir, first_hour, timesteps)
# Submit job to output queue for writing
p.submit([outdir, subname])

except Exception as e:
p.cancel()
raise e

finally:
# Stop output process once job is complete
p.stop()

tx = time.time()
outputs.run_KPIs(outdir)
Expand Down
38 changes: 38 additions & 0 deletions pylesa/process.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
"""Run jobs in separate process"""
from multiprocessing import Process, Queue
import logging
from tqdm import tqdm
from typing import Callable, Dict, List, Any

LOG = logging.getLogger(__name__)

SENTINEL = "STOP_JOB"

class OutputProcess:
def __init__(self):
self._queue = Queue()
self._process = None

def _run_job(self, func: Callable, queue: Queue):
while True:
job = queue.get()
if job == SENTINEL:
break
func(*job)

def start(self, func: Callable) -> None:
# Start process
self._process = Process(target=self._run_job, args=(func, self._queue))
self._process.start()

def submit(self, args: List[Any]):
self._queue.put(args)

def stop(self):
if self._process:
self._queue.put(SENTINEL)
self._process.join()

def cancel(self):
self._queue.empty()
self.stop()