From ee6e66fdd05d835b6ed62dff2e99498fe33e3dbf Mon Sep 17 00:00:00 2001 From: Vishnu Challa Date: Tue, 11 Jun 2024 14:56:20 -0400 Subject: [PATCH] limiting cpu count for multiprocessing Signed-off-by: Vishnu Challa --- backend/app/api/v1/endpoints/cpt/cptJobs.py | 15 +++++++++++---- frontend/src/store/reducers/Utils.js | 3 --- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/backend/app/api/v1/endpoints/cpt/cptJobs.py b/backend/app/api/v1/endpoints/cpt/cptJobs.py index fd8c424f..8b8dc1bb 100644 --- a/backend/app/api/v1/endpoints/cpt/cptJobs.py +++ b/backend/app/api/v1/endpoints/cpt/cptJobs.py @@ -1,6 +1,7 @@ import json import asyncio -import multiprocessing +from concurrent.futures import ProcessPoolExecutor, as_completed +from multiprocessing import cpu_count from fastapi import Response import pandas as pd from datetime import datetime, timedelta, date @@ -45,9 +46,15 @@ async def jobs(start_date: date = Query(None, description="Start date for search return Response(content=json.dumps({'error': "invalid date format, start_date must be less than end_date"}), status_code=422) results_df = pd.DataFrame() - with multiprocessing.Pool() as pool: - results = [pool.apply(fetch_product, args=(product, start_date, end_date)) for product in products] - results_df = pd.concat(results) + with ProcessPoolExecutor(max_workers=cpu_count()) as executor: + futures = {executor.submit(fetch_product, product, start_date, end_date): product for product in products} + for future in as_completed(futures): + product = futures[future] + try: + result = future.result() + results_df = pd.concat([results_df, result]) + except Exception as e: + print(f"Error fetching data for product {product}: {e}") response = { 'startDate': start_date.__str__(), diff --git a/frontend/src/store/reducers/Utils.js b/frontend/src/store/reducers/Utils.js index a826b4c2..fb51adbe 100644 --- a/frontend/src/store/reducers/Utils.js +++ b/frontend/src/store/reducers/Utils.js @@ -63,11 +63,8 @@ const getTelcoUpdatedData = (data, benchmark, version, releaseStream, ciSystem, "nodeName": nodeName, } let filteredData = data - console.log(filteredData) for (let [keyName, value] of Object.entries(filterValues)) filteredData = getFilteredData(filteredData, value, keyName) - console.log(filterValues) - console.log(filteredData) return filteredData }