From 0b393bc6dea0293ae0b477844acd81102731d088 Mon Sep 17 00:00:00 2001 From: Young Date: Mon, 16 Dec 2024 09:54:45 +0000 Subject: [PATCH] Fix Async Call --- qlib/utils/paral.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/qlib/utils/paral.py b/qlib/utils/paral.py index 7b06d9f7f4..3b6671ddbe 100644 --- a/qlib/utils/paral.py +++ b/qlib/utils/paral.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import threading from functools import partial from threading import Thread from typing import Callable, Text, Union @@ -9,7 +10,7 @@ from joblib._parallel_backends import MultiprocessingBackend import pandas as pd -from queue import Queue +from queue import Empty, Queue import concurrent from qlib.config import C, QlibConfig @@ -85,7 +86,17 @@ def close(self): def run(self): while True: - data = self._q.get() + # NOTE: + # atexit will only trigger when all the threads ended. So it may results in deadlock. + # So the child-threading should actively watch the status of main threading to stop itself. + main_thread = threading.main_thread() + if not main_thread.is_alive(): + break + try: + data = self._q.get(timeout=1) + except Empty: + # NOTE: avoid deadlock. make checking main thread possible + continue if data == self.STOP_MARK: break data()