From a000d38ebc5512f7c415ad035963e137d8e92983 Mon Sep 17 00:00:00 2001 From: Mike Tian-Jian Jiang Date: Sun, 12 Sep 2021 22:10:32 +0900 Subject: [PATCH] fix: process pool's fork safety issue since macOS High Sierra --- promptsource/app.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/promptsource/app.py b/promptsource/app.py index 8a4d4421c..c3e2a43f1 100644 --- a/promptsource/app.py +++ b/promptsource/app.py @@ -1,6 +1,8 @@ import argparse import textwrap -from multiprocessing import Manager, Pool +from itertools import repeat +from multiprocessing import Manager +from multiprocessing.pool import ThreadPool import pandas as pd import plotly.express as px @@ -138,11 +140,11 @@ def show_text(t, width=WIDTH, with_markdown=False): all_infos = manager.dict() all_datasets = list(set([t[0] for t in template_collection.keys])) - def get_infos(d_name): + def get_infos(d_name, all_infos): all_infos[d_name] = get_dataset_infos(d_name) - pool = Pool(processes=len(all_datasets)) - pool.map(get_infos, all_datasets) + pool = ThreadPool(processes=len(all_datasets)) + pool.starmap(get_infos, zip(all_datasets, repeat(all_infos))) pool.close() pool.join()