Skip to content

Commit

Permalink
fix: process pool's fork safety issue since macOS High Sierra
Browse files Browse the repository at this point in the history
  • Loading branch information
tianjianjiang committed Oct 12, 2021
1 parent 8a518eb commit f649fc7
Showing 1 changed file with 6 additions and 4 deletions.
10 changes: 6 additions & 4 deletions promptsource/app.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import argparse
import textwrap
from multiprocessing import Manager, Pool
from itertools import repeat
from multiprocessing import Manager
from multiprocessing.pool import ThreadPool

import pandas as pd
import plotly.express as px
Expand Down Expand Up @@ -130,11 +132,11 @@ def show_text(t, width=WIDTH):
all_infos = manager.dict()
all_datasets = list(set([t[0] for t in template_collection.keys]))

def get_infos(d_name):
def get_infos(d_name, all_infos):
all_infos[d_name] = get_dataset_infos(d_name)

pool = Pool(processes=len(all_datasets))
pool.map(get_infos, all_datasets)
pool = ThreadPool(processes=len(all_datasets))
pool.starmap(get_infos, zip(all_datasets, repeat(all_infos)))
pool.close()
pool.join()

Expand Down

0 comments on commit f649fc7

Please sign in to comment.