diff --git a/geo_filter.py b/geo_filter.py index 18f360d..2463de8 100644 --- a/geo_filter.py +++ b/geo_filter.py @@ -57,8 +57,11 @@ def process_files(path, dirs, files, geo_filtered_dict): os.makedirs('processed') except FileExistsError: pass + processes = [] for i in range(100): - multiprocessing.Process(target=worker, args=(q, geo_filtered_dict)).start() + p = multiprocessing.Process(target=worker, args=(q, geo_filtered_dict)) + processes.append(p) + p.start() os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'twitter-sentiment-analysis-f22ce784b0a8.json' try: for path, dirs, files in os.walk('untarred'): @@ -69,3 +72,6 @@ def process_files(path, dirs, files, geo_filtered_dict): finally: with open('geo_filtered.pkl', 'wb') as f: pickle.dump(geo_filtered, f) + for p in processes: + p.terminate() +