From d87852cde6e9332a9da1340773219b969a2f1bd3 Mon Sep 17 00:00:00 2001 From: DE0CH Date: Mon, 22 Jun 2020 00:55:53 +0800 Subject: [PATCH] updated goe_filter so it doesn't use google trans --- geo_filter.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/geo_filter.py b/geo_filter.py index 8bcbcbf..67b1c22 100644 --- a/geo_filter.py +++ b/geo_filter.py @@ -53,18 +53,22 @@ def process_files(path, dirs, files, geo_filtered_dict): os.makedirs('processed') except FileExistsError: pass + processes = [] for i in range(100): - multiprocessing.Process(target=worker, args=(q, geo_filtered_dict)).start() + p = multiprocessing.Process(target=worker, args=(q, geo_filtered_dict)) + processes.append(p) + p.start() os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'twitter-sentiment-analysis-f22ce784b0a8.json' try: for path, dirs, files in os.walk('untarred'): q.put((path, dirs, files)) q.join() - except KeyboardInterrupt: - pass + except KeyboardInterrupt as e: + raise e finally: with open('geo_filtered.pkl', 'wb') as f: pickle.dump(geo_filtered, f) - print('done, press ctrl-c again to quit') + for p in processes: + p.terminate()