updated goe_filter so it doesn't use google trans

DE0CH · Jun 21, 2020 · 723668b · 723668b
1 parent 0161ce9
commit 723668b
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/geo_filter.py b/geo_filter.py
@@ -57,8 +57,11 @@ def process_files(path, dirs, files, geo_filtered_dict):
         os.makedirs('processed')
     except FileExistsError:
         pass
+    processes = []
     for i in range(100):
-        multiprocessing.Process(target=worker, args=(q, geo_filtered_dict)).start()
+        p = multiprocessing.Process(target=worker, args=(q, geo_filtered_dict))
+        processes.append(p)
+        p.start()
     os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'twitter-sentiment-analysis-f22ce784b0a8.json'
     try:
         for path, dirs, files in os.walk('untarred'):
@@ -69,3 +72,6 @@ def process_files(path, dirs, files, geo_filtered_dict):
     finally:
         with open('geo_filtered.pkl', 'wb') as f:
             pickle.dump(geo_filtered, f)
+        for p in processes:
+            p.terminate()
+