Skip to content

Commit

Permalink
updated goe_filter so it doesn't use google trans
Browse files Browse the repository at this point in the history
  • Loading branch information
DE0CH committed Jun 21, 2020
1 parent 5cf2482 commit d87852c
Showing 1 changed file with 8 additions and 4 deletions.
12 changes: 8 additions & 4 deletions geo_filter.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,18 +53,22 @@ def process_files(path, dirs, files, geo_filtered_dict):
os.makedirs('processed')
except FileExistsError:
pass
processes = []
for i in range(100):
multiprocessing.Process(target=worker, args=(q, geo_filtered_dict)).start()
p = multiprocessing.Process(target=worker, args=(q, geo_filtered_dict))
processes.append(p)
p.start()
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'twitter-sentiment-analysis-f22ce784b0a8.json'
try:
for path, dirs, files in os.walk('untarred'):
q.put((path, dirs, files))
q.join()
except KeyboardInterrupt:
pass
except KeyboardInterrupt as e:
raise e
finally:
with open('geo_filtered.pkl', 'wb') as f:
pickle.dump(geo_filtered, f)
print('done, press ctrl-c again to quit')
for p in processes:
p.terminate()


0 comments on commit d87852c

Please sign in to comment.