Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move to pandas based reads and writes #17

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Move to pandas based reads and writes #17

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0f05661
Add pandas as dependency in preprocess.py
chaiitanyasangani88 Oct 18, 2018
1bbc3f1
Moving to pandas based reads and writes in preprocess_csv function
chaiitanyasangani88 Oct 19, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 16 additions & 22 deletions code/preprocess.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import sys
from utils import write_status
from nltk.stem.porter import PorterStemmer

import pandas as pd

def preprocess_word(word):
# Remove punctuation
Expand Down Expand Up @@ -69,28 +69,22 @@ def preprocess_tweet(tweet):


def preprocess_csv(csv_file_name, processed_file_name, test_file=False):
save_to_file = open(processed_file_name, 'w')

with open(csv_file_name, 'r') as csv:
lines = csv.readlines()
total = len(lines)
for i, line in enumerate(lines):
tweet_id = line[:line.find(',')]
if not test_file:
line = line[1 + line.find(','):]
positive = int(line[:line.find(',')])
line = line[1 + line.find(','):]
tweet = line
processed_tweet = preprocess_tweet(tweet)
if not test_file:
save_to_file.write('%s,%d,%s\n' %
(tweet_id, positive, processed_tweet))
else:
save_to_file.write('%s,%s\n' %
(tweet_id, processed_tweet))
write_status(i + 1, total)
save_to_file.close()

#Reading CSV without headers
unprocessed_data = pd.read_csv(csv_file_name, header=None)

if not test_file:
# Take third column in train file
processed_data = unprocessed_data.iloc[:,2].apply(lambda x: preprocess_tweet(x))
else:
# Take second column for test file
processed_data = unprocessed_data.iloc[:, 1].apply(lambda x: preprocess_tweet(x))

final_df = pd.concat([unprocessed_data, processed_data], axis=0)

final_df.to_csv(processed_file_name, index=False, header=False)
print '\nSaved processed tweets to: %s' % processed_file_name

return processed_file_name


Expand Down