Cleaned_EP_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  gender  num_groups  num_entries  num_fans  num_friends     created  \\\n",
      "0      F         210           54        25            2  2014-12-20   \n",
      "1      F          25            9         2            0  2014-08-28   \n",
      "2      U           0        65535      5672            0  2008-04-21   \n",
      "3      M         153            2         6            5  2014-12-13   \n",
      "4      F          29            8        83           60  2014-12-27   \n",
      "\n",
      "            last_login  num_logins   last_wall_activity last_circle_activity  \\\n",
      "0  2015-04-03 12:45:01           6  0000-00-00 00:00:00  0000-00-00 00:00:00   \n",
      "1  2015-04-15 22:32:36          19  0000-00-00 00:00:00  0000-00-00 00:00:00   \n",
      "2  2012-11-13 18:03:23           5  2009-03-27 21:22:12  0000-00-00 00:00:00   \n",
      "3  2015-06-01 20:34:52          15  0000-00-00 00:00:00  2014-12-13 08:37:11   \n",
      "4  2015-05-14 13:25:09          19  0000-00-00 00:00:00  2015-05-14 13:25:34   \n",
      "\n",
      "    birthdate     gid                                            content  \\\n",
      "0  1997-03-31  124249  I just want my mom to hug me and say that she ...   \n",
      "1  1994-03-31    5276  I don't know why this is happening right now. ...   \n",
      "2  1989-03-09   49025  I accept the wisdom of all religions and disca...   \n",
      "3  1993-11-30   58852  HAPPY NEW  YEARS !!!!!!!!!!!!!!!     !!!!!!!!!...   \n",
      "4  1977-08-31  970365  Showing now if anyone is interested...message ...   \n",
      "\n",
      "           date_posted  votes  num_comments  \n",
      "0  2015-01-01 00:00:10      0             0  \n",
      "1  2015-01-01 00:00:46      0             0  \n",
      "2  2015-01-01 00:00:49     10             1  \n",
      "3  2015-01-01 00:02:39      1             2  \n",
      "4  2015-01-01 00:02:45      0             0  \n"
     ]
    }
   ],
   "source": [
    "# The following iPython notebook shows how to take a series of text inputs\n",
    "# (in this case status updates on the Experience Project), pre-process the\n",
    "# text to remove extraneous words, HTML, and punctuation, and create a \n",
    "# bag of words that allows the user to further train random forest models.\n",
    "# In this example, I stop at a point where the most common words in the \n",
    "# test set have been found.\n",
    "\n",
    "#import the whole data file\n",
    "import pandas as pd       \n",
    "extext = pd.read_csv(\"EP_data.csv\", header=0)\n",
    "#id_text = pd.read_csv(\"EP_data.csv\", header=0, usecols=[11,12])\n",
    "extext.shape\n",
    "print extext.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "534305"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extext.shape[0] #what size is our dataset?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 3) (524305, 3)\n"
     ]
    }
   ],
   "source": [
    "#I can select only the columns I want\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "data = extext[['gid','content','gender']]\n",
    "ntrain = 10000\n",
    "data_train, data_test = train_test_split(data, train_size=ntrain)\n",
    "\n",
    "subset= data_train\n",
    "subset.index = range(subset.shape[0])\n",
    "print subset.shape, data_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inbox meh friends\n"
     ]
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import nltk\n",
    "#nltk.download()\n",
    "from nltk.corpus import stopwords # Import the stop word list\n",
    "def status_to_words( raw_status ):\n",
    "    # Function to convert a raw status to a string of words\n",
    "    # The input is a single string (a raw status update), and \n",
    "    # the output is a single string (a preprocessed status update)\n",
    "    #\n",
    "    # 1. Remove HTML\n",
    "    status_text = BeautifulSoup(raw_status).get_text() \n",
    "    #\n",
    "    # 2. Remove non-letters        \n",
    "    letters_only = re.sub(\"[^a-zA-Z]\", \" \", status_text)\n",
    "    # remove http URLs\n",
    "    letters_only = re.sub(r'^https?:\\/\\/.*[\\r\\n]*', '', letters_only)\n",
    "    #\n",
    "    # 3. Convert to lower case, split into individual words\n",
    "    words = letters_only.lower().split()                             \n",
    "    #\n",
    "    # 4. In Python, searching a set is much faster than searching\n",
    "    #   a list, so convert the stop words to a set\n",
    "    stops = set(stopwords.words(\"english\"))\n",
    "    #Remove 'm' and 'hasn' from the text\n",
    "    stops = stops.union([u'hasn',u'm',u've',u'll',u're',u'didn',u'us',u'im',u'doesn',u'couldn',u'won',u'isn',u'http',u'www']) \n",
    "    # \n",
    "    # 5. Remove stop words\n",
    "    meaningful_words = [w for w in words if not w in stops]   \n",
    "    #\n",
    "    # 6. Join the words back into one string separated by space, \n",
    "    # and return the result.\n",
    "    return( \" \".join( meaningful_words ))\n",
    "\n",
    "\n",
    "#Let's see an example of what status_to_words does\n",
    "clean_status = status_to_words( subset['content'][1] )\n",
    "print (clean_status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000\n"
     ]
    }
   ],
   "source": [
    "#How many statuses do we have? Should be same as row number of extext.shape above\n",
    "num_status = subset[\"content\"].size\n",
    "print num_status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaning and parsing the status updates...\n",
      "\n",
      "Status 1000 of 10000\n",
      "\n",
      "Status 2000 of 10000\n",
      "\n",
      "Status 3000 of 10000\n",
      "\n",
      "Status 4000 of 10000\n",
      "\n",
      "Status 5000 of 10000\n",
      "\n",
      "Status 6000 of 10000\n",
      "\n",
      "Status 7000 of 10000\n",
      "\n",
      "Status 8000 of 10000\n",
      "\n",
      "Status 9000 of 10000\n",
      "\n",
      "Status 10000 of 10000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Now run status_to_words on all the statuses (or a subset) for preprocessing\n",
    "print \"Cleaning and parsing the status updates...\\n\"\n",
    "clean_statuses = []\n",
    "#num_status = subset[\"content\"].size #for full set\n",
    "num_status = ntrain #for subset of all statuses\n",
    "for i in xrange( 0, num_status ):\n",
    "    # If the index is evenly divisible by 1000, print a message\n",
    "    if( (i+1)%1000 == 0 ):\n",
    "        print \"Status %d of %d\\n\" % ( i+1, num_status )                                                                    \n",
    "    clean_statuses.append( status_to_words( subset[\"content\"][i] ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating the bag of words...\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# create the Bag o' words. Each word that appears in the statuses after preprocessing gets a place\n",
    "# in the bag (list). Each status gets a list denoting the number of times a particular word\n",
    "# shows up in that status.  Check out \n",
    "# https://www.kaggle.com/c/word2vec-nlp-tutorial/details/part-1-for-beginners-bag-of-words\n",
    "\n",
    "print \"Creating the bag of words...\\n\"\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "# Initialize the \"CountVectorizer\" object, which is scikit-learn's\n",
    "# bag of words tool. Note that some of the preprocessing steps above\n",
    "# can be done directly with this tool; we may want to do that.\n",
    "vectorizer = CountVectorizer(analyzer = \"word\",   \\\n",
    "                             tokenizer = None,    \\\n",
    "                             preprocessor = None, \\\n",
    "                             stop_words = None,   \\\n",
    "                             max_features = 500) \n",
    "\n",
    "# fit_transform() does two functions: First, it fits the model\n",
    "# and learns the vocabulary; second, it transforms our training data\n",
    "# into feature vectors. The input to fit_transform should be a list of \n",
    "# strings.\n",
    "train_data_features = vectorizer.fit_transform(clean_statuses)\n",
    "\n",
    "# Numpy arrays are easy to work with, so convert the result to an \n",
    "# array\n",
    "train_data_features = train_data_features.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "389 able\n",
      "429 actually\n",
      "143 afraid\n",
      "358 age\n",
      "149 agemeet\n",
      "321 ago\n",
      "168 air\n",
      "365 almost\n",
      "501 alone\n",
      "186 along\n",
      "293 already\n",
      "927 also\n",
      "146 although\n",
      "1266 always\n",
      "195 amazing\n",
      "543 another\n",
      "179 answer\n",
      "151 anxiety\n",
      "327 anymore\n",
      "788 anyone\n",
      "762 anything\n",
      "142 anyway\n",
      "842 around\n",
      "337 ask\n",
      "460 asked\n",
      "172 attention\n",
      "741 away\n",
      "217 baby\n",
      "1467 back\n",
      "623 bad\n",
      "147 barb\n",
      "349 beautiful\n",
      "151 became\n",
      "308 become\n",
      "323 bed\n",
      "178 behind\n",
      "475 believe\n",
      "698 best\n",
      "623 better\n",
      "432 big\n",
      "339 bit\n",
      "196 black\n",
      "424 body\n",
      "184 book\n",
      "216 boy\n",
      "253 boyfriend\n",
      "162 break\n",
      "151 bring\n",
      "188 brother\n",
      "201 business\n",
      "283 call\n",
      "301 called\n",
      "460 came\n",
      "159 cannot\n",
      "174 cant\n",
      "269 car\n",
      "499 care\n",
      "270 cause\n",
      "149 chance\n",
      "315 change\n",
      "155 chat\n",
      "157 check\n",
      "248 child\n",
      "291 children\n",
      "180 choose\n",
      "147 church\n",
      "306 close\n",
      "143 clothes\n",
      "147 cold\n",
      "504 com\n",
      "645 come\n",
      "250 comes\n",
      "197 coming\n",
      "175 company\n",
      "218 completely\n",
      "143 connected\n",
      "157 contact\n",
      "187 control\n",
      "1411 could\n",
      "221 couple\n",
      "215 course\n",
      "190 crazy\n",
      "167 cry\n",
      "163 cut\n",
      "319 dad\n",
      "199 date\n",
      "368 dating\n",
      "159 daughter\n",
      "1466 day\n",
      "496 days\n",
      "385 de\n",
      "176 deal\n",
      "226 decided\n",
      "170 deep\n",
      "171 depression\n",
      "161 die\n",
      "412 different\n",
      "143 dog\n",
      "441 done\n",
      "339 dont\n",
      "241 door\n",
      "214 dream\n",
      "158 dreams\n",
      "155 easy\n",
      "177 eat\n",
      "227 either\n",
      "471 else\n",
      "144 emotional\n",
      "166 en\n",
      "422 end\n",
      "148 ended\n",
      "197 engine\n",
      "222 enjoy\n",
      "460 enough\n",
      "200 ep\n",
      "155 especially\n",
      "152 et\n",
      "142 etc\n",
      "1711 even\n",
      "812 ever\n",
      "803 every\n",
      "202 everyday\n",
      "559 everyone\n",
      "668 everything\n",
      "147 ex\n",
      "296 experience\n",
      "423 eyes\n",
      "408 face\n",
      "240 fact\n",
      "181 fall\n",
      "628 family\n",
      "277 far\n",
      "260 father\n",
      "162 fear\n",
      "2003 feel\n",
      "530 feeling\n",
      "211 feelings\n",
      "248 feels\n",
      "540 felt\n",
      "307 finally\n",
      "856 find\n",
      "901 first\n",
      "143 food\n",
      "161 forever\n",
      "411 found\n",
      "333 free\n",
      "668 friend\n",
      "1029 friends\n",
      "192 front\n",
      "250 full\n",
      "246 fun\n",
      "172 future\n",
      "232 gave\n",
      "2441 get\n",
      "225 gets\n",
      "624 getting\n",
      "630 girl\n",
      "143 girlfriend\n",
      "372 girls\n",
      "547 give\n",
      "150 given\n",
      "1576 go\n",
      "678 god\n",
      "185 goes\n",
      "1227 going\n",
      "252 gone\n",
      "150 gonna\n",
      "1371 good\n",
      "144 google\n",
      "1238 got\n",
      "508 great\n",
      "223 group\n",
      "297 guess\n",
      "494 guy\n",
      "385 guys\n",
      "246 hair\n",
      "191 half\n",
      "358 hand\n",
      "232 hands\n",
      "221 happen\n",
      "268 happened\n",
      "620 happy\n",
      "574 hard\n",
      "496 hate\n",
      "252 haven\n",
      "395 head\n",
      "276 hear\n",
      "169 heard\n",
      "560 heart\n",
      "188 hell\n",
      "881 help\n",
      "336 high\n",
      "176 hit\n",
      "239 hold\n",
      "751 home\n",
      "397 hope\n",
      "179 hot\n",
      "200 hours\n",
      "443 house\n",
      "202 however\n",
      "179 huge\n",
      "212 human\n",
      "363 hurt\n",
      "332 husband\n",
      "192 idea\n",
      "196 important\n",
      "395 inside\n",
      "172 instead\n",
      "151 interested\n",
      "207 internet\n",
      "150 issues\n",
      "288 job\n",
      "583 keep\n",
      "191 kept\n",
      "306 kids\n",
      "312 kind\n",
      "144 kiss\n",
      "384 knew\n",
      "2721 know\n",
      "168 knowing\n",
      "217 knows\n",
      "195 la\n",
      "619 last\n",
      "284 later\n",
      "149 le\n",
      "148 learn\n",
      "253 least\n",
      "362 leave\n",
      "463 left\n",
      "219 less\n",
      "791 let\n",
      "2078 life\n",
      "204 light\n",
      "4029 like\n",
      "167 listen\n",
      "826 little\n",
      "520 live\n",
      "173 lives\n",
      "287 living\n",
      "325 lol\n",
      "793 long\n",
      "190 longer\n",
      "728 look\n",
      "307 looked\n",
      "637 looking\n",
      "331 lost\n",
      "566 lot\n",
      "2692 love\n",
      "299 loved\n",
      "215 loving\n",
      "747 made\n",
      "1353 make\n",
      "507 makes\n",
      "312 making\n",
      "719 man\n",
      "843 many\n",
      "153 marketing\n",
      "142 marriage\n",
      "218 married\n",
      "274 matter\n",
      "693 may\n",
      "422 maybe\n",
      "338 mean\n",
      "283 meet\n",
      "909 men\n",
      "330 message\n",
      "283 met\n",
      "356 might\n",
      "557 mind\n",
      "192 mine\n",
      "161 minutes\n",
      "242 miss\n",
      "413 mom\n",
      "241 moment\n",
      "284 money\n",
      "172 month\n",
      "402 months\n",
      "214 morning\n",
      "434 mother\n",
      "159 mouth\n",
      "219 move\n",
      "164 moved\n",
      "1376 much\n",
      "215 music\n",
      "250 must\n",
      "173 naked\n",
      "279 name\n",
      "1022 need\n",
      "188 needed\n",
      "202 needs\n",
      "1642 never\n",
      "816 new\n",
      "423 next\n",
      "348 nice\n",
      "638 night\n",
      "148 normal\n",
      "617 nothing\n",
      "225 often\n",
      "282 oh\n",
      "197 ok\n",
      "172 okay\n",
      "578 old\n",
      "627 older\n",
      "2817 one\n",
      "156 ones\n",
      "247 online\n",
      "254 open\n",
      "440 others\n",
      "159 outside\n",
      "332 pain\n",
      "372 parents\n",
      "358 part\n",
      "329 past\n",
      "2278 people\n",
      "283 perfect\n",
      "855 person\n",
      "214 phone\n",
      "426 place\n",
      "147 platform\n",
      "254 play\n",
      "157 playing\n",
      "465 please\n",
      "325 point\n",
      "155 possible\n",
      "180 post\n",
      "353 pretty\n",
      "259 probably\n",
      "219 problem\n",
      "143 problems\n",
      "495 put\n",
      "181 question\n",
      "249 quite\n",
      "169 rather\n",
      "304 read\n",
      "182 reading\n",
      "182 ready\n",
      "429 real\n",
      "152 realize\n",
      "1760 really\n",
      "302 reason\n",
      "143 recently\n",
      "494 relationship\n",
      "384 remember\n",
      "170 rest\n",
      "971 right\n",
      "359 room\n",
      "194 run\n",
      "141 running\n",
      "234 sad\n",
      "1048 said\n",
      "280 saw\n",
      "1003 say\n",
      "258 saying\n",
      "257 says\n",
      "227 scared\n",
      "635 school\n",
      "328 search\n",
      "196 second\n",
      "1299 see\n",
      "159 seeing\n",
      "164 seem\n",
      "236 seems\n",
      "220 seen\n",
      "319 self\n",
      "161 sense\n",
      "142 serious\n",
      "191 set\n",
      "173 several\n",
      "335 sex\n",
      "260 share\n",
      "213 short\n",
      "325 show\n",
      "163 sick\n",
      "254 side\n",
      "649 since\n",
      "219 single\n",
      "223 sister\n",
      "156 sit\n",
      "391 site\n",
      "195 skin\n",
      "287 sleep\n",
      "249 small\n",
      "199 smile\n",
      "182 social\n",
      "1215 someone\n",
      "991 something\n",
      "556 sometimes\n",
      "331 son\n",
      "157 song\n",
      "246 soon\n",
      "230 sorry\n",
      "163 soul\n",
      "171 special\n",
      "177 stand\n",
      "373 start\n",
      "680 started\n",
      "148 starting\n",
      "298 stay\n",
      "1100 still\n",
      "411 stop\n",
      "161 stopped\n",
      "142 stories\n",
      "304 story\n",
      "204 strong\n",
      "266 stuff\n",
      "183 stupid\n",
      "177 support\n",
      "488 sure\n",
      "811 take\n",
      "260 taking\n",
      "788 talk\n",
      "333 talking\n",
      "719 tell\n",
      "177 telling\n",
      "201 thank\n",
      "701 thing\n",
      "1191 things\n",
      "1506 think\n",
      "328 thinking\n",
      "517 though\n",
      "682 thought\n",
      "200 thoughts\n",
      "213 three\n",
      "2297 time\n",
      "519 times\n",
      "235 tired\n",
      "526 today\n",
      "481 together\n",
      "745 told\n",
      "379 took\n",
      "220 top\n",
      "171 touch\n",
      "145 towards\n",
      "317 tried\n",
      "291 true\n",
      "187 truly\n",
      "258 trust\n",
      "178 truth\n",
      "542 try\n",
      "498 trying\n",
      "195 turn\n",
      "214 turned\n",
      "636 two\n",
      "164 un\n",
      "366 understand\n",
      "435 use\n",
      "373 used\n",
      "184 usually\n",
      "222 wait\n",
      "146 waiting\n",
      "218 walk\n",
      "150 walked\n",
      "146 walking\n",
      "232 wanna\n",
      "2445 want\n",
      "581 wanted\n",
      "317 wants\n",
      "362 wasn\n",
      "234 watch\n",
      "180 watching\n",
      "177 water\n",
      "1272 way\n",
      "166 ways\n",
      "172 wear\n",
      "168 wearing\n",
      "250 web\n",
      "450 website\n",
      "276 week\n",
      "209 weeks\n",
      "165 weight\n",
      "160 weird\n",
      "1009 well\n",
      "682 went\n",
      "160 whatever\n",
      "213 white\n",
      "290 whole\n",
      "379 wife\n",
      "446 wish\n",
      "186 within\n",
      "516 without\n",
      "436 woman\n",
      "801 women\n",
      "172 wonder\n",
      "175 word\n",
      "306 words\n",
      "705 work\n",
      "221 working\n",
      "707 world\n",
      "187 worse\n",
      "2368 would\n",
      "260 wouldn\n",
      "192 write\n",
      "371 wrong\n",
      "643 year\n",
      "1051 years\n",
      "343 yes\n",
      "150 yesterday\n",
      "343 yet\n",
      "271 young\n",
      "406 younger\n"
     ]
    }
   ],
   "source": [
    "# so we see it's 15000 rows (corresponging to the the 15000 statuses)\n",
    "# and 5000 columns (corresponding to the top 5000 words used in the\n",
    "# statuses). Increase \"max_features\" in CountVectorizer to increase\n",
    "# the number of words.\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "#get the column names (the words in the bag)\n",
    "vocab = vectorizer.get_feature_names() \n",
    "\n",
    "# Sum up the counts of each vocabulary word\n",
    "dist = np.sum(train_data_features, axis=0)\n",
    "\n",
    "# For each, print the vocabulary word and the number of times it \n",
    "# appears in the training set\n",
    "for tag, count in zip(vocab, dist):\n",
    "    print count, tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4029 like\n",
      "2817 one\n",
      "2721 know\n",
      "2692 love\n",
      "2445 want\n",
      "2441 get\n",
      "2368 would\n",
      "2297 time\n",
      "2278 people\n",
      "2078 life\n",
      "2003 feel\n",
      "1760 really\n",
      "1711 even\n",
      "1642 never\n",
      "1576 go\n",
      "1506 think\n",
      "1467 back\n",
      "1466 day\n",
      "1411 could\n",
      "1376 much\n",
      "1371 good\n",
      "1353 make\n",
      "1299 see\n",
      "1272 way\n",
      "1266 always\n",
      "1238 got\n",
      "1227 going\n",
      "1215 someone\n",
      "1191 things\n",
      "1100 still\n",
      "1051 years\n",
      "1048 said\n",
      "1029 friends\n",
      "1022 need\n",
      "1009 well\n",
      "1003 say\n",
      "991 something\n",
      "971 right\n",
      "927 also\n",
      "909 men\n",
      "901 first\n",
      "881 help\n",
      "856 find\n",
      "855 person\n",
      "843 many\n",
      "842 around\n",
      "826 little\n",
      "816 new\n",
      "812 ever\n",
      "811 take\n",
      "803 every\n",
      "801 women\n",
      "793 long\n",
      "791 let\n",
      "788 anyone\n",
      "788 talk\n",
      "762 anything\n",
      "751 home\n",
      "747 made\n",
      "745 told\n",
      "741 away\n",
      "728 look\n",
      "719 man\n",
      "719 tell\n",
      "707 world\n",
      "705 work\n",
      "701 thing\n",
      "698 best\n",
      "693 may\n",
      "682 thought\n",
      "682 went\n",
      "680 started\n",
      "678 god\n",
      "668 everything\n",
      "668 friend\n",
      "649 since\n",
      "645 come\n",
      "643 year\n",
      "638 night\n",
      "637 looking\n",
      "636 two\n",
      "635 school\n",
      "630 girl\n",
      "628 family\n",
      "627 older\n",
      "624 getting\n",
      "623 bad\n",
      "623 better\n",
      "620 happy\n",
      "619 last\n",
      "617 nothing\n",
      "583 keep\n",
      "581 wanted\n",
      "578 old\n",
      "574 hard\n",
      "566 lot\n",
      "560 heart\n",
      "559 everyone\n",
      "557 mind\n",
      "556 sometimes\n",
      "547 give\n",
      "543 another\n",
      "542 try\n",
      "540 felt\n",
      "530 feeling\n",
      "526 today\n",
      "520 live\n",
      "519 times\n",
      "517 though\n",
      "516 without\n",
      "508 great\n",
      "507 makes\n",
      "504 com\n",
      "501 alone\n",
      "499 care\n",
      "498 trying\n",
      "496 days\n",
      "496 hate\n",
      "495 put\n",
      "494 guy\n",
      "494 relationship\n",
      "488 sure\n",
      "481 together\n",
      "475 believe\n",
      "471 else\n",
      "465 please\n",
      "463 left\n",
      "460 asked\n",
      "460 came\n",
      "460 enough\n",
      "450 website\n",
      "446 wish\n",
      "443 house\n",
      "441 done\n",
      "440 others\n",
      "436 woman\n",
      "435 use\n",
      "434 mother\n",
      "432 big\n",
      "429 actually\n",
      "429 real\n",
      "426 place\n",
      "424 body\n",
      "423 eyes\n",
      "423 next\n",
      "422 end\n",
      "422 maybe\n",
      "413 mom\n",
      "412 different\n",
      "411 found\n",
      "411 stop\n",
      "408 face\n",
      "406 younger\n",
      "402 months\n",
      "397 hope\n",
      "395 head\n",
      "395 inside\n",
      "391 site\n",
      "389 able\n",
      "385 de\n",
      "385 guys\n",
      "384 knew\n",
      "384 remember\n",
      "379 took\n",
      "379 wife\n",
      "373 start\n",
      "373 used\n",
      "372 girls\n",
      "372 parents\n",
      "371 wrong\n",
      "368 dating\n",
      "366 understand\n",
      "365 almost\n",
      "363 hurt\n",
      "362 leave\n",
      "362 wasn\n",
      "359 room\n",
      "358 age\n",
      "358 hand\n",
      "358 part\n",
      "356 might\n",
      "353 pretty\n",
      "349 beautiful\n",
      "348 nice\n",
      "343 yes\n",
      "343 yet\n",
      "339 bit\n",
      "339 dont\n",
      "338 mean\n",
      "337 ask\n",
      "336 high\n",
      "335 sex\n",
      "333 free\n",
      "333 talking\n",
      "332 husband\n",
      "332 pain\n",
      "331 lost\n",
      "331 son\n",
      "330 message\n",
      "329 past\n",
      "328 search\n",
      "328 thinking\n",
      "327 anymore\n",
      "325 lol\n",
      "325 point\n",
      "325 show\n",
      "323 bed\n",
      "321 ago\n",
      "319 dad\n",
      "319 self\n",
      "317 tried\n",
      "317 wants\n",
      "315 change\n",
      "312 kind\n",
      "312 making\n",
      "308 become\n",
      "307 finally\n",
      "307 looked\n",
      "306 close\n",
      "306 kids\n",
      "306 words\n",
      "304 read\n",
      "304 story\n",
      "302 reason\n",
      "301 called\n",
      "299 loved\n",
      "298 stay\n",
      "297 guess\n",
      "296 experience\n",
      "293 already\n",
      "291 children\n",
      "291 true\n",
      "290 whole\n",
      "288 job\n",
      "287 living\n",
      "287 sleep\n",
      "284 later\n",
      "284 money\n",
      "283 call\n",
      "283 meet\n",
      "283 met\n",
      "283 perfect\n",
      "282 oh\n",
      "280 saw\n",
      "279 name\n",
      "277 far\n",
      "276 hear\n",
      "276 week\n",
      "274 matter\n",
      "271 young\n",
      "270 cause\n",
      "269 car\n",
      "268 happened\n",
      "266 stuff\n",
      "260 father\n",
      "260 share\n",
      "260 taking\n",
      "260 wouldn\n",
      "259 probably\n",
      "258 saying\n",
      "258 trust\n",
      "257 says\n",
      "254 open\n",
      "254 play\n",
      "254 side\n",
      "253 boyfriend\n",
      "253 least\n",
      "252 gone\n",
      "252 haven\n",
      "250 comes\n",
      "250 full\n",
      "250 must\n",
      "250 web\n",
      "249 quite\n",
      "249 small\n",
      "248 child\n",
      "248 feels\n",
      "247 online\n",
      "246 fun\n",
      "246 hair\n",
      "246 soon\n",
      "242 miss\n",
      "241 door\n",
      "241 moment\n",
      "240 fact\n",
      "239 hold\n",
      "236 seems\n",
      "235 tired\n",
      "234 sad\n",
      "234 watch\n",
      "232 gave\n",
      "232 hands\n",
      "232 wanna\n",
      "230 sorry\n",
      "227 either\n",
      "227 scared\n",
      "226 decided\n",
      "225 gets\n",
      "225 often\n",
      "223 group\n",
      "223 sister\n",
      "222 enjoy\n",
      "222 wait\n",
      "221 couple\n",
      "221 happen\n",
      "221 working\n",
      "220 seen\n",
      "220 top\n",
      "219 less\n",
      "219 move\n",
      "219 problem\n",
      "219 single\n",
      "218 completely\n",
      "218 married\n",
      "218 walk\n",
      "217 baby\n",
      "217 knows\n",
      "216 boy\n",
      "215 course\n",
      "215 loving\n",
      "215 music\n",
      "214 dream\n",
      "214 morning\n",
      "214 phone\n",
      "214 turned\n",
      "213 short\n",
      "213 three\n",
      "213 white\n",
      "212 human\n",
      "211 feelings\n",
      "209 weeks\n",
      "207 internet\n",
      "204 light\n",
      "204 strong\n",
      "202 everyday\n",
      "202 however\n",
      "202 needs\n",
      "201 business\n",
      "201 thank\n",
      "200 ep\n",
      "200 hours\n",
      "200 thoughts\n",
      "199 date\n",
      "199 smile\n",
      "197 coming\n",
      "197 engine\n",
      "197 ok\n",
      "196 black\n",
      "196 important\n",
      "196 second\n",
      "195 amazing\n",
      "195 la\n",
      "195 skin\n",
      "195 turn\n",
      "194 run\n",
      "192 front\n",
      "192 idea\n",
      "192 mine\n",
      "192 write\n",
      "191 half\n",
      "191 kept\n",
      "191 set\n",
      "190 crazy\n",
      "190 longer\n",
      "188 brother\n",
      "188 hell\n",
      "188 needed\n",
      "187 control\n",
      "187 truly\n",
      "187 worse\n",
      "186 along\n",
      "186 within\n",
      "185 goes\n",
      "184 book\n",
      "184 usually\n",
      "183 stupid\n",
      "182 reading\n",
      "182 ready\n",
      "182 social\n",
      "181 fall\n",
      "181 question\n",
      "180 choose\n",
      "180 post\n",
      "180 watching\n",
      "179 answer\n",
      "179 hot\n",
      "179 huge\n",
      "178 behind\n",
      "178 truth\n",
      "177 eat\n",
      "177 stand\n",
      "177 support\n",
      "177 telling\n",
      "177 water\n",
      "176 deal\n",
      "176 hit\n",
      "175 company\n",
      "175 word\n",
      "174 cant\n",
      "173 lives\n",
      "173 naked\n",
      "173 several\n",
      "172 attention\n",
      "172 future\n",
      "172 instead\n",
      "172 month\n",
      "172 okay\n",
      "172 wear\n",
      "172 wonder\n",
      "171 depression\n",
      "171 special\n",
      "171 touch\n",
      "170 deep\n",
      "170 rest\n",
      "169 heard\n",
      "169 rather\n",
      "168 air\n",
      "168 knowing\n",
      "168 wearing\n",
      "167 cry\n",
      "167 listen\n",
      "166 en\n",
      "166 ways\n",
      "165 weight\n",
      "164 moved\n",
      "164 seem\n",
      "164 un\n",
      "163 cut\n",
      "163 sick\n",
      "163 soul\n",
      "162 break\n",
      "162 fear\n",
      "161 die\n",
      "161 forever\n",
      "161 minutes\n",
      "161 sense\n",
      "161 stopped\n",
      "160 weird\n",
      "160 whatever\n",
      "159 cannot\n",
      "159 daughter\n",
      "159 mouth\n",
      "159 outside\n",
      "159 seeing\n",
      "158 dreams\n",
      "157 check\n",
      "157 contact\n",
      "157 playing\n",
      "157 song\n",
      "156 ones\n",
      "156 sit\n",
      "155 chat\n",
      "155 easy\n",
      "155 especially\n",
      "155 possible\n",
      "153 marketing\n",
      "152 et\n",
      "152 realize\n",
      "151 anxiety\n",
      "151 became\n",
      "151 bring\n",
      "151 interested\n",
      "150 given\n",
      "150 gonna\n",
      "150 issues\n",
      "150 walked\n",
      "150 yesterday\n",
      "149 agemeet\n",
      "149 chance\n",
      "149 le\n",
      "148 ended\n",
      "148 learn\n",
      "148 normal\n",
      "148 starting\n",
      "147 barb\n",
      "147 church\n",
      "147 cold\n",
      "147 ex\n",
      "147 platform\n",
      "146 although\n",
      "146 waiting\n",
      "146 walking\n",
      "145 towards\n",
      "144 emotional\n",
      "144 google\n",
      "144 kiss\n",
      "143 afraid\n",
      "143 clothes\n",
      "143 connected\n",
      "143 dog\n",
      "143 food\n",
      "143 girlfriend\n",
      "143 problems\n",
      "143 recently\n",
      "142 anyway\n",
      "142 etc\n",
      "142 marriage\n",
      "142 serious\n",
      "142 stories\n",
      "141 running\n"
     ]
    }
   ],
   "source": [
    "#let's order them by frequency\n",
    "\n",
    "testwords = zip(vocab, dist)\n",
    "for tag, count in sorted(testwords, key=lambda testwords: testwords[1], reverse=True):\n",
    "    print count, tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "forest = RandomForestClassifier(n_estimators = 100) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "forest = forest.fit( train_data_features, subset[\"gender\"] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gid</th>\n",
       "      <th>content</th>\n",
       "      <th>gender</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>828259</td>\n",
       "      <td>go ahead ,I'll be as honest as possible</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1025005</td>\n",
       "      <td>Anyone ever tried this?,  multiple times?,  ma...</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>968499</td>\n",
       "      <td>I got this game for my birthday last year and ...</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>381536</td>\n",
       "      <td>throw me against the wall and choke me</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>353615</td>\n",
       "      <td>i have a Fit body so i hope i can find boyfrie...</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       gid                                            content gender\n",
       "0   828259            go ahead ,I'll be as honest as possible      M\n",
       "1  1025005  Anyone ever tried this?,  multiple times?,  ma...      M\n",
       "2   968499  I got this game for my birthday last year and ...      M\n",
       "3   381536             throw me against the wall and choke me      F\n",
       "4   353615  i have a Fit body so i hope i can find boyfrie...      F"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ntest = 20000\n",
    "data_test = data_test[:ntest]\n",
    "data_test.index = range(data_test.shape[0])\n",
    "data_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaning and parsing the test statuses...\n",
      "\n",
      "Status 1000 of 20000\n",
      "\n",
      "Status 2000 of 20000\n",
      "\n",
      "Status 3000 of 20000\n",
      "\n",
      "Status 4000 of 20000\n",
      "\n",
      "Status 5000 of 20000\n",
      "\n",
      "Status 6000 of 20000\n",
      "\n",
      "Status 7000 of 20000\n",
      "\n",
      "Status 8000 of 20000\n",
      "\n",
      "Status 9000 of 20000\n",
      "\n",
      "Status 10000 of 20000\n",
      "\n",
      "354 able\n",
      "394 actually\n",
      "153 afraid\n",
      "328 age\n",
      "115 agemeet\n",
      "334 ago\n",
      "105 air\n",
      "362 almost\n",
      "470 alone\n",
      "186 along\n",
      "270 already\n",
      "881 also\n",
      "134 although\n",
      "1179 always\n",
      "219 amazing\n",
      "539 another\n",
      "148 answer\n",
      "112 anxiety\n",
      "298 anymore\n",
      "777 anyone\n",
      "681 anything\n",
      "162 anyway\n",
      "789 around\n",
      "369 ask\n",
      "374 asked\n",
      "159 attention\n",
      "679 away\n",
      "213 baby\n",
      "1357 back\n",
      "576 bad\n",
      "24 barb\n",
      "384 beautiful\n",
      "148 became\n",
      "262 become\n",
      "323 bed\n",
      "177 behind\n",
      "462 believe\n",
      "605 best\n",
      "621 better\n",
      "405 big\n",
      "315 bit\n",
      "270 black\n",
      "406 body\n",
      "119 book\n",
      "161 boy\n",
      "256 boyfriend\n",
      "146 break\n",
      "152 bring\n",
      "192 brother\n",
      "167 business\n",
      "314 call\n",
      "254 called\n",
      "479 came\n",
      "147 cannot\n",
      "143 cant\n",
      "220 car\n",
      "515 care\n",
      "259 cause\n",
      "174 chance\n",
      "322 change\n",
      "192 chat\n",
      "136 check\n",
      "231 child\n",
      "205 children\n",
      "147 choose\n",
      "93 church\n",
      "292 close\n",
      "154 clothes\n",
      "125 cold\n",
      "351 com\n",
      "657 come\n",
      "240 comes\n",
      "188 coming\n",
      "157 company\n",
      "173 completely\n",
      "115 connected\n",
      "142 contact\n",
      "183 control\n",
      "1289 could\n",
      "245 couple\n",
      "200 course\n",
      "194 crazy\n",
      "175 cry\n",
      "133 cut\n",
      "328 dad\n",
      "157 date\n",
      "277 dating\n",
      "156 daughter\n",
      "1498 day\n",
      "484 days\n",
      "227 de\n",
      "145 deal\n",
      "212 decided\n",
      "157 deep\n",
      "151 depression\n",
      "177 die\n",
      "380 different\n",
      "94 dog\n",
      "426 done\n",
      "324 dont\n",
      "184 door\n",
      "278 dream\n",
      "174 dreams\n",
      "169 easy\n",
      "189 eat\n",
      "200 either\n",
      "450 else\n",
      "107 emotional\n",
      "102 en\n",
      "438 end\n",
      "159 ended\n",
      "118 engine\n",
      "230 enjoy\n",
      "422 enough\n",
      "315 ep\n",
      "197 especially\n",
      "33 et\n",
      "150 etc\n",
      "1602 even\n",
      "752 ever\n",
      "810 every\n",
      "129 everyday\n",
      "556 everyone\n",
      "620 everything\n",
      "129 ex\n",
      "303 experience\n",
      "371 eyes\n",
      "399 face\n",
      "243 fact\n",
      "186 fall\n",
      "594 family\n",
      "263 far\n",
      "217 father\n",
      "144 fear\n",
      "1981 feel\n",
      "548 feeling\n",
      "213 feelings\n",
      "201 feels\n",
      "538 felt\n",
      "292 finally\n",
      "843 find\n",
      "808 first\n",
      "173 food\n",
      "154 forever\n",
      "437 found\n",
      "353 free\n",
      "707 friend\n",
      "976 friends\n",
      "162 front\n",
      "248 full\n",
      "303 fun\n",
      "160 future\n",
      "225 gave\n",
      "2232 get\n",
      "195 gets\n",
      "549 getting\n",
      "641 girl\n",
      "141 girlfriend\n",
      "416 girls\n",
      "576 give\n",
      "143 given\n",
      "1450 go\n",
      "476 god\n",
      "192 goes\n",
      "1052 going\n",
      "222 gone\n",
      "181 gonna\n",
      "1218 good\n",
      "122 google\n",
      "1128 got\n",
      "462 great\n",
      "185 group\n",
      "273 guess\n",
      "420 guy\n",
      "354 guys\n",
      "231 hair\n",
      "176 half\n",
      "307 hand\n",
      "211 hands\n",
      "198 happen\n",
      "256 happened\n",
      "594 happy\n",
      "570 hard\n",
      "543 hate\n",
      "211 haven\n",
      "336 head\n",
      "233 hear\n",
      "189 heard\n",
      "506 heart\n",
      "158 hell\n",
      "886 help\n",
      "296 high\n",
      "145 hit\n",
      "219 hold\n",
      "664 home\n",
      "359 hope\n",
      "170 hot\n",
      "237 hours\n",
      "447 house\n",
      "189 however\n",
      "118 huge\n",
      "147 human\n",
      "366 hurt\n",
      "314 husband\n",
      "183 idea\n",
      "171 important\n",
      "334 inside\n",
      "172 instead\n",
      "170 interested\n",
      "163 internet\n",
      "143 issues\n",
      "241 job\n",
      "513 keep\n",
      "192 kept\n",
      "302 kids\n",
      "303 kind\n",
      "152 kiss\n",
      "366 knew\n",
      "2562 know\n",
      "140 knowing\n",
      "183 knows\n",
      "125 la\n",
      "644 last\n",
      "284 later\n",
      "18 le\n",
      "144 learn\n",
      "240 least\n",
      "365 leave\n",
      "496 left\n",
      "210 less\n",
      "736 let\n",
      "1787 life\n",
      "179 light\n",
      "3825 like\n",
      "144 listen\n",
      "780 little\n",
      "518 live\n",
      "161 lives\n",
      "265 living\n",
      "353 lol\n",
      "712 long\n",
      "176 longer\n",
      "680 look\n",
      "239 looked\n",
      "597 looking\n",
      "309 lost\n",
      "571 lot\n",
      "2481 love\n",
      "296 loved\n",
      "201 loving\n",
      "721 made\n",
      "1240 make\n",
      "434 makes\n",
      "295 making\n",
      "609 man\n",
      "832 many\n",
      "75 marketing\n",
      "140 marriage\n",
      "231 married\n",
      "270 matter\n",
      "500 may\n",
      "466 maybe\n",
      "337 mean\n",
      "285 meet\n",
      "736 men\n",
      "341 message\n",
      "248 met\n",
      "321 might\n",
      "477 mind\n",
      "187 mine\n",
      "166 minutes\n",
      "273 miss\n",
      "514 mom\n",
      "232 moment\n",
      "271 money\n",
      "172 month\n",
      "316 months\n",
      "265 morning\n",
      "318 mother\n",
      "113 mouth\n",
      "209 move\n",
      "137 moved\n",
      "1333 much\n",
      "210 music\n",
      "207 must\n",
      "163 naked\n",
      "276 name\n",
      "1067 need\n",
      "184 needed\n",
      "183 needs\n",
      "1539 never\n",
      "737 new\n",
      "406 next\n",
      "343 nice\n",
      "624 night\n",
      "173 normal\n",
      "562 nothing\n",
      "251 often\n",
      "313 oh\n",
      "228 ok\n",
      "176 okay\n",
      "597 old\n",
      "473 older\n",
      "2877 one\n",
      "157 ones\n",
      "216 online\n",
      "305 open\n",
      "417 others\n",
      "190 outside\n",
      "342 pain\n",
      "301 parents\n",
      "315 part\n",
      "296 past\n",
      "2215 people\n",
      "236 perfect\n",
      "778 person\n",
      "215 phone\n",
      "401 place\n",
      "124 platform\n",
      "247 play\n",
      "159 playing\n",
      "427 please\n",
      "308 point\n",
      "159 possible\n",
      "209 post\n",
      "281 pretty\n",
      "251 probably\n",
      "233 problem\n",
      "143 problems\n",
      "505 put\n",
      "140 question\n",
      "216 quite\n",
      "165 rather\n",
      "273 read\n",
      "124 reading\n",
      "147 ready\n",
      "375 real\n",
      "167 realize\n",
      "1708 really\n",
      "318 reason\n",
      "163 recently\n",
      "455 relationship\n",
      "370 remember\n",
      "178 rest\n",
      "895 right\n",
      "360 room\n",
      "176 run\n",
      "115 running\n",
      "238 sad\n",
      "1065 said\n",
      "277 saw\n",
      "925 say\n",
      "242 saying\n",
      "415 says\n",
      "186 scared\n",
      "594 school\n",
      "211 search\n",
      "195 second\n",
      "1323 see\n",
      "153 seeing\n",
      "179 seem\n",
      "183 seems\n",
      "245 seen\n",
      "332 self\n",
      "136 sense\n",
      "98 serious\n",
      "145 set\n",
      "171 several\n",
      "307 sex\n",
      "194 share\n",
      "162 short\n",
      "269 show\n",
      "165 sick\n",
      "223 side\n",
      "618 since\n",
      "244 single\n",
      "215 sister\n",
      "161 sit\n",
      "271 site\n",
      "167 skin\n",
      "319 sleep\n",
      "221 small\n",
      "203 smile\n",
      "148 social\n",
      "1172 someone\n",
      "989 something\n",
      "481 sometimes\n",
      "165 son\n",
      "151 song\n",
      "241 soon\n",
      "182 sorry\n",
      "165 soul\n",
      "131 special\n",
      "162 stand\n",
      "377 start\n",
      "613 started\n",
      "130 starting\n",
      "268 stay\n",
      "1122 still\n",
      "421 stop\n",
      "141 stopped\n",
      "129 stories\n",
      "270 story\n",
      "151 strong\n",
      "232 stuff\n",
      "170 stupid\n",
      "188 support\n",
      "441 sure\n",
      "810 take\n",
      "221 taking\n",
      "684 talk\n",
      "330 talking\n",
      "710 tell\n",
      "156 telling\n",
      "154 thank\n",
      "650 thing\n",
      "1164 things\n",
      "1434 think\n",
      "332 thinking\n",
      "527 though\n",
      "587 thought\n",
      "218 thoughts\n",
      "242 three\n",
      "2159 time\n",
      "478 times\n",
      "207 tired\n",
      "572 today\n",
      "435 together\n",
      "767 told\n",
      "329 took\n",
      "199 top\n",
      "185 touch\n",
      "128 towards\n",
      "313 tried\n",
      "257 true\n",
      "174 truly\n",
      "172 trust\n",
      "166 truth\n",
      "552 try\n",
      "438 trying\n",
      "206 turn\n",
      "190 turned\n",
      "588 two\n",
      "49 un\n",
      "345 understand\n",
      "410 use\n",
      "395 used\n",
      "205 usually\n",
      "184 wait\n",
      "137 waiting\n",
      "224 walk\n",
      "159 walked\n",
      "133 walking\n",
      "253 wanna\n",
      "2310 want\n",
      "517 wanted\n",
      "239 wants\n",
      "301 wasn\n",
      "241 watch\n",
      "182 watching\n",
      "184 water\n",
      "1185 way\n",
      "124 ways\n",
      "216 wear\n",
      "179 wearing\n",
      "216 web\n",
      "358 website\n",
      "293 week\n",
      "212 weeks\n",
      "170 weight\n",
      "170 weird\n",
      "880 well\n",
      "683 went\n",
      "147 whatever\n",
      "148 white\n",
      "294 whole\n",
      "332 wife\n",
      "510 wish\n",
      "168 within\n",
      "487 without\n",
      "453 woman\n",
      "752 women\n",
      "172 wonder\n",
      "157 word\n",
      "235 words\n",
      "708 work\n",
      "231 working\n",
      "646 world\n",
      "153 worse\n",
      "2264 would\n",
      "232 wouldn\n",
      "170 write\n",
      "367 wrong\n",
      "571 year\n",
      "1006 years\n",
      "322 yes\n",
      "129 yesterday\n",
      "352 yet\n",
      "241 young\n",
      "344 younger\n"
     ]
    }
   ],
   "source": [
    "clean_test_statuses = [] \n",
    "\n",
    "print \"Cleaning and parsing the test statuses...\\n\"\n",
    "for i in xrange(0,num_test):\n",
    "    if( (i+1) % 1000 == 0 ):\n",
    "        print \"Status %d of %d\\n\" % (i+1, ntest)\n",
    "    clean_status = status_to_words( data_test[\"content\"][i] )\n",
    "    clean_test_statuses.append( clean_status )\n",
    "\n",
    "# Get a bag of words for the test set, and convert to a numpy array\n",
    "test_data_features = vectorizer.transform(clean_test_statuses)\n",
    "test_data_features = test_data_features.toarray()\n",
    "vocab = vectorizer.get_feature_names() \n",
    "\n",
    "# Sum up the counts of each vocabulary word\n",
    "dist = np.sum(test_data_features, axis=0)\n",
    "\n",
    "# For each, print the vocabulary word and the number of times it \n",
    "# appears in the training set\n",
    "for tag, count in zip(vocab, dist):\n",
    "    print count, tag\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Use the random forest to make sentiment label predictions\n",
    "result = forest.predict(test_data_features)\n",
    "\n",
    "# Copy the results to a pandas dataframe with an \"id\" column and\n",
    "# a \"sentiment\" column\n",
    "output = pd.DataFrame( data={\"gid\":data_test[\"gid\"], \"P_gender\":result,\"R_gender\":data_test['gender']} )\n",
    "\n",
    "# Use pandas to write the comma-separated output file\n",
    "#output.to_csv( \"Bag_of_Words_model.csv\", index=False, quoting=3 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.256"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "P_correct = round(sum(output['P_gender']==output['R_gender']),2)/ntest\n",
    "P_correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ntest= 20000\n"
     ]
    }
   ],
   "source": [
    "print \"ntest=\",ntest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import random_forest as rf\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"http://www.thenational.ae/world/middle-east/video-shows-isil-executions-of-ethiopian-christians-in-libya?\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n",
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"http://www.cam4.com/dab1gd0g\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n",
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"https://m.youtube.com/watch?v=uF16uJwIE0U\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n",
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"http://timesofindia.indiatimes.com/entertainment/bengali/movies/news/The-making-of-Pather-Panchali-Q-to-play-Satyajit-Ray/articleshow/47323279.cms?\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaning and parsing train statuses...\n",
      "\n",
      "Status 1000 of 10000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"http://youtu.be/JFg3JuwxKKA\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n",
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"http://youtu.be/ZZ_rYTU8xm0\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Status 2000 of 10000\n",
      "\n",
      "Status 3000 of 10000\n",
      "\n",
      "Status 4000 of 10000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"https://www.youtube.com/watch?v=sc5WPpqn13k\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n",
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"http://sexylovestories.wordpress.com/2015/01/08/love-is-dead/\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Status 5000 of 10000\n",
      "\n",
      "Status 6000 of 10000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"https://m.youtube.com/watch?v=XVdbEgyxKgs\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n",
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"https://www.youtube.com/watch?v=mvCgSqPZ4EM\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n",
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"https://youtu.be/59GQbd8CEO4?t=1m51s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n",
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"https://youtu.be/vkWOBW7GA54\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Status 7000 of 10000\n",
      "\n",
      "Status 8000 of 10000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"https://www.youtube.com/watch?v=57WvGQWYAdY\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n",
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"http://youtu.be/s-XolL_1dN0\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Status 9000 of 10000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"https://m.youtube.com/watch?v=p9OBf8f55tU\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n",
      "/Users/Jade/anaconda/envs/python2/lib/python2.7/site-packages/bs4/__init__.py:207: UserWarning: \"http://youtu.be/39wszj0l3U0?list=LLx1fqbKXY8VTqKpw-GLcRJA\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  '\"%s\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Status 10000 of 10000\n",
      "\n",
      "Creating the bag of words...\n",
      "\n",
      "Fitting to a random forest with "
     ]
    }
   ],
   "source": [
    "#EP = pd.read_csv(\"EP_data.csv\", header=0).dropna()\n",
    "target = 'gid'\n",
    "ntrain = 10000\n",
    "ntest = 5000\n",
    "result = rf.rand_forest_predict(EP,'content',target,ntrain,ntest,nfeature=1000,nestimator=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The proportion of correct prediction for  gender  is "
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.44"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print \"The proportion of correct prediction for \",target, \" is \", \n",
    "round(sum(result['Prediction']==result['True Value']),2)/ntest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}