scikit_opts_python2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Preparing Data\n",
    "=============="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import cPickle as pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import factorize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bossa/control_tasks.json       bossa/tasks_export.json\r\n",
      "bossa/control_tasks_runs.json  bossa/tasks_runs_export.json\r\n"
     ]
    }
   ],
   "source": [
    "!ls bossa/*json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BOSSA Results\n",
    "-------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Processing `results_bossa.json` to get a *dictionary* with keys the task ids, and values in as the average value of the scores. To do that, we first convert scores from categorical (`neg`, `neu`, `pos`) to a numeric scale."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>result_id</th>\n",
       "      <th>seconds</th>\n",
       "      <th>task_id</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>11203</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>52775</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    result_id   seconds  task_id score\n",
       "50      11203  0.000025    52775     1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bossa_results = pd.read_json(\"bossa/tasks_runs_export.json\")\n",
    "bossa_results.rename(columns={\"created\": \"start_time\", \"id\": \"result_id\", \"info\": \"score\"}, inplace=True)\n",
    "bossa_results[['start_time']]= bossa_results[['start_time']].apply(pd.to_datetime, dayfirst=True)\n",
    "bossa_results[['finish_time']]= bossa_results[['finish_time']].apply(pd.to_datetime, dayfirst=True)\n",
    "bossa_results['score'] = pd.Categorical(bossa_results['score'], categories=['vneg', 'neg', 'neu', 'pos', 'vpos'])\n",
    "bossa_results['score'].cat.rename_categories([-2, -1, 0, 1, 2], inplace=True)\n",
    "# Normalize everything to -1, 0, 1\n",
    "# bossa_results['score'] = bossa_results['score'].astype(float).apply(lambda x: -1 if x < 0 else 1 if x > 0 else 0)\n",
    "bossa_results[\"seconds\"] = (bossa_results[\"finish_time\"] - bossa_results[\"start_time\"]).astype('timedelta64[us]') / 1e6\n",
    "bossa_results = bossa_results[[\"result_id\", \"seconds\", \"task_id\", \"score\"]]\n",
    "bossa_results.ix[[50]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The information about the sentence comes in a dictionary inside the cells of the serie `info`, so we expand it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task_id</th>\n",
       "      <th>info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>52851</td>\n",
       "      <td>{u'search_words': u'founder', u'appears_in_sen...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    task_id                                               info\n",
       "50    52851  {u'search_words': u'founder', u'appears_in_sen..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bossa_tasks = pd.read_json(\"bossa/tasks_export.json\")\n",
    "bossa_tasks[['created']]= bossa_tasks[['created']].apply(pd.to_datetime, dayfirst=True)\n",
    "bossa_tasks.rename(columns={'id': 'task_id'}, inplace=True)\n",
    "bossa_tasks = bossa_tasks[['task_id', 'info']]\n",
    "bossa_tasks.ix[[50]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And finally we merge the `DataFrame` with the scores with the one containing the sentences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>result_id</th>\n",
       "      <th>seconds</th>\n",
       "      <th>task_id</th>\n",
       "      <th>score</th>\n",
       "      <th>info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>11195</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>52776</td>\n",
       "      <td>2</td>\n",
       "      <td>{u'search_words': u'executive', u'appears_in_s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    result_id   seconds  task_id  score  \\\n",
       "50      11195  0.000021    52776      2   \n",
       "\n",
       "                                                 info  \n",
       "50  {u'search_words': u'executive', u'appears_in_s...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bossa_tasks_scores = pd.merge(bossa_results, bossa_tasks, on='task_id')\n",
    "bossa_tasks_scores.ix[[50]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's now expand the column `info` into as many new columns as keys has the dictionary `info`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'search_words',\n",
       " u'appears_in_sentence',\n",
       " u'url',\n",
       " u'media',\n",
       " u'appears_in_noun_phrases',\n",
       " u'noun_phrases',\n",
       " u'sentence_id',\n",
       " u'text',\n",
       " u'sentence',\n",
       " u'pub_date',\n",
       " u'is_company']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bossa_tasks_scores.ix[50].info.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>result_id</th>\n",
       "      <th>seconds</th>\n",
       "      <th>task_id</th>\n",
       "      <th>score</th>\n",
       "      <th>search_words</th>\n",
       "      <th>appears_in_sentence</th>\n",
       "      <th>url</th>\n",
       "      <th>media</th>\n",
       "      <th>appears_in_noun_phrases</th>\n",
       "      <th>noun_phrases</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>text</th>\n",
       "      <th>sentence</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>is_company</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>11195</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>52776</td>\n",
       "      <td>2</td>\n",
       "      <td>executive</td>\n",
       "      <td>0</td>\n",
       "      <td>http://dealbook.nytimes.com/2013/05/17/a-toeho...</td>\n",
       "      <td>nyt</td>\n",
       "      <td>0</td>\n",
       "      <td>[chinese investors, overseas companies, politi...</td>\n",
       "      <td>14</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>2013-05-17T11:47:51Z</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>11205</td>\n",
       "      <td>0.000018</td>\n",
       "      <td>52776</td>\n",
       "      <td>-1</td>\n",
       "      <td>executive</td>\n",
       "      <td>0</td>\n",
       "      <td>http://dealbook.nytimes.com/2013/05/17/a-toeho...</td>\n",
       "      <td>nyt</td>\n",
       "      <td>0</td>\n",
       "      <td>[chinese investors, overseas companies, politi...</td>\n",
       "      <td>14</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>2013-05-17T11:47:51Z</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>11207</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>52776</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>0</td>\n",
       "      <td>http://dealbook.nytimes.com/2013/05/17/a-toeho...</td>\n",
       "      <td>nyt</td>\n",
       "      <td>0</td>\n",
       "      <td>[chinese investors, overseas companies, politi...</td>\n",
       "      <td>14</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>2013-05-17T11:47:51Z</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>11209</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>52776</td>\n",
       "      <td>-2</td>\n",
       "      <td>executive</td>\n",
       "      <td>0</td>\n",
       "      <td>http://dealbook.nytimes.com/2013/05/17/a-toeho...</td>\n",
       "      <td>nyt</td>\n",
       "      <td>0</td>\n",
       "      <td>[chinese investors, overseas companies, politi...</td>\n",
       "      <td>14</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>2013-05-17T11:47:51Z</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    result_id   seconds  task_id  score search_words appears_in_sentence  \\\n",
       "50      11195  0.000021    52776      2    executive                   0   \n",
       "51      11205  0.000018    52776     -1    executive                   0   \n",
       "52      11207  0.000017    52776      1    executive                   0   \n",
       "53      11209  0.000017    52776     -2    executive                   0   \n",
       "\n",
       "                                                  url media  \\\n",
       "50  http://dealbook.nytimes.com/2013/05/17/a-toeho...   nyt   \n",
       "51  http://dealbook.nytimes.com/2013/05/17/a-toeho...   nyt   \n",
       "52  http://dealbook.nytimes.com/2013/05/17/a-toeho...   nyt   \n",
       "53  http://dealbook.nytimes.com/2013/05/17/a-toeho...   nyt   \n",
       "\n",
       "   appears_in_noun_phrases                                       noun_phrases  \\\n",
       "50                       0  [chinese investors, overseas companies, politi...   \n",
       "51                       0  [chinese investors, overseas companies, politi...   \n",
       "52                       0  [chinese investors, overseas companies, politi...   \n",
       "53                       0  [chinese investors, overseas companies, politi...   \n",
       "\n",
       "   sentence_id                                               text  \\\n",
       "50          14  Chinese investors are increasingly opting to b...   \n",
       "51          14  Chinese investors are increasingly opting to b...   \n",
       "52          14  Chinese investors are increasingly opting to b...   \n",
       "53          14  Chinese investors are increasingly opting to b...   \n",
       "\n",
       "                                             sentence              pub_date  \\\n",
       "50  Chinese investors are increasingly opting to b...  2013-05-17T11:47:51Z   \n",
       "51  Chinese investors are increasingly opting to b...  2013-05-17T11:47:51Z   \n",
       "52  Chinese investors are increasingly opting to b...  2013-05-17T11:47:51Z   \n",
       "53  Chinese investors are increasingly opting to b...  2013-05-17T11:47:51Z   \n",
       "\n",
       "   is_company  \n",
       "50          0  \n",
       "51          0  \n",
       "52          0  \n",
       "53          0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def json_to_series(info):\n",
    "    keys, values = zip(*info.iteritems())\n",
    "    return pd.Series(values, index=keys)\n",
    "\n",
    "bossa_info = bossa_tasks_scores[\"info\"].apply(json_to_series)\n",
    "bossa_info.reset_index()\n",
    "bossa = pd.concat([bossa_tasks_scores, bossa_info], axis=1)\n",
    "bossa.pop(\"info\")\n",
    "# bossa['id'] = bossa['id'].astype(float)\n",
    "bossa.ix[50:53]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Aggregate\n",
    "---------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now aggregate calculating the average per `sentence_id` using a group by. In the process, we lose the source of the data, that's why we first have to save it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bossa.to_csv(\"sentiment/scores_ungrouped.csv\", encoding=\"utf8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we aggregate and create a new `DataFrame` for the different sentences and their score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score    8996\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sentence</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>'We must hope after so much prevarication that this time Google's proposals represent a genuine attempt to address the concerns identified,' said David Wood, the legal counsel for Icomp, an industry group backed by Microsoft and a number of other companies.</th>\n",
       "      <td>-0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>'We must push our leaders to step up and commit to action,' said Hugh Evans, the founder and chief executive of the charity.</th>\n",
       "      <td>-0.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>'We need them to tell the story of how we are making decisions and putting the organization together,' said George Postolos, the Astros' president and chief executive, who added that the team would not want a broadcaster who was uncomfortable explaining the front office's strategy.</th>\n",
       "      <td>-0.666667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                               score\n",
       "sentence                                                                                                                                                                                                                                                                                            \n",
       "'We must hope after so much prevarication that this time Google's proposals represent a genuine attempt to address the concerns identified,' said David Wood, the legal counsel for Icomp, an industry group backed by Microsoft and a number of other companies.                          -0.333333\n",
       "'We must push our leaders to step up and commit to action,' said Hugh Evans, the founder and chief executive of the charity.                                                                                                                                                               -0.285714\n",
       "'We need them to tell the story of how we are making decisions and putting the organization together,' said George Postolos, the Astros' president and chief executive, who added that the team would not want a broadcaster who was uncomfortable explaining the front office's strategy. -0.666667"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences = bossa.groupby(['sentence'])[['score']].aggregate(np.average)\n",
    "sentences.to_csv(\"sentiment/scores.csv\", encoding=\"utf8\")\n",
    "print(sentences.count())\n",
    "sentences[1001:1004]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "def majority(series): #receives a Pandas Series\n",
    "    return Counter(map(lambda x: 1 if x > 0 else -1 if x < 0 else 0, series)).most_common(1)[0][0]\n",
    "\n",
    "# score_calculate = np.average\n",
    "score_calculate = majority\n",
    "\n",
    "sentences = bossa.groupby(['sentence'])[['score']].aggregate(score_calculate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create the tranining and testing sets (data and labels) from a randomized version of the set of assessed sentences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sentence    8996\n",
       "score       8996\n",
       "dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences.reset_index().count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Zero:', 2415)\n",
      "('Non-zero:', 6581)\n"
     ]
    }
   ],
   "source": [
    "raw_scores = sentences.reset_index()\n",
    "scores = raw_scores\n",
    "print('Zero:', len(scores[scores.score==0]))\n",
    "print('Non-zero:', len(scores[scores.score!=0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We could consider 3 classes, but it toruns out that using binary classficication seems to produce better results. Still, try multi-classs classifiers is something worth trying."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/antonio/.virtualenvs/lexisnexis/lib/python2.7/site-packages/IPython/kernel/__main__.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "raw_scores = sentences.reset_index()\n",
    "scores = raw_scores\n",
    "scores = scores[scores.score!=0]  # We ignore the neutral sentences\n",
    "scores['sentiment'] = scores['score'].apply(lambda s: 'pos' if s > 0 else 'neg')\n",
    "# scores['sentiment'] = scores['score'].apply(lambda s: 'pos' if s > 0 else 'neg' if s < 0 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6581"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sentences_df = scores[['sentence', 'sentiment']]\n",
    "sentences_df = sentences_df.reindex(np.random.permutation(sentences_df.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3171</th>\n",
       "      <td>Eric Feinberg, founder of the advocacy group F...</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6509</th>\n",
       "      <td>Shares in Apple closed up slightly on Thursday...</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4833</th>\n",
       "      <td>Joseph Petro, an executive with Morningstar's ...</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7703</th>\n",
       "      <td>The guide, titled 'Champions of Respect,' was ...</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4267</th>\n",
       "      <td>In addition to the flexibility of being a clow...</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               sentence sentiment\n",
       "3171  Eric Feinberg, founder of the advocacy group F...       neg\n",
       "6509  Shares in Apple closed up slightly on Thursday...       pos\n",
       "4833  Joseph Petro, an executive with Morningstar's ...       neg\n",
       "7703  The guide, titled 'Champions of Respect,' was ...       pos\n",
       "4267  In addition to the flexibility of being a clow...       pos"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#SCIKIT-LEARN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import logging\n",
    "import numpy as np\n",
    "from optparse import OptionParser\n",
    "import sys\n",
    "from time import time\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn import metrics\n",
    "from sklearn.cross_validation import cross_val_score, StratifiedKFold,train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "from sklearn.linear_model import PassiveAggressiveClassifier, Perceptron, RidgeClassifier, SGDClassifier\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn.naive_bayes import BernoulliNB, GaussianNB, MultinomialNB\n",
    "from sklearn.neighbors import KNeighborsClassifier, NearestCentroid\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.utils.extmath import density"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Display progress logs on stdout\n",
    "logging.basicConfig(level=logging.INFO,\n",
    "                    format='%(asctime)s %(levelname)s %(message)s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "# categories = [\n",
    "#     'pos',\n",
    "#     'neg',\n",
    "# ]\n",
    "# print(\"Loading sentences for categories:\")\n",
    "# print(categories)\n",
    "# data_train, data_test = train_test_split(document_df, train_size=0.9, test_size=0.1, random_state=100)\n",
    "# print('Data loaded:')\n",
    "# print('Train set: {} samples'.format(len(data_train)))\n",
    "# print('Test set: {} samples'.format(len(data_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[6, 1, 5, 4, 2, 0, 3, 9, 8], [7], [0, 0, 0, 1, 0, 0, 0, 1, 0], [0]]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_test_split([0,1,2,3,4,5,6,7,8,9], [0,0,0,0,1,0,0,0,0,1], train_size=0.9, test_size=0.1, random_state=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# len(document_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# data_train['sentiment'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# data_test['sentiment'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# train_labels = np.array(map(lambda x: 1 if x == 'pos' else 0, data_train['sentiment']))\n",
    "# train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# test_labels = np.array(map(lambda x: 1 if x == 'pos' else 0, data_test['sentiment']))\n",
    "# test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def size_mb(docs):\n",
    "    return sum(len(s.encode('utf-8')) for s in docs) / 1e6\n",
    "\n",
    "# data_train_size_mb = size_mb(data_train['sentence'])\n",
    "# data_test_size_mb = size_mb(data_test['sentence'])\n",
    "\n",
    "# print(\"%d documents - %0.3fMB (training set)\" % (\n",
    "#     len(data_train), data_train_size_mb))\n",
    "# print(\"%d documents - %0.3fMB (test set)\" % (\n",
    "#     len(data_test), data_test_size_mb))\n",
    "# print(\"%d categories\" % len(categories))\n",
    "# print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "###############################################################################\n",
    "# Benchmark classifiers\n",
    "def benchmark(clf, X_train, X_test, y_train, y_test):\n",
    "    print(\"_\" * 80)\n",
    "    print(\"Training: \")\n",
    "    print(clf)\n",
    "    \n",
    "    t0 = time()\n",
    "    clf.fit(X_train, y_train)\n",
    "    train_time = time() - t0\n",
    "    print(\"train time: %0.3fs\" % train_time)\n",
    "\n",
    "    t0 = time()\n",
    "    pred = clf.predict(X_test)\n",
    "    test_time = time() - t0\n",
    "    print(\"test time:  %0.3fs\" % test_time)\n",
    "\n",
    "    score = metrics.accuracy_score(y_test, pred)\n",
    "    print(\"accuracy:   %0.3f\" % score)\n",
    "\n",
    "    precision = metrics.precision_score(y_test, pred)\n",
    "    print(\"precision:   %0.3f\" % score)\n",
    "\n",
    "    recall = metrics.recall_score(y_test, pred)\n",
    "    print(\"recall:   %0.3f\" % score)\n",
    "\n",
    "    print(\"classification report:\")\n",
    "    print(metrics.classification_report(y_test, pred, target_names=['pos', 'neg']))\n",
    "\n",
    "    print(\"confusion matrix:\")\n",
    "    print(metrics.confusion_matrix(y_test, pred))\n",
    "\n",
    "    print()\n",
    "    clf_descr = str(clf).split(\"(\")[0]\n",
    "    return score, precision, recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Params\n",
    "K = 10\n",
    "parameters = {\n",
    "    'vect__max_df': (0.5, 0.75, 1.0),\n",
    "#     'vect__max_features': (None, 1000, 5000, 10000, 50000),\n",
    "#     'vect__ngram_range': ((1, 1), (1, 2), (2, 2), (1, 3), (2, 3), (3, 3)),  # unigrams or bigrams or trigrams\n",
    "#     'vect__stop_words': (None, stopwords.words('english')),\n",
    "#     'tfidf__use_idf': (True, False),\n",
    "#     'tfidf__norm': ('l1', 'l2'),\n",
    "#     'clf__alpha': (1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6),\n",
    "#     'clf__penalty': ('l1', 'l2', 'elasticnet'),\n",
    "#     'clf__n_iter': (10, 50, 80),\n",
    "#     'clf__loss': ('log', 'modified_huber'),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Classifiers\n",
    "clf_list = [\n",
    "    (RidgeClassifier(tol=1e-2, solver=\"lsqr\"), \"Ridge classifier\"),\n",
    "#     (Perceptron(n_iter=50), \"Perceptron\"),\n",
    "#     (PassiveAggressiveClassifier(n_iter=50), \"Passive-aggressive\"),\n",
    "#     (KNeighborsClassifier(n_neighbors=10), \"kNN\"),\n",
    "#     (RandomForestClassifier(n_estimators=100), \"Random forest\")\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "________________________________________________________________________________\n",
      "Training: \n",
      "RidgeClassifier(alpha=1.0, class_weight=None, copy_X=True, fit_intercept=True,\n",
      "        max_iter=None, normalize=False, solver='lsqr', tol=0.01)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "could not convert string to float: 'This is a major moment in children's lives, because they're going into formal education in a formal setting,' said Juliet Gray-Moli�re, the museum's senior manager of early childhood education.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-50-937df3c6909a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     21\u001b[0m         ])\n\u001b[0;32m     22\u001b[0m \u001b[1;31m#         grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m         \u001b[0mbenchmark_results\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbenchmark\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m         \u001b[0mresults\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbenchmark_results\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-28-67f9be98174a>\u001b[0m in \u001b[0;36mbenchmark\u001b[1;34m(clf, X_train, X_test, y_train, y_test)\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[0mt0\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m     \u001b[0mclf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m     \u001b[0mtrain_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mt0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m     \u001b[1;32mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"train time: %0.3fs\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mtrain_time\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/antonio/.virtualenvs/lexisnexis/local/lib/python2.7/site-packages/sklearn/linear_model/ridge.pyc\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    604\u001b[0m             \u001b[0msample_weight\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    605\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 606\u001b[1;33m         \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mRidgeClassifier\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    607\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    608\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/antonio/.virtualenvs/lexisnexis/local/lib/python2.7/site-packages/sklearn/linear_model/ridge.pyc\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[0;32m    380\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    381\u001b[0m         X, y = check_X_y(X, y, ['csr', 'csc', 'coo'], dtype=np.float,\n\u001b[1;32m--> 382\u001b[1;33m                          multi_output=True, y_numeric=True)\n\u001b[0m\u001b[0;32m    383\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    384\u001b[0m         if ((sample_weight is not None) and\n",
      "\u001b[1;32m/home/antonio/.virtualenvs/lexisnexis/local/lib/python2.7/site-packages/sklearn/utils/validation.pyc\u001b[0m in \u001b[0;36mcheck_X_y\u001b[1;34m(X, y, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric)\u001b[0m\n\u001b[0;32m    442\u001b[0m     X = check_array(X, accept_sparse, dtype, order, copy, force_all_finite,\n\u001b[0;32m    443\u001b[0m                     \u001b[0mensure_2d\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mallow_nd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mensure_min_samples\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 444\u001b[1;33m                     ensure_min_features)\n\u001b[0m\u001b[0;32m    445\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mmulti_output\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    446\u001b[0m         y = check_array(y, 'csr', force_all_finite=True, ensure_2d=False,\n",
      "\u001b[1;32m/home/antonio/.virtualenvs/lexisnexis/local/lib/python2.7/site-packages/sklearn/utils/validation.pyc\u001b[0m in \u001b[0;36mcheck_array\u001b[1;34m(array, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features)\u001b[0m\n\u001b[0;32m    342\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    343\u001b[0m                 \u001b[0mdtype\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 344\u001b[1;33m         \u001b[0marray\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    345\u001b[0m         \u001b[1;31m# make sure we actually converted to numeric:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    346\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mdtype_numeric\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0marray\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkind\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"O\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: could not convert string to float: 'This is a major moment in children's lives, because they're going into formal education in a formal setting,' said Juliet Gray-Moli�re, the museum's senior manager of early childhood education."
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "# results = defaultdict(lambda: defaultdict(list))\n",
    "results = defaultdict(list)\n",
    "\n",
    "X = sentences_df['sentence']\n",
    "y = sentences_df['sentiment']\n",
    "X_data, X_val, y_data, y_val = train_test_split(X, y, train_size=0.9, test_size=0.1, random_state=100)\n",
    "X_data = X_data.reset_index(drop=True)\n",
    "X_val = X_val.reset_index(drop=True)\n",
    "y_data = y_data.reset_index(drop=True)\n",
    "y_val = y_val.reset_index(drop=True)\n",
    "for train_index, test_index in StratifiedKFold(y_data, K):\n",
    "    X_train, X_test = X_data[train_index], X_data[test_index]\n",
    "    y_train, y_test = y_data[train_index], y_data[test_index]\n",
    "    \n",
    "    for clf, name in clf_list:\n",
    "        pipeline = Pipeline([\n",
    "            ('vect', CountVectorizer()),\n",
    "            ('tfidf', TfidfTransformer()),\n",
    "            ('clf', clf),\n",
    "        ])\n",
    "#         grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\n",
    "        benchmark_results = benchmark(clf, X_train, X_test, y_train, y_test)\n",
    "        results[name].append(benchmark_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    Three years ago, according to a previous NPD s...\n",
       "1    While it might be counter-intuitive to expect ...\n",
       "2    'I've seen five movies today, and it has been ...\n",
       "3    Large checks came in from Pfizer, Caterpillar,...\n",
       "4    The group stressed it had not employed any sen...\n",
       "Name: sentence, dtype: object"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_data.reset_index(drop=True).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    pos\n",
       "1    pos\n",
       "2    pos\n",
       "3    pos\n",
       "4    neg\n",
       "Name: sentiment, dtype: object"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_data.reset_index(drop=True).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5329"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "582     'Our results for the fourth quarter of 2012 co...\n",
       "594     'People enjoy the humor on board, and that we ...\n",
       "595     'People have been bragging about booking six f...\n",
       "596     'People have been expressing interest in this ...\n",
       "597                                                   NaN\n",
       "598     'People say you can't change China, but I woul...\n",
       "599     'People were driving by daily and not realizin...\n",
       "600                                                   NaN\n",
       "601     'People who are trying too hard or people who ...\n",
       "602                                                   NaN\n",
       "603                                                   NaN\n",
       "604     'President Obama is on pace to break the two m...\n",
       "605     'President Obama's plan to cut Social Security...\n",
       "606                                                   NaN\n",
       "607                                                   NaN\n",
       "608                                                   NaN\n",
       "609     'Quality content has never been more important...\n",
       "610                                                   NaN\n",
       "611                                                   NaN\n",
       "612                                                   NaN\n",
       "613     'Regular Singing' (in previews; opens on Frida...\n",
       "614     'Regular Singing' (previews start on Saturday;...\n",
       "615                                                   NaN\n",
       "616     'Resale Royalty,' at 9, follows Sue McCarthy a...\n",
       "617                                                   NaN\n",
       "618     'Right now we have a situation where the execu...\n",
       "619     'Right now, Apple is being priced as though it...\n",
       "620     'Right now, our plan is to hold back the spot ...\n",
       "621     'Robert is one of those great social entrepren...\n",
       "622                                                   NaN\n",
       "                              ...                        \n",
       "5892                                                  NaN\n",
       "5893    On Saturday, the police said Boeing confirmed ...\n",
       "5894    On Sunday at MetLife Stadium, their top priori...\n",
       "5895    On Thursday evening, police officers opened fi...\n",
       "5896    On Thursday night, more than 100 members of th...\n",
       "5897    On Thursday, Berkshire Hathaway and the invest...\n",
       "5898    On Thursday, Berkshire Hathaway, the conglomer...\n",
       "5899                                                  NaN\n",
       "5900                                                  NaN\n",
       "5901                                                  NaN\n",
       "5902    On Thursday, Nike became the latest corporate ...\n",
       "5903    On Thursday, it was the top-selling book on Am...\n",
       "5904    On Thursday, shares in the Aramark Holdings Co...\n",
       "5905                                                  NaN\n",
       "5906    On Tuesday evening, thousands of young Shiites...\n",
       "5907    On Tuesday morning asset manager Ashmore annou...\n",
       "5908                                                  NaN\n",
       "5909    On Tuesday, China will release its services pu...\n",
       "5910    On Tuesday, Exxon Mobil said it was cutting ba...\n",
       "5911                                                  NaN\n",
       "5912    On Tuesday, SoftBank's chief executive, Masayo...\n",
       "5913    On Tuesday, Timothy D. Cook, Apple's chief exe...\n",
       "5914                                                  NaN\n",
       "5915    On Tuesday, the United States International Tr...\n",
       "5916    On Twitter yesterday, I engaged in some discus...\n",
       "5917    On Wednesday, 'we're breaking the mold of trad...\n",
       "5918    On Wednesday, JPMorgan Chase also reported a s...\n",
       "5919    On Wednesday, Navalny's mayoral campaign manag...\n",
       "5920    On Wednesday, Target and Co.Labs plan to annou...\n",
       "5921                                                  NaN\n",
       "Name: sentence, dtype: object"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "61      \"The tapering of quantitative easing has only ...\n",
       "5264    More and more Chinese parents apparently disag...\n",
       "5267    More recently, General Motors offered shares t...\n",
       "3719    He managed Lewis until 2010, when tensions ari...\n",
       "4226    In a recent and noteworthy post at Realclimate...\n",
       "399     'If you're buying something in an Apple store,...\n",
       "8370    Three individuals with different levels of obj...\n",
       "7080    The French choreographer and founder of the L....\n",
       "8166    There is also some bad blood behind this story...\n",
       "6235    Raiding the endowment 'was a suicidal thing to...\n",
       "7348    The business's founders promote it as a low-pr...\n",
       "3075    Don Steinbrugge, managing partner with Agecrof...\n",
       "3963    Hyundai is offering a $750 credit for people w...\n",
       "4437    In time, music became a sideline, as he took a...\n",
       "7283    The authorities said the fraud was carried out...\n",
       "4107    In 1998, Research in Motion sought professiona...\n",
       "650     'Students' social media and digital footprint ...\n",
       "4332    In its lawsuit, Apple said Psystar had violate...\n",
       "640     'Since the '80s, the days of the dictator Gen....\n",
       "5233    Midway through the Bush administration, the ex...\n",
       "4540    It is a shotgun wedding that has consigned Rya...\n",
       "1505    According to the memo, 'existing and new staff...\n",
       "1818    And Mr. Loeb is expected to argue that Sony's ...\n",
       "7896    The panda cameras are funded by a grant from t...\n",
       "7909    The phrase 'middle out' itself seems to have g...\n",
       "5537    Mr. Pratchett, whose books have sold more than...\n",
       "1613    All the while, beginning with a quiet deal soo...\n",
       "349     'I think these brands are cheapening the town,...\n",
       "8907    Xiaomi has generated considerable attention be...\n",
       "8917    Yet McCarthy has found more than 70 additional...\n",
       "                              ...                        \n",
       "7478    The company, which is no longer controlled by ...\n",
       "1280    A combination with Dish Networks would pose mo...\n",
       "6845    TEHRAN -- All over this city of 12 million peo...\n",
       "1569    After the closing bell, shares of Cisco System...\n",
       "7060    The Dutch company sued General Motors for dama...\n",
       "832     'These investors want to diversify their wealt...\n",
       "987     'We have a real problem, and that's, 'How do w...\n",
       "7231    The administration passed an executive order l...\n",
       "8908    Yahoo's lack of growth exasperated Yahoo share...\n",
       "7103    The Internet and North Korea have been togethe...\n",
       "4633                      It won't make Apple very happy.\n",
       "238     'Facebook and Instagram are spiritual brothers...\n",
       "3141    Economists predict the yen to weaken further, ...\n",
       "3689    He indicated that he would carry on the mainly...\n",
       "3042    Despite the assault on senior management, no e...\n",
       "4338    In later years, after Mattel reduced and then ...\n",
       "7326    The bride's father is the chairman and chief e...\n",
       "8035    The sting operation, executed by the F.B.I. a ...\n",
       "3469    Gary Willoughby, executive director of the Tol...\n",
       "5268    More than 1,300 federal and state policies, ex...\n",
       "0        General Motors will recall nearly 3,200 manua...\n",
       "7566    The executive director of Hawaii's exchange, C...\n",
       "328     'I never wanted there to be any lingering anim...\n",
       "2015    Apple hires brand converts for its stores beca...\n",
       "5166    Microsoft Office 2010 (or Office 2011 for the ...\n",
       "3136    Earlier this year, AT&T said that 9 out of 10 ...\n",
       "4871    Ken Ehrlich, the show's executive producer, ha...\n",
       "2230    Asked about the implications of the app, Rajee...\n",
       "2949    Craig Jackson, chairman and chief executive of...\n",
       "4528    It has now received backing from investors inc...\n",
       "Name: sentence, dtype: object"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_data.reindex()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "597",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-73-1f8ff5eaed02>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mX_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mix\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m597\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m/home/antonio/.virtualenvs/lexisnexis/local/lib/python2.7/site-packages/pandas/core/indexing.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m     71\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_tuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     72\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 73\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_axis\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     74\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     75\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_get_label\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/antonio/.virtualenvs/lexisnexis/local/lib/python2.7/site-packages/pandas/core/indexing.pyc\u001b[0m in \u001b[0;36m_getitem_axis\u001b[1;34m(self, key, axis)\u001b[0m\n\u001b[0;32m    922\u001b[0m                     \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    923\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 924\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_label\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    925\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    926\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_getitem_iterable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/antonio/.virtualenvs/lexisnexis/local/lib/python2.7/site-packages/pandas/core/indexing.pyc\u001b[0m in \u001b[0;36m_get_label\u001b[1;34m(self, label, axis)\u001b[0m\n\u001b[0;32m     82\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_xs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     83\u001b[0m             \u001b[1;32mexcept\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 84\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     85\u001b[0m         elif (isinstance(label, tuple) and\n\u001b[0;32m     86\u001b[0m                 isinstance(label[axis], slice)):\n",
      "\u001b[1;32m/home/antonio/.virtualenvs/lexisnexis/local/lib/python2.7/site-packages/pandas/core/series.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m    512\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    513\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 514\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    515\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    516\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misscalar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/antonio/.virtualenvs/lexisnexis/local/lib/python2.7/site-packages/pandas/core/index.pyc\u001b[0m in \u001b[0;36mget_value\u001b[1;34m(self, series, key)\u001b[0m\n\u001b[0;32m   1458\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1459\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1460\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1461\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1462\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minferred_type\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'integer'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'boolean'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas/index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_value (pandas/index.c:3113)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_value (pandas/index.c:2844)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_loc (pandas/index.c:3704)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/hashtable.pyx\u001b[0m in \u001b[0;36mpandas.hashtable.Int64HashTable.get_item (pandas/hashtable.c:7255)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/hashtable.pyx\u001b[0m in \u001b[0;36mpandas.hashtable.Int64HashTable.get_item (pandas/hashtable.c:7193)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 597"
     ]
    }
   ],
   "source": [
    "X_data.ix[597]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.ix[597]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# K = 10\n",
    "# parameters = {\n",
    "#     'vect__max_df': (0.5, 0.75, 1.0),\n",
    "#     'vect__max_features': (None, 1000, 5000, 10000, 50000),\n",
    "#     'vect__ngram_range': ((1, 1), (1, 2), (2, 2), (1, 3), (2, 3), (3, 3)),  # unigrams or bigrams or trigrams\n",
    "#     'vect__stop_words': (None, stopwords.words('english')),\n",
    "#     'tfidf__use_idf': (True, False),\n",
    "#     'tfidf__norm': ('l1', 'l2'),\n",
    "# #     'clf__alpha': (1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6),\n",
    "# #     'clf__penalty': ('l1', 'l2', 'elasticnet'),\n",
    "# #     'clf__n_iter': (10, 50, 80),\n",
    "# #     'clf__loss': ('log', 'modified_huber'),\n",
    "# }\n",
    "# csf_list = [\n",
    "#     (RidgeClassifier(tol=1e-2, solver=\"lsqr\"), \"Ridge Classifier\"),\n",
    "#     (Perceptron(n_iter=50), \"Perceptron\"),\n",
    "#     (PassiveAggressiveClassifier(n_iter=50), \"Passive-Aggressive\"),\n",
    "#     (KNeighborsClassifier(n_neighbors=10), \"kNN\"),\n",
    "#     (RandomForestClassifier(n_estimators=100), \"Random forest\")\n",
    "# ]\n",
    "# results = {}\n",
    "# for clf, name in csf_list:\n",
    "#     pipeline = Pipeline([\n",
    "#         ('vect', CountVectorizer()),\n",
    "#         ('tfidf', TfidfTransformer()),\n",
    "#         ('clf', clf),\n",
    "#     ])\n",
    "#     grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\n",
    "#     skf = StratifiedKFold(y, K)\n",
    "#     results[name] = cross_val_score(grid_search, X, y, cv=skf, n_jobs=-1, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting features from the training data using a sparse vectorizer\n",
      "done in 0.398901s at 2.688MB/s\n",
      "n_samples: 5922, n_features: 18088\n",
      "\n",
      "Extracting features from the test data using the same vectorizer\n",
      "done in 0.027828s at 4.383MB/s\n",
      "n_samples: 659, n_features: 18088\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# split a training set and a test set\n",
    "y_train = train_labels\n",
    "y_test = test_labels\n",
    "\n",
    "print(\"Extracting features from the training data using a sparse vectorizer\")\n",
    "t0 = time()\n",
    "vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,\n",
    "                             stop_words='english')\n",
    "X_train = vectorizer.fit_transform(data_train.sentence.tolist())\n",
    "duration = time() - t0\n",
    "print(\"done in %fs at %0.3fMB/s\" % (duration, data_train_size_mb / duration))\n",
    "print(\"n_samples: %d, n_features: %d\" % X_train.shape)\n",
    "print()\n",
    "\n",
    "print(\"Extracting features from the test data using the same vectorizer\")\n",
    "t0 = time()\n",
    "X_test = vectorizer.transform(data_test.sentence.tolist())\n",
    "duration = time() - t0\n",
    "print(\"done in %fs at %0.3fMB/s\" % (duration, data_test_size_mb / duration))\n",
    "print(\"n_samples: %d, n_features: %d\" % X_test.shape)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting 1000 best features by a chi-squared test\n",
      "done in 0.022103s\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# mapping from integer feature name to original token string\n",
    "feature_names = vectorizer.get_feature_names()\n",
    "opts_select_chi2 = 100\n",
    "\n",
    "print(\"Extracting %d best features by a chi-squared test\" %\n",
    "      opts_select_chi2)\n",
    "t0 = time()\n",
    "ch2 = SelectKBest(chi2, k=opts_select_chi2)\n",
    "X_train = ch2.fit_transform(X_train, y_train)\n",
    "X_test = ch2.transform(X_test)\n",
    "if feature_names:\n",
    "    # keep selected feature names\n",
    "    feature_names = [feature_names[i] for i\n",
    "                     in ch2.get_support(indices=True)]\n",
    "print(\"done in %fs\" % (time() - t0))\n",
    "print()\n",
    "\n",
    "feature_names = np.asarray(feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# feature_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def trim(s):\n",
    "    \"\"\"Trim string to fit on terminal (assuming 80-column display)\"\"\"\n",
    "    return s if len(s) < 80 else s[:75] + \"...\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}