03_nlp_lda_extra_stopwords.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLP: Use LDA to identify additional stop words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mattm\\Anaconda3\\lib\\site-packages\\gensim\\utils.py:865: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from nltk.corpus import stopwords\n",
    "from gensim import corpora, models, similarities, matutils\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def remove_stop_words(raw_text, stop_list):\n",
    "    words = raw_text.split()\n",
    "    cleaned = [w for w in words if not w in stop_list]\n",
    "    return( \" \".join(cleaned))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('news_articles_cleaned_trimmed.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_text</th>\n",
       "      <th>datetime</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>cleaned_article_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1743</th>\n",
       "      <td>\\nThe crisis in Greece, a slowdown in China an...</td>\n",
       "      <td>2015-03-26 13:58:21</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>Bank of England warns of danger to markets fro...</td>\n",
       "      <td>https://www.theguardian.com/business/2015/mar/...</td>\n",
       "      <td>The crisis Greece slowdown China eurozone main...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9352</th>\n",
       "      <td>\\nIn a chapter of their new book, The Body Eco...</td>\n",
       "      <td>2013-06-09 21:51:02</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>IMF and Greece: Institutional Monstrous Failure</td>\n",
       "      <td>https://www.theguardian.com/commentisfree/2013...</td>\n",
       "      <td>In chapter book The_Body_Economic_academics_Da...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39682</th>\n",
       "      <td>(Adds close of U.S. markets) * Stronger U.S. Q...</td>\n",
       "      <td>2015-09-25 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>GLOBAL MARKETS-Dollar, shares gain on Yellen s...</td>\n",
       "      <td>http://uk.reuters.com/article/markets-global-i...</td>\n",
       "      <td>World equity dollar advanced Friday_to_end_a_r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27579</th>\n",
       "      <td>* European shares fall for 2nd straight sessi...</td>\n",
       "      <td>2010-05-03 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Europe shares fall; Greece aid fails to calm n...</td>\n",
       "      <td>http://uk.reuters.com/article/markets-europe-s...</td>\n",
       "      <td>European shares edged concerns massive bailout...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49345</th>\n",
       "      <td>TOKYO, March 10 (Reuters) - Japan's Nikkei av...</td>\n",
       "      <td>2011-03-10</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Nikkei falls but supported by dividend prospects</td>\n",
       "      <td>http://uk.reuters.com/article/markets-japan-st...</td>\n",
       "      <td>Japan Nikkei U_S stocks weaker buying stocks d...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            article_text             datetime  \\\n",
       "1743   \\nThe crisis in Greece, a slowdown in China an...  2015-03-26 13:58:21   \n",
       "9352   \\nIn a chapter of their new book, The Body Eco...  2013-06-09 21:51:02   \n",
       "39682  (Adds close of U.S. markets) * Stronger U.S. Q...  2015-09-25 00:00:00   \n",
       "27579   * European shares fall for 2nd straight sessi...  2010-05-03 00:00:00   \n",
       "49345   TOKYO, March 10 (Reuters) - Japan's Nikkei av...           2011-03-10   \n",
       "\n",
       "         source                                              title  \\\n",
       "1743   Guardian  Bank of England warns of danger to markets fro...   \n",
       "9352   Guardian    IMF and Greece: Institutional Monstrous Failure   \n",
       "39682   Reuters  GLOBAL MARKETS-Dollar, shares gain on Yellen s...   \n",
       "27579   Reuters  Europe shares fall; Greece aid fails to calm n...   \n",
       "49345   Reuters   Nikkei falls but supported by dividend prospects   \n",
       "\n",
       "                                                     url  \\\n",
       "1743   https://www.theguardian.com/business/2015/mar/...   \n",
       "9352   https://www.theguardian.com/commentisfree/2013...   \n",
       "39682  http://uk.reuters.com/article/markets-global-i...   \n",
       "27579  http://uk.reuters.com/article/markets-europe-s...   \n",
       "49345  http://uk.reuters.com/article/markets-japan-st...   \n",
       "\n",
       "                                    cleaned_article_text  \n",
       "1743   The crisis Greece slowdown China eurozone main...  \n",
       "9352   In chapter book The_Body_Economic_academics_Da...  \n",
       "39682  World equity dollar advanced Friday_to_end_a_r...  \n",
       "27579  European shares edged concerns massive bailout...  \n",
       "49345  Japan Nikkei U_S stocks weaker buying stocks d...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LDA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Take a sample from the entire dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_df = df.sample(10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None, stop_words='english',\n",
       "        strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "        tokenizer=None, vocabulary=None)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vectorizer = CountVectorizer(stop_words='english')\n",
    "count_vectorizer.fit(sample_df['cleaned_article_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "counts = count_vectorizer.transform(sample_df['cleaned_article_text']).transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(99162, 10000)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corpus = matutils.Sparse2Corpus(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "id2word = dict((v, k) for k, v in count_vectorizer.vocabulary_.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99162"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(id2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lda = models.LdaMulticore(corpus=corpus, num_topics=10, id2word=id2word, passes=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.005*\"growth\" + 0.004*\"rate\" + 0.004*\"economy\" + 0.004*\"bank\" + 0.004*\"rates\" + 0.003*\"financial\" + 0.003*\"global\" + 0.003*\"banks\" + 0.003*\"government\" + 0.003*\"fed\"'),\n",
       " (1,\n",
       "  '0.006*\"prices\" + 0.006*\"shares\" + 0.005*\"economy\" + 0.005*\"investors\" + 0.005*\"stocks\" + 0.004*\"european\" + 0.003*\"u_s\" + 0.003*\"yen\" + 0.003*\"rates\" + 0.003*\"financial\"'),\n",
       " (2,\n",
       "  '0.006*\"shares\" + 0.005*\"stocks\" + 0.004*\"economy\" + 0.004*\"investors\" + 0.004*\"financial\" + 0.003*\"bank\" + 0.003*\"economic\" + 0.003*\"dollar\" + 0.003*\"stock\" + 0.003*\"u_s\"'),\n",
       " (3,\n",
       "  '0.006*\"shares\" + 0.005*\"stocks\" + 0.004*\"growth\" + 0.004*\"investors\" + 0.004*\"economic\" + 0.004*\"prices\" + 0.004*\"european\" + 0.003*\"expected\" + 0.003*\"u_s\" + 0.003*\"economy\"'),\n",
       " (4,\n",
       "  '0.004*\"government\" + 0.004*\"growth\" + 0.004*\"oil\" + 0.004*\"expected\" + 0.004*\"economic\" + 0.003*\"yen\" + 0.003*\"tax\" + 0.003*\"business\" + 0.003*\"shares\" + 0.003*\"european\"'),\n",
       " (5,\n",
       "  '0.006*\"shares\" + 0.004*\"government\" + 0.004*\"growth\" + 0.003*\"yen\" + 0.003*\"investors\" + 0.003*\"dollar\" + 0.003*\"financial\" + 0.003*\"sector\" + 0.003*\"stocks\" + 0.003*\"nikkei\"'),\n",
       " (6,\n",
       "  '0.004*\"economy\" + 0.004*\"growth\" + 0.004*\"government\" + 0.003*\"european\" + 0.003*\"oil\" + 0.003*\"economic\" + 0.003*\"world\" + 0.003*\"uk\" + 0.003*\"rate\" + 0.003*\"financial\"'),\n",
       " (7,\n",
       "  '0.006*\"shares\" + 0.005*\"economic\" + 0.004*\"economy\" + 0.004*\"growth\" + 0.003*\"government\" + 0.003*\"oil\" + 0.003*\"trade\" + 0.003*\"companies\" + 0.003*\"rate\" + 0.003*\"global\"'),\n",
       " (8,\n",
       "  '0.005*\"growth\" + 0.004*\"economic\" + 0.004*\"nikkei\" + 0.004*\"government\" + 0.003*\"expected\" + 0.003*\"investors\" + 0.003*\"u_s\" + 0.003*\"china\" + 0.003*\"stocks\" + 0.003*\"shares\"'),\n",
       " (9,\n",
       "  '0.005*\"shares\" + 0.005*\"stocks\" + 0.005*\"investors\" + 0.005*\"growth\" + 0.005*\"economy\" + 0.004*\"european\" + 0.004*\"economic\" + 0.003*\"government\" + 0.003*\"debt\" + 0.003*\"uk\"')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda.print_topics()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove extra stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "extra_stopwords = ['shares', 'economy', 'economic', 'prices', 'investors']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "153"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_list = stopwords.words('english')\n",
    "len(stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "158"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_list = stop_list + extra_stopwords\n",
    "len(stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_df['article_text_extra_stops'] = sample_df['cleaned_article_text'].apply(remove_stop_words, stop_list=stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_text</th>\n",
       "      <th>datetime</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>cleaned_article_text</th>\n",
       "      <th>article_text_extra_stops</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>68176</th>\n",
       "      <td>* June ADP employment number rises to 172,000 ...</td>\n",
       "      <td>2016-07-07 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>US STOCKS-S&amp;P, Dow little changed ahead of cru...</td>\n",
       "      <td>http://uk.reuters.com/article/usa-stocks-idUKL...</td>\n",
       "      <td>The S P Dow little changed late_morning tradin...</td>\n",
       "      <td>The S P Dow little changed late_morning tradin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60875</th>\n",
       "      <td>(Updates to midsession)   TOKYO, May 25 (Reut...</td>\n",
       "      <td>2007-05-25 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Tokyo stocks down broadly on lower U.S., China...</td>\n",
       "      <td>http://uk.reuters.com/article/markets-japan-st...</td>\n",
       "      <td>Japanese stocks broadly investors concerned fa...</td>\n",
       "      <td>Japanese stocks broadly concerned falls U_S Ch...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41671</th>\n",
       "      <td>(Updates to U.S. market open) * U.S. dollar se...</td>\n",
       "      <td>2017-02-03</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>GLOBAL MARKETS-U.S. dollar and bond yields low...</td>\n",
       "      <td>http://uk.reuters.com/article/global-markets-i...</td>\n",
       "      <td>The U_S dollar headed fourth straight weekly l...</td>\n",
       "      <td>The U_S dollar headed fourth straight weekly l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79440</th>\n",
       "      <td>Wall Street eased overnight but it was the dol...</td>\n",
       "      <td>2016-11-16 23:23:00</td>\n",
       "      <td>FT</td>\n",
       "      <td>Fast Asia Open: Dollar daze, Aussie jobs</td>\n",
       "      <td>https://www.ft.com/content/62d851e6-14c5-31f9-...</td>\n",
       "      <td>Wall Street eased overnight dollar capturing a...</td>\n",
       "      <td>Wall Street eased overnight dollar capturing a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60680</th>\n",
       "      <td>ZURICH, Sept 25 (Reuters) - The Swiss blue-chi...</td>\n",
       "      <td>2015-09-25 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Swiss stocks - Factors to watch on Sept 25</td>\n",
       "      <td>http://uk.reuters.com/article/markets-swiss-st...</td>\n",
       "      <td>The Swiss blue chip SMI seen opening up_percen...</td>\n",
       "      <td>The Swiss blue chip SMI seen opening up_percen...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            article_text             datetime  \\\n",
       "68176  * June ADP employment number rises to 172,000 ...  2016-07-07 00:00:00   \n",
       "60875   (Updates to midsession)   TOKYO, May 25 (Reut...  2007-05-25 00:00:00   \n",
       "41671  (Updates to U.S. market open) * U.S. dollar se...           2017-02-03   \n",
       "79440  Wall Street eased overnight but it was the dol...  2016-11-16 23:23:00   \n",
       "60680  ZURICH, Sept 25 (Reuters) - The Swiss blue-chi...  2015-09-25 00:00:00   \n",
       "\n",
       "        source                                              title  \\\n",
       "68176  Reuters  US STOCKS-S&P, Dow little changed ahead of cru...   \n",
       "60875  Reuters  Tokyo stocks down broadly on lower U.S., China...   \n",
       "41671  Reuters  GLOBAL MARKETS-U.S. dollar and bond yields low...   \n",
       "79440       FT           Fast Asia Open: Dollar daze, Aussie jobs   \n",
       "60680  Reuters         Swiss stocks - Factors to watch on Sept 25   \n",
       "\n",
       "                                                     url  \\\n",
       "68176  http://uk.reuters.com/article/usa-stocks-idUKL...   \n",
       "60875  http://uk.reuters.com/article/markets-japan-st...   \n",
       "41671  http://uk.reuters.com/article/global-markets-i...   \n",
       "79440  https://www.ft.com/content/62d851e6-14c5-31f9-...   \n",
       "60680  http://uk.reuters.com/article/markets-swiss-st...   \n",
       "\n",
       "                                    cleaned_article_text  \\\n",
       "68176  The S P Dow little changed late_morning tradin...   \n",
       "60875  Japanese stocks broadly investors concerned fa...   \n",
       "41671  The U_S dollar headed fourth straight weekly l...   \n",
       "79440  Wall Street eased overnight dollar capturing a...   \n",
       "60680  The Swiss blue chip SMI seen opening up_percen...   \n",
       "\n",
       "                                article_text_extra_stops  \n",
       "68176  The S P Dow little changed late_morning tradin...  \n",
       "60875  Japanese stocks broadly concerned falls U_S Ch...  \n",
       "41671  The U_S dollar headed fourth straight weekly l...  \n",
       "79440  Wall Street eased overnight dollar capturing a...  \n",
       "60680  The Swiss blue chip SMI seen opening up_percen...  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4826"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sample_df.iloc[2028]['cleaned_article_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4734"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sample_df.iloc[2028]['article_text_extra_stops'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Repeat process"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(99162, 10000)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vectorizer = CountVectorizer(stop_words='english')\n",
    "count_vectorizer.fit(sample_df['article_text_extra_stops'])\n",
    "counts = count_vectorizer.transform(sample_df['article_text_extra_stops']).transpose()\n",
    "counts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99162"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus = matutils.Sparse2Corpus(counts)\n",
    "id2word = dict((v, k) for k, v in count_vectorizer.vocabulary_.items())\n",
    "len(id2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.005*\"yen\" + 0.004*\"oil\" + 0.004*\"stocks\" + 0.004*\"growth\" + 0.003*\"company\" + 0.003*\"china\" + 0.003*\"government\" + 0.003*\"trading\" + 0.003*\"sector\" + 0.003*\"stock\"'),\n",
       " (1,\n",
       "  '0.004*\"government\" + 0.004*\"financial\" + 0.004*\"growth\" + 0.004*\"u_s\" + 0.004*\"rates\" + 0.004*\"stocks\" + 0.004*\"uk\" + 0.003*\"european\" + 0.003*\"oil\" + 0.003*\"tax\"'),\n",
       " (2,\n",
       "  '0.006*\"growth\" + 0.005*\"european\" + 0.003*\"government\" + 0.003*\"global\" + 0.003*\"stocks\" + 0.003*\"policy\" + 0.003*\"uk\" + 0.003*\"china\" + 0.003*\"euro\" + 0.002*\"tax\"'),\n",
       " (3,\n",
       "  '0.006*\"stocks\" + 0.005*\"growth\" + 0.004*\"government\" + 0.004*\"european\" + 0.003*\"yen\" + 0.003*\"financial\" + 0.003*\"bank\" + 0.003*\"rate\" + 0.003*\"sector\" + 0.003*\"global\"'),\n",
       " (4,\n",
       "  '0.005*\"growth\" + 0.004*\"fed\" + 0.003*\"stocks\" + 0.003*\"bank\" + 0.003*\"financial\" + 0.003*\"european\" + 0.003*\"expected\" + 0.003*\"yen\" + 0.003*\"rate\" + 0.003*\"oil\"'),\n",
       " (5,\n",
       "  '0.004*\"bank\" + 0.004*\"nikkei\" + 0.003*\"banks\" + 0.003*\"stocks\" + 0.003*\"growth\" + 0.003*\"companies\" + 0.003*\"u_s\" + 0.003*\"european\" + 0.003*\"rates\" + 0.003*\"expected\"'),\n",
       " (6,\n",
       "  '0.005*\"growth\" + 0.003*\"government\" + 0.003*\"stocks\" + 0.003*\"rates\" + 0.003*\"jobs\" + 0.003*\"global\" + 0.003*\"u_s\" + 0.003*\"long\" + 0.003*\"expected\" + 0.003*\"oil\"'),\n",
       " (7,\n",
       "  '0.005*\"banks\" + 0.004*\"growth\" + 0.004*\"european\" + 0.004*\"uk\" + 0.003*\"rate\" + 0.003*\"financial\" + 0.003*\"debt\" + 0.003*\"stocks\" + 0.003*\"government\" + 0.003*\"bank\"'),\n",
       " (8,\n",
       "  '0.004*\"government\" + 0.003*\"uk\" + 0.003*\"expected\" + 0.003*\"financial\" + 0.003*\"stocks\" + 0.003*\"china\" + 0.003*\"rate\" + 0.003*\"oil\" + 0.003*\"global\" + 0.003*\"european\"'),\n",
       " (9,\n",
       "  '0.005*\"u_s\" + 0.005*\"stocks\" + 0.004*\"dollar\" + 0.004*\"rate\" + 0.004*\"nikkei\" + 0.004*\"yen\" + 0.004*\"european\" + 0.004*\"expected\" + 0.003*\"rates\" + 0.003*\"bank\"')]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = models.LdaMulticore(corpus=corpus, num_topics=10, id2word=id2word, passes=1)\n",
    "lda.print_topics()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extra_stopwords = extra_stopwords + ['government', 'growth', 'banks']\n",
    "len(extra_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "161"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_list = stopwords.words('english')\n",
    "stop_list = stop_list + extra_stopwords\n",
    "len(stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_df['article_text_extra_stops_v2'] = sample_df['cleaned_article_text'].apply(remove_stop_words, stop_list=stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_text</th>\n",
       "      <th>datetime</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>cleaned_article_text</th>\n",
       "      <th>article_text_extra_stops</th>\n",
       "      <th>article_text_extra_stops_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>724</th>\n",
       "      <td>\\nThe United States' reputation for sound econ...</td>\n",
       "      <td>2013-12-10 13:54:54</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>America's reputation for sound economic policy...</td>\n",
       "      <td>https://www.theguardian.com/business/economics...</td>\n",
       "      <td>The_United_States reputation sound economic po...</td>\n",
       "      <td>The_United_States reputation sound policymakin...</td>\n",
       "      <td>The_United_States reputation sound policymakin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38832</th>\n",
       "      <td>* Blue-chip FTSE 100 index steadies in late tr...</td>\n",
       "      <td>2014-11-06 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>FTSE steadies after gaining on Draghi's comments</td>\n",
       "      <td>http://uk.reuters.com/article/markets-stocks-f...</td>\n",
       "      <td>Britain steadied late trading surrendering ear...</td>\n",
       "      <td>Britain steadied late trading surrendering ear...</td>\n",
       "      <td>Britain steadied late trading surrendering ear...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54869</th>\n",
       "      <td>* Nikkei slips on profit-taking after bounce ...</td>\n",
       "      <td>2009-08-07</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Nikkei slips on nerves as jobs data looms</td>\n",
       "      <td>http://uk.reuters.com/article/markets-japan-st...</td>\n",
       "      <td>Japan Nikkei stock edged investors nervous ahe...</td>\n",
       "      <td>Japan Nikkei stock edged nervous ahead crucial...</td>\n",
       "      <td>Japan Nikkei stock edged nervous ahead crucial...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27070</th>\n",
       "      <td>* FTSEurofirst 300 closes 0.6 pct lower; 3rd ...</td>\n",
       "      <td>2010-09-29 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Europe shares close at three-wk low; retailers...</td>\n",
       "      <td>http://uk.reuters.com/article/markets-europe-s...</td>\n",
       "      <td>European shares slipped three_week closing Hen...</td>\n",
       "      <td>European slipped three_week closing Hennes_Mau...</td>\n",
       "      <td>European slipped three_week closing Hennes_Mau...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81278</th>\n",
       "      <td>Shares in Monster Beverage led the S&amp;P 500 on ...</td>\n",
       "      <td>2017-03-02 21:25:00</td>\n",
       "      <td>FT</td>\n",
       "      <td>Monster shares lead Wall Street gainers after ...</td>\n",
       "      <td>https://www.ft.com/content/48be67d6-ff69-11e6-...</td>\n",
       "      <td>Shares Monster_Beverage led S P posted biggest...</td>\n",
       "      <td>Shares Monster_Beverage led S P posted biggest...</td>\n",
       "      <td>Shares Monster_Beverage led S P posted biggest...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            article_text             datetime  \\\n",
       "724    \\nThe United States' reputation for sound econ...  2013-12-10 13:54:54   \n",
       "38832  * Blue-chip FTSE 100 index steadies in late tr...  2014-11-06 00:00:00   \n",
       "54869   * Nikkei slips on profit-taking after bounce ...           2009-08-07   \n",
       "27070   * FTSEurofirst 300 closes 0.6 pct lower; 3rd ...  2010-09-29 00:00:00   \n",
       "81278  Shares in Monster Beverage led the S&P 500 on ...  2017-03-02 21:25:00   \n",
       "\n",
       "         source                                              title  \\\n",
       "724    Guardian  America's reputation for sound economic policy...   \n",
       "38832   Reuters   FTSE steadies after gaining on Draghi's comments   \n",
       "54869   Reuters          Nikkei slips on nerves as jobs data looms   \n",
       "27070   Reuters  Europe shares close at three-wk low; retailers...   \n",
       "81278        FT  Monster shares lead Wall Street gainers after ...   \n",
       "\n",
       "                                                     url  \\\n",
       "724    https://www.theguardian.com/business/economics...   \n",
       "38832  http://uk.reuters.com/article/markets-stocks-f...   \n",
       "54869  http://uk.reuters.com/article/markets-japan-st...   \n",
       "27070  http://uk.reuters.com/article/markets-europe-s...   \n",
       "81278  https://www.ft.com/content/48be67d6-ff69-11e6-...   \n",
       "\n",
       "                                    cleaned_article_text  \\\n",
       "724    The_United_States reputation sound economic po...   \n",
       "38832  Britain steadied late trading surrendering ear...   \n",
       "54869  Japan Nikkei stock edged investors nervous ahe...   \n",
       "27070  European shares slipped three_week closing Hen...   \n",
       "81278  Shares Monster_Beverage led S P posted biggest...   \n",
       "\n",
       "                                article_text_extra_stops  \\\n",
       "724    The_United_States reputation sound policymakin...   \n",
       "38832  Britain steadied late trading surrendering ear...   \n",
       "54869  Japan Nikkei stock edged nervous ahead crucial...   \n",
       "27070  European slipped three_week closing Hennes_Mau...   \n",
       "81278  Shares Monster_Beverage led S P posted biggest...   \n",
       "\n",
       "                             article_text_extra_stops_v2  \n",
       "724    The_United_States reputation sound policymakin...  \n",
       "38832  Britain steadied late trading surrendering ear...  \n",
       "54869  Japan Nikkei stock edged nervous ahead crucial...  \n",
       "27070  European slipped three_week closing Hennes_Mau...  \n",
       "81278  Shares Monster_Beverage led S P posted biggest...  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4826\n",
      "4734\n",
      "4673\n"
     ]
    }
   ],
   "source": [
    "print(len(sample_df.iloc[2028]['cleaned_article_text']))\n",
    "print(len(sample_df.iloc[2028]['article_text_extra_stops']))\n",
    "print(len(sample_df.iloc[2028]['article_text_extra_stops_v2']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## And again.."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(99162, 10000)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vectorizer = CountVectorizer(stop_words='english')\n",
    "count_vectorizer.fit(sample_df['article_text_extra_stops_v2'])\n",
    "counts = count_vectorizer.transform(sample_df['article_text_extra_stops_v2']).transpose()\n",
    "counts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99162"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus = matutils.Sparse2Corpus(counts)\n",
    "id2word = dict((v, k) for k, v in count_vectorizer.vocabulary_.items())\n",
    "len(id2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.004*\"expected\" + 0.004*\"rate\" + 0.003*\"rates\" + 0.003*\"sector\" + 0.003*\"u_s\" + 0.003*\"financial\" + 0.003*\"uk\" + 0.003*\"spending\" + 0.003*\"stocks\" + 0.003*\"trading\"'),\n",
       " (1,\n",
       "  '0.005*\"rate\" + 0.004*\"earnings\" + 0.004*\"fed\" + 0.004*\"european\" + 0.003*\"rates\" + 0.003*\"yen\" + 0.003*\"dollar\" + 0.003*\"inflation\" + 0.003*\"bank\" + 0.003*\"profit\"'),\n",
       " (2,\n",
       "  '0.004*\"rates\" + 0.004*\"rate\" + 0.003*\"debt\" + 0.003*\"european\" + 0.003*\"financial\" + 0.003*\"global\" + 0.003*\"stocks\" + 0.003*\"oil\" + 0.003*\"bank\" + 0.003*\"business\"'),\n",
       " (3,\n",
       "  '0.003*\"stocks\" + 0.003*\"yen\" + 0.003*\"expected\" + 0.003*\"european\" + 0.003*\"financial\" + 0.002*\"bank\" + 0.002*\"rates\" + 0.002*\"earnings\" + 0.002*\"debt\" + 0.002*\"china\"'),\n",
       " (4,\n",
       "  '0.006*\"european\" + 0.005*\"stocks\" + 0.005*\"u_s\" + 0.003*\"uk\" + 0.003*\"bank\" + 0.003*\"rate\" + 0.003*\"expected\" + 0.003*\"company\" + 0.003*\"sector\" + 0.003*\"rates\"'),\n",
       " (5,\n",
       "  '0.004*\"u_s\" + 0.004*\"stocks\" + 0.003*\"china\" + 0.003*\"stock\" + 0.003*\"expected\" + 0.003*\"financial\" + 0.003*\"yen\" + 0.003*\"sector\" + 0.003*\"earnings\" + 0.003*\"companies\"'),\n",
       " (6,\n",
       "  '0.007*\"oil\" + 0.005*\"european\" + 0.005*\"stocks\" + 0.003*\"global\" + 0.003*\"u_s\" + 0.003*\"china\" + 0.003*\"world\" + 0.003*\"bank\" + 0.003*\"dollar\" + 0.003*\"sector\"'),\n",
       " (7,\n",
       "  '0.004*\"european\" + 0.004*\"yen\" + 0.003*\"uk\" + 0.003*\"nikkei\" + 0.003*\"stocks\" + 0.003*\"expected\" + 0.003*\"financial\" + 0.003*\"euro\" + 0.003*\"global\" + 0.003*\"gains\"'),\n",
       " (8,\n",
       "  '0.005*\"stocks\" + 0.004*\"financial\" + 0.003*\"dollar\" + 0.003*\"trade\" + 0.003*\"china\" + 0.003*\"bank\" + 0.003*\"companies\" + 0.003*\"global\" + 0.003*\"stock\" + 0.002*\"inflation\"'),\n",
       " (9,\n",
       "  '0.004*\"stocks\" + 0.004*\"u_s\" + 0.003*\"debt\" + 0.003*\"nikkei\" + 0.003*\"yen\" + 0.003*\"financial\" + 0.003*\"uk\" + 0.003*\"stock\" + 0.003*\"policy\" + 0.003*\"rates\"')]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = models.LdaMulticore(corpus=corpus, num_topics=10, id2word=id2word, passes=1)\n",
    "lda.print_topics()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extra_stopwords = extra_stopwords + ['expected', 'rate', 'rates', 'stocks', 'financial']\n",
    "len(extra_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "166"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_list = stopwords.words('english')\n",
    "stop_list = stop_list + extra_stopwords\n",
    "len(stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_df['article_text_extra_stops_v3'] = sample_df['cleaned_article_text'].apply(remove_stop_words, stop_list=stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_text</th>\n",
       "      <th>datetime</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>cleaned_article_text</th>\n",
       "      <th>article_text_extra_stops</th>\n",
       "      <th>article_text_extra_stops_v2</th>\n",
       "      <th>article_text_extra_stops_v3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>31197</th>\n",
       "      <td>* Pan-European FTSEurofirst 300 index falls 0....</td>\n",
       "      <td>2015-09-10 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>European shares down, Brazil-exposed stocks un...</td>\n",
       "      <td>http://uk.reuters.com/article/markets-stocks-e...</td>\n",
       "      <td>European shares retreated companies exposed Br...</td>\n",
       "      <td>European retreated companies exposed Brazil ca...</td>\n",
       "      <td>European retreated companies exposed Brazil ca...</td>\n",
       "      <td>European retreated companies exposed Brazil ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35836</th>\n",
       "      <td>* FTSEurofirst 300 up 1.1 percent * Germany's ...</td>\n",
       "      <td>2013-09-19 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>European shares surge as Fed leaves stimulus i...</td>\n",
       "      <td>http://uk.reuters.com/article/markets-europe-s...</td>\n",
       "      <td>European shares strongly the_U_S_Federal_Reser...</td>\n",
       "      <td>European strongly the_U_S_Federal_Reserve surp...</td>\n",
       "      <td>European strongly the_U_S_Federal_Reserve surp...</td>\n",
       "      <td>European strongly the_U_S_Federal_Reserve surp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45977</th>\n",
       "      <td>(Updates to close)   TOKYO, Jan 22 (Reuters) ...</td>\n",
       "      <td>2007-01-22</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Nikkei at 9-mth high as Olympus,security firms...</td>\n",
       "      <td>http://uk.reuters.com/article/markets-japan-st...</td>\n",
       "      <td>The Nikkei highest nine_months shares precisio...</td>\n",
       "      <td>The Nikkei highest nine_months precision elect...</td>\n",
       "      <td>The Nikkei highest nine_months precision elect...</td>\n",
       "      <td>The Nikkei highest nine_months precision elect...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35959</th>\n",
       "      <td>LONDON, April 12 (Reuters) - European shares ...</td>\n",
       "      <td>2010-04-12 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>European shares turn flat; banks up, miners fall</td>\n",
       "      <td>http://uk.reuters.com/article/markets-europe-s...</td>\n",
       "      <td>European shares turned flat morning stronger b...</td>\n",
       "      <td>European turned flat morning stronger banking ...</td>\n",
       "      <td>European turned flat morning stronger banking ...</td>\n",
       "      <td>European turned flat morning stronger banking ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20942</th>\n",
       "      <td>PARIS, Oct 1 (Reuters) - European stocks inch...</td>\n",
       "      <td>2007-10-01 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Airlines, miners help Europe stocks recover fr...</td>\n",
       "      <td>http://uk.reuters.com/article/markets-europe-s...</td>\n",
       "      <td>European stocks inched early_on reversing earl...</td>\n",
       "      <td>European stocks inched early_on reversing earl...</td>\n",
       "      <td>European stocks inched early_on reversing earl...</td>\n",
       "      <td>European inched early_on reversing early losse...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            article_text             datetime  \\\n",
       "31197  * Pan-European FTSEurofirst 300 index falls 0....  2015-09-10 00:00:00   \n",
       "35836  * FTSEurofirst 300 up 1.1 percent * Germany's ...  2013-09-19 00:00:00   \n",
       "45977   (Updates to close)   TOKYO, Jan 22 (Reuters) ...           2007-01-22   \n",
       "35959   LONDON, April 12 (Reuters) - European shares ...  2010-04-12 00:00:00   \n",
       "20942   PARIS, Oct 1 (Reuters) - European stocks inch...  2007-10-01 00:00:00   \n",
       "\n",
       "        source                                              title  \\\n",
       "31197  Reuters  European shares down, Brazil-exposed stocks un...   \n",
       "35836  Reuters  European shares surge as Fed leaves stimulus i...   \n",
       "45977  Reuters  Nikkei at 9-mth high as Olympus,security firms...   \n",
       "35959  Reuters   European shares turn flat; banks up, miners fall   \n",
       "20942  Reuters  Airlines, miners help Europe stocks recover fr...   \n",
       "\n",
       "                                                     url  \\\n",
       "31197  http://uk.reuters.com/article/markets-stocks-e...   \n",
       "35836  http://uk.reuters.com/article/markets-europe-s...   \n",
       "45977  http://uk.reuters.com/article/markets-japan-st...   \n",
       "35959  http://uk.reuters.com/article/markets-europe-s...   \n",
       "20942  http://uk.reuters.com/article/markets-europe-s...   \n",
       "\n",
       "                                    cleaned_article_text  \\\n",
       "31197  European shares retreated companies exposed Br...   \n",
       "35836  European shares strongly the_U_S_Federal_Reser...   \n",
       "45977  The Nikkei highest nine_months shares precisio...   \n",
       "35959  European shares turned flat morning stronger b...   \n",
       "20942  European stocks inched early_on reversing earl...   \n",
       "\n",
       "                                article_text_extra_stops  \\\n",
       "31197  European retreated companies exposed Brazil ca...   \n",
       "35836  European strongly the_U_S_Federal_Reserve surp...   \n",
       "45977  The Nikkei highest nine_months precision elect...   \n",
       "35959  European turned flat morning stronger banking ...   \n",
       "20942  European stocks inched early_on reversing earl...   \n",
       "\n",
       "                             article_text_extra_stops_v2  \\\n",
       "31197  European retreated companies exposed Brazil ca...   \n",
       "35836  European strongly the_U_S_Federal_Reserve surp...   \n",
       "45977  The Nikkei highest nine_months precision elect...   \n",
       "35959  European turned flat morning stronger banking ...   \n",
       "20942  European stocks inched early_on reversing earl...   \n",
       "\n",
       "                             article_text_extra_stops_v3  \n",
       "31197  European retreated companies exposed Brazil ca...  \n",
       "35836  European strongly the_U_S_Federal_Reserve surp...  \n",
       "45977  The Nikkei highest nine_months precision elect...  \n",
       "35959  European turned flat morning stronger banking ...  \n",
       "20942  European inched early_on reversing early losse...  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4826\n",
      "4734\n",
      "4673\n",
      "4641\n"
     ]
    }
   ],
   "source": [
    "print(len(sample_df.iloc[2028]['cleaned_article_text']))\n",
    "print(len(sample_df.iloc[2028]['article_text_extra_stops']))\n",
    "print(len(sample_df.iloc[2028]['article_text_extra_stops_v2']))\n",
    "print(len(sample_df.iloc[2028]['article_text_extra_stops_v3']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Again.."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(99162, 10000)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vectorizer = CountVectorizer(stop_words='english')\n",
    "count_vectorizer.fit(sample_df['article_text_extra_stops_v3'])\n",
    "counts = count_vectorizer.transform(sample_df['article_text_extra_stops_v3']).transpose()\n",
    "counts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99162"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus = matutils.Sparse2Corpus(counts)\n",
    "id2word = dict((v, k) for k, v in count_vectorizer.vocabulary_.items())\n",
    "len(id2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.005*\"uk\" + 0.005*\"european\" + 0.003*\"u_s\" + 0.002*\"tax\" + 0.002*\"britain\" + 0.002*\"inflation\" + 0.002*\"country\" + 0.002*\"bank\" + 0.002*\"business\" + 0.002*\"spending\"'),\n",
       " (1,\n",
       "  '0.004*\"dollar\" + 0.003*\"oil\" + 0.003*\"u_s\" + 0.003*\"company\" + 0.003*\"stock\" + 0.003*\"yen\" + 0.003*\"fed\" + 0.002*\"britain\" + 0.002*\"business\" + 0.002*\"bank\"'),\n",
       " (2,\n",
       "  '0.004*\"oil\" + 0.003*\"earnings\" + 0.003*\"business\" + 0.003*\"sector\" + 0.003*\"european\" + 0.003*\"u_s\" + 0.003*\"uk\" + 0.002*\"inflation\" + 0.002*\"yen\" + 0.002*\"company\"'),\n",
       " (3,\n",
       "  '0.005*\"u_s\" + 0.004*\"china\" + 0.004*\"nikkei\" + 0.004*\"yen\" + 0.004*\"stock\" + 0.003*\"global\" + 0.003*\"trading\" + 0.003*\"bank\" + 0.003*\"oil\" + 0.003*\"european\"'),\n",
       " (4,\n",
       "  '0.005*\"european\" + 0.004*\"dollar\" + 0.003*\"bank\" + 0.003*\"global\" + 0.003*\"policy\" + 0.003*\"world\" + 0.003*\"u_s\" + 0.003*\"uk\" + 0.003*\"debt\" + 0.003*\"china\"'),\n",
       " (5,\n",
       "  '0.005*\"european\" + 0.004*\"earnings\" + 0.003*\"world\" + 0.003*\"u_s\" + 0.003*\"biggest\" + 0.003*\"uk\" + 0.003*\"stock\" + 0.003*\"trade\" + 0.002*\"debt\" + 0.002*\"bank\"'),\n",
       " (6,\n",
       "  '0.004*\"yen\" + 0.003*\"bank\" + 0.003*\"oil\" + 0.003*\"sector\" + 0.003*\"uk\" + 0.003*\"earnings\" + 0.003*\"companies\" + 0.003*\"dollar\" + 0.003*\"company\" + 0.003*\"price\"'),\n",
       " (7,\n",
       "  '0.004*\"china\" + 0.003*\"u_s\" + 0.003*\"sector\" + 0.003*\"debt\" + 0.003*\"work\" + 0.003*\"oil\" + 0.003*\"global\" + 0.002*\"fed\" + 0.002*\"european\" + 0.002*\"yen\"'),\n",
       " (8,\n",
       "  '0.004*\"oil\" + 0.003*\"uk\" + 0.003*\"bank\" + 0.003*\"tax\" + 0.003*\"european\" + 0.003*\"yen\" + 0.003*\"nikkei\" + 0.003*\"policy\" + 0.003*\"business\" + 0.003*\"profit\"'),\n",
       " (9,\n",
       "  '0.004*\"european\" + 0.003*\"bank\" + 0.003*\"europe\" + 0.003*\"nikkei\" + 0.003*\"oil\" + 0.003*\"global\" + 0.002*\"companies\" + 0.002*\"crisis\" + 0.002*\"stock\" + 0.002*\"company\"')]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = models.LdaMulticore(corpus=corpus, num_topics=10, id2word=id2word, passes=1)\n",
    "lda.print_topics()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "115"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extra_stopwords = extra_stopwords + ['uk', 'european', 'u_s', 'tax', 'britain', 'inflation', 'country', 'bank', \n",
    "                                    'business', 'spending', 'dollar', 'oil', 'company', 'stock', 'yen', 'fed', 'earnings',\n",
    "                                    'sector', 'global', 'policy', 'world', 'debt', 'china', 'biggest', 'trade', 'bank',\n",
    "                                    'work', 'nikkei', 'profit', 'business', 'European', 'UK', 'U_S', 'Bank', 'Nikkei', 'Fed', \n",
    "                                     'Britain', 'Dollar', 'Yen', 'Global', 'World', 'China']\n",
    "len(extra_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "268"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_list = stopwords.words('english')\n",
    "stop_list = stop_list + extra_stopwords\n",
    "len(stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "206"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_list = set(stop_list)\n",
    "len(stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_df['article_text_extra_stops_v4'] = sample_df['cleaned_article_text'].apply(remove_stop_words, stop_list=stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2357\n",
      "2289\n",
      "2269\n",
      "2253\n",
      "2106\n"
     ]
    }
   ],
   "source": [
    "print(len(sample_df.iloc[2000]['cleaned_article_text']))\n",
    "print(len(sample_df.iloc[2000]['article_text_extra_stops']))\n",
    "print(len(sample_df.iloc[2000]['article_text_extra_stops_v2']))\n",
    "print(len(sample_df.iloc[2000]['article_text_extra_stops_v3']))\n",
    "print(len(sample_df.iloc[2000]['article_text_extra_stops_v4']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Again"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(99160, 10000)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vectorizer = CountVectorizer(stop_words='english')\n",
    "count_vectorizer.fit(sample_df['article_text_extra_stops_v4'])\n",
    "counts = count_vectorizer.transform(sample_df['article_text_extra_stops_v4']).transpose()\n",
    "counts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99160"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus = matutils.Sparse2Corpus(counts)\n",
    "id2word = dict((v, k) for k, v in count_vectorizer.vocabulary_.items())\n",
    "len(id2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.003*\"crisis\" + 0.002*\"euro\" + 0.002*\"europe\" + 0.002*\"long\" + 0.002*\"countries\" + 0.002*\"money\" + 0.002*\"japan\" + 0.002*\"central\" + 0.002*\"public\" + 0.002*\"says\"'),\n",
       " (1,\n",
       "  '0.003*\"companies\" + 0.003*\"hit\" + 0.003*\"japan\" + 0.002*\"crisis\" + 0.002*\"cut\" + 0.002*\"central\" + 0.002*\"level\" + 0.002*\"billion\" + 0.002*\"gained\" + 0.002*\"gains\"'),\n",
       " (2,\n",
       "  '0.003*\"companies\" + 0.002*\"jobs\" + 0.002*\"price\" + 0.002*\"euro\" + 0.002*\"eu\" + 0.002*\"europe\" + 0.002*\"trading\" + 0.002*\"hit\" + 0.002*\"reporting\" + 0.002*\"showed\"'),\n",
       " (3,\n",
       "  '0.002*\"eu\" + 0.002*\"euro\" + 0.002*\"companies\" + 0.002*\"chief\" + 0.002*\"crisis\" + 0.002*\"hit\" + 0.002*\"term\" + 0.002*\"public\" + 0.002*\"trading\" + 0.002*\"deal\"'),\n",
       " (4,\n",
       "  '0.002*\"trading\" + 0.002*\"europe\" + 0.002*\"hit\" + 0.002*\"gains\" + 0.002*\"companies\" + 0.002*\"investment\" + 0.002*\"expectations\" + 0.002*\"japan\" + 0.002*\"according\" + 0.002*\"cut\"'),\n",
       " (5,\n",
       "  '0.003*\"sales\" + 0.003*\"trading\" + 0.002*\"companies\" + 0.002*\"billion\" + 0.002*\"japan\" + 0.002*\"euro\" + 0.002*\"price\" + 0.002*\"cut\" + 0.002*\"strong\" + 0.002*\"gains\"'),\n",
       " (6,\n",
       "  '0.003*\"europe\" + 0.002*\"long\" + 0.002*\"companies\" + 0.002*\"hit\" + 0.002*\"crisis\" + 0.002*\"chief\" + 0.002*\"sales\" + 0.002*\"euro\" + 0.002*\"according\" + 0.002*\"trading\"'),\n",
       " (7,\n",
       "  '0.003*\"euro\" + 0.002*\"japan\" + 0.002*\"jobs\" + 0.002*\"energy\" + 0.002*\"gained\" + 0.002*\"crisis\" + 0.002*\"added\" + 0.002*\"gains\" + 0.002*\"investment\" + 0.002*\"price\"'),\n",
       " (8,\n",
       "  '0.003*\"euro\" + 0.003*\"gains\" + 0.003*\"trading\" + 0.002*\"strong\" + 0.002*\"investment\" + 0.002*\"ftse\" + 0.002*\"cut\" + 0.002*\"billion\" + 0.002*\"chief\" + 0.002*\"price\"'),\n",
       " (9,\n",
       "  '0.003*\"like\" + 0.002*\"price\" + 0.002*\"companies\" + 0.002*\"gains\" + 0.002*\"crisis\" + 0.002*\"president\" + 0.002*\"expectations\" + 0.002*\"money\" + 0.002*\"hit\" + 0.002*\"public\"')]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = models.LdaMulticore(corpus=corpus, num_topics=10, id2word=id2word, passes=1)\n",
    "lda.print_topics()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "128"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extra_stopwords = extra_stopwords + ['Euro', 'euro', 'EU', 'eu', 'Europe', 'europe', 'long', 'Japan', 'japan', 'says', \n",
    "                                    'hit', 'reporting', 'Reporting']\n",
    "len(extra_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "281"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_list = stopwords.words('english')\n",
    "stop_list = stop_list + extra_stopwords\n",
    "len(stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_df['article_text_extra_stops_v5'] = sample_df['cleaned_article_text'].apply(remove_stop_words, stop_list=stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2357\n",
      "2289\n",
      "2269\n",
      "2253\n",
      "2106\n",
      "2056\n"
     ]
    }
   ],
   "source": [
    "print(len(sample_df.iloc[2000]['cleaned_article_text']))\n",
    "print(len(sample_df.iloc[2000]['article_text_extra_stops']))\n",
    "print(len(sample_df.iloc[2000]['article_text_extra_stops_v2']))\n",
    "print(len(sample_df.iloc[2000]['article_text_extra_stops_v3']))\n",
    "print(len(sample_df.iloc[2000]['article_text_extra_stops_v4']))\n",
    "print(len(sample_df.iloc[2000]['article_text_extra_stops_v5']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Again"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(99159, 10000)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vectorizer = CountVectorizer(stop_words='english')\n",
    "count_vectorizer.fit(sample_df['article_text_extra_stops_v5'])\n",
    "counts = count_vectorizer.transform(sample_df['article_text_extra_stops_v5']).transpose()\n",
    "counts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99159"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus = matutils.Sparse2Corpus(counts)\n",
    "id2word = dict((v, k) for k, v in count_vectorizer.vocabulary_.items())\n",
    "len(id2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.003*\"companies\" + 0.003*\"trading\" + 0.002*\"ftse\" + 0.002*\"gains\" + 0.002*\"sales\" + 0.002*\"reported\" + 0.002*\"chief\" + 0.002*\"greece\" + 0.002*\"expectations\" + 0.002*\"crisis\"'),\n",
       " (1,\n",
       "  '0.002*\"crisis\" + 0.002*\"public\" + 0.002*\"gains\" + 0.002*\"record\" + 0.002*\"cut\" + 0.002*\"gained\" + 0.002*\"countries\" + 0.002*\"according\" + 0.002*\"central\" + 0.002*\"major\"'),\n",
       " (2,\n",
       "  '0.003*\"crisis\" + 0.002*\"investment\" + 0.002*\"price\" + 0.002*\"credit\" + 0.002*\"like\" + 0.002*\"level\" + 0.002*\"trading\" + 0.002*\"going\" + 0.002*\"companies\" + 0.002*\"gains\"'),\n",
       " (3,\n",
       "  '0.002*\"public\" + 0.002*\"chancellor\" + 0.002*\"labour\" + 0.002*\"price\" + 0.002*\"jobs\" + 0.002*\"companies\" + 0.002*\"crisis\" + 0.002*\"need\" + 0.002*\"make\" + 0.002*\"countries\"'),\n",
       " (4,\n",
       "  '0.003*\"billion\" + 0.002*\"money\" + 0.002*\"pay\" + 0.002*\"bond\" + 0.002*\"central\" + 0.002*\"jobs\" + 0.002*\"chief\" + 0.002*\"demand\" + 0.002*\"fund\" + 0.002*\"sales\"'),\n",
       " (5,\n",
       "  '0.003*\"gains\" + 0.003*\"trading\" + 0.002*\"companies\" + 0.002*\"jobs\" + 0.002*\"price\" + 0.002*\"seen\" + 0.002*\"energy\" + 0.002*\"central\" + 0.002*\"billion\" + 0.002*\"sales\"'),\n",
       " (6,\n",
       "  '0.003*\"companies\" + 0.002*\"trading\" + 0.002*\"results\" + 0.002*\"futures\" + 0.002*\"gains\" + 0.002*\"added\" + 0.002*\"crisis\" + 0.002*\"losses\" + 0.002*\"cut\" + 0.002*\"following\"'),\n",
       " (7,\n",
       "  '0.002*\"sales\" + 0.002*\"cut\" + 0.002*\"showed\" + 0.002*\"countries\" + 0.002*\"chief\" + 0.002*\"industrial\" + 0.002*\"expectations\" + 0.002*\"bond\" + 0.002*\"labour\" + 0.002*\"according\"'),\n",
       " (8,\n",
       "  '0.003*\"trading\" + 0.003*\"gains\" + 0.002*\"sales\" + 0.002*\"price\" + 0.002*\"investment\" + 0.002*\"president\" + 0.002*\"cut\" + 0.002*\"gained\" + 0.002*\"crude\" + 0.002*\"crisis\"'),\n",
       " (9,\n",
       "  '0.002*\"companies\" + 0.002*\"strong\" + 0.002*\"price\" + 0.002*\"money\" + 0.002*\"cut\" + 0.002*\"consumer\" + 0.002*\"greece\" + 0.002*\"cuts\" + 0.002*\"crisis\" + 0.002*\"billion\"')]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda = models.LdaMulticore(corpus=corpus, num_topics=10, id2word=id2word, passes=1)\n",
    "lda.print_topics()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "extra_stopwords = extra_stopwords + ['strong', 'financial', 'Strong', 'Financial']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['shares',\n",
       " 'economy',\n",
       " 'economic',\n",
       " 'prices',\n",
       " 'investors',\n",
       " 'government',\n",
       " 'growth',\n",
       " 'banks',\n",
       " 'expected',\n",
       " 'rate',\n",
       " 'rates',\n",
       " 'stocks',\n",
       " 'financial',\n",
       " 'uk',\n",
       " 'european',\n",
       " 'u_s',\n",
       " 'tax',\n",
       " 'britain',\n",
       " 'inflation',\n",
       " 'country',\n",
       " 'bank',\n",
       " 'business',\n",
       " 'spending',\n",
       " 'dollar',\n",
       " 'oil',\n",
       " 'company',\n",
       " 'stock',\n",
       " 'yen',\n",
       " 'fed',\n",
       " 'earnings',\n",
       " 'sector',\n",
       " 'global',\n",
       " 'policy',\n",
       " 'world',\n",
       " 'debt',\n",
       " 'china',\n",
       " 'biggest',\n",
       " 'trade',\n",
       " 'bank',\n",
       " 'work',\n",
       " 'nikkei',\n",
       " 'profit',\n",
       " 'business',\n",
       " 'uk',\n",
       " 'european',\n",
       " 'u_s',\n",
       " 'tax',\n",
       " 'britain',\n",
       " 'inflation',\n",
       " 'country',\n",
       " 'bank',\n",
       " 'business',\n",
       " 'spending',\n",
       " 'dollar',\n",
       " 'oil',\n",
       " 'company',\n",
       " 'stock',\n",
       " 'yen',\n",
       " 'fed',\n",
       " 'earnings',\n",
       " 'sector',\n",
       " 'global',\n",
       " 'policy',\n",
       " 'world',\n",
       " 'debt',\n",
       " 'china',\n",
       " 'biggest',\n",
       " 'trade',\n",
       " 'bank',\n",
       " 'work',\n",
       " 'nikkei',\n",
       " 'profit',\n",
       " 'business',\n",
       " 'uk',\n",
       " 'european',\n",
       " 'u_s',\n",
       " 'tax',\n",
       " 'britain',\n",
       " 'inflation',\n",
       " 'country',\n",
       " 'bank',\n",
       " 'business',\n",
       " 'spending',\n",
       " 'dollar',\n",
       " 'oil',\n",
       " 'company',\n",
       " 'stock',\n",
       " 'yen',\n",
       " 'fed',\n",
       " 'earnings',\n",
       " 'sector',\n",
       " 'global',\n",
       " 'policy',\n",
       " 'world',\n",
       " 'debt',\n",
       " 'china',\n",
       " 'biggest',\n",
       " 'trade',\n",
       " 'bank',\n",
       " 'work',\n",
       " 'nikkei',\n",
       " 'profit',\n",
       " 'business',\n",
       " 'European',\n",
       " 'UK',\n",
       " 'U_S',\n",
       " 'Bank',\n",
       " 'Nikkei',\n",
       " 'Fed',\n",
       " 'Britain',\n",
       " 'Dollar',\n",
       " 'Yen',\n",
       " 'Global',\n",
       " 'World',\n",
       " 'China',\n",
       " 'Euro',\n",
       " 'euro',\n",
       " 'EU',\n",
       " 'eu',\n",
       " 'Europe',\n",
       " 'europe',\n",
       " 'long',\n",
       " 'Japan',\n",
       " 'japan',\n",
       " 'says',\n",
       " 'hit',\n",
       " 'reporting',\n",
       " 'Reporting']"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "['shares',\n",
    " 'economy',\n",
    " 'economic',\n",
    " 'prices',\n",
    " 'investors',\n",
    " 'government',\n",
    " 'growth',\n",
    " 'banks',\n",
    " 'expected',\n",
    " 'rate',\n",
    " 'rates',\n",
    " 'stocks',\n",
    " 'financial',\n",
    " 'uk',\n",
    " 'european',\n",
    " 'u_s',\n",
    " 'tax',\n",
    " 'britain',\n",
    " 'inflation',\n",
    " 'country',\n",
    " 'bank',\n",
    " 'business',\n",
    " 'spending',\n",
    " 'dollar',\n",
    " 'oil',\n",
    " 'company',\n",
    " 'stock',\n",
    " 'yen',\n",
    " 'fed',\n",
    " 'earnings',\n",
    " 'sector',\n",
    " 'global',\n",
    " 'policy',\n",
    " 'world',\n",
    " 'debt',\n",
    " 'china',\n",
    " 'biggest',\n",
    " 'trade',\n",
    " 'bank',\n",
    " 'work',\n",
    " 'nikkei',\n",
    " 'profit',\n",
    " 'business',\n",
    " 'uk',\n",
    " 'european',\n",
    " 'u_s',\n",
    " 'tax',\n",
    " 'britain',\n",
    " 'inflation',\n",
    " 'country',\n",
    " 'bank',\n",
    " 'business',\n",
    " 'spending',\n",
    " 'dollar',\n",
    " 'oil',\n",
    " 'company',\n",
    " 'stock',\n",
    " 'yen',\n",
    " 'fed',\n",
    " 'earnings',\n",
    " 'sector',\n",
    " 'global',\n",
    " 'policy',\n",
    " 'world',\n",
    " 'debt',\n",
    " 'china',\n",
    " 'biggest',\n",
    " 'trade',\n",
    " 'bank',\n",
    " 'work',\n",
    " 'nikkei',\n",
    " 'profit',\n",
    " 'business',\n",
    " 'uk',\n",
    " 'european',\n",
    " 'u_s',\n",
    " 'tax',\n",
    " 'britain',\n",
    " 'inflation',\n",
    " 'country',\n",
    " 'bank',\n",
    " 'business',\n",
    " 'spending',\n",
    " 'dollar',\n",
    " 'oil',\n",
    " 'company',\n",
    " 'stock',\n",
    " 'yen',\n",
    " 'fed',\n",
    " 'earnings',\n",
    " 'sector',\n",
    " 'global',\n",
    " 'policy',\n",
    " 'world',\n",
    " 'debt',\n",
    " 'china',\n",
    " 'biggest',\n",
    " 'trade',\n",
    " 'bank',\n",
    " 'work',\n",
    " 'nikkei',\n",
    " 'profit',\n",
    " 'business',\n",
    " 'European',\n",
    " 'UK',\n",
    " 'U_S',\n",
    " 'Bank',\n",
    " 'Nikkei',\n",
    " 'Fed',\n",
    " 'Britain',\n",
    " 'Dollar',\n",
    " 'Yen',\n",
    " 'Global',\n",
    " 'World',\n",
    " 'China',\n",
    " 'Euro',\n",
    " 'euro',\n",
    " 'EU',\n",
    " 'eu',\n",
    " 'Europe',\n",
    " 'europe',\n",
    " 'long',\n",
    " 'Japan',\n",
    " 'japan',\n",
    " 'says',\n",
    " 'hit',\n",
    " 'reporting',\n",
    " 'Reporting']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "extra_stopwords = list(set(extra_stopwords))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Apply extra stop word removal to full data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "218"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_list = stopwords.words('english')\n",
    "stop_list = stop_list + extra_stopwords\n",
    "len(stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['article_text_extra_stops'] = df['cleaned_article_text'].apply(remove_stop_words, stop_list=stop_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_text</th>\n",
       "      <th>datetime</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>cleaned_article_text</th>\n",
       "      <th>article_text_extra_stops</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>57471</th>\n",
       "      <td>(Refiles to correct typographical error in fir...</td>\n",
       "      <td>2009-10-26</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>REFILE-Late selloff extends European stocks' l...</td>\n",
       "      <td>http://uk.reuters.com/article/markets-europe-s...</td>\n",
       "      <td>European stocks fell third session row Monday ...</td>\n",
       "      <td>European third session row sharp drop oil rebo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1932</th>\n",
       "      <td>\\nThe US president, Barack Obama, has signalle...</td>\n",
       "      <td>2016-11-13 19:39:08</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>Barack Obama calls for 'meaningful debt relief...</td>\n",
       "      <td>https://www.theguardian.com/us-news/2016/nov/1...</td>\n",
       "      <td>The US president Barack_Obama signalled use cr...</td>\n",
       "      <td>The US president Barack_Obama signalled use cr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4653</th>\n",
       "      <td>\\nHamlet without the prince. That, bluntly, is...</td>\n",
       "      <td>2017-01-17 10:29:43</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>Davos without Donald Trump is like Hamlet with...</td>\n",
       "      <td>https://www.theguardian.com/business/2017/jan/...</td>\n",
       "      <td>Hamlet without prince That bluntly Davos feels...</td>\n",
       "      <td>Hamlet without prince That bluntly Davos feels...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11583</th>\n",
       "      <td>\\nMark Carney has backed the prime minister’s ...</td>\n",
       "      <td>2016-10-06 19:18:21</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>Mark Carney backs prime minister's call for ec...</td>\n",
       "      <td>https://www.theguardian.com/politics/2016/oct/...</td>\n",
       "      <td>Mark_Carney backed prime minister call rethink...</td>\n",
       "      <td>Mark_Carney backed prime minister call rethink...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72597</th>\n",
       "      <td>Hedge funds may not like Donald Trump, but th...</td>\n",
       "      <td>2016-09-26 00:00:00</td>\n",
       "      <td>CNBC</td>\n",
       "      <td>\\n    This trade will tell you whether Tru...</td>\n",
       "      <td>https://www.cnbc.com/2016/09/26/this-trade-wil...</td>\n",
       "      <td>Hedge funds may like Donald_Trump mean try mak...</td>\n",
       "      <td>Hedge funds may like Donald_Trump mean try mak...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            article_text             datetime  \\\n",
       "57471  (Refiles to correct typographical error in fir...           2009-10-26   \n",
       "1932   \\nThe US president, Barack Obama, has signalle...  2016-11-13 19:39:08   \n",
       "4653   \\nHamlet without the prince. That, bluntly, is...  2017-01-17 10:29:43   \n",
       "11583  \\nMark Carney has backed the prime minister’s ...  2016-10-06 19:18:21   \n",
       "72597   Hedge funds may not like Donald Trump, but th...  2016-09-26 00:00:00   \n",
       "\n",
       "         source                                              title  \\\n",
       "57471   Reuters  REFILE-Late selloff extends European stocks' l...   \n",
       "1932   Guardian  Barack Obama calls for 'meaningful debt relief...   \n",
       "4653   Guardian  Davos without Donald Trump is like Hamlet with...   \n",
       "11583  Guardian  Mark Carney backs prime minister's call for ec...   \n",
       "72597      CNBC      \\n    This trade will tell you whether Tru...   \n",
       "\n",
       "                                                     url  \\\n",
       "57471  http://uk.reuters.com/article/markets-europe-s...   \n",
       "1932   https://www.theguardian.com/us-news/2016/nov/1...   \n",
       "4653   https://www.theguardian.com/business/2017/jan/...   \n",
       "11583  https://www.theguardian.com/politics/2016/oct/...   \n",
       "72597  https://www.cnbc.com/2016/09/26/this-trade-wil...   \n",
       "\n",
       "                                    cleaned_article_text  \\\n",
       "57471  European stocks fell third session row Monday ...   \n",
       "1932   The US president Barack_Obama signalled use cr...   \n",
       "4653   Hamlet without prince That bluntly Davos feels...   \n",
       "11583  Mark_Carney backed prime minister call rethink...   \n",
       "72597  Hedge funds may like Donald_Trump mean try mak...   \n",
       "\n",
       "                                article_text_extra_stops  \n",
       "57471  European third session row sharp drop oil rebo...  \n",
       "1932   The US president Barack_Obama signalled use cr...  \n",
       "4653   Hamlet without prince That bluntly Davos feels...  \n",
       "11583  Mark_Carney backed prime minister call rethink...  \n",
       "72597  Hedge funds may like Donald_Trump mean try mak...  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('news_articles_extra_stops.csv', index=None, encoding='utf-8')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}