robomax-election-tweets-bot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# !unzip data/election-day-tweets.zip -d data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 751
    },
    "colab_type": "code",
    "id": "dNiN7FobI9WN",
    "outputId": "e728b8e2-ffad-49f9-988a-ad434b7c8cdc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: whoosh in /Users/nellex/anaconda3/lib/python3.6/site-packages\n",
      "\u001b[33mYou are using pip version 9.0.1, however version 18.0 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already satisfied: gensim in /Users/nellex/anaconda3/lib/python3.6/site-packages\n",
      "Requirement already satisfied: six>=1.5.0 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from gensim)\n",
      "Requirement already satisfied: numpy>=1.11.3 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from gensim)\n",
      "Requirement already satisfied: scipy>=0.18.1 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from gensim)\n",
      "Requirement already satisfied: smart-open>=1.2.1 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from gensim)\n",
      "Requirement already satisfied: boto>=2.32 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: requests in /Users/nellex/anaconda3/lib/python3.6/site-packages (from smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: bz2file in /Users/nellex/anaconda3/lib/python3.6/site-packages (from smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: boto3 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from requests->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: idna<2.7,>=2.5 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from requests->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from requests->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from requests->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: s3transfer<0.2.0,>=0.1.10 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from boto3->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: botocore<1.10.0,>=1.9.6 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from boto3->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from boto3->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: docutils>=0.10 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from botocore<1.10.0,>=1.9.6->boto3->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /Users/nellex/.local/lib/python3.6/site-packages (from botocore<1.10.0,>=1.9.6->boto3->smart-open>=1.2.1->gensim)\n",
      "\u001b[33mYou are using pip version 9.0.1, however version 18.0 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already satisfied: sumy in /Users/nellex/anaconda3/lib/python3.6/site-packages\n",
      "Requirement already satisfied: docopt<0.7,>=0.6.1 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from sumy)\n",
      "Requirement already satisfied: nltk>=3.0.2 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from sumy)\n",
      "Requirement already satisfied: breadability>=0.1.20 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from sumy)\n",
      "Requirement already satisfied: requests>=2.7.0 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from sumy)\n",
      "Requirement already satisfied: six in /Users/nellex/anaconda3/lib/python3.6/site-packages (from nltk>=3.0.2->sumy)\n",
      "Requirement already satisfied: chardet in /Users/nellex/anaconda3/lib/python3.6/site-packages (from breadability>=0.1.20->sumy)\n",
      "Requirement already satisfied: lxml>=2.0 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from breadability>=0.1.20->sumy)\n",
      "Requirement already satisfied: idna<2.7,>=2.5 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from requests>=2.7.0->sumy)\n",
      "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from requests>=2.7.0->sumy)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from requests>=2.7.0->sumy)\n",
      "\u001b[33mYou are using pip version 9.0.1, however version 18.0 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Requirement already satisfied: textblob in /Users/nellex/anaconda3/lib/python3.6/site-packages\n",
      "Requirement already satisfied: nltk>=3.1 in /Users/nellex/anaconda3/lib/python3.6/site-packages (from textblob)\n",
      "Requirement already satisfied: six in /Users/nellex/anaconda3/lib/python3.6/site-packages (from nltk>=3.1->textblob)\n",
      "\u001b[33mYou are using pip version 9.0.1, however version 18.0 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install whoosh\n",
    "!pip install gensim\n",
    "!pip install sumy\n",
    "!pip install textblob\n",
    "!pip install flask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 153
    },
    "colab_type": "code",
    "id": "z8TQiDLSCFgr",
    "outputId": "832232a3-b774-4d8b-a05f-8dfb9295b8ad"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /Users/nellex/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /Users/nellex/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package wordnet to /Users/nellex/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import unicodedata\n",
    "import gensim\n",
    "import nltk\n",
    "import html\n",
    "import json\n",
    "import os\n",
    "\n",
    "from IPython.display import display, HTML\n",
    "from whoosh.fields import Schema, ID, KEYWORD, TEXT\n",
    "from whoosh.index import create_in\n",
    "from whoosh.query import Term\n",
    "from IPython.display import display, HTML\n",
    "from tqdm import tqdm\n",
    "\n",
    "pd.set_option('display.max_colwidth', 500)\n",
    "nltk.download('punkt')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('stopwords')\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1986
    },
    "colab_type": "code",
    "id": "QEWbuoW0B7Yc",
    "outputId": "0fbe78da-25d3-49e2-9a95-0ab4424f1ed3"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tweet_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>136241</th>\n",
       "      <td>796198280476708864</td>\n",
       "      <td>Any regrets? https://t.co/5m1TwFOpd7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62702</th>\n",
       "      <td>796234506365640704</td>\n",
       "      <td>House &amp; Senate will hold as GOP controlled, SC will be conservative for decades... I wouldn't be too certain https://t.co/VT6XFJ0IBY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>366061</th>\n",
       "      <td>796014295184093189</td>\n",
       "      <td>#Election2016 fingers crossed.... https://t.co/BCMnxcHIMe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288852</th>\n",
       "      <td>795988106625290240</td>\n",
       "      <td>One more day until people go back to normal, and the office stops talking about politics for 4 years! #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248613</th>\n",
       "      <td>795971202321944577</td>\n",
       "      <td>In line...at the polls...at 6:43 am! Let's do this! #ElectionDay #election2016 #HillaryClinton #trump https://t.co/1jzFZGEz5F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212440</th>\n",
       "      <td>795937294448553984</td>\n",
       "      <td>Top Trends Sweden-Nov08 11:19 CET Sanna Nielsen #Election2016 Ilmar Reepalu #ImWithHer Stevie Wonder  https://t.co/75j1lqqN7Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88893</th>\n",
       "      <td>796325001288908800</td>\n",
       "      <td>Its funny how people are getting so salt over trump winning even though the president doesnt make a lot of the choices congress does</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109644</th>\n",
       "      <td>796068056346886144</td>\n",
       "      <td>Look I always vote democrat. But if Trump was running democrat, I'd vote for a republican. That's how full of nonsense this bs is</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51077</th>\n",
       "      <td>796215704420749315</td>\n",
       "      <td>@dccommonsense @bhtru yes, because a president Trump with SCOTUS, and a unified Congress will just be ITCHING to reform elections</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263609</th>\n",
       "      <td>795978906486669312</td>\n",
       "      <td>RT RelatabIetxts: Who are you going to vote?  #ElectionDay #Election2016 #Vote2016 #USADecides (Pls retweet after voting)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30255</th>\n",
       "      <td>796325587245821957</td>\n",
       "      <td>#Senate Florida's Marco Rubio wins re-election in Senate race https://t.co/gkEGbiVgn5 MarcoRubio NTANEWS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265531</th>\n",
       "      <td>795979628733231104</td>\n",
       "      <td>On a day that will divide America, @tacobueno knows what's up and brings back tortilla soup. #Election2016 #VoteForSoup</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235767</th>\n",
       "      <td>795957538357387265</td>\n",
       "      <td>#Election2016 #cartoon #USADecides https://t.co/0saQpqmOqL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94960</th>\n",
       "      <td>795879005320871936</td>\n",
       "      <td>Agree, no one owns anyone or their vote but Her vote is first for her God? https://t.co/BO4xoomSX0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257503</th>\n",
       "      <td>795976577683193857</td>\n",
       "      <td>Good morning America. Tonight will be better than game 7. Go get it! #Election2016 https://t.co/zhwHVrOWXE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178032</th>\n",
       "      <td>795858033586933769</td>\n",
       "      <td>\"There is hope for everyone until there isn't\" #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159931</th>\n",
       "      <td>796306629843156992</td>\n",
       "      <td>Silver linings, people, silver linings https://t.co/MxBVeUcK9A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257574</th>\n",
       "      <td>795976609782112256</td>\n",
       "      <td>#Election2016 #ElectionNight #EleccionesEEUU2016 https://t.co/VFbfwr2MYv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369916</th>\n",
       "      <td>796017046211403776</td>\n",
       "      <td>#Election2016 https://t.co/a03ukSgNkB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92014</th>\n",
       "      <td>796336038100828160</td>\n",
       "      <td>Me, too. https://t.co/XTnsbonTrO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396996</th>\n",
       "      <td>796037146469658624</td>\n",
       "      <td>#ivoted #pansuitnation #imwithher #election2016 https://t.co/2v11jjPqOT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288067</th>\n",
       "      <td>795987835463626752</td>\n",
       "      <td>Ridiculously long line to vote in DC &amp; growing by the second. Don't mind a bit! #VOTE #Election2016 https://t.co/Jjp5ACCPk4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62483</th>\n",
       "      <td>796234067595427840</td>\n",
       "      <td>If CNN MSNBC 86d polls &amp; detailed GOP v Dem work &amp; agendas in Congress, Senate, &amp; Judicial Branch &amp; impact on lives, this'd b VERY different</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326159</th>\n",
       "      <td>796000726925398016</td>\n",
       "      <td>LOL you are #Number2  #WakeUpAmerica #NeverHillary #CrookedHillary #Benghazi #Wikileaks #DncLeaks2 https://t.co/p57QtlYWlB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88514</th>\n",
       "      <td>796323850950676480</td>\n",
       "      <td>@mattklewis @DailyCaller Obama delivered Oval Office and both houses of Congress to GOP for the first time in almost 90 years.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383179</th>\n",
       "      <td>796027360545886208</td>\n",
       "      <td>This, too! https://t.co/4svl2wddUQ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240132</th>\n",
       "      <td>795962179610615809</td>\n",
       "      <td>Go vote! @HillaryClinton #Election2016 #ImWithHer https://t.co/9a7JZEQLGO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>392041</th>\n",
       "      <td>796033903505063936</td>\n",
       "      <td>Tonight! #Complex is taking over #YouTube for #Election2016 #FlexTheVote ~ Get Ready to 'Flex the Vote' With Complex https://t.co/A9kYNv6i0S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72491</th>\n",
       "      <td>796257115031470084</td>\n",
       "      <td>Let's hope congress decides to veto all the poor decisions and policies that are going to be introduced.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41143</th>\n",
       "      <td>796193870442889217</td>\n",
       "      <td>@rationalwalk I used to say that about Obama. He wrecked health care. Trump could do damage if he has the whole congress</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>390458</th>\n",
       "      <td>796032813426294784</td>\n",
       "      <td>It was busy, but no line at 8 am in Rexburg. 22nd vote at my precinct this morning. #voted #Election2016 https://t.co/a2OAZclXIx</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209348</th>\n",
       "      <td>795931434645147652</td>\n",
       "      <td>Oylar EKMELEDDIN IHSANOGLU' NA ! #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>370917</th>\n",
       "      <td>796017855921852416</td>\n",
       "      <td>Rock That Vote Fools!!! #election2016 #vote #imwithher https://t.co/xU0ynziIKV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318649</th>\n",
       "      <td>795998175114842112</td>\n",
       "      <td>Good luck America #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247796</th>\n",
       "      <td>795970359631564800</td>\n",
       "      <td>Just got reminded regarding #Election2016 that The Simpsons already did it https://t.co/5j3kpXzBLl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34106</th>\n",
       "      <td>796164603688329216</td>\n",
       "      <td>@LitzKyle  If he wins and Congress swings right, it would be the thirty year conservative backlash we get here in America.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243132</th>\n",
       "      <td>795965400802611200</td>\n",
       "      <td>The face you make when you're standing in front of people who aren't sure who they want to vote for yet. https://t.co/mShXKC7OO2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31863</th>\n",
       "      <td>796147036689223680</td>\n",
       "      <td>Why doesn't Congress appoint a Special Prosecutor to investigate the Benghazi &amp; email scandals?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34055</th>\n",
       "      <td>796164321776697345</td>\n",
       "      <td>@AshlynNicole31 like hello congress?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162081</th>\n",
       "      <td>796331857797386240</td>\n",
       "      <td>people like this make me feel bad being a democrat.  https://t.co/iSrlyaWR3j</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>372903</th>\n",
       "      <td>796019297323782145</td>\n",
       "      <td>#NotasDeImpacto Dos mujeres fueron arrestadas por protestar en toplees en lugar donde Trump votara https://t.co/SsIDDAF7wb</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253447</th>\n",
       "      <td>795974940675768320</td>\n",
       "      <td>Hey girl. Is your name America? Cause you're gettin fucked tonight. (   ) #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41803</th>\n",
       "      <td>796195507702026240</td>\n",
       "      <td>YOU GUYS DO REALIZE THERES MORE REPUBLICAN THAN DEMOCRATIC IN CONGRESS SOOOO HILARY CANT MAKE HER PROMISES COME TRUE IF SHE WINS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186220</th>\n",
       "      <td>795868760313888768</td>\n",
       "      <td>MTVs #VirtualReality Art Show Brings To Life Everything That's At Stake In The  #Election2016 https://t.co/yKb1i9kJfd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157994</th>\n",
       "      <td>796280480073220096</td>\n",
       "      <td>They won't find a better candidate than her so why not? Btw good thing none of Trump's wives would be allowed to ru https://t.co/SeysogfYqb</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238416</th>\n",
       "      <td>795960257520553984</td>\n",
       "      <td>C'MON AMERICA - DON'T F**K THIS UP  #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17641</th>\n",
       "      <td>796041600925401089</td>\n",
       "      <td>yo can't really do anything, it's laws the president won't attempt to establish because he know congress won't let https://t.co/TNs6Zr4HrM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>359470</th>\n",
       "      <td>796011729515909121</td>\n",
       "      <td>Who will be the President? Chi vincera? #Election2016 #USA2016 #SkyElezioniUSA #ElezioniUsa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106112</th>\n",
       "      <td>796045264599580672</td>\n",
       "      <td>@wolfblitzer WikiLeaks emails tell the tale Wolf,U R not a journalist but simply a whore for the Democrat party.Disgrace to ur profession</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>346474</th>\n",
       "      <td>796007358937911296</td>\n",
       "      <td>'When the dollar depreciates that means US travellers have less money to travel abroad' https://t.co/4HYYVxHYjv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123884</th>\n",
       "      <td>796158354653646848</td>\n",
       "      <td>Duckworth's election would add one more Democrat to US Senate. Less than 50 women have ever served in US Senate https://t.co/SjqeQJjbqG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>388768</th>\n",
       "      <td>796031596482609152</td>\n",
       "      <td>Happy voting. #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257676</th>\n",
       "      <td>795976650932580352</td>\n",
       "      <td>#Election2016 https://t.co/fayyVWZ1mJ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333287</th>\n",
       "      <td>796003215561080834</td>\n",
       "      <td>Make sure to get out and vote today! https://t.co/GZyXVM00oC #ElectionDay #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256037</th>\n",
       "      <td>795975981274173440</td>\n",
       "      <td>I can't think of a better theme song for today. #ImWithHer #Election2016 https://t.co/YzKfdLdAuU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135433</th>\n",
       "      <td>796195870085697536</td>\n",
       "      <td>@KokueiAi nah, every member of the lgbt community needs to vote straight democrat until SCOTUS flips in a couple decades.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5614</th>\n",
       "      <td>795955297541976065</td>\n",
       "      <td>#fuck #SCOTUS #fuck #POTUS #fuck #Congress #fuck #Pentagon and #fuck the #bourgeois little horse they road in on. They have all your money.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38159</th>\n",
       "      <td>796184575252844544</td>\n",
       "      <td>@khuxford @JeremyMJordan You think a Republican Congress will stand in his way?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53376</th>\n",
       "      <td>796219139010469888</td>\n",
       "      <td>Scared your open borders is in danger? https://t.co/0gbxXC8DFC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169305</th>\n",
       "      <td>795847077385408512</td>\n",
       "      <td>GET out and VOTE! Show your opinion where it counts! No matter what your opinion is, vote! #Election2016 https://t.co/3KbmNzl5KL</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>39763 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        id  \\\n",
       "136241  796198280476708864   \n",
       "62702   796234506365640704   \n",
       "366061  796014295184093189   \n",
       "288852  795988106625290240   \n",
       "248613  795971202321944577   \n",
       "212440  795937294448553984   \n",
       "88893   796325001288908800   \n",
       "109644  796068056346886144   \n",
       "51077   796215704420749315   \n",
       "263609  795978906486669312   \n",
       "30255   796325587245821957   \n",
       "265531  795979628733231104   \n",
       "235767  795957538357387265   \n",
       "94960   795879005320871936   \n",
       "257503  795976577683193857   \n",
       "178032  795858033586933769   \n",
       "159931  796306629843156992   \n",
       "257574  795976609782112256   \n",
       "369916  796017046211403776   \n",
       "92014   796336038100828160   \n",
       "396996  796037146469658624   \n",
       "288067  795987835463626752   \n",
       "62483   796234067595427840   \n",
       "326159  796000726925398016   \n",
       "88514   796323850950676480   \n",
       "383179  796027360545886208   \n",
       "240132  795962179610615809   \n",
       "392041  796033903505063936   \n",
       "72491   796257115031470084   \n",
       "41143   796193870442889217   \n",
       "...                    ...   \n",
       "390458  796032813426294784   \n",
       "209348  795931434645147652   \n",
       "370917  796017855921852416   \n",
       "318649  795998175114842112   \n",
       "247796  795970359631564800   \n",
       "34106   796164603688329216   \n",
       "243132  795965400802611200   \n",
       "31863   796147036689223680   \n",
       "34055   796164321776697345   \n",
       "162081  796331857797386240   \n",
       "372903  796019297323782145   \n",
       "253447  795974940675768320   \n",
       "41803   796195507702026240   \n",
       "186220  795868760313888768   \n",
       "157994  796280480073220096   \n",
       "238416  795960257520553984   \n",
       "17641   796041600925401089   \n",
       "359470  796011729515909121   \n",
       "106112  796045264599580672   \n",
       "346474  796007358937911296   \n",
       "123884  796158354653646848   \n",
       "388768  796031596482609152   \n",
       "257676  795976650932580352   \n",
       "333287  796003215561080834   \n",
       "256037  795975981274173440   \n",
       "135433  796195870085697536   \n",
       "5614    795955297541976065   \n",
       "38159   796184575252844544   \n",
       "53376   796219139010469888   \n",
       "169305  795847077385408512   \n",
       "\n",
       "                                                                                                                                          tweet_text  \n",
       "136241                                                                                                          Any regrets? https://t.co/5m1TwFOpd7  \n",
       "62702           House & Senate will hold as GOP controlled, SC will be conservative for decades... I wouldn't be too certain https://t.co/VT6XFJ0IBY  \n",
       "366061                                                                                     #Election2016 fingers crossed.... https://t.co/BCMnxcHIMe  \n",
       "288852                           One more day until people go back to normal, and the office stops talking about politics for 4 years! #Election2016  \n",
       "248613                 In line...at the polls...at 6:43 am! Let's do this! #ElectionDay #election2016 #HillaryClinton #trump https://t.co/1jzFZGEz5F  \n",
       "212440                 Top Trends Sweden-Nov08 11:19 CET Sanna Nielsen #Election2016 Ilmar Reepalu #ImWithHer Stevie Wonder  https://t.co/75j1lqqN7Y  \n",
       "88893           Its funny how people are getting so salt over trump winning even though the president doesnt make a lot of the choices congress does  \n",
       "109644             Look I always vote democrat. But if Trump was running democrat, I'd vote for a republican. That's how full of nonsense this bs is  \n",
       "51077              @dccommonsense @bhtru yes, because a president Trump with SCOTUS, and a unified Congress will just be ITCHING to reform elections  \n",
       "263609                     RT RelatabIetxts: Who are you going to vote?  #ElectionDay #Election2016 #Vote2016 #USADecides (Pls retweet after voting)  \n",
       "30255                                       #Senate Florida's Marco Rubio wins re-election in Senate race https://t.co/gkEGbiVgn5 MarcoRubio NTANEWS  \n",
       "265531                       On a day that will divide America, @tacobueno knows what's up and brings back tortilla soup. #Election2016 #VoteForSoup  \n",
       "235767                                                                                    #Election2016 #cartoon #USADecides https://t.co/0saQpqmOqL  \n",
       "94960                                             Agree, no one owns anyone or their vote but Her vote is first for her God? https://t.co/BO4xoomSX0  \n",
       "257503                                    Good morning America. Tonight will be better than game 7. Go get it! #Election2016 https://t.co/zhwHVrOWXE  \n",
       "178032                                                                                  \"There is hope for everyone until there isn't\" #Election2016  \n",
       "159931                                                                                Silver linings, people, silver linings https://t.co/MxBVeUcK9A  \n",
       "257574                                                                      #Election2016 #ElectionNight #EleccionesEEUU2016 https://t.co/VFbfwr2MYv  \n",
       "369916                                                                                                         #Election2016 https://t.co/a03ukSgNkB  \n",
       "92014                                                                                                               Me, too. https://t.co/XTnsbonTrO  \n",
       "396996                                                                       #ivoted #pansuitnation #imwithher #election2016 https://t.co/2v11jjPqOT  \n",
       "288067                   Ridiculously long line to vote in DC & growing by the second. Don't mind a bit! #VOTE #Election2016 https://t.co/Jjp5ACCPk4  \n",
       "62483   If CNN MSNBC 86d polls & detailed GOP v Dem work & agendas in Congress, Senate, & Judicial Branch & impact on lives, this'd b VERY different  \n",
       "326159                    LOL you are #Number2  #WakeUpAmerica #NeverHillary #CrookedHillary #Benghazi #Wikileaks #DncLeaks2 https://t.co/p57QtlYWlB  \n",
       "88514                 @mattklewis @DailyCaller Obama delivered Oval Office and both houses of Congress to GOP for the first time in almost 90 years.  \n",
       "383179                                                                                                            This, too! https://t.co/4svl2wddUQ  \n",
       "240132                                                                     Go vote! @HillaryClinton #Election2016 #ImWithHer https://t.co/9a7JZEQLGO  \n",
       "392041  Tonight! #Complex is taking over #YouTube for #Election2016 #FlexTheVote ~ Get Ready to 'Flex the Vote' With Complex https://t.co/A9kYNv6i0S  \n",
       "72491                                       Let's hope congress decides to veto all the poor decisions and policies that are going to be introduced.  \n",
       "41143                       @rationalwalk I used to say that about Obama. He wrecked health care. Trump could do damage if he has the whole congress  \n",
       "...                                                                                                                                              ...  \n",
       "390458              It was busy, but no line at 8 am in Rexburg. 22nd vote at my precinct this morning. #voted #Election2016 https://t.co/a2OAZclXIx  \n",
       "209348                                                                                                Oylar EKMELEDDIN IHSANOGLU' NA ! #Election2016  \n",
       "370917                                                                Rock That Vote Fools!!! #election2016 #vote #imwithher https://t.co/xU0ynziIKV  \n",
       "318649                                                                                                               Good luck America #Election2016  \n",
       "247796                                            Just got reminded regarding #Election2016 that The Simpsons already did it https://t.co/5j3kpXzBLl  \n",
       "34106                     @LitzKyle  If he wins and Congress swings right, it would be the thirty year conservative backlash we get here in America.  \n",
       "243132              The face you make when you're standing in front of people who aren't sure who they want to vote for yet. https://t.co/mShXKC7OO2  \n",
       "31863                                                Why doesn't Congress appoint a Special Prosecutor to investigate the Benghazi & email scandals?  \n",
       "34055                                                                                                           @AshlynNicole31 like hello congress?  \n",
       "162081                                                                  people like this make me feel bad being a democrat.  https://t.co/iSrlyaWR3j  \n",
       "372903                    #NotasDeImpacto Dos mujeres fueron arrestadas por protestar en toplees en lugar donde Trump votara https://t.co/SsIDDAF7wb  \n",
       "253447                                                       Hey girl. Is your name America? Cause you're gettin fucked tonight. (   ) #Election2016  \n",
       "41803               YOU GUYS DO REALIZE THERES MORE REPUBLICAN THAN DEMOCRATIC IN CONGRESS SOOOO HILARY CANT MAKE HER PROMISES COME TRUE IF SHE WINS  \n",
       "186220                         MTVs #VirtualReality Art Show Brings To Life Everything That's At Stake In The  #Election2016 https://t.co/yKb1i9kJfd  \n",
       "157994   They won't find a better candidate than her so why not? Btw good thing none of Trump's wives would be allowed to ru https://t.co/SeysogfYqb  \n",
       "238416                                                                                             C'MON AMERICA - DON'T F**K THIS UP  #Election2016  \n",
       "17641     yo can't really do anything, it's laws the president won't attempt to establish because he know congress won't let https://t.co/TNs6Zr4HrM  \n",
       "359470                                                   Who will be the President? Chi vincera? #Election2016 #USA2016 #SkyElezioniUSA #ElezioniUsa  \n",
       "106112     @wolfblitzer WikiLeaks emails tell the tale Wolf,U R not a journalist but simply a whore for the Democrat party.Disgrace to ur profession  \n",
       "346474                               'When the dollar depreciates that means US travellers have less money to travel abroad' https://t.co/4HYYVxHYjv  \n",
       "123884       Duckworth's election would add one more Democrat to US Senate. Less than 50 women have ever served in US Senate https://t.co/SjqeQJjbqG  \n",
       "388768                                                                                                                   Happy voting. #Election2016  \n",
       "257676                                                                                                         #Election2016 https://t.co/fayyVWZ1mJ  \n",
       "333287                                                       Make sure to get out and vote today! https://t.co/GZyXVM00oC #ElectionDay #Election2016  \n",
       "256037                                              I can't think of a better theme song for today. #ImWithHer #Election2016 https://t.co/YzKfdLdAuU  \n",
       "135433                     @KokueiAi nah, every member of the lgbt community needs to vote straight democrat until SCOTUS flips in a couple decades.  \n",
       "5614     #fuck #SCOTUS #fuck #POTUS #fuck #Congress #fuck #Pentagon and #fuck the #bourgeois little horse they road in on. They have all your money.  \n",
       "38159                                                                @khuxford @JeremyMJordan You think a Republican Congress will stand in his way?  \n",
       "53376                                                                                 Scared your open borders is in danger? https://t.co/0gbxXC8DFC  \n",
       "169305              GET out and VOTE! Show your opinion where it counts! No matter what your opinion is, vote! #Election2016 https://t.co/3KbmNzl5KL  \n",
       "\n",
       "[39763 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tweet_para_df = pd.read_csv('data/election-day-tweets.csv', sep=',', header=0)\n",
    "tweet_para_df = tweet_para_df.sample(frac=0.1, random_state=42)\n",
    "\n",
    "tweet_para_df = tweet_para_df[['id', 'text']]\n",
    "tweet_para_df['tweet_text'] = tweet_para_df.apply(\n",
    "    lambda row: \n",
    "    str(unicodedata.normalize('NFD', html.unescape(row['text'].replace('\\n', ' ')))\n",
    "        .encode('ascii', 'ignore'), \n",
    "        'utf-8'), \n",
    "    axis=1)\n",
    "tweet_para_df.drop(['text'], inplace=True, axis=1)\n",
    "display(tweet_para_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "CE42ZaTqB7Yj",
    "outputId": "888e1950-bf21-460a-d77f-01731123ba2a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "39763it [00:15, 2595.92it/s]\n"
     ]
    }
   ],
   "source": [
    "# Index for Election Tweets\n",
    "\n",
    "ELECTION_TWEET_INDEX_DIR = \"election_tweet_index\"\n",
    "\n",
    "schema = Schema(content=TEXT, idx=ID(unique=True, stored=True))\n",
    " \n",
    "# Create index dir if it does not exists.\n",
    "if not os.path.exists(ELECTION_TWEET_INDEX_DIR):\n",
    "    os.mkdir(ELECTION_TWEET_INDEX_DIR)\n",
    "\n",
    "# Initialize index\n",
    "search_index = create_in(ELECTION_TWEET_INDEX_DIR, schema)\n",
    "\n",
    "# Fill index with posts from DB\n",
    "writer = search_index.writer()\n",
    "\n",
    "for sqIdx, row in tqdm(tweet_para_df.iterrows()):\n",
    "    writer.update_document(content=row['tweet_text'], idx=str(row['id']))\n",
    "\n",
    "writer.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "zn2F97-HB7Ym"
   },
   "outputs": [],
   "source": [
    "# Load the pre-generated index instead\n",
    "\n",
    "from whoosh.fields import Schema, ID, KEYWORD, TEXT\n",
    "from whoosh.index import create_in, open_dir\n",
    "from whoosh.query import Term\n",
    "from IPython.display import display, HTML\n",
    "from tqdm import tqdm\n",
    "\n",
    "ELECTION_TWEET_INDEX_DIR = \"election_tweet_index\"\n",
    "\n",
    "search_index = open_dir(ELECTION_TWEET_INDEX_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "uvBjDhIeB7Yo",
    "outputId": "3ba2bd61-030b-4ba9-e114-72cc2b1389ef"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hillary', 'promise']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_to_process = ('JJ', 'NN')\n",
    "\n",
    "def fetch_intent_from_question(text, tags=tags_to_process):\n",
    "    tokenized = nltk.word_tokenize(text)\n",
    "    tagged = nltk.pos_tag(tokenized)\n",
    "    entities = [word for word, tag in tagged if tag.startswith(tags)]\n",
    "    if not entities:\n",
    "        return tokenized\n",
    "    return entities\n",
    "\n",
    "fetch_intent_from_question(\"Will Hillary stand by her promise?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1969
    },
    "colab_type": "code",
    "id": "7mydH0HTB7Yq",
    "outputId": "62529c84-3bbb-40c8-eff4-922135aa1b86"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tweet_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>246422</th>\n",
       "      <td>795968823572754432</td>\n",
       "      <td>Prediction for #election2016 is either a comfortable Hillary Clinton win or a marginal Donald Trump win - hopefully I can get a call right!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225797</th>\n",
       "      <td>795949870653771776</td>\n",
       "      <td>Donald Trump is everything that's wrong with our culture. Hillary Clinton is everything that's wrong with our government.  #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189172</th>\n",
       "      <td>795872814486142976</td>\n",
       "      <td>Donald Trump or Hillary Clinton? #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122681</th>\n",
       "      <td>796153703145291776</td>\n",
       "      <td>Republican Donald Trump and Democrat Hillary Clinton have scored early victories in their bitter presidential... https://t.co/yyi0EthGCw</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156198</th>\n",
       "      <td>796266957263818752</td>\n",
       "      <td>#Elections2016 : Republican Donald Trump wins 288 electoral colleges against Democrat Hillary Clinton's 215 to emer https://t.co/r9WA0niWFW</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264563</th>\n",
       "      <td>795979274696200192</td>\n",
       "      <td>#Election2016 my gut instinct says donald trump will win,,, even though hillary has a more chance of winning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307873</th>\n",
       "      <td>795994539483918336</td>\n",
       "      <td>Plot twist: Donald trump is voting for Hillary #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246807</th>\n",
       "      <td>795969255959302145</td>\n",
       "      <td>I'm so worried for you guys &amp; I'm not even American. Vote wisely. Vote for Hillary. Don't let Donald Trump ruins America. #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231518</th>\n",
       "      <td>795953873273884672</td>\n",
       "      <td>Imagine being American right now and having to choose either Donald Trump or Hillary Clinton to run your country  #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>321340</th>\n",
       "      <td>795999053251280896</td>\n",
       "      <td>Half Naked Women protesting against Donald Trump in polling area. These are the nuts that support Hillary Clinton. https://t.co/YkoO4BCy5d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102366</th>\n",
       "      <td>796016018133057537</td>\n",
       "      <td>Voters finally head to the polls to decide between Democrat Hillary Clinton or Republican Donald Trump. Read more: https://t.co/IUkTBHCLnB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160147</th>\n",
       "      <td>796309377980440576</td>\n",
       "      <td>Republican Donald J. Trump upsets Democrat Hillary  Clinton for U.S. President in November 8, 2016 Presidential Election.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198883</th>\n",
       "      <td>795896673167966208</td>\n",
       "      <td>Hillary Clinton is like a headache while Donald Trump is like a migraine. #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160726</th>\n",
       "      <td>796317056484438016</td>\n",
       "      <td>#ElectionNight LIKE @JimmiJourney on #FB if U LIKE  Hillary Clinton or Donald Trump  #p2 #democrat https://t.co/qVkSiSjECK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132519</th>\n",
       "      <td>796186470646067202</td>\n",
       "      <td>US election: Republican Donald Trump leads over Democrat Hillary Clinton 140-104 in the race to 270 electoral votes.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228808</th>\n",
       "      <td>795952000840192001</td>\n",
       "      <td>There's nothing that Jill Stein or Donald Trump has done that's worse than what Hillary has been up to.  #AmericaDecides #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167329</th>\n",
       "      <td>795844448357851136</td>\n",
       "      <td>Donald Trump and Hillary Clinton can both eat my ass. #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165021</th>\n",
       "      <td>795841312729677824</td>\n",
       "      <td>Devil : Donald Trump - Deep Sea : Hillary Clinton  That Awkward Moment When USA Fails Wisdom  #Election2016 #philadelphia #ElectionNight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97790</th>\n",
       "      <td>795976595395620864</td>\n",
       "      <td>Voting for Republican Donald Trump is today only, November 8th. Voting for Democrat Hillary Clinton is tomorrow only, Novemeber 9th.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264511</th>\n",
       "      <td>795979257570873359</td>\n",
       "      <td>To my American friends - vote for Hillary. Hillary is a voice for the voiceless. Donald Trump is a voice for Donald Trump. #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175589</th>\n",
       "      <td>795855264905732096</td>\n",
       "      <td>Realistically Hillary Clinton And Donald Trump Have Played Ecco The Dolphin.  #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151835</th>\n",
       "      <td>796244861905092608</td>\n",
       "      <td>Hillary's Presidential dreams are now over. She will always be known as the Democrat who lost to Donald Trump #electionday</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>375949</th>\n",
       "      <td>796021384107397120</td>\n",
       "      <td>#Election2016 and if will win HILLARY TRUMP or DONALD CLINTON !!!!!  @REALDONALDTRUMP @HILLARYCLINTON</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159843</th>\n",
       "      <td>796305520759967745</td>\n",
       "      <td>Donald Trump will become the 45th US president after a stunning victory over Democrat Hillary Clinton.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124540</th>\n",
       "      <td>796160780127399936</td>\n",
       "      <td>Trump, Clinton in tight battles in Ohio, Florida and elsewhere https://t.co/nGKCbLl5BS Republican Donald Trump and Democrat Hillary Clinto</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157531</th>\n",
       "      <td>796276178956873728</td>\n",
       "      <td>Oakland reacts to the election of Donald Trump: Democrat Hillary Clinton conceded the presidential race at 11:40 pm https://t.co/z5Inf3IeT4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157701</th>\n",
       "      <td>796277658459832320</td>\n",
       "      <td>Donald Trump will become the 45th US President after a stunning victory over Democrat Hillary Clinton. Great Victory.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94161</th>\n",
       "      <td>795860125991006208</td>\n",
       "      <td>Democrat  Hillary Clinton and Tim Kaine  17 Republican  Donald J. Trump and Michael R. Pence  14 in Hart's Location, NH #ImWithHer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84019</th>\n",
       "      <td>796301674176479232</td>\n",
       "      <td>Hillary Clinton lets Donald Trump deal with a tanking economy and a do-nothing Republican Congress #Election2016 https://t.co/hTEfeZv9TS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>380229</th>\n",
       "      <td>796024903992483845</td>\n",
       "      <td>RT for Donald Trump LIKE for Hillary Clinton #ElectionDay #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382519</th>\n",
       "      <td>796026888032555008</td>\n",
       "      <td>God Bless Donald Trump  #ElectionDay #Election2016 #Elecciones Bill Hillary Clinton vs Florida Latinos   https://t.co/ImnpCxycne</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242444</th>\n",
       "      <td>795964612873224192</td>\n",
       "      <td>Hillary Clinton / Timothy Kaine o Donald Trump / Michael Pence, la boleta  #ElectionDay #Election2016 https://t.co/LBjrcAK5R6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>346119</th>\n",
       "      <td>796007244223627264</td>\n",
       "      <td>Who do you believe will be the next president of the United States? Donald Trump or Hillary Clinton? Explain your a https://t.co/ZMZ8wIkaHU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158969</th>\n",
       "      <td>796292496166940672</td>\n",
       "      <td>Donald Trump will become the 45th US president after a stunning victory over Democrat Hillary Clinton</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127434</th>\n",
       "      <td>796170479488765952</td>\n",
       "      <td>Predictably, Trump wins Indiana, Kentucky as Clinton grabs Vermont: Republican Donald Trump and Democrat Hillary https://t.co/w1KIVht0Bh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>337337</th>\n",
       "      <td>796004529405919232</td>\n",
       "      <td>#Election2016 Quien cree usted ganara las #EleccionesEEUU2016  Donald Trump @realDonaldTrump  o Hillary Clinton @HillaryClinton #MyVote2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155671</th>\n",
       "      <td>796264236477480960</td>\n",
       "      <td>#Clinton In massive upset, Donald Trump elected US PresidentUS Republican Donald Trump beat Democrat Hillary Clint https://t.co/X2brhJLI91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167800</th>\n",
       "      <td>795845037712142336</td>\n",
       "      <td>Donald Trump and Hillary Clinton             Same shit          Different toilet                     #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137257</th>\n",
       "      <td>796201332201975810</td>\n",
       "      <td>Republican candidate Donald Trump is advancing further ahead of Democrat rival Hillary Clinton in the race for... https://t.co/qD0duifSG5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190442</th>\n",
       "      <td>795874758592172032</td>\n",
       "      <td>Hillary Clinton and Donald Trump have wrapped up their US Presidential election campaigns - voting opens in a matte https://t.co/ODUeUOyEfF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313503</th>\n",
       "      <td>795996457438474241</td>\n",
       "      <td>How Donald Trump Dominated Twitter Against Hillary Clinton: https://t.co/dtP1a7DygU #Election2016 #112Blog https://t.co/JYcLNBYzlG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174712</th>\n",
       "      <td>795854286349549568</td>\n",
       "      <td>Donald Trump &amp; Hillary Clinton They are Tow Faces for One Coin! #DonaldTrump  #HillaryClinton  #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156144</th>\n",
       "      <td>796266643483561984</td>\n",
       "      <td>Hillary Clinton supporters in dismay as the Democrat candidate calls to congratulate Donald Trump https://t.co/wHLICH2upd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260991</th>\n",
       "      <td>795977920103776256</td>\n",
       "      <td>If Hillary wins, Donald Trump's first tweet to her should be \"You're welcome.\" #Election2016 #ElectionDay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97214</th>\n",
       "      <td>795969405641494528</td>\n",
       "      <td>The Democrat Hillary Clinton and the Republican Donald Trump, putting an end to a campaign that has around the world. #ElectionDay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197829</th>\n",
       "      <td>795894977113063424</td>\n",
       "      <td>Hillary Clinton and her so called celebs supporters do it for money Donald Trump does it for the American people #Election2016 #ElectionDay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138956</th>\n",
       "      <td>796205526157824002</td>\n",
       "      <td>Electoral College Vote tally (270 to win):   Hillary Clinton (Democrat) 209   Donald Trump (Republican) 222   #ElectionNight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234959</th>\n",
       "      <td>795956588376752128</td>\n",
       "      <td>AMERICANS if you vote for DONALD TRUMP today you will prevent World War III and Hillary's war with Russia.  TRUMP 2 https://t.co/1dW3nrf3Uz</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175670</th>\n",
       "      <td>795855343838433282</td>\n",
       "      <td>Dixville Notch, NH #Election2016 results: Hillary Clinton 4 votes, Donald Trump 2, Gary Johnson 1, &amp; a Mitt Romney write-in. LOL!!!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249378</th>\n",
       "      <td>795972055711842306</td>\n",
       "      <td>Donald #Trump is the real face of #America. Hillary #Clinton is a mask. The question: Are Americans ready to face the reality? #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156183</th>\n",
       "      <td>796266872786325504</td>\n",
       "      <td>Hillary Clinton supporters in dismay as the Democrat candidate calls to congratulate Donald Trump on victory https://t.co/MElOz136yD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206437</th>\n",
       "      <td>795925977960513536</td>\n",
       "      <td>Donald Trump #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158320</th>\n",
       "      <td>796283762673876992</td>\n",
       "      <td>Woke up like this...Donald Trump has beaten Democrat Hillary Clinton in the #USElection #ElectionNight #ElectionDay https://t.co/Fboq0EFZOe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>389517</th>\n",
       "      <td>796032154404814849</td>\n",
       "      <td>God Bless Donald Trump #ElectionDay #Election2016 #Elecciones Bill Hillary Clinton vs Florida Latinos  https://t.co/zWwB2AQzvv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103400</th>\n",
       "      <td>796024453662699520</td>\n",
       "      <td>Voters finally head to the polls to decide between Democrat Hillary Clinton or Republican Donald Trump. Read more</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182399</th>\n",
       "      <td>795863626552733697</td>\n",
       "      <td>#Election2016 Donald Trump</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95969</th>\n",
       "      <td>795943972510150656</td>\n",
       "      <td>At long last, Americans vote: Democrat Hillary Clinton and Republican Donald Trump face the judgment of the voters https://t.co/URjDH188uq</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285893</th>\n",
       "      <td>795987047517274112</td>\n",
       "      <td>I hope Hillary Clinton wins today and Donald Trump doesnt get what he wants  America will be better  #Election2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209524</th>\n",
       "      <td>795931780318691328</td>\n",
       "      <td>Prossimo Presidente degli Stati Uniti d'America ? Hillary Clinton o Donald Trump ?  #ElectionDay #Election2016 #vote #SondaggiVeloci</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241620</th>\n",
       "      <td>795963816249032708</td>\n",
       "      <td>#Election2016 Donald Trump mi terrorizza. Hillary Clinton invece mi spaventa.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        id  \\\n",
       "246422  795968823572754432   \n",
       "225797  795949870653771776   \n",
       "189172  795872814486142976   \n",
       "122681  796153703145291776   \n",
       "156198  796266957263818752   \n",
       "264563  795979274696200192   \n",
       "307873  795994539483918336   \n",
       "246807  795969255959302145   \n",
       "231518  795953873273884672   \n",
       "321340  795999053251280896   \n",
       "102366  796016018133057537   \n",
       "160147  796309377980440576   \n",
       "198883  795896673167966208   \n",
       "160726  796317056484438016   \n",
       "132519  796186470646067202   \n",
       "228808  795952000840192001   \n",
       "167329  795844448357851136   \n",
       "165021  795841312729677824   \n",
       "97790   795976595395620864   \n",
       "264511  795979257570873359   \n",
       "175589  795855264905732096   \n",
       "151835  796244861905092608   \n",
       "375949  796021384107397120   \n",
       "159843  796305520759967745   \n",
       "124540  796160780127399936   \n",
       "157531  796276178956873728   \n",
       "157701  796277658459832320   \n",
       "94161   795860125991006208   \n",
       "84019   796301674176479232   \n",
       "380229  796024903992483845   \n",
       "...                    ...   \n",
       "382519  796026888032555008   \n",
       "242444  795964612873224192   \n",
       "346119  796007244223627264   \n",
       "158969  796292496166940672   \n",
       "127434  796170479488765952   \n",
       "337337  796004529405919232   \n",
       "155671  796264236477480960   \n",
       "167800  795845037712142336   \n",
       "137257  796201332201975810   \n",
       "190442  795874758592172032   \n",
       "313503  795996457438474241   \n",
       "174712  795854286349549568   \n",
       "156144  796266643483561984   \n",
       "260991  795977920103776256   \n",
       "97214   795969405641494528   \n",
       "197829  795894977113063424   \n",
       "138956  796205526157824002   \n",
       "234959  795956588376752128   \n",
       "175670  795855343838433282   \n",
       "249378  795972055711842306   \n",
       "156183  796266872786325504   \n",
       "206437  795925977960513536   \n",
       "158320  796283762673876992   \n",
       "389517  796032154404814849   \n",
       "103400  796024453662699520   \n",
       "182399  795863626552733697   \n",
       "95969   795943972510150656   \n",
       "285893  795987047517274112   \n",
       "209524  795931780318691328   \n",
       "241620  795963816249032708   \n",
       "\n",
       "                                                                                                                                          tweet_text  \n",
       "246422   Prediction for #election2016 is either a comfortable Hillary Clinton win or a marginal Donald Trump win - hopefully I can get a call right!  \n",
       "225797      Donald Trump is everything that's wrong with our culture. Hillary Clinton is everything that's wrong with our government.  #Election2016  \n",
       "189172                                                                                                Donald Trump or Hillary Clinton? #Election2016  \n",
       "122681      Republican Donald Trump and Democrat Hillary Clinton have scored early victories in their bitter presidential... https://t.co/yyi0EthGCw  \n",
       "156198   #Elections2016 : Republican Donald Trump wins 288 electoral colleges against Democrat Hillary Clinton's 215 to emer https://t.co/r9WA0niWFW  \n",
       "264563                                  #Election2016 my gut instinct says donald trump will win,,, even though hillary has a more chance of winning  \n",
       "307873                                                                                  Plot twist: Donald trump is voting for Hillary #Election2016  \n",
       "246807       I'm so worried for you guys & I'm not even American. Vote wisely. Vote for Hillary. Don't let Donald Trump ruins America. #Election2016  \n",
       "231518               Imagine being American right now and having to choose either Donald Trump or Hillary Clinton to run your country  #Election2016  \n",
       "321340    Half Naked Women protesting against Donald Trump in polling area. These are the nuts that support Hillary Clinton. https://t.co/YkoO4BCy5d  \n",
       "102366    Voters finally head to the polls to decide between Democrat Hillary Clinton or Republican Donald Trump. Read more: https://t.co/IUkTBHCLnB  \n",
       "160147                     Republican Donald J. Trump upsets Democrat Hillary  Clinton for U.S. President in November 8, 2016 Presidential Election.  \n",
       "198883                                                       Hillary Clinton is like a headache while Donald Trump is like a migraine. #Election2016  \n",
       "160726                    #ElectionNight LIKE @JimmiJourney on #FB if U LIKE  Hillary Clinton or Donald Trump  #p2 #democrat https://t.co/qVkSiSjECK  \n",
       "132519                          US election: Republican Donald Trump leads over Democrat Hillary Clinton 140-104 in the race to 270 electoral votes.  \n",
       "228808        There's nothing that Jill Stein or Donald Trump has done that's worse than what Hillary has been up to.  #AmericaDecides #Election2016  \n",
       "167329                                                                           Donald Trump and Hillary Clinton can both eat my ass. #Election2016  \n",
       "165021      Devil : Donald Trump - Deep Sea : Hillary Clinton  That Awkward Moment When USA Fails Wisdom  #Election2016 #philadelphia #ElectionNight  \n",
       "97790           Voting for Republican Donald Trump is today only, November 8th. Voting for Democrat Hillary Clinton is tomorrow only, Novemeber 9th.  \n",
       "264511      To my American friends - vote for Hillary. Hillary is a voice for the voiceless. Donald Trump is a voice for Donald Trump. #Election2016  \n",
       "175589                                                   Realistically Hillary Clinton And Donald Trump Have Played Ecco The Dolphin.  #Election2016  \n",
       "151835                    Hillary's Presidential dreams are now over. She will always be known as the Democrat who lost to Donald Trump #electionday  \n",
       "375949                                         #Election2016 and if will win HILLARY TRUMP or DONALD CLINTON !!!!!  @REALDONALDTRUMP @HILLARYCLINTON  \n",
       "159843                                        Donald Trump will become the 45th US president after a stunning victory over Democrat Hillary Clinton.  \n",
       "124540    Trump, Clinton in tight battles in Ohio, Florida and elsewhere https://t.co/nGKCbLl5BS Republican Donald Trump and Democrat Hillary Clinto  \n",
       "157531   Oakland reacts to the election of Donald Trump: Democrat Hillary Clinton conceded the presidential race at 11:40 pm https://t.co/z5Inf3IeT4  \n",
       "157701                         Donald Trump will become the 45th US President after a stunning victory over Democrat Hillary Clinton. Great Victory.  \n",
       "94161             Democrat  Hillary Clinton and Tim Kaine  17 Republican  Donald J. Trump and Michael R. Pence  14 in Hart's Location, NH #ImWithHer  \n",
       "84019       Hillary Clinton lets Donald Trump deal with a tanking economy and a do-nothing Republican Congress #Election2016 https://t.co/hTEfeZv9TS  \n",
       "380229                                                                       RT for Donald Trump LIKE for Hillary Clinton #ElectionDay #Election2016  \n",
       "...                                                                                                                                              ...  \n",
       "382519              God Bless Donald Trump  #ElectionDay #Election2016 #Elecciones Bill Hillary Clinton vs Florida Latinos   https://t.co/ImnpCxycne  \n",
       "242444                 Hillary Clinton / Timothy Kaine o Donald Trump / Michael Pence, la boleta  #ElectionDay #Election2016 https://t.co/LBjrcAK5R6  \n",
       "346119   Who do you believe will be the next president of the United States? Donald Trump or Hillary Clinton? Explain your a https://t.co/ZMZ8wIkaHU  \n",
       "158969                                         Donald Trump will become the 45th US president after a stunning victory over Democrat Hillary Clinton  \n",
       "127434      Predictably, Trump wins Indiana, Kentucky as Clinton grabs Vermont: Republican Donald Trump and Democrat Hillary https://t.co/w1KIVht0Bh  \n",
       "337337   #Election2016 Quien cree usted ganara las #EleccionesEEUU2016  Donald Trump @realDonaldTrump  o Hillary Clinton @HillaryClinton #MyVote2016  \n",
       "155671    #Clinton In massive upset, Donald Trump elected US PresidentUS Republican Donald Trump beat Democrat Hillary Clint https://t.co/X2brhJLI91  \n",
       "167800                            Donald Trump and Hillary Clinton             Same shit          Different toilet                     #Election2016  \n",
       "137257     Republican candidate Donald Trump is advancing further ahead of Democrat rival Hillary Clinton in the race for... https://t.co/qD0duifSG5  \n",
       "190442   Hillary Clinton and Donald Trump have wrapped up their US Presidential election campaigns - voting opens in a matte https://t.co/ODUeUOyEfF  \n",
       "313503            How Donald Trump Dominated Twitter Against Hillary Clinton: https://t.co/dtP1a7DygU #Election2016 #112Blog https://t.co/JYcLNBYzlG  \n",
       "174712                                  Donald Trump & Hillary Clinton They are Tow Faces for One Coin! #DonaldTrump  #HillaryClinton  #Election2016  \n",
       "156144                     Hillary Clinton supporters in dismay as the Democrat candidate calls to congratulate Donald Trump https://t.co/wHLICH2upd  \n",
       "260991                                     If Hillary wins, Donald Trump's first tweet to her should be \"You're welcome.\" #Election2016 #ElectionDay  \n",
       "97214             The Democrat Hillary Clinton and the Republican Donald Trump, putting an end to a campaign that has around the world. #ElectionDay  \n",
       "197829   Hillary Clinton and her so called celebs supporters do it for money Donald Trump does it for the American people #Election2016 #ElectionDay  \n",
       "138956                  Electoral College Vote tally (270 to win):   Hillary Clinton (Democrat) 209   Donald Trump (Republican) 222   #ElectionNight  \n",
       "234959   AMERICANS if you vote for DONALD TRUMP today you will prevent World War III and Hillary's war with Russia.  TRUMP 2 https://t.co/1dW3nrf3Uz  \n",
       "175670           Dixville Notch, NH #Election2016 results: Hillary Clinton 4 votes, Donald Trump 2, Gary Johnson 1, & a Mitt Romney write-in. LOL!!!  \n",
       "249378  Donald #Trump is the real face of #America. Hillary #Clinton is a mask. The question: Are Americans ready to face the reality? #Election2016  \n",
       "156183          Hillary Clinton supporters in dismay as the Democrat candidate calls to congratulate Donald Trump on victory https://t.co/MElOz136yD  \n",
       "206437                                                                                                                    Donald Trump #Election2016  \n",
       "158320   Woke up like this...Donald Trump has beaten Democrat Hillary Clinton in the #USElection #ElectionNight #ElectionDay https://t.co/Fboq0EFZOe  \n",
       "389517                God Bless Donald Trump #ElectionDay #Election2016 #Elecciones Bill Hillary Clinton vs Florida Latinos  https://t.co/zWwB2AQzvv  \n",
       "103400                             Voters finally head to the polls to decide between Democrat Hillary Clinton or Republican Donald Trump. Read more  \n",
       "182399                                                                                                                    #Election2016 Donald Trump  \n",
       "95969     At long last, Americans vote: Democrat Hillary Clinton and Republican Donald Trump face the judgment of the voters https://t.co/URjDH188uq  \n",
       "285893                            I hope Hillary Clinton wins today and Donald Trump doesnt get what he wants  America will be better  #Election2016  \n",
       "209524          Prossimo Presidente degli Stati Uniti d'America ? Hillary Clinton o Donald Trump ?  #ElectionDay #Election2016 #vote #SondaggiVeloci  \n",
       "241620                                                                 #Election2016 Donald Trump mi terrorizza. Hillary Clinton invece mi spaventa.  \n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from whoosh.qparser import QueryParser\n",
    "from whoosh import scoring\n",
    "from whoosh import qparser\n",
    "\n",
    "# Search inside index for post containing \"test\", then it displays\n",
    "# results.\n",
    "searcher = search_index.searcher()\n",
    "og = qparser.OrGroup.factory(0.95)\n",
    "qp = QueryParser(\"content\", schema=search_index.schema, group=og)\n",
    "\n",
    "\n",
    "def find_candidate_answer_set(Q):\n",
    "    intent = fetch_intent_from_question(Q)\n",
    "    query = qp.parse(\" \".join(intent))\n",
    "\n",
    "    result = searcher.search(query, limit=100)    \n",
    "    result_index_list = []\n",
    "    for r in result: result_index_list.append(int(r['idx']))\n",
    "    \n",
    "    return result_index_list\n",
    "\n",
    "Q = \"Donald Trump will crush Hillary?\"\n",
    "result_index_list = find_candidate_answer_set(Q)\n",
    "display(tweet_para_df[tweet_para_df.id.isin(result_index_list)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "N7qdYkcwB7Yu"
   },
   "outputs": [],
   "source": [
    "# Load Google's pre-trained Word2Vec model.\n",
    "\n",
    "w2v_model = gensim.models.KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', \n",
    "                                                            binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "k4CXN1H0B7Yx"
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "import nltk\n",
    "import numpy as np\n",
    "\n",
    "def feature_alignment_score(row):\n",
    "    global counter_value\n",
    "    intents = [x for x in fetch_intent_from_question(row['question']) if x in w2v_model.vocab]\n",
    "    tokens = [x for x in nltk.word_tokenize(row['sentence']) if x in w2v_model.vocab]\n",
    "    total_items = len(intents) * len(tokens)\n",
    "    iter_items = itertools.product(intents, tokens)\n",
    "    alignment_score = 0.0\n",
    "    for a, b in iter_items:\n",
    "        alignment_score += w2v_model.distance(a, b)\n",
    "    if total_items == 0: return float('NaN')\n",
    "    return alignment_score / total_items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "v63UlftvB7Yz"
   },
   "outputs": [],
   "source": [
    "def feature_max_alignment(row):\n",
    "    intents = [x for x in fetch_intent_from_question(row['question']) if x in w2v_model.vocab]\n",
    "    tokens =  [x for x in fetch_intent_from_question(row['sentence']) if x in w2v_model.vocab]\n",
    "    if len(intents) * len(tokens) == 0: return float('NaN')\n",
    "    max_alignment_score = 0.0\n",
    "    for intent in intents:\n",
    "        max_alignment_score += np.min([w2v_model.distance(x, intent) for x in tokens])\n",
    "    return max_alignment_score / len(intents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "Xj6X3kGUB7Y2"
   },
   "outputs": [],
   "source": [
    "def feature_number_of_intent(row):\n",
    "    intents = [x for x in fetch_intent_from_question(row['question']) if x in w2v_model.vocab]\n",
    "    return len(intents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "Pw95VR3hB7Y4"
   },
   "outputs": [],
   "source": [
    "from nltk import word_tokenize          \n",
    "from nltk.stem import WordNetLemmatizer \n",
    "class LemmaTokenizer(object):\n",
    "    def __init__(self):\n",
    "        self.wnl = WordNetLemmatizer()\n",
    "    def __call__(self, articles):\n",
    "        return [self.wnl.lemmatize(t) for t in word_tokenize(articles)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "G2yB17ViB7Y6"
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from nltk.stem import PorterStemmer\n",
    "\n",
    "def feature_exact_match(row):\n",
    "    ps = PorterStemmer()\n",
    "    return sum(\n",
    "        (\n",
    "            Counter([ps.stem(x) for x in row['sentence'].lower().split()]) \n",
    "            & \n",
    "            Counter([ps.stem(x) for x in row['question'].lower().split()])\n",
    "        ).values()\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "8HACcWvMB7Y8"
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "def feature_count_vectorizer(feature_df):\n",
    "    ctv = CountVectorizer(tokenizer=LemmaTokenizer(), analyzer='word',token_pattern=r'\\w{1,}', \n",
    "                          ngram_range=(1, 2), stop_words = 'english')\n",
    "    ctv.fit(feature_df.sentence.values + feature_df.question.values)\n",
    "\n",
    "    sent_ctv = ctv.transform(feature_df.sentence.values)\n",
    "    ques_ctv = ctv.transform(feature_df.question.values)\n",
    "\n",
    "    ctv_score = []\n",
    "    for s, q in tqdm(zip(sent_ctv, ques_ctv)):\n",
    "        dot = s.dot(q.T).data\n",
    "        val = dot[0] if len(dot) else 0\n",
    "        ctv_score.append(val)\n",
    "\n",
    "    return pd.Series(ctv_score).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "O-SJx1vEB7ZA"
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import pandas as pd\n",
    "\n",
    "def feature_tfidf(feature_df):\n",
    "    tfv = TfidfVectorizer(tokenizer=LemmaTokenizer(), min_df=3,  max_features=None, \n",
    "                strip_accents='unicode', analyzer='word',token_pattern=r'\\w{1,}',\n",
    "                ngram_range=(1, 2), use_idf=1,smooth_idf=1,sublinear_tf=1,\n",
    "                stop_words = 'english')\n",
    "    tfv.fit(feature_df.sentence.values)\n",
    "\n",
    "    sent_tfv = tfv.transform(feature_df.sentence.values)\n",
    "    ques_tfv = tfv.transform(feature_df.question.values)\n",
    "\n",
    "    tfv_score = []\n",
    "    for s, q in tqdm(zip(sent_tfv, ques_tfv)):\n",
    "        dot = s.dot(q.T).data\n",
    "        val = dot[0] if len(dot) else 0.0\n",
    "        tfv_score.append(val)\n",
    "        \n",
    "    return pd.Series(tfv_score).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 3995
    },
    "colab_type": "code",
    "id": "YPamS7glB7ZD",
    "outputId": "d6c6ff1a-a2ab-4e48-9b01-55766e9f1b95"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 2210.34it/s]\n",
      "100it [00:00, 2279.98it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>sentence</th>\n",
       "      <th>question</th>\n",
       "      <th>alignment_score</th>\n",
       "      <th>max_alignment</th>\n",
       "      <th>number_of_intent</th>\n",
       "      <th>question_words</th>\n",
       "      <th>exact_match</th>\n",
       "      <th>count_vectorizer_match</th>\n",
       "      <th>tfidf_match</th>\n",
       "      <th>coverage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>103818</th>\n",
       "      <td>796027043951603712</td>\n",
       "      <td>@GoFishLynn Sandler now give to CF, &amp; to the Democrat Party &amp; Hillary campaign - more dodgy characters that Hillary &amp; co love to know.</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.801519</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.039670</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266103</th>\n",
       "      <td>795979846635634689</td>\n",
       "      <td>Hillary, Hillary, I wish you become the first Lady US President https://t.co/SOcibpazLl</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.728329</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.044751</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197896</th>\n",
       "      <td>795895103554514944</td>\n",
       "      <td>Hillary better win  #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.662548</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.061801</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189172</th>\n",
       "      <td>795872814486142976</td>\n",
       "      <td>Donald Trump or Hillary Clinton? #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.607569</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.332669</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177192</th>\n",
       "      <td>795857044054179840</td>\n",
       "      <td>Trump or Hillary, I'm gonna miss Obama #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.713200</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.033993</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255012</th>\n",
       "      <td>795975559964725249</td>\n",
       "      <td>GO HILLARY, GO! #Election2016  @HillaryClinton</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.845974</td>\n",
       "      <td>5.595390e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.034375</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266227</th>\n",
       "      <td>795979890935889920</td>\n",
       "      <td>#Election2016 good luck (hillary must win)</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.827696</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.045636</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307873</th>\n",
       "      <td>795994539483918336</td>\n",
       "      <td>Plot twist: Donald trump is voting for Hillary #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.811614</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.029126</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262092</th>\n",
       "      <td>795978335851520001</td>\n",
       "      <td>#NastyWomen4Hillary #iWillVote! #HillaryClinton #NastyWomen #debate #election2016 #Hillary</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.762607</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.034698</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233568</th>\n",
       "      <td>795955237890695168</td>\n",
       "      <td>Great!!! It's over!!!. Hillary won!!! https://t.co/tcvNw7ds0q</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.776320</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.019479</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379662</th>\n",
       "      <td>796024437221036033</td>\n",
       "      <td>Hiring Hillary probably wont end the world. But electing Trump just might #ElectionDay #Election2016 #Vote2016 #Hillary #govote</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.779435</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.045603</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126583</th>\n",
       "      <td>796167355160207361</td>\n",
       "      <td>@PurpleDerg @Grahckheuhl also, Hillary benefits from that setup because she's a household name, every democrat in america knows Hillary--</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.815356</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.043238</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214562</th>\n",
       "      <td>795939903351599104</td>\n",
       "      <td>go Hillary! #cantsleep  #Election2016 https://t.co/j5Zhn1oHru</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.697253</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.035501</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147147</th>\n",
       "      <td>796227545343528965</td>\n",
       "      <td>Again: I voted for Hillary Clinton. I am a democrat. But I did not choose Hillary Clinton. And that's all the difference.</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.723209</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.040450</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308768</th>\n",
       "      <td>795994826013544449</td>\n",
       "      <td>Man, Hillary is always trippin'! https://t.co/HcnYJvDurm</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.737116</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.036805</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130614</th>\n",
       "      <td>796180580664832002</td>\n",
       "      <td>@Ethxn_10 i know, i said this bc the Democrat is hillary.</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.839247</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.036011</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272087</th>\n",
       "      <td>795982043234570240</td>\n",
       "      <td>I'm voting for Hillary Clit. #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.719178</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.034588</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>277458</th>\n",
       "      <td>795983971096129536</td>\n",
       "      <td>Come on, you have to vote for Hillary!!! Vamos, hay que votar a Hillary!!!  #Election2016  #GoHillary  https://t.co/6E8PwYgZnU</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.748153</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.033132</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348324</th>\n",
       "      <td>796007990843363328</td>\n",
       "      <td>So, Hillary is Batman? https://t.co/bv6kZ1vzfj</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.714419</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.493514</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250760</th>\n",
       "      <td>795973445511217152</td>\n",
       "      <td>Good luck Hillary!  #Election2016 #ElectionDay #TheWorldDoesntWantTrump</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.730966</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.029161</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355199</th>\n",
       "      <td>796010236171718656</td>\n",
       "      <td>@SportsCenter @espn Going to vote Hillary no.  #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.783481</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.033106</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164209</th>\n",
       "      <td>795840067193503744</td>\n",
       "      <td>\"I'm with Hillary because the Kardashians are for her\" - @brittney_dybala #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.816590</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.041404</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331108</th>\n",
       "      <td>796002459013574656</td>\n",
       "      <td>Hillary ha votato https://t.co/3VfN9SxeU0</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.596377</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.048919</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369212</th>\n",
       "      <td>796016528323964932</td>\n",
       "      <td>@LindaDono, come on Hillary! https://t.co/f0KVkYadPN</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.734084</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.036658</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249359</th>\n",
       "      <td>795972042772258817</td>\n",
       "      <td>Oh! A Hillary shirt! #TurnArizonaBlue #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.726891</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.049269</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264511</th>\n",
       "      <td>795979257570873359</td>\n",
       "      <td>To my American friends - vote for Hillary. Hillary is a voice for the voiceless. Donald Trump is a voice for Donald Trump. #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.801065</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.031424</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>276814</th>\n",
       "      <td>795983750987399168</td>\n",
       "      <td>Go Hillary! #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.585948</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.053735</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290445</th>\n",
       "      <td>795988680754298880</td>\n",
       "      <td>Please vote for Hillary Please #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.744362</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.051246</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199388</th>\n",
       "      <td>795897387009118208</td>\n",
       "      <td>i have voted for hillary https://t.co/rN76G1dkTI</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.835285</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.038208</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335684</th>\n",
       "      <td>796004023220535296</td>\n",
       "      <td>#Election2016 #StrongerTogether #Hillary #NeverTrump #USorELSE</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.720438</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.044093</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134090</th>\n",
       "      <td>796191597293686784</td>\n",
       "      <td>@systxmatyc @vinnygael Hillary, it's a Democrat state</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.713274</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.033865</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233613</th>\n",
       "      <td>795955270111424512</td>\n",
       "      <td>#Hillary                https://t.co/xSbb9nl5sc</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.580553</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.044954</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254665</th>\n",
       "      <td>795975426799796225</td>\n",
       "      <td>Just voted for Hillary!  #ImWithHer #HillaryForPresident #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.762170</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.028601</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205126</th>\n",
       "      <td>795923271736819712</td>\n",
       "      <td>will vote hillary for nudes https://t.co/H6ZPT6V38D</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.843688</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.050713</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360648</th>\n",
       "      <td>796012172140802048</td>\n",
       "      <td>The election of 2016. Hillary or Trump Hillary or Trump choose Choose CHOOSE #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.710167</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.067195</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364861</th>\n",
       "      <td>796013694773854208</td>\n",
       "      <td>Hillary  #Election2016 https://t.co/2yg9LqaD0L</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.580553</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.040930</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>325266</th>\n",
       "      <td>796000428483899392</td>\n",
       "      <td>Hillary will win today!!!  #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.753197</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.027985</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6552</th>\n",
       "      <td>795968415320129536</td>\n",
       "      <td>Hillary is a criminal https://t.co/62CXgzwpj9</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.695364</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.070790</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211416</th>\n",
       "      <td>795935384278536192</td>\n",
       "      <td>Espero que a Hillary Clinton Ganhe  #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.581416</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.057771</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209313</th>\n",
       "      <td>795931395516416000</td>\n",
       "      <td>USA please vote for Hillary!!! #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.738303</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.026857</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237897</th>\n",
       "      <td>795959671349649409</td>\n",
       "      <td>if you're reading this and you're from America please vote Hillary. #ImWithHer #Election2016 #Hillary</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.759167</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.042061</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225194</th>\n",
       "      <td>795949393190993921</td>\n",
       "      <td>If you have voted for Hillary Clinton retweet. ..#Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.723077</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.038225</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224452</th>\n",
       "      <td>795948845956038656</td>\n",
       "      <td>be smart and vote Hillary! #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.689871</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.043209</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386889</th>\n",
       "      <td>796030312035676160</td>\n",
       "      <td>Hillary is like the Nickelback of politicians #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.772657</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.109528</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234451</th>\n",
       "      <td>795956058850111489</td>\n",
       "      <td>Hillary!! https://t.co/MC9qc4AyC5</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.438567</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.035190</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309910</th>\n",
       "      <td>795995226536083456</td>\n",
       "      <td>Go go Hillary  #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.659480</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.072265</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212454</th>\n",
       "      <td>795937309233442816</td>\n",
       "      <td>Hillary or Trump ? #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.612044</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.540917</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260485</th>\n",
       "      <td>795977715744653312</td>\n",
       "      <td>I  belive in Hillary victory #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.722638</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.109528</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>312436</th>\n",
       "      <td>795996085730820097</td>\n",
       "      <td>I voted #Election2016 #HillaryClintonForPresident #HilaryClinton #Hillary</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.729549</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.043688</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159315</th>\n",
       "      <td>796297527805046785</td>\n",
       "      <td>Is the democratic woman Hillary the democratic leader still for the democrat party so can Hillary covern the democratic states ? RT</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.805727</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.776727</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>390472</th>\n",
       "      <td>796032829058490368</td>\n",
       "      <td>#Election2016 #IVOTED #MyVote2016 I'm rooting for Hillary Clinton</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.701384</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.039140</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335376</th>\n",
       "      <td>796003922200633344</td>\n",
       "      <td>VOTE FOR HILLARY #ImWithHer #VoteHillary #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.819147</td>\n",
       "      <td>5.595390e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.032446</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185854</th>\n",
       "      <td>795868326857703424</td>\n",
       "      <td>Hillary got the first win. #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.760742</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.046974</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>340216</th>\n",
       "      <td>796005428748570624</td>\n",
       "      <td>Hillary Clinton for President! #ImWithHer #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.636588</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.028664</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261342</th>\n",
       "      <td>795978037510729730</td>\n",
       "      <td>GO HILLARY!!!!!!!!!!! https://t.co/3rNBJSrohB</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.793524</td>\n",
       "      <td>5.595390e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.018930</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223775</th>\n",
       "      <td>795948299543080961</td>\n",
       "      <td>Go hillary https://t.co/4xXbzsHeIP</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.749799</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.051433</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327387</th>\n",
       "      <td>796001143319134208</td>\n",
       "      <td>Hillary ate https://t.co/eKHbgoB2NG</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.598453</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.070790</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187413</th>\n",
       "      <td>795870315716845568</td>\n",
       "      <td>Hillary is full of SHIT #ImWithHer #Election2016 #VoteTrump</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.778631</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.036487</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197652</th>\n",
       "      <td>795894701652148224</td>\n",
       "      <td>Will anti-Trump immigrant wave prove strong enough for Hillary? #election2016 #hillary #trump #usa #usele https://t.co/3GnwXYzgdu</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.814100</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.352648</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>349679</th>\n",
       "      <td>796008419635314688</td>\n",
       "      <td>Let Hillary Be Hillary | Full Frontal with Samantha Bee | TBS #Election2016 #ElectionFinalThoughts  https://t.co/FJYIbO4z79</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.747621</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.081270</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        id  \\\n",
       "103818  796027043951603712   \n",
       "266103  795979846635634689   \n",
       "197896  795895103554514944   \n",
       "189172  795872814486142976   \n",
       "177192  795857044054179840   \n",
       "255012  795975559964725249   \n",
       "266227  795979890935889920   \n",
       "307873  795994539483918336   \n",
       "262092  795978335851520001   \n",
       "233568  795955237890695168   \n",
       "379662  796024437221036033   \n",
       "126583  796167355160207361   \n",
       "214562  795939903351599104   \n",
       "147147  796227545343528965   \n",
       "308768  795994826013544449   \n",
       "130614  796180580664832002   \n",
       "272087  795982043234570240   \n",
       "277458  795983971096129536   \n",
       "348324  796007990843363328   \n",
       "250760  795973445511217152   \n",
       "355199  796010236171718656   \n",
       "164209  795840067193503744   \n",
       "331108  796002459013574656   \n",
       "369212  796016528323964932   \n",
       "249359  795972042772258817   \n",
       "264511  795979257570873359   \n",
       "276814  795983750987399168   \n",
       "290445  795988680754298880   \n",
       "199388  795897387009118208   \n",
       "335684  796004023220535296   \n",
       "...                    ...   \n",
       "134090  796191597293686784   \n",
       "233613  795955270111424512   \n",
       "254665  795975426799796225   \n",
       "205126  795923271736819712   \n",
       "360648  796012172140802048   \n",
       "364861  796013694773854208   \n",
       "325266  796000428483899392   \n",
       "6552    795968415320129536   \n",
       "211416  795935384278536192   \n",
       "209313  795931395516416000   \n",
       "237897  795959671349649409   \n",
       "225194  795949393190993921   \n",
       "224452  795948845956038656   \n",
       "386889  796030312035676160   \n",
       "234451  795956058850111489   \n",
       "309910  795995226536083456   \n",
       "212454  795937309233442816   \n",
       "260485  795977715744653312   \n",
       "312436  795996085730820097   \n",
       "159315  796297527805046785   \n",
       "390472  796032829058490368   \n",
       "335376  796003922200633344   \n",
       "185854  795868326857703424   \n",
       "340216  796005428748570624   \n",
       "261342  795978037510729730   \n",
       "223775  795948299543080961   \n",
       "327387  796001143319134208   \n",
       "187413  795870315716845568   \n",
       "197652  795894701652148224   \n",
       "349679  796008419635314688   \n",
       "\n",
       "                                                                                                                                         sentence  \\\n",
       "103818     @GoFishLynn Sandler now give to CF, & to the Democrat Party & Hillary campaign - more dodgy characters that Hillary & co love to know.   \n",
       "266103                                                    Hillary, Hillary, I wish you become the first Lady US President https://t.co/SOcibpazLl   \n",
       "197896                                                                                                          Hillary better win  #Election2016   \n",
       "189172                                                                                             Donald Trump or Hillary Clinton? #Election2016   \n",
       "177192                                                                                       Trump or Hillary, I'm gonna miss Obama #Election2016   \n",
       "255012                                                                                             GO HILLARY, GO! #Election2016  @HillaryClinton   \n",
       "266227                                                                                                 #Election2016 good luck (hillary must win)   \n",
       "307873                                                                               Plot twist: Donald trump is voting for Hillary #Election2016   \n",
       "262092                                                 #NastyWomen4Hillary #iWillVote! #HillaryClinton #NastyWomen #debate #election2016 #Hillary   \n",
       "233568                                                                              Great!!! It's over!!!. Hillary won!!! https://t.co/tcvNw7ds0q   \n",
       "379662            Hiring Hillary probably wont end the world. But electing Trump just might #ElectionDay #Election2016 #Vote2016 #Hillary #govote   \n",
       "126583  @PurpleDerg @Grahckheuhl also, Hillary benefits from that setup because she's a household name, every democrat in america knows Hillary--   \n",
       "214562                                                                              go Hillary! #cantsleep  #Election2016 https://t.co/j5Zhn1oHru   \n",
       "147147                  Again: I voted for Hillary Clinton. I am a democrat. But I did not choose Hillary Clinton. And that's all the difference.   \n",
       "308768                                                                                   Man, Hillary is always trippin'! https://t.co/HcnYJvDurm   \n",
       "130614                                                                                  @Ethxn_10 i know, i said this bc the Democrat is hillary.   \n",
       "272087                                                                                                 I'm voting for Hillary Clit. #Election2016   \n",
       "277458             Come on, you have to vote for Hillary!!! Vamos, hay que votar a Hillary!!!  #Election2016  #GoHillary  https://t.co/6E8PwYgZnU   \n",
       "348324                                                                                             So, Hillary is Batman? https://t.co/bv6kZ1vzfj   \n",
       "250760                                                                    Good luck Hillary!  #Election2016 #ElectionDay #TheWorldDoesntWantTrump   \n",
       "355199                                                                               @SportsCenter @espn Going to vote Hillary no.  #Election2016   \n",
       "164209                                                    \"I'm with Hillary because the Kardashians are for her\" - @brittney_dybala #Election2016   \n",
       "331108                                                                                                  Hillary ha votato https://t.co/3VfN9SxeU0   \n",
       "369212                                                                                       @LindaDono, come on Hillary! https://t.co/f0KVkYadPN   \n",
       "249359                                                                                        Oh! A Hillary shirt! #TurnArizonaBlue #Election2016   \n",
       "264511   To my American friends - vote for Hillary. Hillary is a voice for the voiceless. Donald Trump is a voice for Donald Trump. #Election2016   \n",
       "276814                                                                                                                  Go Hillary! #Election2016   \n",
       "290445                                                                                               Please vote for Hillary Please #Election2016   \n",
       "199388                                                                                           i have voted for hillary https://t.co/rN76G1dkTI   \n",
       "335684                                                                             #Election2016 #StrongerTogether #Hillary #NeverTrump #USorELSE   \n",
       "...                                                                                                                                           ...   \n",
       "134090                                                                                      @systxmatyc @vinnygael Hillary, it's a Democrat state   \n",
       "233613                                                                                            #Hillary                https://t.co/xSbb9nl5sc   \n",
       "254665                                                                     Just voted for Hillary!  #ImWithHer #HillaryForPresident #Election2016   \n",
       "205126                                                                                        will vote hillary for nudes https://t.co/H6ZPT6V38D   \n",
       "360648                                                 The election of 2016. Hillary or Trump Hillary or Trump choose Choose CHOOSE #Election2016   \n",
       "364861                                                                                             Hillary  #Election2016 https://t.co/2yg9LqaD0L   \n",
       "325266                                                                                                   Hillary will win today!!!  #Election2016   \n",
       "6552                                                                                                Hillary is a criminal https://t.co/62CXgzwpj9   \n",
       "211416                                                                                          Espero que a Hillary Clinton Ganhe  #Election2016   \n",
       "209313                                                                                               USA please vote for Hillary!!! #Election2016   \n",
       "237897                                      if you're reading this and you're from America please vote Hillary. #ImWithHer #Election2016 #Hillary   \n",
       "225194                                                                             If you have voted for Hillary Clinton retweet. ..#Election2016   \n",
       "224452                                                                                                   be smart and vote Hillary! #Election2016   \n",
       "386889                                                                                Hillary is like the Nickelback of politicians #Election2016   \n",
       "234451                                                                                                          Hillary!! https://t.co/MC9qc4AyC5   \n",
       "309910                                                                                                               Go go Hillary  #Election2016   \n",
       "212454                                                                                                           Hillary or Trump ? #Election2016   \n",
       "260485                                                                                                 I  belive in Hillary victory #Election2016   \n",
       "312436                                                                  I voted #Election2016 #HillaryClintonForPresident #HilaryClinton #Hillary   \n",
       "159315        Is the democratic woman Hillary the democratic leader still for the democrat party so can Hillary covern the democratic states ? RT   \n",
       "390472                                                                          #Election2016 #IVOTED #MyVote2016 I'm rooting for Hillary Clinton   \n",
       "335376                                                                                     VOTE FOR HILLARY #ImWithHer #VoteHillary #Election2016   \n",
       "185854                                                                                                   Hillary got the first win. #Election2016   \n",
       "340216                                                                                    Hillary Clinton for President! #ImWithHer #Election2016   \n",
       "261342                                                                                              GO HILLARY!!!!!!!!!!! https://t.co/3rNBJSrohB   \n",
       "223775                                                                                                         Go hillary https://t.co/4xXbzsHeIP   \n",
       "327387                                                                                                        Hillary ate https://t.co/eKHbgoB2NG   \n",
       "187413                                                                                Hillary is full of SHIT #ImWithHer #Election2016 #VoteTrump   \n",
       "197652          Will anti-Trump immigrant wave prove strong enough for Hillary? #election2016 #hillary #trump #usa #usele https://t.co/3GnwXYzgdu   \n",
       "349679                Let Hillary Be Hillary | Full Frontal with Samantha Bee | TBS #Election2016 #ElectionFinalThoughts  https://t.co/FJYIbO4z79   \n",
       "\n",
       "               question  alignment_score  max_alignment  number_of_intent  \\\n",
       "103818  How is Hillary?         0.801519   2.220446e-16                 1   \n",
       "266103  How is Hillary?         0.728329   2.220446e-16                 1   \n",
       "197896  How is Hillary?         0.662548   2.220446e-16                 1   \n",
       "189172  How is Hillary?         0.607569   2.220446e-16                 1   \n",
       "177192  How is Hillary?         0.713200   2.220446e-16                 1   \n",
       "255012  How is Hillary?         0.845974   5.595390e-01                 1   \n",
       "266227  How is Hillary?         0.827696   4.789451e-01                 1   \n",
       "307873  How is Hillary?         0.811614   2.220446e-16                 1   \n",
       "262092  How is Hillary?         0.762607   2.220446e-16                 1   \n",
       "233568  How is Hillary?         0.776320   2.220446e-16                 1   \n",
       "379662  How is Hillary?         0.779435   2.220446e-16                 1   \n",
       "126583  How is Hillary?         0.815356   2.220446e-16                 1   \n",
       "214562  How is Hillary?         0.697253   2.220446e-16                 1   \n",
       "147147  How is Hillary?         0.723209   2.220446e-16                 1   \n",
       "308768  How is Hillary?         0.737116   2.220446e-16                 1   \n",
       "130614  How is Hillary?         0.839247   4.789451e-01                 1   \n",
       "272087  How is Hillary?         0.719178   2.220446e-16                 1   \n",
       "277458  How is Hillary?         0.748153   2.220446e-16                 1   \n",
       "348324  How is Hillary?         0.714419   2.220446e-16                 1   \n",
       "250760  How is Hillary?         0.730966   2.220446e-16                 1   \n",
       "355199  How is Hillary?         0.783481   2.220446e-16                 1   \n",
       "164209  How is Hillary?         0.816590   2.220446e-16                 1   \n",
       "331108  How is Hillary?         0.596377   2.220446e-16                 1   \n",
       "369212  How is Hillary?         0.734084   2.220446e-16                 1   \n",
       "249359  How is Hillary?         0.726891   2.220446e-16                 1   \n",
       "264511  How is Hillary?         0.801065   2.220446e-16                 1   \n",
       "276814  How is Hillary?         0.585948   2.220446e-16                 1   \n",
       "290445  How is Hillary?         0.744362   2.220446e-16                 1   \n",
       "199388  How is Hillary?         0.835285   4.789451e-01                 1   \n",
       "335684  How is Hillary?         0.720438   2.220446e-16                 1   \n",
       "...                 ...              ...            ...               ...   \n",
       "134090  How is Hillary?         0.713274   2.220446e-16                 1   \n",
       "233613  How is Hillary?         0.580553   2.220446e-16                 1   \n",
       "254665  How is Hillary?         0.762170   2.220446e-16                 1   \n",
       "205126  How is Hillary?         0.843688   4.789451e-01                 1   \n",
       "360648  How is Hillary?         0.710167   2.220446e-16                 1   \n",
       "364861  How is Hillary?         0.580553   2.220446e-16                 1   \n",
       "325266  How is Hillary?         0.753197   2.220446e-16                 1   \n",
       "6552    How is Hillary?         0.695364   2.220446e-16                 1   \n",
       "211416  How is Hillary?         0.581416   2.220446e-16                 1   \n",
       "209313  How is Hillary?         0.738303   2.220446e-16                 1   \n",
       "237897  How is Hillary?         0.759167   2.220446e-16                 1   \n",
       "225194  How is Hillary?         0.723077   2.220446e-16                 1   \n",
       "224452  How is Hillary?         0.689871   2.220446e-16                 1   \n",
       "386889  How is Hillary?         0.772657   2.220446e-16                 1   \n",
       "234451  How is Hillary?         0.438567   2.220446e-16                 1   \n",
       "309910  How is Hillary?         0.659480   2.220446e-16                 1   \n",
       "212454  How is Hillary?         0.612044   2.220446e-16                 1   \n",
       "260485  How is Hillary?         0.722638   2.220446e-16                 1   \n",
       "312436  How is Hillary?         0.729549   2.220446e-16                 1   \n",
       "159315  How is Hillary?         0.805727   2.220446e-16                 1   \n",
       "390472  How is Hillary?         0.701384   2.220446e-16                 1   \n",
       "335376  How is Hillary?         0.819147   5.595390e-01                 1   \n",
       "185854  How is Hillary?         0.760742   2.220446e-16                 1   \n",
       "340216  How is Hillary?         0.636588   2.220446e-16                 1   \n",
       "261342  How is Hillary?         0.793524   5.595390e-01                 1   \n",
       "223775  How is Hillary?         0.749799   4.789451e-01                 1   \n",
       "327387  How is Hillary?         0.598453   2.220446e-16                 1   \n",
       "187413  How is Hillary?         0.778631   2.220446e-16                 1   \n",
       "197652  How is Hillary?         0.814100   2.220446e-16                 1   \n",
       "349679  How is Hillary?         0.747621   2.220446e-16                 1   \n",
       "\n",
       "        question_words  exact_match  count_vectorizer_match  tfidf_match  \\\n",
       "103818               3            0                       2     0.039670   \n",
       "266103               3            0                       2     0.044751   \n",
       "197896               3            0                       1     0.061801   \n",
       "189172               3            0                       2     0.332669   \n",
       "177192               3            0                       1     0.033993   \n",
       "255012               3            0                       1     0.034375   \n",
       "266227               3            0                       1     0.045636   \n",
       "307873               3            1                       1     0.029126   \n",
       "262092               3            0                       1     0.034698   \n",
       "233568               3            0                       1     0.019479   \n",
       "379662               3            0                       2     0.045603   \n",
       "126583               3            0                       2     0.043238   \n",
       "214562               3            0                       1     0.035501   \n",
       "147147               3            0                       2     0.040450   \n",
       "308768               3            1                       1     0.036805   \n",
       "130614               3            1                       1     0.036011   \n",
       "272087               3            0                       1     0.034588   \n",
       "277458               3            0                       2     0.033132   \n",
       "348324               3            1                       2     0.493514   \n",
       "250760               3            0                       1     0.029161   \n",
       "355199               3            0                       1     0.033106   \n",
       "164209               3            0                       1     0.041404   \n",
       "331108               3            0                       1     0.048919   \n",
       "369212               3            0                       1     0.036658   \n",
       "249359               3            0                       1     0.049269   \n",
       "264511               3            1                       2     0.031424   \n",
       "276814               3            0                       1     0.053735   \n",
       "290445               3            0                       1     0.051246   \n",
       "199388               3            0                       1     0.038208   \n",
       "335684               3            0                       1     0.044093   \n",
       "...                ...          ...                     ...          ...   \n",
       "134090               3            0                       1     0.033865   \n",
       "233613               3            0                       1     0.044954   \n",
       "254665               3            0                       1     0.028601   \n",
       "205126               3            0                       1     0.050713   \n",
       "360648               3            0                       2     0.067195   \n",
       "364861               3            0                       1     0.040930   \n",
       "325266               3            0                       1     0.027985   \n",
       "6552                 3            1                       1     0.070790   \n",
       "211416               3            0                       1     0.057771   \n",
       "209313               3            0                       1     0.026857   \n",
       "237897               3            0                       2     0.042061   \n",
       "225194               3            0                       1     0.038225   \n",
       "224452               3            0                       1     0.043209   \n",
       "386889               3            1                       1     0.109528   \n",
       "234451               3            0                       1     0.035190   \n",
       "309910               3            0                       1     0.072265   \n",
       "212454               3            0                       2     0.540917   \n",
       "260485               3            0                       1     0.109528   \n",
       "312436               3            0                       1     0.043688   \n",
       "159315               3            1                       3     0.776727   \n",
       "390472               3            0                       1     0.039140   \n",
       "335376               3            0                       1     0.032446   \n",
       "185854               3            0                       1     0.046974   \n",
       "340216               3            0                       1     0.028664   \n",
       "261342               3            0                       1     0.018930   \n",
       "223775               3            0                       1     0.051433   \n",
       "327387               3            0                       1     0.070790   \n",
       "187413               3            1                       1     0.036487   \n",
       "197652               3            1                       4     0.352648   \n",
       "349679               3            0                       2     0.081270   \n",
       "\n",
       "        coverage  \n",
       "103818  0.000000  \n",
       "266103  0.000000  \n",
       "197896  0.000000  \n",
       "189172  0.000000  \n",
       "177192  0.000000  \n",
       "255012  0.000000  \n",
       "266227  0.000000  \n",
       "307873  0.333333  \n",
       "262092  0.000000  \n",
       "233568  0.000000  \n",
       "379662  0.000000  \n",
       "126583  0.000000  \n",
       "214562  0.000000  \n",
       "147147  0.000000  \n",
       "308768  0.333333  \n",
       "130614  0.333333  \n",
       "272087  0.000000  \n",
       "277458  0.000000  \n",
       "348324  0.333333  \n",
       "250760  0.000000  \n",
       "355199  0.000000  \n",
       "164209  0.000000  \n",
       "331108  0.000000  \n",
       "369212  0.000000  \n",
       "249359  0.000000  \n",
       "264511  0.333333  \n",
       "276814  0.000000  \n",
       "290445  0.000000  \n",
       "199388  0.000000  \n",
       "335684  0.000000  \n",
       "...          ...  \n",
       "134090  0.000000  \n",
       "233613  0.000000  \n",
       "254665  0.000000  \n",
       "205126  0.000000  \n",
       "360648  0.000000  \n",
       "364861  0.000000  \n",
       "325266  0.000000  \n",
       "6552    0.333333  \n",
       "211416  0.000000  \n",
       "209313  0.000000  \n",
       "237897  0.000000  \n",
       "225194  0.000000  \n",
       "224452  0.000000  \n",
       "386889  0.333333  \n",
       "234451  0.000000  \n",
       "309910  0.000000  \n",
       "212454  0.000000  \n",
       "260485  0.000000  \n",
       "312436  0.000000  \n",
       "159315  0.333333  \n",
       "390472  0.000000  \n",
       "335376  0.000000  \n",
       "185854  0.000000  \n",
       "340216  0.000000  \n",
       "261342  0.000000  \n",
       "223775  0.000000  \n",
       "327387  0.000000  \n",
       "187413  0.333333  \n",
       "197652  0.333333  \n",
       "349679  0.000000  \n",
       "\n",
       "[100 rows x 11 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def generate_feature(Q):\n",
    "    result_index_list = find_candidate_answer_set(Q)\n",
    "    feature_df = tweet_para_df[tweet_para_df.id.isin(result_index_list)].copy()\n",
    "    \n",
    "    feature_df.rename(columns={'tweet_text': 'sentence'}, inplace=True)\n",
    "    feature_df['question'] = Q\n",
    "    \n",
    "    feature_df['alignment_score'] = feature_df.apply(feature_alignment_score, axis=1)\n",
    "    feature_df.alignment_score.fillna(-1.0, inplace=True)\n",
    "\n",
    "    feature_df['max_alignment'] = feature_df.apply(feature_max_alignment, axis=1)\n",
    "    feature_df.max_alignment.fillna(-1.0, inplace=True)\n",
    "\n",
    "    feature_df['number_of_intent'] = feature_df.apply(feature_number_of_intent, axis=1)\n",
    "    feature_df['question_words'] = feature_df.question.apply(lambda x: len(x.split()))\n",
    "    feature_df['exact_match'] = feature_df.apply(feature_exact_match, axis=1)\n",
    "\n",
    "    feature_df['count_vectorizer_match'] = feature_count_vectorizer(feature_df)\n",
    "    feature_df['tfidf_match'] = feature_tfidf(feature_df)\n",
    "    \n",
    "    feature_df['coverage'] = feature_df.exact_match / feature_df.question_words\n",
    "    \n",
    "    return feature_df\n",
    "    \n",
    "generate_feature(\"How is Hillary?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 4454
    },
    "colab_type": "code",
    "id": "OtGXBEghB7ZG",
    "outputId": "5d4d39d1-11d2-40ca-b668-50a387009049"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 2551.47it/s]\n",
      "100it [00:00, 2535.89it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>sentence</th>\n",
       "      <th>question</th>\n",
       "      <th>alignment_score</th>\n",
       "      <th>max_alignment</th>\n",
       "      <th>number_of_intent</th>\n",
       "      <th>question_words</th>\n",
       "      <th>exact_match</th>\n",
       "      <th>count_vectorizer_match</th>\n",
       "      <th>tfidf_match</th>\n",
       "      <th>coverage</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>348324</th>\n",
       "      <td>796007990843363328</td>\n",
       "      <td>So, Hillary is Batman? https://t.co/bv6kZ1vzfj</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.714419</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.493514</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.809667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8584</th>\n",
       "      <td>795986326495645698</td>\n",
       "      <td>@CBSNews If Hillary wins-Can Congress stop the scandalous nature of Hillary?Can they protect us from being her prey? Vote Trump!</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.786155</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.656810</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.793776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197652</th>\n",
       "      <td>795894701652148224</td>\n",
       "      <td>Will anti-Trump immigrant wave prove strong enough for Hillary? #election2016 #hillary #trump #usa #usele https://t.co/3GnwXYzgdu</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.814100</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.352648</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.789133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134171</th>\n",
       "      <td>796191862562308097</td>\n",
       "      <td>#3Novices : EXCLUSIVE: Hillary wins Florida? Republican chief reveals Democrats win in vital county https://t.co/svskluqjYo  HILLARY Clint</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.813238</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.347694</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.700904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159315</th>\n",
       "      <td>796297527805046785</td>\n",
       "      <td>Is the democratic woman Hillary the democratic leader still for the democrat party so can Hillary covern the democratic states ? RT</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.805727</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.776727</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.676888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189172</th>\n",
       "      <td>795872814486142976</td>\n",
       "      <td>Donald Trump or Hillary Clinton? #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.607569</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.332669</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.471353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212454</th>\n",
       "      <td>795937309233442816</td>\n",
       "      <td>Hillary or Trump ? #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.612044</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.540917</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.433865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386889</th>\n",
       "      <td>796030312035676160</td>\n",
       "      <td>Hillary is like the Nickelback of politicians #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.772657</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.109528</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.431349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6552</th>\n",
       "      <td>795968415320129536</td>\n",
       "      <td>Hillary is a criminal https://t.co/62CXgzwpj9</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.695364</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.070790</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.428553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169068</th>\n",
       "      <td>795846761692528640</td>\n",
       "      <td>Related...we are all voting for #hillary right? #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.864070</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.463318</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.421538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308768</th>\n",
       "      <td>795994826013544449</td>\n",
       "      <td>Man, Hillary is always trippin'! https://t.co/HcnYJvDurm</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.737116</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.036805</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.415186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187413</th>\n",
       "      <td>795870315716845568</td>\n",
       "      <td>Hillary is full of SHIT #ImWithHer #Election2016 #VoteTrump</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.778631</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.036487</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.415186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307873</th>\n",
       "      <td>795994539483918336</td>\n",
       "      <td>Plot twist: Donald trump is voting for Hillary #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.811614</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.029126</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.390497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318853</th>\n",
       "      <td>795998250046201856</td>\n",
       "      <td>Team Hillary tutta la vita #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.743299</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.109528</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.384421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260485</th>\n",
       "      <td>795977715744653312</td>\n",
       "      <td>I  belive in Hillary victory #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.722638</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.109528</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.384421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>380517</th>\n",
       "      <td>796025166035873793</td>\n",
       "      <td>Hillary pees standing up #election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.722998</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.109528</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.384421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351927</th>\n",
       "      <td>796009170998874113</td>\n",
       "      <td>Hillary killed Abraham Lincoln #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.698832</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.109528</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.384421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264511</th>\n",
       "      <td>795979257570873359</td>\n",
       "      <td>To my American friends - vote for Hillary. Hillary is a voice for the voiceless. Donald Trump is a voice for Donald Trump. #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.801065</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.031424</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.366763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>349679</th>\n",
       "      <td>796008419635314688</td>\n",
       "      <td>Let Hillary Be Hillary | Full Frontal with Samantha Bee | TBS #Election2016 #ElectionFinalThoughts  https://t.co/FJYIbO4z79</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.747621</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.081270</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.343854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274212</th>\n",
       "      <td>795982814835601408</td>\n",
       "      <td>Hillary Clinton is my surest Bet. Hillary(Maywhether/Hillary  VS. Pacman/Trump)   #USADecides #Election2016 #usaelections2016 #VOTEHILLARY</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.679253</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.059039</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.337205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156004</th>\n",
       "      <td>796265954686615552</td>\n",
       "      <td>i did not vote for hillary because she is a woman or a democrat. i voted for hillary because the people i support needed her</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.835642</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.063192</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.328382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130614</th>\n",
       "      <td>796180580664832002</td>\n",
       "      <td>@Ethxn_10 i know, i said this bc the Democrat is hillary.</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.839247</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.036011</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.323850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360648</th>\n",
       "      <td>796012172140802048</td>\n",
       "      <td>The election of 2016. Hillary or Trump Hillary or Trump choose Choose CHOOSE #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.710167</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.067195</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.321101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113345</th>\n",
       "      <td>796090684998459392</td>\n",
       "      <td>Black College Students Think Hillary Is a Liar In New Poll-While Democrat presidential nominee Hillary Clinton... https://t.co/sjv83ED99p</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.673008</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.072381</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.312051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272087</th>\n",
       "      <td>795982043234570240</td>\n",
       "      <td>I'm voting for Hillary Clit. #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.719178</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.034588</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.307940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>390472</th>\n",
       "      <td>796032829058490368</td>\n",
       "      <td>#Election2016 #IVOTED #MyVote2016 I'm rooting for Hillary Clinton</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.701384</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.039140</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.307940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341722</th>\n",
       "      <td>796005881502646272</td>\n",
       "      <td>Not voting for #Hillary #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.728734</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.045082</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.307940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180620</th>\n",
       "      <td>795861255785508864</td>\n",
       "      <td>#Election2016 #Hillary #trump</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.697680</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.041931</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.307940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177192</th>\n",
       "      <td>795857044054179840</td>\n",
       "      <td>Trump or Hillary, I'm gonna miss Obama #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.713200</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.033993</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.307940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251363</th>\n",
       "      <td>795974043946127360</td>\n",
       "      <td>#pantsuitnation #Hillary #Election2016 https://t.co/0npwU6GJJ3</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.694142</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.036098</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.307940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>340216</th>\n",
       "      <td>796005428748570624</td>\n",
       "      <td>Hillary Clinton for President! #ImWithHer #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.636588</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.028664</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.255978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>385424</th>\n",
       "      <td>796029309639151616</td>\n",
       "      <td>#ElectionDay #Hillary https://t.co/dpKqcuID7p</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.651547</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.028970</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.255978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98260</th>\n",
       "      <td>795980750336258049</td>\n",
       "      <td>@EricTrump @LaraLeaTrump VOTE HILLARY! VOTE DEMOCRAT!!!</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.781567</td>\n",
       "      <td>5.595390e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.024571</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.239963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335376</th>\n",
       "      <td>796003922200633344</td>\n",
       "      <td>VOTE FOR HILLARY #ImWithHer #VoteHillary #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.819147</td>\n",
       "      <td>5.595390e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.032446</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.234937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255408</th>\n",
       "      <td>795975717284642817</td>\n",
       "      <td>#Election2016 Hillary</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.432263</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.057573</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.222020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245879</th>\n",
       "      <td>795968243060117504</td>\n",
       "      <td>Hillary sucks https://t.co/krltisTndZ</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.578938</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.070790</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.222020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211416</th>\n",
       "      <td>795935384278536192</td>\n",
       "      <td>Espero que a Hillary Clinton Ganhe  #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.581416</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.057771</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.222020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327387</th>\n",
       "      <td>796001143319134208</td>\n",
       "      <td>Hillary ate https://t.co/eKHbgoB2NG</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.598453</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.070790</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.222020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205126</th>\n",
       "      <td>795923271736819712</td>\n",
       "      <td>will vote hillary for nudes https://t.co/H6ZPT6V38D</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.843688</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.050713</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.221601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234451</th>\n",
       "      <td>795956058850111489</td>\n",
       "      <td>Hillary!! https://t.co/MC9qc4AyC5</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.438567</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.035190</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.208654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176177</th>\n",
       "      <td>795855877634985984</td>\n",
       "      <td>Vote Hillary Clinton. #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.468611</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.038340</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.208654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364861</th>\n",
       "      <td>796013694773854208</td>\n",
       "      <td>Hillary  #Election2016 https://t.co/2yg9LqaD0L</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.580553</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.040930</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.208654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331108</th>\n",
       "      <td>796002459013574656</td>\n",
       "      <td>Hillary ha votato https://t.co/3VfN9SxeU0</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.596377</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.048919</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.208654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209955</th>\n",
       "      <td>795932671176282112</td>\n",
       "      <td>Hillary Hoyeeee !! #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.432263</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.044156</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.208654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>276814</th>\n",
       "      <td>795983750987399168</td>\n",
       "      <td>Go Hillary! #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.585948</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.053735</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.208654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256811</th>\n",
       "      <td>795976299160498176</td>\n",
       "      <td>Hillary Clinton has voted. #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.612237</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.034261</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.208654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352157</th>\n",
       "      <td>796009259330863108</td>\n",
       "      <td>#Election2016 Hillary!</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.432263</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.046324</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.208654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254026</th>\n",
       "      <td>795975181839859712</td>\n",
       "      <td>Hillary Clinton voting now. #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.584335</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.039063</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.208654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263812</th>\n",
       "      <td>795978988866965504</td>\n",
       "      <td>Go Hillary!  #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.585948</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.053735</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.208654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233613</th>\n",
       "      <td>795955270111424512</td>\n",
       "      <td>#Hillary                https://t.co/xSbb9nl5sc</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.580553</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.044954</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.208654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179792</th>\n",
       "      <td>795860232782036992</td>\n",
       "      <td>Hillary Clinton for president! #Election2016</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.597086</td>\n",
       "      <td>2.220446e-16</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.208654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261342</th>\n",
       "      <td>795978037510729730</td>\n",
       "      <td>GO HILLARY!!!!!!!!!!! https://t.co/3rNBJSrohB</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.793524</td>\n",
       "      <td>5.595390e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.018930</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.201843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77857</th>\n",
       "      <td>796270845245038592</td>\n",
       "      <td>@kuledud3 hillary hates lgbts too i mean i hate both but it was hillary telling congress to not vote for gay marriage</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.836268</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.080007</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.199030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199388</th>\n",
       "      <td>795897387009118208</td>\n",
       "      <td>i have voted for hillary https://t.co/rN76G1dkTI</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.835285</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.038208</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.197185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266227</th>\n",
       "      <td>795979890935889920</td>\n",
       "      <td>#Election2016 good luck (hillary must win)</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.827696</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.045636</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.197185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223775</th>\n",
       "      <td>795948299543080961</td>\n",
       "      <td>Go hillary https://t.co/4xXbzsHeIP</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.749799</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.051433</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.197185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137280</th>\n",
       "      <td>796201374002319362</td>\n",
       "      <td>@ClCCO i'm a democrat but fuck hillary</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.774648</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.049861</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.197185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16600</th>\n",
       "      <td>796036463922188288</td>\n",
       "      <td>Congress to investigate hillary https://t.co/YkC4FkCalv</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.804468</td>\n",
       "      <td>4.789451e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.040950</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.197185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255012</th>\n",
       "      <td>795975559964725249</td>\n",
       "      <td>GO HILLARY, GO! #Election2016  @HillaryClinton</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.845974</td>\n",
       "      <td>5.595390e-01</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.034375</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.161323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196687</th>\n",
       "      <td>795893349123244032</td>\n",
       "      <td>#Election2016 Hillary'nin askerleriyiz</td>\n",
       "      <td>How is Hillary?</td>\n",
       "      <td>0.864526</td>\n",
       "      <td>-1.000000e+00</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.100576</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        id  \\\n",
       "348324  796007990843363328   \n",
       "8584    795986326495645698   \n",
       "197652  795894701652148224   \n",
       "134171  796191862562308097   \n",
       "159315  796297527805046785   \n",
       "189172  795872814486142976   \n",
       "212454  795937309233442816   \n",
       "386889  796030312035676160   \n",
       "6552    795968415320129536   \n",
       "169068  795846761692528640   \n",
       "308768  795994826013544449   \n",
       "187413  795870315716845568   \n",
       "307873  795994539483918336   \n",
       "318853  795998250046201856   \n",
       "260485  795977715744653312   \n",
       "380517  796025166035873793   \n",
       "351927  796009170998874113   \n",
       "264511  795979257570873359   \n",
       "349679  796008419635314688   \n",
       "274212  795982814835601408   \n",
       "156004  796265954686615552   \n",
       "130614  796180580664832002   \n",
       "360648  796012172140802048   \n",
       "113345  796090684998459392   \n",
       "272087  795982043234570240   \n",
       "390472  796032829058490368   \n",
       "341722  796005881502646272   \n",
       "180620  795861255785508864   \n",
       "177192  795857044054179840   \n",
       "251363  795974043946127360   \n",
       "...                    ...   \n",
       "340216  796005428748570624   \n",
       "385424  796029309639151616   \n",
       "98260   795980750336258049   \n",
       "335376  796003922200633344   \n",
       "255408  795975717284642817   \n",
       "245879  795968243060117504   \n",
       "211416  795935384278536192   \n",
       "327387  796001143319134208   \n",
       "205126  795923271736819712   \n",
       "234451  795956058850111489   \n",
       "176177  795855877634985984   \n",
       "364861  796013694773854208   \n",
       "331108  796002459013574656   \n",
       "209955  795932671176282112   \n",
       "276814  795983750987399168   \n",
       "256811  795976299160498176   \n",
       "352157  796009259330863108   \n",
       "254026  795975181839859712   \n",
       "263812  795978988866965504   \n",
       "233613  795955270111424512   \n",
       "179792  795860232782036992   \n",
       "261342  795978037510729730   \n",
       "77857   796270845245038592   \n",
       "199388  795897387009118208   \n",
       "266227  795979890935889920   \n",
       "223775  795948299543080961   \n",
       "137280  796201374002319362   \n",
       "16600   796036463922188288   \n",
       "255012  795975559964725249   \n",
       "196687  795893349123244032   \n",
       "\n",
       "                                                                                                                                          sentence  \\\n",
       "348324                                                                                              So, Hillary is Batman? https://t.co/bv6kZ1vzfj   \n",
       "8584              @CBSNews If Hillary wins-Can Congress stop the scandalous nature of Hillary?Can they protect us from being her prey? Vote Trump!   \n",
       "197652           Will anti-Trump immigrant wave prove strong enough for Hillary? #election2016 #hillary #trump #usa #usele https://t.co/3GnwXYzgdu   \n",
       "134171  #3Novices : EXCLUSIVE: Hillary wins Florida? Republican chief reveals Democrats win in vital county https://t.co/svskluqjYo  HILLARY Clint   \n",
       "159315         Is the democratic woman Hillary the democratic leader still for the democrat party so can Hillary covern the democratic states ? RT   \n",
       "189172                                                                                              Donald Trump or Hillary Clinton? #Election2016   \n",
       "212454                                                                                                            Hillary or Trump ? #Election2016   \n",
       "386889                                                                                 Hillary is like the Nickelback of politicians #Election2016   \n",
       "6552                                                                                                 Hillary is a criminal https://t.co/62CXgzwpj9   \n",
       "169068                                                                               Related...we are all voting for #hillary right? #Election2016   \n",
       "308768                                                                                    Man, Hillary is always trippin'! https://t.co/HcnYJvDurm   \n",
       "187413                                                                                 Hillary is full of SHIT #ImWithHer #Election2016 #VoteTrump   \n",
       "307873                                                                                Plot twist: Donald trump is voting for Hillary #Election2016   \n",
       "318853                                                                                                    Team Hillary tutta la vita #Election2016   \n",
       "260485                                                                                                  I  belive in Hillary victory #Election2016   \n",
       "380517                                                                                                      Hillary pees standing up #election2016   \n",
       "351927                                                                                                Hillary killed Abraham Lincoln #Election2016   \n",
       "264511    To my American friends - vote for Hillary. Hillary is a voice for the voiceless. Donald Trump is a voice for Donald Trump. #Election2016   \n",
       "349679                 Let Hillary Be Hillary | Full Frontal with Samantha Bee | TBS #Election2016 #ElectionFinalThoughts  https://t.co/FJYIbO4z79   \n",
       "274212  Hillary Clinton is my surest Bet. Hillary(Maywhether/Hillary  VS. Pacman/Trump)   #USADecides #Election2016 #usaelections2016 #VOTEHILLARY   \n",
       "156004                i did not vote for hillary because she is a woman or a democrat. i voted for hillary because the people i support needed her   \n",
       "130614                                                                                   @Ethxn_10 i know, i said this bc the Democrat is hillary.   \n",
       "360648                                                  The election of 2016. Hillary or Trump Hillary or Trump choose Choose CHOOSE #Election2016   \n",
       "113345   Black College Students Think Hillary Is a Liar In New Poll-While Democrat presidential nominee Hillary Clinton... https://t.co/sjv83ED99p   \n",
       "272087                                                                                                  I'm voting for Hillary Clit. #Election2016   \n",
       "390472                                                                           #Election2016 #IVOTED #MyVote2016 I'm rooting for Hillary Clinton   \n",
       "341722                                                                                                       Not voting for #Hillary #Election2016   \n",
       "180620                                                                                                               #Election2016 #Hillary #trump   \n",
       "177192                                                                                        Trump or Hillary, I'm gonna miss Obama #Election2016   \n",
       "251363                                                                              #pantsuitnation #Hillary #Election2016 https://t.co/0npwU6GJJ3   \n",
       "...                                                                                                                                            ...   \n",
       "340216                                                                                     Hillary Clinton for President! #ImWithHer #Election2016   \n",
       "385424                                                                                               #ElectionDay #Hillary https://t.co/dpKqcuID7p   \n",
       "98260                                                                                      @EricTrump @LaraLeaTrump VOTE HILLARY! VOTE DEMOCRAT!!!   \n",
       "335376                                                                                      VOTE FOR HILLARY #ImWithHer #VoteHillary #Election2016   \n",
       "255408                                                                                                                      #Election2016 Hillary    \n",
       "245879                                                                                                       Hillary sucks https://t.co/krltisTndZ   \n",
       "211416                                                                                           Espero que a Hillary Clinton Ganhe  #Election2016   \n",
       "327387                                                                                                         Hillary ate https://t.co/eKHbgoB2NG   \n",
       "205126                                                                                         will vote hillary for nudes https://t.co/H6ZPT6V38D   \n",
       "234451                                                                                                           Hillary!! https://t.co/MC9qc4AyC5   \n",
       "176177                                                                                                         Vote Hillary Clinton. #Election2016   \n",
       "364861                                                                                              Hillary  #Election2016 https://t.co/2yg9LqaD0L   \n",
       "331108                                                                                                   Hillary ha votato https://t.co/3VfN9SxeU0   \n",
       "209955                                                                                                            Hillary Hoyeeee !! #Election2016   \n",
       "276814                                                                                                                   Go Hillary! #Election2016   \n",
       "256811                                                                                                    Hillary Clinton has voted. #Election2016   \n",
       "352157                                                                                                                      #Election2016 Hillary!   \n",
       "254026                                                                                                   Hillary Clinton voting now. #Election2016   \n",
       "263812                                                                                                                  Go Hillary!  #Election2016   \n",
       "233613                                                                                             #Hillary                https://t.co/xSbb9nl5sc   \n",
       "179792                                                                                                Hillary Clinton for president! #Election2016   \n",
       "261342                                                                                               GO HILLARY!!!!!!!!!!! https://t.co/3rNBJSrohB   \n",
       "77857                        @kuledud3 hillary hates lgbts too i mean i hate both but it was hillary telling congress to not vote for gay marriage   \n",
       "199388                                                                                            i have voted for hillary https://t.co/rN76G1dkTI   \n",
       "266227                                                                                                  #Election2016 good luck (hillary must win)   \n",
       "223775                                                                                                          Go hillary https://t.co/4xXbzsHeIP   \n",
       "137280                                                                                                      @ClCCO i'm a democrat but fuck hillary   \n",
       "16600                                                                                      Congress to investigate hillary https://t.co/YkC4FkCalv   \n",
       "255012                                                                                              GO HILLARY, GO! #Election2016  @HillaryClinton   \n",
       "196687                                                                                                      #Election2016 Hillary'nin askerleriyiz   \n",
       "\n",
       "               question  alignment_score  max_alignment  number_of_intent  \\\n",
       "348324  How is Hillary?         0.714419   2.220446e-16                 1   \n",
       "8584    How is Hillary?         0.786155   2.220446e-16                 1   \n",
       "197652  How is Hillary?         0.814100   2.220446e-16                 1   \n",
       "134171  How is Hillary?         0.813238   2.220446e-16                 1   \n",
       "159315  How is Hillary?         0.805727   2.220446e-16                 1   \n",
       "189172  How is Hillary?         0.607569   2.220446e-16                 1   \n",
       "212454  How is Hillary?         0.612044   2.220446e-16                 1   \n",
       "386889  How is Hillary?         0.772657   2.220446e-16                 1   \n",
       "6552    How is Hillary?         0.695364   2.220446e-16                 1   \n",
       "169068  How is Hillary?         0.864070   4.789451e-01                 1   \n",
       "308768  How is Hillary?         0.737116   2.220446e-16                 1   \n",
       "187413  How is Hillary?         0.778631   2.220446e-16                 1   \n",
       "307873  How is Hillary?         0.811614   2.220446e-16                 1   \n",
       "318853  How is Hillary?         0.743299   2.220446e-16                 1   \n",
       "260485  How is Hillary?         0.722638   2.220446e-16                 1   \n",
       "380517  How is Hillary?         0.722998   2.220446e-16                 1   \n",
       "351927  How is Hillary?         0.698832   2.220446e-16                 1   \n",
       "264511  How is Hillary?         0.801065   2.220446e-16                 1   \n",
       "349679  How is Hillary?         0.747621   2.220446e-16                 1   \n",
       "274212  How is Hillary?         0.679253   2.220446e-16                 1   \n",
       "156004  How is Hillary?         0.835642   4.789451e-01                 1   \n",
       "130614  How is Hillary?         0.839247   4.789451e-01                 1   \n",
       "360648  How is Hillary?         0.710167   2.220446e-16                 1   \n",
       "113345  How is Hillary?         0.673008   2.220446e-16                 1   \n",
       "272087  How is Hillary?         0.719178   2.220446e-16                 1   \n",
       "390472  How is Hillary?         0.701384   2.220446e-16                 1   \n",
       "341722  How is Hillary?         0.728734   2.220446e-16                 1   \n",
       "180620  How is Hillary?         0.697680   2.220446e-16                 1   \n",
       "177192  How is Hillary?         0.713200   2.220446e-16                 1   \n",
       "251363  How is Hillary?         0.694142   2.220446e-16                 1   \n",
       "...                 ...              ...            ...               ...   \n",
       "340216  How is Hillary?         0.636588   2.220446e-16                 1   \n",
       "385424  How is Hillary?         0.651547   2.220446e-16                 1   \n",
       "98260   How is Hillary?         0.781567   5.595390e-01                 1   \n",
       "335376  How is Hillary?         0.819147   5.595390e-01                 1   \n",
       "255408  How is Hillary?         0.432263   2.220446e-16                 1   \n",
       "245879  How is Hillary?         0.578938   2.220446e-16                 1   \n",
       "211416  How is Hillary?         0.581416   2.220446e-16                 1   \n",
       "327387  How is Hillary?         0.598453   2.220446e-16                 1   \n",
       "205126  How is Hillary?         0.843688   4.789451e-01                 1   \n",
       "234451  How is Hillary?         0.438567   2.220446e-16                 1   \n",
       "176177  How is Hillary?         0.468611   2.220446e-16                 1   \n",
       "364861  How is Hillary?         0.580553   2.220446e-16                 1   \n",
       "331108  How is Hillary?         0.596377   2.220446e-16                 1   \n",
       "209955  How is Hillary?         0.432263   2.220446e-16                 1   \n",
       "276814  How is Hillary?         0.585948   2.220446e-16                 1   \n",
       "256811  How is Hillary?         0.612237   2.220446e-16                 1   \n",
       "352157  How is Hillary?         0.432263   2.220446e-16                 1   \n",
       "254026  How is Hillary?         0.584335   2.220446e-16                 1   \n",
       "263812  How is Hillary?         0.585948   2.220446e-16                 1   \n",
       "233613  How is Hillary?         0.580553   2.220446e-16                 1   \n",
       "179792  How is Hillary?         0.597086   2.220446e-16                 1   \n",
       "261342  How is Hillary?         0.793524   5.595390e-01                 1   \n",
       "77857   How is Hillary?         0.836268   4.789451e-01                 1   \n",
       "199388  How is Hillary?         0.835285   4.789451e-01                 1   \n",
       "266227  How is Hillary?         0.827696   4.789451e-01                 1   \n",
       "223775  How is Hillary?         0.749799   4.789451e-01                 1   \n",
       "137280  How is Hillary?         0.774648   4.789451e-01                 1   \n",
       "16600   How is Hillary?         0.804468   4.789451e-01                 1   \n",
       "255012  How is Hillary?         0.845974   5.595390e-01                 1   \n",
       "196687  How is Hillary?         0.864526  -1.000000e+00                 1   \n",
       "\n",
       "        question_words  exact_match  count_vectorizer_match  tfidf_match  \\\n",
       "348324               3            1                       2     0.493514   \n",
       "8584                 3            0                       5     0.656810   \n",
       "197652               3            1                       4     0.352648   \n",
       "134171               3            0                       3     0.347694   \n",
       "159315               3            1                       3     0.776727   \n",
       "189172               3            0                       2     0.332669   \n",
       "212454               3            0                       2     0.540917   \n",
       "386889               3            1                       1     0.109528   \n",
       "6552                 3            1                       1     0.070790   \n",
       "169068               3            0                       2     0.463318   \n",
       "308768               3            1                       1     0.036805   \n",
       "187413               3            1                       1     0.036487   \n",
       "307873               3            1                       1     0.029126   \n",
       "318853               3            0                       1     0.109528   \n",
       "260485               3            0                       1     0.109528   \n",
       "380517               3            0                       1     0.109528   \n",
       "351927               3            0                       1     0.109528   \n",
       "264511               3            1                       2     0.031424   \n",
       "349679               3            0                       2     0.081270   \n",
       "274212               3            1                       2     0.059039   \n",
       "156004               3            1                       2     0.063192   \n",
       "130614               3            1                       1     0.036011   \n",
       "360648               3            0                       2     0.067195   \n",
       "113345               3            1                       2     0.072381   \n",
       "272087               3            0                       1     0.034588   \n",
       "390472               3            0                       1     0.039140   \n",
       "341722               3            0                       1     0.045082   \n",
       "180620               3            0                       1     0.041931   \n",
       "177192               3            0                       1     0.033993   \n",
       "251363               3            0                       1     0.036098   \n",
       "...                ...          ...                     ...          ...   \n",
       "340216               3            0                       1     0.028664   \n",
       "385424               3            0                       1     0.028970   \n",
       "98260                3            0                       1     0.024571   \n",
       "335376               3            0                       1     0.032446   \n",
       "255408               3            0                       1     0.057573   \n",
       "245879               3            0                       1     0.070790   \n",
       "211416               3            0                       1     0.057771   \n",
       "327387               3            0                       1     0.070790   \n",
       "205126               3            0                       1     0.050713   \n",
       "234451               3            0                       1     0.035190   \n",
       "176177               3            0                       1     0.038340   \n",
       "364861               3            0                       1     0.040930   \n",
       "331108               3            0                       1     0.048919   \n",
       "209955               3            0                       1     0.044156   \n",
       "276814               3            0                       1     0.053735   \n",
       "256811               3            0                       1     0.034261   \n",
       "352157               3            0                       1     0.046324   \n",
       "254026               3            0                       1     0.039063   \n",
       "263812               3            0                       1     0.053735   \n",
       "233613               3            0                       1     0.044954   \n",
       "179792               3            0                       1     0.038284   \n",
       "261342               3            0                       1     0.018930   \n",
       "77857                3            0                       2     0.080007   \n",
       "199388               3            0                       1     0.038208   \n",
       "266227               3            0                       1     0.045636   \n",
       "223775               3            0                       1     0.051433   \n",
       "137280               3            0                       1     0.049861   \n",
       "16600                3            0                       1     0.040950   \n",
       "255012               3            0                       1     0.034375   \n",
       "196687               3            0                       0     0.000000   \n",
       "\n",
       "        coverage     score  \n",
       "348324  0.333333  0.809667  \n",
       "8584    0.000000  0.793776  \n",
       "197652  0.333333  0.789133  \n",
       "134171  0.000000  0.700904  \n",
       "159315  0.333333  0.676888  \n",
       "189172  0.000000  0.471353  \n",
       "212454  0.000000  0.433865  \n",
       "386889  0.333333  0.431349  \n",
       "6552    0.333333  0.428553  \n",
       "169068  0.000000  0.421538  \n",
       "308768  0.333333  0.415186  \n",
       "187413  0.333333  0.415186  \n",
       "307873  0.333333  0.390497  \n",
       "318853  0.000000  0.384421  \n",
       "260485  0.000000  0.384421  \n",
       "380517  0.000000  0.384421  \n",
       "351927  0.000000  0.384421  \n",
       "264511  0.333333  0.366763  \n",
       "349679  0.000000  0.343854  \n",
       "274212  0.333333  0.337205  \n",
       "156004  0.333333  0.328382  \n",
       "130614  0.333333  0.323850  \n",
       "360648  0.000000  0.321101  \n",
       "113345  0.333333  0.312051  \n",
       "272087  0.000000  0.307940  \n",
       "390472  0.000000  0.307940  \n",
       "341722  0.000000  0.307940  \n",
       "180620  0.000000  0.307940  \n",
       "177192  0.000000  0.307940  \n",
       "251363  0.000000  0.307940  \n",
       "...          ...       ...  \n",
       "340216  0.000000  0.255978  \n",
       "385424  0.000000  0.255978  \n",
       "98260   0.000000  0.239963  \n",
       "335376  0.000000  0.234937  \n",
       "255408  0.000000  0.222020  \n",
       "245879  0.000000  0.222020  \n",
       "211416  0.000000  0.222020  \n",
       "327387  0.000000  0.222020  \n",
       "205126  0.000000  0.221601  \n",
       "234451  0.000000  0.208654  \n",
       "176177  0.000000  0.208654  \n",
       "364861  0.000000  0.208654  \n",
       "331108  0.000000  0.208654  \n",
       "209955  0.000000  0.208654  \n",
       "276814  0.000000  0.208654  \n",
       "256811  0.000000  0.208654  \n",
       "352157  0.000000  0.208654  \n",
       "254026  0.000000  0.208654  \n",
       "263812  0.000000  0.208654  \n",
       "233613  0.000000  0.208654  \n",
       "179792  0.000000  0.208654  \n",
       "261342  0.000000  0.201843  \n",
       "77857   0.000000  0.199030  \n",
       "199388  0.000000  0.197185  \n",
       "266227  0.000000  0.197185  \n",
       "223775  0.000000  0.197185  \n",
       "137280  0.000000  0.197185  \n",
       "16600   0.000000  0.197185  \n",
       "255012  0.000000  0.161323  \n",
       "196687  0.000000  0.100576  \n",
       "\n",
       "[100 rows x 12 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load classifier model\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.externals import joblib\n",
    "\n",
    "def fetch_resuts_for_question(Q):\n",
    "    filename = 'model/random_forest_model_sample.sav'\n",
    "    model = joblib.load(filename)\n",
    "\n",
    "    feature_df = generate_feature(Q)\n",
    "    features = feature_df[['question_words', 'exact_match', 'coverage', 'number_of_intent', \\\n",
    "                           'alignment_score', 'max_alignment', 'count_vectorizer_match', 'tfidf_match']]\n",
    "    predicted = model.predict_proba(features)[:,1]\n",
    "    feature_df['score'] = pd.Series(predicted).values\n",
    "    feature_df.sort_values(by='score', ascending=False, inplace=True)\n",
    "\n",
    "    return (feature_df)\n",
    "\n",
    "fetch_resuts_for_question(\"How is Hillary?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 368
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "ZGjuIFwhB7ZI",
    "outputId": "66c01863-9f43-4ccb-de3e-bc7c65b48981"
   },
   "outputs": [],
   "source": [
    "from sumy.parsers.plaintext import PlaintextParser \n",
    "from sumy.nlp.tokenizers import Tokenizer \n",
    "from sumy.summarizers.lex_rank import LexRankSummarizer\n",
    "from sumy.summarizers.lsa import LsaSummarizer\n",
    "from sumy.summarizers.edmundson import EdmundsonSummarizer\n",
    "from sumy.nlp.stemmers import Stemmer\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "\n",
    "def summarize_results(Q, feature_df):\n",
    "    feature_df = feature_df[~feature_df.sentence.str.contains(\"\\?\")]\n",
    "    text = \". \".join(feature_df[feature_df.score > 0.4]\n",
    "                     .head(5).sentence.tolist())\n",
    "    text = re.sub(r\"([#@])(\\w+)\\b\", '', text)\n",
    "    text = re.sub(r'http([a-zA-Z0-9/:\\.]+)\\b', '', text)\n",
    "    parser = PlaintextParser(text, Tokenizer(\"english\"))\n",
    "    \n",
    "    # print (\"Text:\", text)\n",
    "    \n",
    "    summarizer = EdmundsonSummarizer(Stemmer('english'))\n",
    "    summarizer.stop_words = stopwords.words('english')\n",
    "    summarizer.bonus_words = fetch_intent_from_question(text, ('JJ')) + nltk.word_tokenize(Q)\n",
    "    summarizer.stigma_words = ['?', 'when', 'how', 'who'] # stopwords.words('english') # nltk.word_tokenize(Q)\n",
    "    summarizer.null_words = stopwords.words('english')\n",
    "\n",
    "\n",
    "    return \" \".join([str(s) for s in summarizer(parser.document, 3)])\n",
    "    \n",
    "# summarize_results(fetch_resuts_for_question(\"Congress will lose this election?\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "7_pDg538B7ZL"
   },
   "outputs": [],
   "source": [
    "from textblob import TextBlob\n",
    "\n",
    "def fetch_opinion(feature_df):\n",
    "    text = \" \".join(feature_df[feature_df.score > 0.5].sentence.tolist())\n",
    "    return TextBlob(text).sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "j7DrVijpB7ZM"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 2372.71it/s]\n",
      "100it [00:00, 2309.79it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'polarity': 0.2252465277777777,\n",
       " 'response': \"Not only did Donald Trump win but the republicans have kept control of Congress. This is even scarier than some may imagine.. Hillary Clinton and Donald Trump have wrapped up their US Presidential election campaigns - voting opens in a matte . US election: Republican Donald Trump leads over Democrat Hillary Clinton 140-104 in the race to 270 electoral votes.. Donald Trump claims US Election machines 'change votes from Republican to Democrat' as\",\n",
       " 'subjectivity': 0.4613888888888889}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_answer(Q):\n",
    "    df = fetch_resuts_for_question(Q)\n",
    "    response = {}\n",
    "    response['response'] = summarize_results(Q, df)\n",
    "    response['polarity'] = fetch_opinion(df).polarity\n",
    "    response['subjectivity'] = fetch_opinion(df).subjectivity\n",
    "    return response\n",
    "\n",
    "get_answer(\"Did Donald Trump win the US election?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Cs6QdS2rB7ZP"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100it [00:00, 2451.52it/s]\n",
      "100it [00:00, 2471.77it/s]\n",
      "127.0.0.1 - - [23/Sep/2018 00:13:02] \"GET /ask?q=Will%20Donald%20Trump%20win%20the%20elections%3F HTTP/1.1\" 200 -\n",
      "100it [00:00, 2616.50it/s]\n",
      "100it [00:00, 2602.73it/s]\n",
      "127.0.0.1 - - [23/Sep/2018 00:13:26] \"GET /ask?q=Who%20won%20in%20California%3F HTTP/1.1\" 200 -\n",
      "100it [00:00, 2544.32it/s]\n",
      "100it [00:00, 2608.19it/s]\n",
      "127.0.0.1 - - [23/Sep/2018 00:13:55] \"GET /ask?q=Are%20people%20voting%20for%20Donald%20Trump%3F HTTP/1.1\" 200 -\n",
      "100it [00:00, 2506.28it/s]\n",
      "100it [00:00, 2550.75it/s]\n",
      "127.0.0.1 - - [23/Sep/2018 00:14:13] \"GET /ask?q=Is%20anyone%20supporting%20Donald%20Trump%3F HTTP/1.1\" 200 -\n",
      "17it [00:00, 2272.39it/s]\n",
      "17it [00:00, 2167.80it/s]\n",
      "127.0.0.1 - - [23/Sep/2018 00:14:51] \"GET /ask?q=Who%20is%20winning%20in%20Utah%3F HTTP/1.1\" 200 -\n",
      "100it [00:00, 2474.94it/s]\n",
      "100it [00:00, 2497.69it/s]\n",
      "127.0.0.1 - - [23/Sep/2018 00:15:20] \"GET /ask?q=Who%20is%20winning%20in%20Texas%3F HTTP/1.1\" 200 -\n",
      "100it [00:00, 2321.21it/s]\n",
      "100it [00:00, 2472.37it/s]\n",
      "127.0.0.1 - - [23/Sep/2018 00:15:42] \"GET /ask?q=Are%20the%20Republicans%20going%20to%20win%20in%20California%3F HTTP/1.1\" 200 -\n",
      "100it [00:00, 2587.30it/s]\n",
      "100it [00:00, 2464.93it/s]\n",
      "127.0.0.1 - - [23/Sep/2018 00:16:16] \"GET /ask?q=Is%20Hillary%20Clinton%20going%20to%20win%3F HTTP/1.1\" 200 -\n",
      "100it [00:00, 2571.22it/s]\n",
      "100it [00:00, 2598.41it/s]\n",
      "127.0.0.1 - - [23/Sep/2018 00:16:41] \"GET /ask?q=What%20do%20you%20think%20about%20Donald%20Trump%3F HTTP/1.1\" 200 -\n",
      "100it [00:00, 2568.84it/s]\n",
      "100it [00:00, 2564.41it/s]\n",
      "127.0.0.1 - - [23/Sep/2018 00:17:14] \"GET /ask?q=Will%20you%20be%20leaving%20America%3F HTTP/1.1\" 200 -\n",
      "100it [00:00, 2383.45it/s]\n",
      "100it [00:00, 2488.98it/s]\n",
      "127.0.0.1 - - [23/Sep/2018 00:17:26] \"GET /ask?q=Would%20you%20be%20leaving%20USA%20now%3F HTTP/1.1\" 200 -\n",
      "100it [00:00, 2495.32it/s]\n",
      "100it [00:00, 2469.81it/s]\n",
      "127.0.0.1 - - [23/Sep/2018 00:18:08] \"GET /ask?q=Who%20is%20winning%20in%20Washington%3F%20 HTTP/1.1\" 200 -\n"
     ]
    }
   ],
   "source": [
    "from flask import Flask, request, jsonify\n",
    "\n",
    "app = Flask(__name__)\n",
    "app.config['TEMPLATES_AUTO_RELOAD'] = True\n",
    "\n",
    "@app.route('/')\n",
    "def index():\n",
    "    return \"Hello from Chatbot!\"\n",
    "\n",
    "@app.route('/ask')\n",
    "def ask():\n",
    "    try:\n",
    "        Q = request.args.get('q')\n",
    "        response = jsonify(get_answer(Q))\n",
    "    except:\n",
    "        response = jsonify(response=\"Well I\\'m certainly clueless about that :|\")\n",
    "    response.headers.add('Access-Control-Allow-Origin', '*')\n",
    "    return response\n",
    " \n",
    "\n",
    "if __name__ == '__main__':\n",
    "    app.run(debug=False, port=8990)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "github-election-tweets.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}