CSC869-TermProject.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SMS Spam Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "messages = [line.rstrip() for line in open('smsspamcollection/SMSSpamCollection')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5574"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(messages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 ham\tGo until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\n",
      "\n",
      "\n",
      "1 ham\tOk lar... Joking wif u oni...\n",
      "\n",
      "\n",
      "2 spam\tFree entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 08452810075over18's\n",
      "\n",
      "\n",
      "3 ham\tU dun say so early hor... U c already then say...\n",
      "\n",
      "\n",
      "4 ham\tNah I don't think he goes to usf, he lives around here though\n",
      "\n",
      "\n",
      "5 spam\tFreeMsg Hey there darling it's been 3 week's now and no word back! I'd like some fun you up for it still? Tb ok! XxX std chgs to send, Â£1.50 to rcv\n",
      "\n",
      "\n",
      "6 ham\tEven my brother is not like to speak with me. They treat me like aids patent.\n",
      "\n",
      "\n",
      "7 ham\tAs per your request 'Melle Melle (Oru Minnaminunginte Nurungu Vettam)' has been set as your callertune for all Callers. Press *9 to copy your friends Callertune\n",
      "\n",
      "\n",
      "8 spam\tWINNER!! As a valued network customer you have been selected to receivea Â£900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only.\n",
      "\n",
      "\n",
      "9 spam\tHad your mobile 11 months or more? U R entitled to Update to the latest colour mobiles with camera for Free! Call The Mobile Update Co FREE on 08002986030\n",
      "\n",
      "\n",
      "10 ham\tI'm gonna be home soon and i don't want to talk about this stuff anymore tonight, k? I've cried enough today.\n",
      "\n",
      "\n",
      "11 spam\tSIX chances to win CASH! From 100 to 20,000 pounds txt> CSH11 and send to 87575. Cost 150p/day, 6days, 16+ TsandCs apply Reply HL 4 info\n",
      "\n",
      "\n",
      "12 spam\tURGENT! You have won a 1 week FREE membership in our Â£100,000 Prize Jackpot! Txt the word: CLAIM to No: 81010 T&C www.dbuk.net LCCLTD POBOX 4403LDNW1A7RW18\n",
      "\n",
      "\n",
      "13 ham\tI've been searching for the right words to thank you for this breather. I promise i wont take your help for granted and will fulfil my promise. You have been wonderful and a blessing at all times.\n",
      "\n",
      "\n",
      "14 ham\tI HAVE A DATE ON SUNDAY WITH WILL!!\n",
      "\n",
      "\n",
      "15 spam\tXXXMobileMovieClub: To use your credit, click the WAP link in the next txt message or click here>> http://wap. xxxmobilemovieclub.com?n=QJKGIGHJJGCBL\n",
      "\n",
      "\n",
      "16 ham\tOh k...i'm watching here:)\n",
      "\n",
      "\n",
      "17 ham\tEh u remember how 2 spell his name... Yes i did. He v naughty make until i v wet.\n",
      "\n",
      "\n",
      "18 ham\tFine if thatÂ’s the way u feel. ThatÂ’s the way its gota b\n",
      "\n",
      "\n",
      "19 spam\tEngland v Macedonia - dont miss the goals/team news. Txt ur national team to 87077 eg ENGLAND to 87077 Try:WALES, SCOTLAND 4txt/Ãº1.20 POBOXox36504W45WQ 16+\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for messno, msg in enumerate(messages[:20]):\n",
    "    print(messno, msg)\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Organize messages into dataframes using pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>spam</td>\n",
       "      <td>FreeMsg Hey there darling it's been 3 week's n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ham</td>\n",
       "      <td>Even my brother is not like to speak with me. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ham</td>\n",
       "      <td>As per your request 'Melle Melle (Oru Minnamin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>spam</td>\n",
       "      <td>WINNER!! As a valued network customer you have...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>spam</td>\n",
       "      <td>Had your mobile 11 months or more? U R entitle...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                            message\n",
       "0   ham  Go until jurong point, crazy.. Available only ...\n",
       "1   ham                      Ok lar... Joking wif u oni...\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3   ham  U dun say so early hor... U c already then say...\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...\n",
       "5  spam  FreeMsg Hey there darling it's been 3 week's n...\n",
       "6   ham  Even my brother is not like to speak with me. ...\n",
       "7   ham  As per your request 'Melle Melle (Oru Minnamin...\n",
       "8  spam  WINNER!! As a valued network customer you have...\n",
       "9  spam  Had your mobile 11 months or more? U R entitle..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "messages = pd.read_csv('smsspamcollection/SMSSpamCollection', sep='\\t', names = ['label', 'message'])\n",
    "\n",
    "messages.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5572</td>\n",
       "      <td>5572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>2</td>\n",
       "      <td>5169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>ham</td>\n",
       "      <td>Sorry, I'll call later</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>4825</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       label                 message\n",
       "count   5572                    5572\n",
       "unique     2                    5169\n",
       "top      ham  Sorry, I'll call later\n",
       "freq    4825                      30"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\">message</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>unique</th>\n",
       "      <th>top</th>\n",
       "      <th>freq</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ham</th>\n",
       "      <td>4825</td>\n",
       "      <td>4516</td>\n",
       "      <td>Sorry, I'll call later</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spam</th>\n",
       "      <td>747</td>\n",
       "      <td>653</td>\n",
       "      <td>Please call our customer service representativ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      message                                                               \n",
       "        count unique                                                top freq\n",
       "label                                                                       \n",
       "ham      4825   4516                             Sorry, I'll call later   30\n",
       "spam      747    653  Please call our customer service representativ...    4"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages.groupby('label').describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# add a new feature called length that stores length of the message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>spam</td>\n",
       "      <td>FreeMsg Hey there darling it's been 3 week's n...</td>\n",
       "      <td>147</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                            message  length\n",
       "0   ham  Go until jurong point, crazy.. Available only ...     111\n",
       "1   ham                      Ok lar... Joking wif u oni...      29\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...     155\n",
       "3   ham  U dun say so early hor... U c already then say...      49\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...      61\n",
       "5  spam  FreeMsg Hey there darling it's been 3 week's n...     147"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages['length'] = messages['message'].apply(len)\n",
    "\n",
    "messages.head(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# distribution of message length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlkAAAFACAYAAACPyWmJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAHVZJREFUeJzt3X+QXWWd5/H3d/JDkFYDKk1LJwSL\nLP4aRbpBXGenusVRwNSEqZEVZxwiGyczUyzj7Axr0K1ydHenMKs1CKVLiaIbZh0DxWqRCoxKRXt3\n3RWGtDD8ECkiGhJIQCQwtqyGH9/94z6R7s5N+nZyn+57u9+vqlt9znOee85z+pvb9ck5554TmYkk\nSZLa6zdmewCSJElzkSFLkiSpAkOWJElSBYYsSZKkCgxZkiRJFRiyJEmSKjBkSZIkVWDIkiRJqsCQ\nJUmSVMHC2R4AwCte8Ypcvnx5tfX/4he/4Kijjqq2ftVl/bqb9etu1q97Wbt6RkdHH8/MV07VryNC\n1vLly9m6dWu19Y+MjDA0NFRt/arL+nU369fdrF/3snb1RMT2Vvp5ulCSJKkCQ5YkSVIFhixJkqQK\nDFmSJEkVGLIkSZIqMGRJkiRVYMiSJEmqwJAlSZJUgSFLkiSpAkOWJElSBYYsSZKkCgxZTfT1LyMi\nJrz6+pfN9rAkSVIX6YgHRHea3Q/v4IR1mye0bV+/cpZGI0mSupFHsiRJkiowZEmSJFVgyJIkSarA\nkCVJklSBIUuSJKkCQ5YkSVIFhixJkqQKDFmSJEkVGLIkSZIqMGRJkiRVYMiSJEmqYMqQFREnR8Sd\n417/HBF/ERHHRMQtEfFA+Xl06R8RcWVEbIuIuyLi1Pq7IUmS1FmmDFmZeX9mnpKZpwADwNPA14FL\ngS2ZuQLYUuYBzgZWlNda4KoaA59xCxYRERNeff3LZntUkiSpQy2cZv8zgR9l5vaIWAUMlfYNwAiw\nDlgFXJuZCdwaEUsioi8zd7VpzLPjuWc4Yd3mCU3b16+cpcFIkqRON91rss4Hvlqme/cFp/Lz2NJ+\nPLBj3Ht2ljZJkqR5IxoHnFroGLEYeAR4fWY+GhFPZuaSccv3ZObREXETcFlmfre0bwE+nJmjk9a3\nlsbpRHp7ewc2btzYnj1qYmxsjJ6enpb7j46Osvi4kya07d29rWnbwMBAW8aoA5tu/dRZrF93s37d\ny9rVMzw8PJqZg1P1m87pwrOB72fmo2X+0X2nASOiD3istO8Elo57Xz+NcDZBZl4NXA0wODiYQ0ND\n0xjK9IyMjDCd9Q8PDzc5NXhJ07ZWQ6oO3XTrp85i/bqb9ete1m72Ted04ft44VQhwCZgdZleDdw4\nrv2C8i3DM4Cnuv56LEmSpGlq6UhWRLwY+B3gT8Y1fxK4PiLWAA8B55X2m4FzgG00vol4YdtGK0mS\n1CVaClmZ+TTw8kltP6PxbcPJfRO4qC2jkyRJ6lLe8V2SJKkCQ5YkSVIFhixJkqQKDFmSJEkVGLIk\nSZIqMGRJkiRVYMiSJEmqwJAlSZJUgSFLkiSpAkOWJElSBYYsSZKkCgxZkiRJFRiyJEmSKjBkSZIk\nVWDIkiRJqsCQJUmSVIEhS5IkqQJDliRJUgWGLEmSpAoMWZIkSRUYsiRJkiowZEmSJFVgyJIkSarA\nkCVJklSBIUuSJKkCQ5YkSVIFLYWsiFgSETdExA8j4r6IeGtEHBMRt0TEA+Xn0aVvRMSVEbEtIu6K\niFPr7oIkSVLnafVI1hXANzLzNcCbgPuAS4EtmbkC2FLmAc4GVpTXWuCqto5YkiSpC0wZsiLipcBv\nA9cAZObezHwSWAVsKN02AOeW6VXAtdlwK7AkIvraPnJJkqQOFpl58A4RpwBXAz+gcRRrFPgQ8HBm\nLhnXb09mHh0Rm4FPZuZ3S/sWYF1mbp203rU0jnTR29s7sHHjxvbt1SRjY2P09PS03H90dJTFx500\noW3v7m1N2wYGBtoyRh3YdOunzmL9upv1617Wrp7h4eHRzBycqt/CFta1EDgVuDgzb4uIK3jh1GAz\n0aRtvySXmVfTCG8MDg7m0NBQC0M5NCMjI0xn/cPDw5ywbvOEtu3rL2naNlVI1eGbbv3UWaxfd7N+\n3cvazb5WrsnaCezMzNvK/A00Qtej+04Dlp+Pjeu/dNz7+4FH2jNcSZKk7jBlyMrM3cCOiDi5NJ1J\n49ThJmB1aVsN3FimNwEXlG8ZngE8lZm72jtsSZKkztbK6UKAi4GvRMRi4EHgQhoB7fqIWAM8BJxX\n+t4MnANsA54ufSVJkuaVlkJWZt4JNLvA68wmfRO46DDHJUmS1NW847skSVIFhixJkqQKDFmSJEkV\nGLIkSZIqMGRJkiRVYMiSJEmqwJAlSZJUgSFLkiSpAkPW4ViwiIiY8OrrXzbbo5IkSR2g1cfqqJnn\nnuGEdZsnNG1fv3KWBiNJkjqJR7IkSZIqMGRJkiRVYMiSJEmqwJAlSZJUgSFLkiSpAkOWJElSBYYs\nSZKkCgxZkiRJFRiyJEmSKjBkSZIkVWDIkiRJqsCQJUmSVIEhS5IkqQJDliRJUgWGLEmSpApaClkR\n8ZOIuDsi7oyIraXtmIi4JSIeKD+PLu0REVdGxLaIuCsiTq25A5IkSZ1oOkeyhjPzlMwcLPOXAlsy\ncwWwpcwDnA2sKK+1wFXtGqwkSVK3OJzThauADWV6A3DuuPZrs+FWYElE9B3GdiRJkrpOqyErgW9F\nxGhErC1tvZm5C6D8PLa0Hw/sGPfenaVNkiRp3ojMnLpTxKsy85GIOBa4BbgY2JSZS8b12ZOZR0fE\nTcBlmfnd0r4F+HBmjk5a51oapxPp7e0d2LhxY9t2arKxsTF6enpa7j86Osri406a0LZ397aW2wYG\nBg59sNrPdOunzmL9upv1617Wrp7h4eHRcZdPHVBLIWvCGyI+DowBfwwMZeaucjpwJDNPjojPl+mv\nlv737+t3oHUODg7m1q1bpzWO6RgZGWFoaKjl/hHBCes2T2jbvn5ly23T/Z3q4KZbP3UW69fdrF/3\nsnb1RERLIWvK04URcVREvGTfNPBO4B5gE7C6dFsN3FimNwEXlG8ZngE8dbCAJUmSNBctbKFPL/D1\niNjX/+8z8xsRcTtwfUSsAR4Cziv9bwbOAbYBTwMXtn3UkiRJHW7KkJWZDwJvatL+M+DMJu0JXNSW\n0UmSJHUp7/guSZJUgSFLkiSpAkOWJElSBYYsSZKkCgxZkiRJFRiyJEmSKjBkSZIkVWDIkiRJqsCQ\nJUmSVIEhS5IkqQJDliRJUgWGLEmSpAoMWZIkSRUYsiRJkiowZEmSJFVgyJIkSarAkNVuCxYRERNe\nff3LZntUkiRphi2c7QHMOc89wwnrNk9o2r5+5SwNRpIkzRaPZEmSJFVgyJIkSarAkCVJklSBIUuS\nJKkCQ5YkSVIFhixJkqQKDFmSJEkVGLIkSZIqaDlkRcSCiLgjIjaX+RMj4raIeCAirouIxaX9RWV+\nW1m+vM7QJUmSOtd0jmR9CLhv3Px64PLMXAHsAdaU9jXAnsw8Cbi89JMkSZpXWgpZEdEPvBv4YpkP\n4O3ADaXLBuDcMr2qzFOWn1n6S5IkzRuRmVN3irgBuAx4CXAJ8AHg1nK0iohYCvxDZr4hIu4BzsrM\nnWXZj4C3ZObjk9a5FlgL0NvbO7Bx48a27dRkY2Nj9PT0tNx/dHSUxcedNKFt7+5th9U2MDAwzVFr\nn+nWT53F+nU369e9rF09w8PDo5k5OFW/KR8QHRErgccyczQihvY1N+maLSx7oSHzauBqgMHBwRwa\nGprcpW1GRkaYzvqHh4ebPOT5ksNqayXMqrnp1k+dxfp1N+vXvazd7JsyZAFvA343Is4BjgBeCnwG\nWBIRCzPzWaAfeKT03wksBXZGxELgZcATbR+5JElSB5vymqzM/Ehm9mfmcuB84NuZ+YfAd4D3lG6r\ngRvL9KYyT1n+7fQwjiRJmmcO5z5Z64C/jIhtwMuBa0r7NcDLS/tfApce3hAlSZK6TyunC38tM0eA\nkTL9IHB6kz6/BM5rw9gkSZK6lnd8lyRJqsCQJUmSVIEhS5IkqQJDliRJUgWGLEmSpAoMWZIkSRUY\nsiRJkiowZEmSJFVgyJIkSarAkKVp6etfRkRMePX1L5vtYUmS1HGm9VgdaffDOzhh3eYJbdvXr5yl\n0UiS1Lk8kiVJklSBIUuSJKkCQ5YkSVIFhixJkqQKDFmSJEkVGLIkSZIqMGRJkiRVYMiSJEmqwJAl\nSZJUgSFLkiSpAkOWJElSBYYsSZKkCgxZkiRJFRiyJEmSKjBkSZIkVTBlyIqIIyLiHyPinyLi3oj4\nRGk/MSJui4gHIuK6iFhc2l9U5reV5cvr7oIkSVLnaeVI1q+At2fmm4BTgLMi4gxgPXB5Zq4A9gBr\nSv81wJ7MPAm4vPSb3xYsIiImvBa+6Mj92vr6l832SCVJUpssnKpDZiYwVmYXlVcCbwf+oLRvAD4O\nXAWsKtMANwCfjYgo65mfnnuGE9ZtntC0ff3Kpm2SJGluiFayT0QsAEaBk4DPAZ8Cbi1Hq4iIpcA/\nZOYbIuIe4KzM3FmW/Qh4S2Y+Pmmda4G1AL29vQMbN25s315NMjY2Rk9PT8v9R0dHWXzcSRPa9u7e\nNiNtAwMDLY9zNjT/3fyIRu5+waJFi3njG3+zLducbv3UWaxfd7N+3cva1TM8PDyamYNT9WspZP26\nc8QS4OvAx4AvTwpZN2fmb0bEvcC7JoWs0zPzZwda7+DgYG7durXlcUzXyMgIQ0NDLfePiJaPPLW7\nrdMP+E3nd9OufZlu/dRZrF93s37dy9rVExEthaxpfbswM58ERoAzgCURse90Yz/wSJneCSwtg1gI\nvAx4YjrbkSRJ6natfLvwleUIFhFxJPAO4D7gO8B7SrfVwI1lelOZpyz/9ry+HkuSJM1LU174DvQB\nG8p1Wb8BXJ+ZmyPiB8DGiPjPwB3ANaX/NcDfRcQ2Gkewzq8wbkmSpI7WyrcL7wLe3KT9QeD0Ju2/\nBM5ry+gkSZK6lHd8lyRJqsCQJUmSVIEhS5IkqQJDliRJUgWGLEmSpAoMWZIkSRUYsiRJkiowZEmS\nJFVgyJIkSarAkCVJklSBIUuSJKkCQ5YkSVIFhixJkqQKDFmSJEkVGLIkSZIqMGRJkiRVYMiSJEmq\nwJDVSRYsIiImvPr6l832qCRJ0iFYONsD0DjPPcMJ6zZPaNq+fuUsDUaSJB0Oj2RJkiRVYMiSJEmq\nwJAlSZJUgSFLkiSpAkOWDqivf9l+33aUJEmt8duFna7c1mG8445fyq6dD1Xf9O6Hd/htR0mSDtGU\nISsilgLXAscBzwNXZ+YVEXEMcB2wHPgJ8K8zc080EsEVwDnA08AHMvP7dYY/D3hbB0mSulIrpwuf\nBf4qM18LnAFcFBGvAy4FtmTmCmBLmQc4G1hRXmuBq9o+akmSpA43ZcjKzF37jkRl5s+B+4DjgVXA\nhtJtA3BumV4FXJsNtwJLIqKv7SOXJEnqYNO68D0ilgNvBm4DejNzFzSCGHBs6XY8sGPc23aWNkmS\npHkjMrO1jhE9wP8E/iYzvxYRT2bmknHL92Tm0RFxE3BZZn63tG8BPpyZo5PWt5bG6UR6e3sHNm7c\n2J49amJsbIyenp6W+4+OjrL4uJMmtO3dva2j2gYGBlren0N1uL+Hdo1xuvVTZ7F+3c36dS9rV8/w\n8PBoZg5O1a+lkBURi4DNwDcz829L2/3AUGbuKqcDRzLz5Ij4fJn+6uR+B1r/4OBgbt26taUdOxQj\nIyMMDQ213D8iml5s3kltrYbjw3G4v4d2jXG69VNnsX7dzfp1L2tXT0S0FLKmPF1Yvi14DXDfvoBV\nbAJWl+nVwI3j2i+IhjOApw4WsCRJkuaiVu6T9Tbgj4C7I+LO0vZR4JPA9RGxBngIOK8su5nG7Ru2\n0biFw4VtHbHarq9/Gbsf3jF1R0mS1LIpQ1a5tupAt/o+s0n/BC46zHFpBjW76Sh4Py5Jkg6Hj9WR\nJEmqwJAlSZJUgSFLkiSpAkOWJElSBYYsSZKkCgxZkiRJFRiyJEmSKjBkSZIkVWDImsP6+pcRERNe\nff3LZntYkiTNC608Vkddqtmd3Ld/+vdoPI5SkiTVZMiab557Zv/g5eNzJElqO08XSpIkVeCRrG60\nYNF+p/wWLD6C5/b+cpYGJEmSJjNkdaMDnPLzNKAkSZ3D04WSJEkVGLIkSZIqMGRJkiRVYMiSJEmq\nwJAlSZJUgSFLkiSpAkOWJElSBYYsSZKkCgxZkiRJFRiyJEmSKjBkSZIkVWDIkiRJqmDKkBURX4qI\nxyLinnFtx0TELRHxQPl5dGmPiLgyIrZFxF0RcWrNwUuSJHWqVo5k/TfgrEltlwJbMnMFsKXMA5wN\nrCivtcBV7RmmJElSd5kyZGXm/wKemNS8CthQpjcA545rvzYbbgWWRERfuwYrSZLULSIzp+4UsRzY\nnJlvKPNPZuaSccv3ZObREbEZ+GRmfre0bwHWZebWJutcS+NoF729vQMbN25sw+40NzY2Rk9PT8v9\nR0dHWXzcSRPa9u7eNq/a2rHOgYGB/dZ5KKZbP3UW69fdrF/3snb1DA8Pj2bm4FT92h2ybgIumxSy\nPpyZowdb/+DgYG7dul8Oa5uRkRGGhoaaLuvrX8buh3fs137Cus0T5revXzmv2tqxzlb+bbXiYPVT\n57N+3c36dS9rV09EtBSyFh7i+h+NiL7M3FVOBz5W2ncCS8f16wceOcRtzIjdD+9oGhAkSZIOx6He\nwmETsLpMrwZuHNd+QfmW4RnAU5m56zDHKEmS1HWmPJIVEV8FhoBXRMRO4K+BTwLXR8Qa4CHgvNL9\nZuAcYBvwNHBhhTFLkiR1vClDVma+7wCLzmzSN4GLDndQkiRJ3c47vkuSJFVgyJIkSarAkCVJklSB\nIUuSJKkCQ5YkSVIFhixJkqQK5k3I6utfRkTs95IkSarhUB+r03WaPT4HfISOJEmqY94cyZIkSZpJ\nhixJkqQKDFmSJEkVGLIkSZIqMGRJkiRVYMiSJEmqwJClOhYs2u+eZH39y2Z7VJIkzZh5c58szbDn\nntnvvmTek0ySNJ94JEuSJKkCQ5YkSVIFhizNHK/TkiTNI16TpZnjdVqSpHnEI1mSJEkVGLIkSZIq\nMGRJkiRVYMiSJEmqwJCl2eU3DiVJc1SVkBURZ0XE/RGxLSIurbENzRHlG4fjX7t3754QukZHR1n4\noiMPOYz19S+bkSA3U9uRJHWHtt/CISIWAJ8DfgfYCdweEZsy8wft3pbmqEm3elh83LM8t/eXLd3+\noa9/Gbsf3rFf+37v/fTvERET2o47fim7dj50yMPe/fCOQx5jq9s+nPdKkmZWjftknQ5sy8wHASJi\nI7AKMGSpulaDTtN7dlUIXvtOh052qPcLa7p/Ncbd4QybkrpBjZB1PDD+r99O4C0VtqP57gAB5pC1\nGLwAFiw+guf2/vLQ1tksUDXZl8PaRpNxN1tfu9ugedhpFoqavb/V90JrQbXVMDYfQ9tMHFE9UO3m\n+u92JszHf7PNdPrvITKzvSuMOA94V2Z+sMz/EXB6Zl48qd9aYG2ZPRm4v60DmegVwOMV16+6rF93\ns37dzfp1L2tXzwmZ+cqpOtU4krUTWDpuvh94ZHKnzLwauLrC9vcTEVszc3AmtqX2s37dzfp1N+vX\nvazd7Kvx7cLbgRURcWJELAbOBzZV2I4kSVLHavuRrMx8NiL+LfBNYAHwpcy8t93bkSRJ6mQ1TheS\nmTcDN9dY9yGakdOSqsb6dTfr192sX/eydrOs7Re+S5IkycfqSJIkVWHIkiRJqmDOhyyfo9jZImJp\nRHwnIu6LiHsj4kOl/ZiIuCUiHig/jy7tERFXlnreFRGnzu4eCBqP04qIOyJic5k/MSJuK/W7rnzT\nmIh4UZnfVpYvn81xCyJiSUTcEBE/LJ/Dt/r56x4R8e/K3857IuKrEXGEn7/OMadD1rjnKJ4NvA54\nX0S8bnZHpUmeBf4qM18LnAFcVGp0KbAlM1cAW8o8NGq5orzWAlfN/JDVxIeA+8bNrwcuL/XbA6wp\n7WuAPZl5EnB56afZdQXwjcx8DfAmGnX089cFIuJ44M+Bwcx8A41v9J+Pn7+OMadDFuOeo5iZe4F9\nz1FUh8jMXZn5/TL9cxp/4I+nUacNpdsG4NwyvQq4NhtuBZZERN8MD1vjREQ/8G7gi2U+gLcDN5Qu\nk+u3r643AGdGW5+NpOmIiJcCvw1cA5CZezPzSfz8dZOFwJERsRB4MbALP38dY66HrGbPUTx+lsai\nKZRD128GbgN6M3MXNIIYcGzpZk07z2eADwPPl/mXA09m5rNlfnyNfl2/svyp0l+z49XAT4Evl9O9\nX4yIo/Dz1xUy82Hg08BDNMLVU8Aofv46xlwPWc0Suves6EAR0QP8D+AvMvOfD9a1SZs1nSURsRJ4\nLDNHxzc36ZotLNPMWwicClyVmW8GfsELpwabsX4dpFwrtwo4EXgVcBSNU7qT+fmbJXM9ZLX0HEXN\nrohYRCNgfSUzv1aaH913GqL8fKy0W9PO8jbgdyPiJzROx7+dxpGtJeX0BUys0a/rV5a/DHhiJges\nCXYCOzPztjJ/A43Q5eevO7wD+HFm/jQznwG+BvxL/Px1jLkesnyOYocr1wNcA9yXmX87btEmYHWZ\nXg3cOK79gvItpzOAp/ad1tDMy8yPZGZ/Zi6n8fn6dmb+IfAd4D2l2+T67avre0p//yc9SzJzN7Aj\nIk4uTWcCP8DPX7d4CDgjIl5c/pbuq5+fvw4x5+/4HhHn0Pif9b7nKP7NLA9J40TEbwH/G7ibF67p\n+SiN67KuB5bR+ENyXmY+Uf6QfBY4C3gauDAzt874wLWfiBgCLsnMlRHxahpHto4B7gDen5m/iogj\ngL+jce3dE8D5mfngbI1ZEBGn0PjSwmLgQeBCGv8B9/PXBSLiE8B7aXxT+w7ggzSuvfLz1wHmfMiS\nJEmaDXP9dKEkSdKsMGRJkiRVYMiSJEmqwJAlSZJUgSFLkiSpAkOWpHktIsYqrPOUcvuYffMfj4hL\n2r0dSZ3NkCVJ7XcKcM6UvSTNaYYsSYcsIpZHxA/Lg4XviYivRMQ7IuL/RMQDEXF66XdURHwpIm4v\nDyJeVdpfHxH/GBF3RsRdEbGi9L0pIv6prPO9pe/HyvvviYiry40xiYjTynu/FxGfioh7SvuCMn97\nWf4nLezPvx/X/xPj9vG+iPhCRNwbEd+KiCMPtO3ydIn/CLy37Nd7y+pfFxEjEfFgRPx5m0shqQMZ\nsiQdrpOAK4A3Aq8B/gD4LeASGnfvB/gPNB7hcRowDHwqIo4C/hS4IjNPAQZpPFvtLOCRzHxTZr4B\n+EZZx2cz87TSdiSwsrR/GfjTzHwr8Ny4ca2h8diX04DTgD+OiBMPtBMR8U5gBXA6jSNRAxHx22Xx\nCuBzmfl64Eng9w+07czcC3wMuC4zT8nM60rf1wDvKuv/6/LMTklzmCFL0uH6cWbenZnPA/cCW8rz\n0O4Glpc+7wQujYg7gRHgCBqPbPke8NGIWAeckJn/r7zvHRGxPiL+VWY+VdYxHBG3RcTdNB5E/fqI\nWAK8JDP/b+nz9+PG9U4az9m7k8Zjml5OIywdyDvL6w7g+zRC0b7+P87MO8v0KLB8im03c1Nm/ioz\nH6fxwOXeKfpL6nILp+4iSQf1q3HTz4+bf54X/sYE8PuZef+k994XEbcB7wa+GREfzMxvR8QAjWua\nLouIbwH/BfivwGBm7oiIj9MIanGQcQVwcWZ+s8X9COCyzPz8hMaI5ZP28TkaR9IOtu1mJq/Dv7/S\nHOeRLEkz4ZvAxeOuo3pz+flq4MHMvBLYBLwxIl4FPJ2Z/x34NHAqjUAF8HhE9ADvAcjMPcDPI+KM\nsvz8Sdv8s32n5SLiX5RTlAcb478p6ycijo+IYw/UeYpt/xx4yUG2JWke8H9SkmbCfwI+A9xVgtZP\naFxT9V7g/RHxDLCbxgXjp9G4Zut54BngzzLzyYj4Ao1TiT8Bbh+37jXAFyLiFzRORe47vfhFGqcr\nv1+2+VPg3AMNMDO/FRGvBb5XsuAY8H4mXuc12YG2/R1eOD162UHeL2kOi8alE5LUnSKiJzPHyvSl\nQF9mfmiub1tS5/NIlqRu9+6I+AiNv2fbgQ/Mk21L6nAeyZIkSarAC98lSZIqMGRJkiRVYMiSJEmq\nwJAlSZJUgSFLkiSpgv8PykNEIMQKw5kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ee64c929b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "plt.xlabel('message length')\n",
    "messages['length'].hist(bins=100, edgecolor='black', figsize=(10,5))\n",
    "plt.savefig('output/Histogram1.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# plot separate histograms for spam and ham to analyze the difference in distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuUAAAFGCAYAAADNWuEjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3X+03HV95/HnywRQUfkZMOYmBCvV\ntrYqvUWq25Yaq2A5BT3S4rY1urjpnmpra8822PYc6u52F7q2qKer21TEuPUHSH9AWatlUevpD9CA\nFgWsRBRygUAsP2prVcD3/jHfwOTmhtzce2c+d2aej3PmzMzn+/3OvGcmud/XfObz/XxTVUiSJElq\n53GtC5AkSZImnaFckiRJasxQLkmSJDVmKJckSZIaM5RLkiRJjRnKJUmSpMYM5Rp5Sb6a5MWt65Ak\nSVooQ7kkSZLUmKFckiRJasxQrnHx3CQ3JHkgySVJHp/kiCRXJtmV5L7u9tTuDZJ8Msl/S/J3Sf4l\nyV8kOSrJ+5P8c5LPJFnf7iVJkg5Eks1J7kjy9ST/mGRDkt9Oclm3b/h6kuuTPKdvm3OTfLlbdlOS\nl/cte02Sv01yYZL7k9ya5AVd+44k9yTZ2ObVatwYyjUufho4FTge+AHgNfT+fV8MHAesA/4N+INZ\n250N/DywBvgu4O+7bY4EbgbOG3zpkqTFSvJM4A3AD1XVk4GXAl/tFp8BfJje3/YPAH+e5KBu2ZeB\nHwEOA94C/HGS1X0P/XzgBuCobtsPAT8EPAP4OeAPkjxpcK9Mk8JQrnHxjqq6s6ruBf4CeG5V/VNV\n/UlVfaOqvg78DvBjs7a7uKq+XFUPAH8JfLmq/l9VPUTvD/jzhvoqJEkL9TBwCPC9SQ6qqq9W1Ze7\nZddV1WVV9SDw+8DjgZMBqurD3f7jO1V1CXALcFLf436lqi6uqoeBS4C1wH+pqm9V1V8B36YX0KVF\nMZRrXOzsu/0N4ElJnpjkD5PcluSfgU8BhydZ0bfu3X23/22O+/Z+SNIIqKrtwK8Avw3ck+RDSZ7W\nLd7Rt953gBngaQBJXp3kc93wlPuBZwNH9z307P0CVeW+QkvOUK5x9mvAM4HnV9VTgB/t2tOuJEnS\noFTVB6rq39EbtljABd2itbvXSfI4YAq4M8lxwB/RG/ZyVFUdDnwB9xNqwFCucfZkej0Y9yc5EseH\nS9LYSvLMJC9KcgjwTXp//x/uFv9gklckWUmvN/1bwDXAofTC+67uMV5Lr6dcGjpDucbZ24AnAF+j\n98f3o23LkSQN0CHA+fT+5u8EjgF+o1t2OfAzwH30Du5/RVU9WFU3Ab9H7yD/u4HvB/52yHVLAKSq\nWtcgSZI0EEl+G3hGVf1c61qkx2JPuSRJktSYoVySJElqzOErkiRJUmP2lEuSlkyS93SnHv9CX9v/\nTPLFJDck+bMkh/cte3OS7d0p0V/apmpJas9QLklaSu8FTp3VdhXw7Kr6AeBLwJsBknwvcDbwfd02\n75x1ci9JmhgrWxfwWI4++uhav3596zIkaV6uu+66r1XVqtZ1tFRVn0qyflbbX/XdvQZ4ZXf7DOBD\nVfUt4CtJttM7vfnfP9ZzuG+QNErmu29Y1qF8/fr1bNu2rXUZkjQvSW5rXcMI+A/AJd3tNfRC+m4z\nXdtekmwCNgGsW7fOfYOkkTHffYPDVyRJQ5HkN4GHgPfvbppjtTlnH6iqLVU1XVXTq1ZN9I8RksbU\nsu4plySNhyQbgdOBDfXotF8zwNq+1aaAO4ddmyQtB/aUS5IGKsmpwGbgp6rqG32LrgDOTnJIkuOB\nE4BPt6hRklqzp1yStGSSfBA4BTg6yQxwHr3ZVg4BrkoCcE1V/aequjHJpcBN9Ia1vL6qHm5TuSS1\nZSiXJC2ZqnrVHM0XPcb6vwP8zuAqkqTR4PAVSZIkqTFDuSRJktSYoVySJElqzFAuSZIkNWYolyRJ\nkhozlEuSJEmNTUwoXz21jiSPXFZPrWtdkiRJ0qKYb8bHxMxTvvOOHRy3+cpH7t92wekNq5EkSVo8\n8834mJieckmSJGm5MpRLkiRJjRnKJUmSpMYM5ZIkSVJj+w3lSd6T5J4kX+hrOzLJVUlu6a6P6NqT\n5B1Jtie5IcmJfdts7Na/JcnGwbwcSZIkafTMp6f8vcCps9rOBa6uqhOAq7v7AKcBJ3SXTcC7oBfi\ngfOA5wMnAeftDvKSJEnSpNtvKK+qTwH3zmo+A9ja3d4KnNnX/r7quQY4PMlq4KXAVVV1b1XdB1zF\n3kFfkiRJmkgLHVN+bFXdBdBdH9O1rwF29K0307Xtq30vSTYl2ZZk265duxZY3jysOGiPyfadcF+S\nJEmtLPXJgzJHWz1G+96NVVuALQDT09NzrrMkHn5wj8n2wQn3JUmS1MZCe8rv7oal0F3f07XPAGv7\n1psC7nyMdkmSJGniLTSUXwHsnkFlI3B5X/uru1lYTgYe6Ia3fAx4SZIjugM8X9K1SZIkSRNvv8NX\nknwQOAU4OskMvVlUzgcuTXIOcDtwVrf6R4CXAduBbwCvBaiqe5P8V+Az3Xr/papmHzwqSZIkTaT9\nhvKqetU+Fm2YY90CXr+Px3kP8J4Dqk6SJEmaAJ7RU5IkSWrMUC5JkiQ1ZiiXJEmSGjOUS5IkjQtP\njjiylvrkQZIkSWrFkyOOLHvKJUmSpMYM5ZIkSVJjhnJJkiSpMUO5JEmS1JihXJIkSWrMUC5JkiQ1\nZiiXJEmSGjOUS5IkSY0ZyiVJkqTGDOWSJElSY4ZySdKSSfKeJPck+UJf25FJrkpyS3d9RNeeJO9I\nsj3JDUlObFe5JLVlKJckLaX3AqfOajsXuLqqTgCu7u4DnAac0F02Ae8aUo2StOwYyiVJS6aqPgXc\nO6v5DGBrd3srcGZf+/uq5xrg8CSrh1OpJC0vhnJJ0qAdW1V3AXTXx3Tta4AdfevNdG17SbIpybYk\n23bt2jXQYiWpBUO5JKmVzNFWc61YVVuqarqqpletWjXgsiRp+AzlkqRBu3v3sJTu+p6ufQZY27fe\nFHDnkGuTpGXBUC5JGrQrgI3d7Y3A5X3tr+5mYTkZeGD3MBdJmjQrWxcgSRofST4InAIcnWQGOA84\nH7g0yTnA7cBZ3eofAV4GbAe+Abx26AVL0jJhKJckLZmqetU+Fm2YY90CXj/YiiRpNDh8RZIkSWrM\nUC5JkiQ1ZiiXJEmSGjOUS5IkSY0ZyiVJkqTGDOWSJElSY4ZySZIkqTFDuSRJktSYoVySJElqzFAu\nSZIkNWYolyRJkhozlEuSJEmNGcolSZKkxgzlkiRJUmOGckmSJKkxQ7kkSZLU2KJCeZJfTXJjki8k\n+WCSxyc5Psm1SW5JckmSg7t1D+nub++Wr1+KFyBJkiSNugWH8iRrgF8Gpqvq2cAK4GzgAuDCqjoB\nuA84p9vkHOC+qnoGcGG3niRJkjTxFjt8ZSXwhCQrgScCdwEvAi7rlm8Fzuxun9Hdp1u+IUkW+fyS\nJEnSyFtwKK+qO4C3ArfTC+MPANcB91fVQ91qM8Ca7vYaYEe37UPd+kct9PklSZKkcbGY4StH0Ov9\nPh54GnAocNocq9buTR5jWf/jbkqyLcm2Xbt2LbQ8SZIkaWQsZvjKi4GvVNWuqnoQ+FPgBcDh3XAW\ngCngzu72DLAWoFt+GHDv7Aetqi1VNV1V06tWrVpEeZIkSdJoWEwovx04OckTu7HhG4CbgE8Ar+zW\n2Qhc3t2+ortPt/zjVbVXT7kkSZI0aRYzpvxaegdsXg98vnusLcBm4E1JttMbM35Rt8lFwFFd+5uA\ncxdRtyRJkjQ2Vu5/lX2rqvOA82Y13wqcNMe63wTOWszzSZIkSePIM3pKkiRJjRnKJUmSpMYM5ZIk\nSVJjhnJJkiSpMUO5JEmS1JihXJIkSWrMUC5JkiQ1ZiiXJEmSGjOUS5IkSY0ZyiVJkqTGDOWSpKFI\n8qtJbkzyhSQfTPL4JMcnuTbJLUkuSXJw6zolqQVDuSRp4JKsAX4ZmK6qZwMrgLOBC4ALq+oE4D7g\nnHZVSlI7hnJJ0rCsBJ6QZCXwROAu4EXAZd3yrcCZjWqTpKYM5ZKkgauqO4C3ArfTC+MPANcB91fV\nQ91qM8CaNhVKUluGcknSwCU5AjgDOB54GnAocNocq9Y+tt+UZFuSbbt27RpcoZLUiKFckjQMLwa+\nUlW7qupB4E+BFwCHd8NZAKaAO+fauKq2VNV0VU2vWrVqOBVL0hAZyiVJw3A7cHKSJyYJsAG4CfgE\n8MpunY3A5Y3qk6SmDOWSpIGrqmvpHdB5PfB5evufLcBm4E1JtgNHARc1K1KSGlq5/1UkSVq8qjoP\nOG9W863ASQ3KkaRlxZ5ySZIkqTFDuSRJktSYoVySJElqzFAuSZIkNWYolyRJkhozlEuSJEmNGcol\nSZKkxgzlkiRJUmOGckmSJKkxQ7kkSZLUmKFckiRJasxQLkmSJDVmKJckSZIaM5RLkiRJjRnKJUmS\npMYM5ZIkSeNsxUEkeeSyempd64o0h5WtC5AkSdIAPfwgx22+8pG7t11wesNitC/2lEuSJEmNGcr7\n+fOOJEmSGnD4Sj9/3pEkSVID9pRLkiRJjRnKJUmSpMYWFcqTHJ7ksiRfTHJzkh9OcmSSq5Lc0l0f\n0a2bJO9Isj3JDUlOXJqXIEmSJI22xfaUvx34aFU9C3gOcDNwLnB1VZ0AXN3dBzgNOKG7bALetcjn\nliRJksbCgkN5kqcAPwpcBFBV366q+4EzgK3daluBM7vbZwDvq55rgMOTrF5w5ZIkSdKYWExP+dOB\nXcDFST6b5N1JDgWOraq7ALrrY7r11wA7+raf6dr2kGRTkm1Jtu3atWsR5UmSJEmjYTGhfCVwIvCu\nqnoe8K88OlRlLpmjrfZqqNpSVdNVNb1q1apFlCdJkiSNhsWE8hlgpqqu7e5fRi+k3717WEp3fU/f\n+mv7tp8C7lzE80uSJEljYcGhvKp2AjuSPLNr2gDcBFwBbOzaNgKXd7evAF7dzcJyMvDA7mEukiRJ\n0iRb7Bk9fwl4f5KDgVuB19IL+pcmOQe4HTirW/cjwMuA7cA3unUlSZKkibeoUF5VnwOm51i0YY51\nC3j9Yp5PkiRJGkee0VOSJGkZWj21jiSPXFZPrWtdkgZoscNXJEmSNAA779jBcZuvfOT+bRec3rAa\nDZo95ZIkSVJjhnJJkiSpMUO5JEmS1JihXJI0FEkOT3JZki8muTnJDyc5MslVSW7pro9oXacktWAo\nlyQNy9uBj1bVs4DnADcD5wJXV9UJwNXdfUmaOIZySdLAJXkK8KPARQBV9e2quh84A9jarbYVOLNN\nhZLUlqFckjQMTwd2ARcn+WySdyc5FDi2qu4C6K6PaVmkJLViKJckDcNK4ETgXVX1POBfOYChKkk2\nJdmWZNuuXbsGVaMkNWMolyQNwwwwU1XXdvcvoxfS706yGqC7vmeujatqS1VNV9X0qlWrhlKwJA2T\noVySNHBVtRPYkeSZXdMG4CbgCmBj17YRuLxBeZLU3MrWBUiSJsYvAe9PcjBwK/Baep1DlyY5B7gd\nOKthfZLUjKFckjQUVfU5YHqORRuGXYskLTcOX5EkSZIaM5RLkiRJjRnKJUmSpMYM5ZIkSVJjhnJJ\nkiSpMUO5JEmS1JihXJIkSWrMUC5JkiQ1ZiiXJEmSGjOUS5IkSY0ZyiVJkqTGDOWPZcVBJNnjsnpq\nXeuqJEmSNGZWti5gWXv4QY7bfOUeTbddcHqjYiRJkjSu7CmXJEmSGjOUS5IkSY0ZyiVJkqTGDOWS\nJElSY4ZySZIkqTFDuSRJktSYoVySJElqzFAuSZIkNWYolyRJkhozlEuSJEmNGcolSZKkxla2LkCS\nJEnzsOIgkrSuQgNiKJckSRoFDz/IcZuv3KPptgtOb1SMltqih68kWZHks0mu7O4fn+TaJLckuSTJ\nwV37Id397d3y9Yt9bkmSJGkcLMWY8jcCN/fdvwC4sKpOAO4DzunazwHuq6pnABd260mSJEkTb1Gh\nPMkU8JPAu7v7AV4EXNatshU4s7t9RnefbvmGODBKkiRJWnRP+duAXwe+090/Cri/qh7q7s8Aa7rb\na4AdAN3yB7r1JUmSpIm24FCe5HTgnqq6rr95jlVrHsv6H3dTkm1Jtu3atWuh5UmSJI2M1VPrSLLH\nRZNlMbOvvBD4qSQvAx4PPIVez/nhSVZ2veFTwJ3d+jPAWmAmyUrgMODe2Q9aVVuALQDT09N7hXZJ\nkqRxs/OOHc6sMuEW3FNeVW+uqqmqWg+cDXy8qn4W+ATwym61jcDl3e0ruvt0yz9eVYZuSZIkTbxB\nnNFzM/CmJNvpjRm/qGu/CDiqa38TcO4AnluSJEkaOUty8qCq+iTwye72rcBJc6zzTeCspXg+SZIk\naZwMoqdckiRJ0gEwlEuShma+Z4GWpEljKJckDdN8zwItSRPFUC5JGooDPAu0JE0UQ7kkaVgO5CzQ\ne/DEcpLGnaFckjRwCzgL9J6NVVuqarqqpletWjWQGiWppSWZElGSpP040LNAS9JEsadckjRwCzgL\ntCRNFEO5JKmlfZ0FWpImisNXDtSKg+hNGNDz1DVruWvm9oYFSdJomc9ZoCVp0hjKD9TDD3Lc5isf\nuXvbBac3LEaSJEnjwOErkiRJS2T11DqS7HFZPbWudVkaAfaUS5IkLZGdd+zY4xd18Fd1zY895ZIk\nSVJjhnJJkiSpMUO5JEmS1JihXJIkSWrMUC5JkiQ1ZiiXJEmSGjOUS5IkSY0ZyiVJkqTGDOWSJElS\nY4ZySZKkQVpxEEn2uKyeWte6Ki0zK1sXIEmSNNYefpDjNl+5R9NtF5zeqBgtV4ZySZKkYet6z6Xd\nDOWSJEnDNqv33J5zOaZckiRJasxQLkmSJDVmKJckSZIaM5RLkiRJjRnKJUmSpMYM5ZIkSVJjhnJJ\nkiSpMUO5JEmS1JihXJIkSWrMUC5JkiQ1ZiiXJEmSGjOUS5IkSY0ZyiVJkqTGDOWLteIgkuxxWT21\nrnVVkiRJGiErWxcw8h5+kOM2X7lH020XnN6oGEmSJI2iBfeUJ1mb5BNJbk5yY5I3du1HJrkqyS3d\n9RFde5K8I8n2JDckOXGpXsSyM6v33J5zSZIkPZbF9JQ/BPxaVV2f5MnAdUmuAl4DXF1V5yc5FzgX\n2AycBpzQXZ4PvKu7Hj+zes/tOZckSdJjWXBPeVXdVVXXd7e/DtwMrAHOALZ2q20FzuxunwG8r3qu\nAQ5PsnrBlY+41VPrHIsuSZIkYInGlCdZDzwPuBY4tqrugl5wT3JMt9oaYEffZjNd212zHmsTsAlg\n3brxDak779ix91j0t76cJI/cf+qatdw1c/uwS5MkSdKQLTqUJ3kS8CfAr1TVP/eHytmrztFWezVU\nbQG2AExPT++1fKw57EXSmEqyFngf8FTgO8CWqnp7kiOBS4D1wFeBn66q+1rVKUmtLGpKxCQH0Qvk\n76+qP+2a7949LKW7vqdrnwHW9m0+Bdy5mOcfGXNMmyhJE2b3cUjfA5wMvD7J99I77ujqqjoBuLq7\nL0kTZ8E95ekly4uAm6vq9/sWXQFsBM7vri/va39Dkg/RO8Dzgd3DXMae0yZKmnDd3/vdQxu/nqT/\nOKRTutW2Ap+kNzmAJE2UxQxfeSHw88Dnk3yua/sNemH80iTnALcDZ3XLPgK8DNgOfAN47SKeW5I0\nouZ5HJIkTZQFh/Kq+hvmHicOsGGO9Qt4/UKfT5I0+g7gOKTZ203EJACSJteixpRLkjRfB3gc0h6q\naktVTVfV9KpVq4ZTsCQNkaFckjRw8zgOCfY8DkmSJsqSzFMuSdJ+HOhxSJI0UQzlkqSBO9DjkCRp\n0jh8RZIkSWrMUC5JkiQ1ZiiXJEmSGjOUS5IkSY0ZyiVJ0sRbPbWOJHtcVk95oioNj7OvSJKkibfz\njh0ct/nKPdpuu+D0RtVoEtlTLkmSJDVmKJckSZIaM5RLkiRJjY1lKJ/rYA1JkiRpuRrLAz09WEOS\nJEmjZCx7yiVJ0mSYz1SGSzXd4VyPs/KQJ/jLvJbEWPaUS5KkyTCfX8eX6hf0fT1Of5u/zGuh7CmX\nJEmSGjOUS5IkSY0ZyiVJkqTGDOWSJElSY4ZySZIE7D27yEJmKBkrKw5yZhUNjbOvSJIkYO/ZRSZ+\nJpGHH/T90NDYUy5JkiQ1ZiiXJEkjYa6T98zJYScaQQ5fkSRJI2HeJwFy2IlGkD3lkiRJk2TWLwke\n1Ls82FMuSZI0SWb9kgD+mrAc2FMuSdIyNdcYans0pfFkT7kkScvUvMdQSxp59pRLkqS5zWPs8SB7\n82c/tjTO7CmXJElzm8fY40H25nsyI00Se8olSZKkxgzlkiRpac0a9jKf4SzzPjGQNKYcvrKcdX/U\n+j11zVrumrm9UUGSJM3DAk7e40GtmnSG8uXMeUQlSUtg9dQ6dt6xY4+21p08c9U0VHN0fGlPsz+j\n1v9mxp2hXJKkMbcce6GbH8S5gN78SdP8M5owjimXJGkRFjol4FzbrTzkCY6pVhtzTH85n3WGOUXm\nuLOnXJKkRVhoL/S+trNnUk3MZ8hs4ykyx5095aNm1rfU2b0qfiOVJM3LfHpG57HdQJ9LmiBD7ylP\ncirwdmAF8O6qOn/YNYy0OcbA7e8b6XI8wEeSdnO/0MhCJxNYyFhsJy6Q9muooTzJCuB/AT8BzACf\nSXJFVd00zDomzUJ/SlroUdcerS1pvtwvSFLPsIevnARsr6pbq+rbwIeAM4Zcw/ibz0+L8zixw+4w\nv/uyc+fOeQ2V2Wu7llNeSVruhrZfmH0A2nyG/y34YMw5hmss2UGc8/j7PbtuadS0PmC0xfMPe/jK\nGqA/oc0Azx9yDeNvPj8tzl7nrS/f/x/uuX5+nM928zgJ0lxDbFYc/Hge/vY397nNfA3ysZeKQ4w0\nwYa2X5hrereFHrR2oH9j573dfMzjb7xT2WnUtT5gtMXzp6oG+gR7PFlyFvDSqnpdd//ngZOq6pf6\n1tkEbOruPhP4xwN8mqOBry1BuaNk0l6zr3e8jfLrPa6qVrUuYpTMZ7/QtS923zAORvn/xlLxPfA9\ngNF7D+a1bxh2T/kMsLbv/hRwZ/8KVbUF2LLQJ0iyraqmF7r9KJq01+zrHW+T9nq1//0CLH7fMA78\nv+F7AL4HML7vwbDHlH8GOCHJ8UkOBs4GrhhyDZKk5cP9giQx5J7yqnooyRuAj9Gb+uo9VXXjMGuQ\nJC0f7hckqWfo85RX1UeAjwzwKSbx581Je82+3vE2aa934g1hvzAu/L/hewC+BzCm78FQD/SUJEmS\ntLdhjymXJEmSNIuhXJIkSWps6GPKl1qSZ9E7+9saoOhNpXVFVd3ctDBJkiRpnkZ6THmSzcCr6J2W\neaZrnqI3pdaHqur8VrUNUpJj6fsSUlV3Ny5p4JIcCVRV3de6lmHwM5Yk6VGTsF8c9VD+JeD7qurB\nWe0HAzdW1QltKhuMJM8F/jdwGHBH1zwF3A/8YlVd36q2QUiyDvhdYAO91xjgKcDHgXOr6qvtqhsM\nP+Px/4yl+UhyGPBm4Exg95kA7wEuB86vqvtb1TZskxDGHkuSACex54iAT9coB7gDMEn7xVEfvvId\n4GnAbbPaV3fLxs17gV+oqmv7G5OcDFwMPKdFUQN0CfA24Ger6mGAJCuAs+j9OnJyw9oG5b34GY/7\nZyzNx6X0vpyeUlU7AZI8FdgIfBj4iYa1DcW+wliSsQtj+5LkJcA7gVvYM5A+I8kvVtVfNStueN7L\nhOwXR72n/FTgD+j9Y93RNa8DngG8oao+2qq2QUhyy756/5Nsr6pnDLumQdrP693nslHmZzy/ZdK4\nS/KPVfXMA102TpJ8jn2HsT+sqrEJY/uS5GbgtNm/GiY5HvhIVX1Pk8KGaJL2iyPdU15VH03y3Tz6\ns07ojS3/zO5etzHzl0n+L/A+Hv0SshZ4NTBWX0A61yV5J7CVPV/vRuCzzaoaLD/j8f+Mpfm4Lcmv\nA1t3D9fohnG8hkf/r4y7Q2cHcoCquibJoS0KamAljx4z1+8O4KAh19LKxOwXR7qnfBIlOY1HZ5vZ\n/SXkiu6MeGOlOzbgHOZ4vcBFVfWthuUNjJ/x+H/G0v4kOQI4l97/jWPpjSW+m97/jQuq6t6G5Q1F\nkncA38XcYewrVfWGVrUNS5I3Az9Nbzhf/3twNnBpVf2PVrUN06TsFw3lkiQtc0l+hN6vwp+fkHHE\nwOSEsceS5HuY+z24qWlhWnKG8hHSdzT+GcAxXfPYHo2fZCW9XtQz2fOo88vp9aI++BibjyQ/4/H/\njKX5SPLpqjqpu/064PXAnwMvAf5iXKf8lWabpP2iZ/QcLZcC9wE/XlVHVdVRwI/Tmxbow00rG4z/\nAzwXeAvwMuAnu9vPAf64YV2D5Gc8/p+xNB/944V/AXhJVb2FXij/2TYlDVeSw5Kcn+TmJP/UXW7u\n2g5vXd8wdBNa7L59WJJ3J7khyQe6YwwmwcTsF+0pHyGTdjT+fl7vl6rqu4dd06D5Ge+xbCw/Y2k+\nkvwDcAq9zrOPVdV037LPVtXzWtU2LEk+Rm9ayK2zpoV8DbChqiZhWsjrq+rE7va7gZ3AHwGvAH6s\nqs5sWd8wTNJ+0Z7y0XJbkl/v/3ac5NjuzKbjeDT+fUnOSvLIv9Mkj0vyM/S+NY8jP+Px/4yl+TgM\nuA7YBhzZhVGSPIneuOJJsL6qLtgdyAGqamc3dGddw7pama6q36qq26rqQmB964KGZGL2i4by0fIz\nwFHAXye5L8m9wCeBI+kdnT1uzgZeCdyd5EtJbqHXS/CKbtk4mtTPeGf3GX+J8f+Mpf2qqvVV9fSq\nOr673h1MvwO8vGVtQzQxYewxHJPkTUl+DXhKkv4vZJOS4SZmv+jwlRGT5Fn0zuZ1TVX9S1/7qeN2\nsqR+SY6i1zv0tqr6udb1DEqS5wNfrKoHkjyR3pRoJwI3Av+9qh5oWuAS66ZEfBW9gzuvB04DXkDv\n9W7xQE9pcs2aFnL3AX67p4U8v6rG/te0JOfNanpnVe3qfjn53ap6dYu6hm1Sso+hfIQk+WV6R+Df\nTO/guDdW1eXdskfGnY2LJFfM0fwiemMMqaqfGm5Fg5fkRuA5VfVQki3AvwJ/Amzo2l/RtMAlluT9\n9E6O8QTgAeBQ4M/ovd5U1cZiFaSXAAABx0lEQVSG5UlappK8tqoubl1HS5PyHkxS9hnpM3pOoP8I\n/GBV/UuS9cBlSdZX1dsZzzGGU8BNwLvpTZUX4IeA32tZ1IA9rqoe6m5P9/2x+Zv0Tjk9br6/qn6g\nmxrxDuBpVfVwkj8G/qFxbZKWr7cAYx9I92NS3oOJyT6G8tGyYvfPNlX11SSn0PvHeRxj9g+zMw28\nEfhN4D9X1eeS/FtV/XXjugbpC329H/+QZLqqtiX5bmAch3I8rhvCcijwRHoHt90LHMLknEJa0hyS\n3LCvRfTOcjr2fA+ACco+hvLRsjPJc6vqcwDdt8bTgfcA39+2tKVXVd8BLkzy4e76bsb/3+zrgLcn\n+S3ga8DfJ9lB76Cm1zWtbDAuAr4IrKD35evDSW4FTqZ3WmlJk+tY4KXsPRNTgL8bfjlN+B5MUPZx\nTPkISTIFPNQ/PVTfshdW1d82KGtokvwk8MKq+o3WtQxakicDT6f3JWSmqu5uXNLAJHkaQFXd2Z0Q\n5MXA7VX16baVSWopyUXAxVX1N3Ms+0BV/fsGZQ2V78FkZR9DuSRJktTYpMxxKUmSJC1bhnJJkiSp\nMUO5JEmS1JihXJIkSWrMUC5JkiQ19v8B2+Q21TPpL8MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ee64c928d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "messages.hist(column='length',by='label',bins=60,figsize=(12,5),\n",
    "              edgecolor='black')\n",
    "plt.savefig('output/Histogram2.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Text pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def text_process(data):\n",
    "    '''\n",
    "    1. remove punc\n",
    "    2. do stemming of words\n",
    "    3. remove stop words\n",
    "    4. return list of clean text words\n",
    "    '''\n",
    "    nopunc = [c for c in data if c not in string.punctuation] #remove punctuations\n",
    "    nopunc = ''.join(nopunc)\n",
    "    \n",
    "    stemmed = ''\n",
    "    nopunc = nopunc.split()\n",
    "    for i in nopunc:\n",
    "        stemmer = SnowballStemmer('english')\n",
    "        stemmed += (stemmer.stem(i)) + ' ' # stemming of words\n",
    "        \n",
    "    clean_msgs = [word for word in stemmed.split() if word.lower() not in stopwords.words('english')] # remove stopwords\n",
    "    \n",
    "    return clean_msgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Rohan\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "\n",
      "################################################## Processed Messages ##################################################\n",
      "\n",
      "     label                                            message  length                                      processed_msg\n",
      "0      ham  Go until jurong point, crazy.. Available only ...     111  [go, jurong, point, crazi, avail, onli, bugi, ...\n",
      "1      ham                      Ok lar... Joking wif u oni...      29                       [ok, lar, joke, wif, u, oni]\n",
      "2     spam  Free entry in 2 a wkly comp to win FA Cup fina...     155  [free, entri, 2, wkli, comp, win, fa, cup, fin...\n",
      "3      ham  U dun say so early hor... U c already then say...      49      [u, dun, say, earli, hor, u, c, alreadi, say]\n",
      "4      ham  Nah I don't think he goes to usf, he lives aro...      61  [nah, dont, think, goe, usf, live, around, tho...\n",
      "5     spam  FreeMsg Hey there darling it's been 3 week's n...     147  [freemsg, hey, darl, 3, week, word, back, id, ...\n",
      "6      ham  Even my brother is not like to speak with me. ...      77  [even, brother, like, speak, treat, like, aid,...\n",
      "7      ham  As per your request 'Melle Melle (Oru Minnamin...     160  [per, request, mell, mell, oru, minnaminungint...\n",
      "8     spam  WINNER!! As a valued network customer you have...     157  [winner, valu, network, custom, select, receiv...\n",
      "9     spam  Had your mobile 11 months or more? U R entitle...     154  [mobil, 11, month, u, r, entitl, updat, latest...\n",
      "10     ham  I'm gonna be home soon and i don't want to tal...     109  [im, gonna, home, soon, dont, want, talk, stuf...\n",
      "11    spam  SIX chances to win CASH! From 100 to 20,000 po...     136  [six, chanc, win, cash, 100, 20000, pound, txt...\n",
      "12    spam  URGENT! You have won a 1 week FREE membership ...     155  [urgent, 1, week, free, membership, £100000, p...\n",
      "13     ham  I've been searching for the right words to tha...     196  [ive, search, right, word, thank, breather, pr...\n",
      "14     ham                I HAVE A DATE ON SUNDAY WITH WILL!!      35                                     [date, sunday]\n",
      "15    spam  XXXMobileMovieClub: To use your credit, click ...     149  [xxxmobilemovieclub, use, credit, click, wap, ...\n",
      "16     ham                         Oh k...i'm watching here:)      26                                   [oh, kim, watch]\n",
      "17     ham  Eh u remember how 2 spell his name... Yes i di...      81  [eh, u, rememb, 2, spell, name, yes, v, naught...\n",
      "18     ham  Fine if thats the way u feel. Thats the way ...      56   [fine, that, way, u, feel, that, way, gota, b]\n",
      "19    spam  England v Macedonia - dont miss the goals/team...     155  [england, v, macedonia, dont, miss, goalsteam,...\n",
      "20     ham          Is that seriously how you spell his name?      41                             [serious, spell, name]\n",
      "21     ham    I‘m going to try for 2 months ha ha only joking      47       [i'm, go, tri, 2, month, ha, ha, onli, joke]\n",
      "22     ham  So ü pay first lar... Then when is da stock co...      52             [ü, pay, first, lar, da, stock, comin]\n",
      "23     ham  Aft i finish my lunch then i go str down lor. ...      88  [aft, finish, lunch, go, str, lor, ard, 3, smt...\n",
      "24     ham  Ffffffffff. Alright no way I can meet up with ...      57           [ffffffffff, alright, way, meet, sooner]\n",
      "25     ham  Just forced myself to eat a slice. I'm really ...     144  [forc, eat, slice, im, realli, hungri, tho, su...\n",
      "26     ham                     Lol your always so convincing.      30                              [lol, alway, convinc]\n",
      "27     ham  Did you catch the bus ? Are you frying an egg ...     134  [catch, bus, fri, egg, make, tea, eat, mom, le...\n",
      "28     ham  I'm back &amp; we're packing the car now, I'll...      75   [im, back, amp, pack, car, ill, let, know, room]\n",
      "29     ham  Ahhh. Work. I vaguely remember that! What does...      64   [ahhh, work, vagu, rememb, doe, feel, like, lol]\n",
      "...    ...                                                ...     ...                                                ...\n",
      "5542   ham           Armand says get your ass over to epsilon      40                   [armand, say, get, ass, epsilon]\n",
      "5543   ham             U still havent got urself a jacket ah?      38        [u, still, havent, got, urself, jacket, ah]\n",
      "5544   ham  I'm taking derek &amp; taylor to walmart, if I...     154  [im, take, derek, amp, taylor, walmart, im, ba...\n",
      "5545   ham      Hi its in durban are you still on this number      45                        [hi, durban, still, number]\n",
      "5546   ham         Ic. There are a lotta childporn cars then.      42                        [ic, lotta, childporn, car]\n",
      "5547  spam  Had your contract mobile 11 Mnths? Latest Moto...     160  [contract, mobil, 11, mnths, latest, motorola,...\n",
      "5548   ham                 No, I was trying it all weekend ;V      34                                  [tri, weekend, v]\n",
      "5549   ham  You know, wot people wear. T shirts, jumpers, ...      87  [know, wot, peopl, wear, shirt, jumper, hat, b...\n",
      "5550   ham        Cool, what time you think you can get here?      43                           [cool, time, think, get]\n",
      "5551   ham  Wen did you get so spiritual and deep. That's ...      51                   [wen, get, spiritu, deep, great]\n",
      "5552   ham  Have a safe trip to Nigeria. Wish you happines...      91  [safe, trip, nigeria, wish, happi, veri, soon,...\n",
      "5553   ham                        Hahaha..use your brain dear      27                            [hahahaus, brain, dear]\n",
      "5554   ham  Well keep in mind I've only got enough gas for...      98  [well, keep, mind, ive, onli, got, enough, gas...\n",
      "5555   ham  Yeh. Indians was nice. Tho it did kane me off ...     153  [yeh, indian, nice, tho, kane, bit, shud, go, ...\n",
      "5556   ham  Yes i have. So that's why u texted. Pshew...mi...      63               [yes, whi, u, text, pshewmiss, much]\n",
      "5557   ham  No. I meant the calculation is the same. That ...     273  [meant, calcul, ltgt, unit, ltgt, school, real...\n",
      "5558   ham                             Sorry, I'll call later      22                          [sorri, ill, call, later]\n",
      "5559   ham  if you aren't here in the next  &lt;#&gt;  hou...      66        [arent, next, ltgt, hour, imma, flip, shit]\n",
      "5560   ham                  Anything lor. Juz both of us lor.      33                         [anyth, lor, juz, us, lor]\n",
      "5561   ham  Get me out of this dump heap. My mom decided t...      70     [get, dump, heap, mom, decid, come, low, bore]\n",
      "5562   ham  Ok lor... Sony ericsson salesman... I ask shuh...      96  [ok, lor, soni, ericsson, salesman, ask, shuhu...\n",
      "5563   ham                                Ard 6 like dat lor.      19                           [ard, 6, like, dat, lor]\n",
      "5564   ham  Why don't you wait 'til at least wednesday to ...      67  [whi, dont, wait, til, least, wednesday, see, ...\n",
      "5565   ham                                       Huh y lei...      12                                         [huh, lei]\n",
      "5566  spam  REMINDER FROM O2: To get 2.50 pounds free call...     147  [remind, o2, get, 250, pound, free, call, cred...\n",
      "5567  spam  This is the 2nd time we have tried 2 contact u...     160  [2nd, time, tri, 2, contact, u, u, £750, pound...\n",
      "5568   ham               Will ü b going to esplanade fr home?      36                     [ü, b, go, esplanad, fr, home]\n",
      "5569   ham  Pity, * was in mood for that. So...any other s...      57                       [piti, mood, soani, suggest]\n",
      "5570   ham  The guy did some bitching but I acted like i'd...     125  [guy, bitch, act, like, id, interest, buy, som...\n",
      "5571   ham                         Rofl. Its true to its name      26                                 [rofl, true, name]\n",
      "\n",
      "[5572 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "# File:        text_preprocess.py\n",
    "#\n",
    "# Author:      Rohan Patel\n",
    "#\n",
    "# Date:        05/09/2018\n",
    "#\n",
    "# Description: This script loads the sms spam data, organizes the data into a pandas dataframe, adds a new \n",
    "#              feature (length of message) to the data, applies some basic text pre-processing techniques \n",
    "#              like stopword removal and punctuation removal, and converts the messages into a list of processed \n",
    "#              tokens. Finally, the processed dataframe is copied into a new csv file processed_msgs.csv\n",
    "\n",
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import SnowballStemmer\n",
    "import string\n",
    "nltk.download('stopwords')\n",
    "\n",
    "def loadData():\n",
    "    \n",
    "    messages = pd.read_csv('smsspamcollection/SMSSpamCollection', sep='\\t', names = ['label', 'message'])\n",
    "    messages['length'] = messages['message'].apply(len)\n",
    "    \n",
    "    return messages\n",
    "\n",
    "def text_process(data):\n",
    "    '''\n",
    "    1. remove punc\n",
    "    2. do stemming of words\n",
    "    3. remove stop words\n",
    "    4. return list of clean text words\n",
    "    '''\n",
    "    nopunc = [c for c in data if c not in string.punctuation] #remove punctuations\n",
    "    nopunc = ''.join(nopunc)\n",
    "    \n",
    "    stemmed = ''\n",
    "    nopunc = nopunc.split()\n",
    "    for i in nopunc:\n",
    "        stemmer = SnowballStemmer('english')\n",
    "        stemmed += (stemmer.stem(i)) + ' ' # stemming of words\n",
    "        \n",
    "    clean_msgs = [word for word in stemmed.split() if word.lower() not in stopwords.words('english')] # remove stopwords\n",
    "    \n",
    "    return clean_msgs\n",
    "\n",
    "def main():\n",
    "    \n",
    "    messages = loadData()\n",
    "    #print(messages)\n",
    "    messages['processed_msg'] = messages['message'].apply(text_process)\n",
    "    \n",
    "    print('\\n################################################## Processed Messages ##################################################\\n')\n",
    "    with pd.option_context('expand_frame_repr', False):\n",
    "        print (messages)\n",
    "    #print(messages)\n",
    "\n",
    "    messages.to_csv('output/processed_msgs.csv', encoding='utf-8', index=False) #copy processed messages dataframe to a new csv file\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Vectorization of processed messages & Initial Text Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "############### SVM ###############\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.88      1.00      0.94      1475\n",
      "       spam       0.00      0.00      0.00       197\n",
      "\n",
      "avg / total       0.78      0.88      0.83      1672\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rohan\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "############### Decision Tree ###############\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.97      0.98      0.98      1475\n",
      "       spam       0.84      0.79      0.81       197\n",
      "\n",
      "avg / total       0.96      0.96      0.96      1672\n",
      "\n",
      "\n",
      "############### Multinomial NB ###############\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.88      1.00      0.94      1475\n",
      "       spam       1.00      0.01      0.01       197\n",
      "\n",
      "avg / total       0.90      0.88      0.83      1672\n",
      "\n",
      "\n",
      "############### KNN ###############\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.96      0.97      0.96      1475\n",
      "       spam       0.75      0.68      0.71       197\n",
      "\n",
      "avg / total       0.93      0.94      0.93      1672\n",
      "\n",
      "\n",
      "############### Random Forest ###############\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.97      1.00      0.98      1475\n",
      "       spam       1.00      0.76      0.86       197\n",
      "\n",
      "avg / total       0.97      0.97      0.97      1672\n",
      "\n",
      "\n",
      "############### AdaBoost ###############\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.97      0.98      0.98      1475\n",
      "       spam       0.86      0.74      0.80       197\n",
      "\n",
      "avg / total       0.95      0.96      0.95      1672\n",
      "\n",
      "\n",
      "############### Bagging Classifier ###############\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.97      0.99      0.98      1475\n",
      "       spam       0.92      0.79      0.85       197\n",
      "\n",
      "avg / total       0.97      0.97      0.97      1672\n",
      "\n",
      "\n",
      "############### Accuracy Scores ###############\n",
      "\n",
      "\n",
      "                    Accuracy Rate\n",
      "SVM                 0.882177     \n",
      "Decision Tree       0.957536     \n",
      "Multinomial NB      0.882775     \n",
      "KNN                 0.935407     \n",
      "Random Forest       0.971292     \n",
      "AdaBoost            0.955742     \n",
      "Bagging Classifier  0.967105     \n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl0AAAGPCAYAAABmnGWSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmcJVV99/HPFwZEZFGG0aiDgApR\nBIZlQBEjKoq44YMYBXFf87gbNQFBUYyRIPFRDJpgRFwQoiQgKgqKIHFn2DdRRJABxQFkBAGB4ff8\nUdVwaXqmG7q7qu/05/169atvnapb93fvLP3tc06dSlUhSZKk6bVK3wVIkiTNBoYuSZKkDhi6JEmS\nOmDokiRJ6oChS5IkqQOGLkmSpA4YuqRZIsm/J3n/FJ3rUUluSrJqu31aktdPxbnb8307yaum6nwD\n5z0yyT9N9Xn7eK328390+/iBSb6RZGmSryXZO8nJ0/Xaku6fOX0XIGnyklwOPAy4A1gGXAR8ETi8\nqu4EqKq/uw/nen1VfW95x1TVb4G1Jlf1Xa/3QeCxVfXygfM/ZyrOPZ2SBHgb8EZgY+CPwE+AA6vq\n/Ol+/aoa/PxfTPPnP7eq7mjbjpruGiTdN/Z0SSuPF1TV2sCGwEHAPwKfm+oXSeIva41PAu8A3g6s\nB2wKHA88r4daNgR+ORC47reR3ktJU8/QJa1kqmppVZ0AvBR4VZLN4Z7DXUnWT/LNJDckuT7J/yZZ\nJcmXgEcB32iHr/4hyUZJKsnrkvwW+P5A22AAe0ySn7dDXF9Psl77Wk9LsniwxiSXJ3lmkl2B9wEv\nbV/v3Hb/XcOVbV37J7kiyR+SfDHJuu2+kTpeleS3Sa5Nst84H9H6Sb6b5MYkP0iyYXuuw5L866g6\nv5HknaNPkGQT4C3AXlX1/ar6S1XdXFVHVdVBYxz/kPbzXpLkj+3j+QP7X53ksram3yTZu21/bFvj\n0va9/dfAc6rd/yHgAwOf4eva8/1w4NjHte/5+iSXJHnJwL4jk3wmyYlJ/gw8fZzPT9L9ZOiSVlJV\n9XNgMfA3Y+x+d7tvHs2w1Puap9QrgN/S9JqtVVUHDzxnJ+DxwLOX85KvBF4LPIJmmPPQCdT4HeCf\ngf9qX2/BGIe9uv16OvBommHNfxt1zFOAvwZ2Bj6Q5PEreNm9gQ8D6wPncPcw3BeAvZKsAk0wbc93\n9Bjn2BlY3H7GE7EK8HmaHqlHAbeMvIckD6L5rJ7T9lQ+ua2Lts6TgYcA84FPjT5xVR3APT/De/Ru\ntuf/LvAV4KHAXsCnkzxh4LCXAR8B1gZ+iKRpYeiSVm5X0wx9jXY78HBgw6q6var+t8a/EesHq+rP\nVXXLcvZ/qaouqKo/A+8HXjJFQ1V7Ax+vqsuq6iZgX2DPUb1sH6qqW6rqXOBcYKzwNuJbVXV6Vf0F\n2A/YIckGbYBaShOoAPYETquqa8Y4x1zgdxN9A1V1XVX9d9sbdiNNwNlp4JA7gc2TPLCqfldVF7bt\nt9MEtUdU1a1VdX8C0fOBy6vq81V1R1WdBfw3zTywEV+vqh9V1Z1Vdev9eA1JE2DoklZujwSuH6P9\nY8ClwMntsNY+EzjXlfdh/xXAajS9SZP1iPZ8g+eeQ9NDN+L3A49vZsWT/O+qsw1x17evAU1v18iE\n/pcDX1rOOa6jCa0TkmTNJP/RDpH+CTgdeHCSVduQ+lLg74DfJflWkse1T/0HIMDPk1yY5LUTfc0B\nGwJPbIeSb0hyA02Q/auBY8b7s5U0BQxd0koqyXY0oetevSNVdWNVvbuqHg28APj7JCM9PMvr8Rqv\nJ2yDgcePoumluRb4M7DmQF2r0gxrTvS8V9MEh8Fz3wGM1QM1EXfVmWQtmp7Aq9umLwMvTLKAZij1\n+OWc4xRgfpKFE3zNd9MMfz6xqtYBnjpSAkBVnVRVz6IJcr8APtu2/76q3lBVjwDeRDMs+NgJvuaI\nK4EfVNWDB77Wqqr/O3DMeH8GkqaAoUtaySRZJ8nzgWOAL4+1fEGS57eTsAP8iWaZiWXt7mto5k7d\nVy9PslmSNYEDgWOrahnwS2CNJM9LshqwP/CAgeddA2w0MpdqDEcD70qycRuSRuYv3d8r9Z6b5ClJ\nVqeZM/WzqroSoKoWA2fQ9HD99/KGUqvqV8CngaPbCwVWT7JGkj2X02u4Ns08rhvaCwwOGNmR5GFJ\ndmvnXv0FuIn2zyLJ3w5MuP8jTThaxn3zTWDTJK9Islr7td04894kTQNDl7Ty+EaSG2l6NvYDPg68\nZjnHbgJ8j+YH/E+AT1fVae2+jwL7t0NR77kPr/8l4Eiaob41aJZSoKqWAm8G/hO4iqbna/Bqxq+1\n369LctYY5z2iPffpwG+AW2nWx7q/vkITeq4HtqUZahv0BWALlj+0OOLtNJPhDwNuAH4N7A58Y4xj\nPwE8kKbn76fAdwb2rULTE3Z1W9NONJ8XwHbAz5LcBJwAvKOqfjPuOxzQziHbhWaO2tU0fz7/wj2D\nr6QOZPy5s5I0eyR5Ks0w40YjC8tK0lSwp0uSWu3w5zuA/zRwSZpq44auJEe0CxJesJz9SXJokkuT\nnJdkm4F9r0ryq/Zryu+jJklTpZ3jdAPNZPZP9FyOpJXQuMOLbVf7TcAXq2rzMfY/l2Z+xXOBJwKf\nrKontpNFFwELaSZ/nglsW1V/nNq3IEmSNPON29NVVacz9jo/I15IE8iqqn5Ks/bMw2lWrf5uVV3f\nBq3vArtORdGSJEnDZirmdD2Sey6st7htW177vSR5Y5JF7deYw5iSJEkzTZLvjH9UY874h4z/emO0\n1Qra791YdThwOMC6665bCxcu9JJKSZI0DP400QOnInQt5p4rUc+nWQtmMfC0Ue2njXeyTTbZhEWL\nFk1BWZIkSdMrya8meuxUDC+eALyyvYrxScDSqvodcBKwS5KHJHkIzeJ8J03B60mSJA2dcXu6khxN\n02O1fpLFNCs5rwZQVf8OnEhz5eKlNDeafU277/okH6a5pQbAgVW1ogn5kiRJK61xQ1dV7TXO/gLe\nspx9R9DcwkOSJGlWm4o5XZIk6X64/fbbWbx4MbfeemvfpWgca6yxBvPnz2e11Va73+cwdEmS1JPF\nixez9tprs9FGG5GMddG/ZoKq4rrrrmPx4sVsvPHG9/s83ntRkqSe3HrrrcydO9fANcMlYe7cuZPu\nkTR0SZLUIwPXcJiKPydDlyRJs9xxxx1HEn7xi1/0Xcr9suqqq7LVVlux+eab84IXvIAbbrhhhcff\ncMMNfPrTn+6ours5p0uSpBlio32+NaXnu/yg503ouKOPPpqnPOUpHHPMMXzwgx+c0hoGLVu2jFVX\nXXXKz/vABz6Qc845B4BXvepVHHbYYey3337LPX4kdL35zW+e8lpWxJ4uSZJmsZtuuokf/ehHfO5z\nn+OYY465x76DDz6YLbbYggULFrDPPvsAcOmll/LMZz6TBQsWsM022/DrX/+a0047jec///l3Pe+t\nb30rRx55JAAbbbQRBx54IE95ylP42te+xmc/+1m22247FixYwB577MHNN98MwDXXXMPuu+/OggUL\nWLBgAT/+8Y95//vfzyc/+cm7zrvffvtx6KGHrvD97LDDDlx11VV3vbedd96ZbbbZhi222IKvf/3r\nAOyzzz78+te/ZquttuK9730vAB/72MfYbrvt2HLLLTnggAMm8Ykunz1dkiTNYscffzy77rorm266\nKeuttx5nnXUW22yzDd/+9rc5/vjj+dnPfsaaa67J9dc365vvvffe7LPPPuy+++7ceuut3HnnnVx5\n5ZUrfI011liDH/7whwBcd911vOENbwBg//3353Of+xxve9vbePvb385OO+3Ecccdx7Jly7jpppt4\nxCMewYte9CLe8Y53cOedd3LMMcfw85//fLmvs2zZMk455RRe97rX3fW6xx13HOussw7XXnstT3rS\nk9htt9046KCDuOCCC+7qHTv55JP51a9+xc9//nOqit12243TTz+dpz71qZP+fAcZuiRJmsWOPvpo\n3vnOdwKw5557cvTRR7PNNtvwve99j9e85jWsueaaAKy33nrceOONXHXVVey+++5AE2om4qUvfeld\njy+44AL2339/brjhBm666Sae/exnA/D973+fL37xi0AzR2vddddl3XXXZe7cuZx99tlcc801bL31\n1sydO/de57/lllvYaqutuPzyy9l222151rOeBTRLPbzvfe/j9NNPZ5VVVuGqq67immuuudfzTz75\nZE4++WS23nproOkh+9WvfmXokiRJU+O6667j+9//PhdccAFJWLZsGUk4+OCDqap7XbHX3ITm3ubM\nmcOdd9551/bopRUe9KAH3fX41a9+NccffzwLFizgyCOP5LTTTlthja9//es58sgj+f3vf89rX/va\nMY8ZmdO1dOlSnv/853PYYYfx9re/naOOOoolS5Zw5plnstpqq7HRRhuNuexDVbHvvvvypje9aYW1\nTJZzuiRJmqWOPfZYXvnKV3LFFVdw+eWXc+WVV7Lxxhvzwx/+kF122YUjjjjirjlX119/Peussw7z\n58/n+OOPB+Avf/kLN998MxtuuCEXXXQRf/nLX1i6dCmnnHLKcl/zxhtv5OEPfzi33347Rx111F3t\nO++8M5/5zGeAZpjwT3/6EwC777473/nOdzjjjDPu6hVbnnXXXZdDDz2UQw45hNtvv52lS5fy0Ic+\nlNVWW41TTz2VK664AoC1116bG2+88a7nPfvZz+aII47gpptuAuCqq67iD3/4w339OMdl6JIkaZY6\n+uij7xoqHLHHHnvwla98hV133ZXddtuNhQsXstVWW3HIIYcA8KUvfYlDDz2ULbfckic/+cn8/ve/\nZ4MNNuAlL3kJW265JXvvvfddw3Rj+fCHP8wTn/hEnvWsZ/G4xz3urvZPfvKTnHrqqWyxxRZsu+22\nXHjhhQCsvvrqPP3pT+clL3nJhK583HrrrVmwYAHHHHMMe++9N4sWLWLhwoUcddRRd73e3Llz2XHH\nHdl8881573vfyy677MLLXvYydthhB7bYYgte/OIX3yOUTZUsr6uwLwsXLqxFixb1XYYkSdPu4osv\n5vGPf3zfZcxod955J9tssw1f+9rX2GSTTXqtZaw/ryRnVtXCiTzfni5JkjQjXXTRRTz2sY9l5513\n7j1wTQUn0kuSpBlps80247LLLuu7jCljT5ckSVIHDF2SJPVops2t1tim4s/J0CVJUk/WWGMNrrvu\nOoPXDFdVXHfddRNeDHZ5nNMlSVJP5s+fz+LFi1myZEnfpWgca6yxBvPnz5/UOQxdkiT1ZLXVVmPj\njTfuuwx1xOFFSZKkDhi6JEmSOmDokiRJ6oChS5IkqQOGLkmSpA4YuiRJkjpg6JIkSerAhEJXkl2T\nXJLk0iT7jLF/wySnJDkvyWlJ5g/sOzjJhUkuTnJokkzlG5AkSRoG44auJKsChwHPATYD9kqy2ajD\nDgG+WFVbAgcCH22f+2RgR2BLYHNgO2CnKatekiRpSEykp2t74NKquqyqbgOOAV446pjNgFPax6cO\n7C9gDWB14AHAasA1ky1akiRp2EwkdD0SuHJge3HbNuhcYI/28e7A2knmVtVPaELY79qvk6rq4smV\nLEmSNHwmErrGmoM1+nbo7wF2SnI2zfDhVcAdSR4LPB6YTxPUnpHkqfd6geSNSRYlWeRNPyVJ0spo\nIqFrMbDBwPZ84OrBA6rq6qp6UVVtDezXti2l6fX6aVXdVFU3Ad8GnjT6Barq8KpaWFUL582bdz/f\niiRJ0sw1kdB1BrBJko2TrA7sCZwweECS9ZOMnGtf4Ij28W9pesDmJFmNphfM4UVJkjTrjBu6quoO\n4K3ASTSB6atVdWGSA5Ps1h72NOCSJL8EHgZ8pG0/Fvg1cD7NvK9zq+obU/sWJEmSZr5UjZ6e1a+F\nCxfWokWL+i5DkiRpXEnOrKqFEznWFeklSZI6YOiSJEnqgKFLkiSpA4YuSZKkDhi6JEmSOmDokiRJ\n6oChS5IkqQOGLkmSpA4YuiRJkjpg6JIkSeqAoUuSJKkDc/ouQJI09Tba51t9l7BClx/0vL5LkDpn\nT5ckSVIHDF2SJEkdMHRJkiR1wNAlSZLUAUOXJElSBwxdkiRJHXDJCEmSNKVcsmRs9nRJkiR1wNAl\nSZLUAUOXJElSBwxdkiRJHTB0SZIkdcDQJUmS1AFDlyRJUgcMXZIkSR0wdEmSJHVgQqErya5JLkly\naZJ9xti/YZJTkpyX5LQk8wf2PSrJyUkuTnJRko2mrnxJkqThMG7oSrIqcBjwHGAzYK8km4067BDg\ni1W1JXAg8NGBfV8EPlZVjwe2B/4wFYVLkiQNk4n0dG0PXFpVl1XVbcAxwAtHHbMZcEr7+NSR/W04\nm1NV3wWoqpuq6uYpqVySJGmITCR0PRK4cmB7cds26Fxgj/bx7sDaSeYCmwI3JPmfJGcn+Vjbc3YP\nSd6YZFGSRUuWLLnv70KSJGmGm0joyhhtNWr7PcBOSc4GdgKuAu4A5gB/0+7fDng08Op7nazq8Kpa\nWFUL582bN/HqJUmShsREQtdiYIOB7fnA1YMHVNXVVfWiqtoa2K9tW9o+9+x2aPIO4HhgmympXJIk\naYhMJHSdAWySZOMkqwN7AicMHpBk/SQj59oXOGLguQ9JMtJ99QzgosmXLUmSNFzmjHdAVd2R5K3A\nScCqwBFVdWGSA4FFVXUC8DTgo0kKOB14S/vcZUneA5ySJMCZwGen561oOmy0z7f6LmGFLj/oeX2X\nIEnShIwbugCq6kTgxFFtHxh4fCxw7HKe+11gy0nUKEmSNPRckV6SJKkDhi5JkqQOTGh4UZKk2cT5\nrJoO9nRJkiR1wNAlSZLUAUOXJElSBwxdkiRJHTB0SZIkdcDQJUmS1AFDlyRJUgcMXZIkSR0wdEmS\nJHXA0CVJktQBQ5ckSVIHDF2SJEkdMHRJkiR1wNAlSZLUAUOXJElSBwxdkiRJHTB0SZIkdcDQJUmS\n1AFDlyRJUgcMXZIkSR0wdEmSJHXA0CVJktQBQ5ckSVIHDF2SJEkdmFDoSrJrkkuSXJpknzH2b5jk\nlCTnJTktyfxR+9dJclWSf5uqwiVJkobJuKEryarAYcBzgM2AvZJsNuqwQ4AvVtWWwIHAR0ft/zDw\ng8mXK0mSNJwm0tO1PXBpVV1WVbcBxwAvHHXMZsAp7eNTB/cn2RZ4GHDy5MuVJEkaThMJXY8ErhzY\nXty2DToX2KN9vDuwdpK5SVYB/hV474peIMkbkyxKsmjJkiUTq1ySJGmITCR0ZYy2GrX9HmCnJGcD\nOwFXAXcAbwZOrKorWYGqOryqFlbVwnnz5k2gJEmSpOEyZwLHLAY2GNieD1w9eEBVXQ28CCDJWsAe\nVbU0yQ7A3yR5M7AWsHqSm6rqXpPxJUmSVmYTCV1nAJsk2ZimB2tP4GWDByRZH7i+qu4E9gWOAKiq\nvQeOeTWw0MAlSZJmo3GHF6vqDuCtwEnAxcBXq+rCJAcm2a097GnAJUl+STNp/iPTVK8kSdJQmkhP\nF1V1InDiqLYPDDw+Fjh2nHMcCRx5nyuUJElaCbgivSRJUgcMXZIkSR0wdEmSJHVgQnO6JKkPG+3z\nrb5LWK7LD3pe3yVIGjL2dEmSJHXA0CVJktQBQ5ckSVIHDF2SJEkdMHRJkiR1wNAlSZLUAUOXJElS\nBwxdkiRJHTB0SZIkdcDQJUmS1AFDlyRJUgcMXZIkSR0wdEmSJHXA0CVJktQBQ5ckSVIHDF2SJEkd\nMHRJkiR1wNAlSZLUAUOXJElSBwxdkiRJHTB0SZIkdcDQJUmS1AFDlyRJUgcmFLqS7JrkkiSXJtln\njP0bJjklyXlJTksyv23fKslPklzY7nvpVL8BSZKkYTBu6EqyKnAY8BxgM2CvJJuNOuwQ4ItVtSVw\nIPDRtv1m4JVV9QRgV+ATSR48VcVLkiQNi4n0dG0PXFpVl1XVbcAxwAtHHbMZcEr7+NSR/VX1y6r6\nVfv4auAPwLypKFySJGmYTCR0PRK4cmB7cds26Fxgj/bx7sDaSeYOHpBke2B14NejXyDJG5MsSrJo\nyZIlE61dkiRpaEwkdGWMthq1/R5gpyRnAzsBVwF33HWC5OHAl4DXVNWd9zpZ1eFVtbCqFs6bZ0eY\nJEla+cyZwDGLgQ0GtucDVw8e0A4dvgggyVrAHlW1tN1eB/gWsH9V/XQqipYkSRo2E+npOgPYJMnG\nSVYH9gROGDwgyfpJRs61L3BE2746cBzNJPuvTV3ZkiRJw2Xc0FVVdwBvBU4CLga+WlUXJjkwyW7t\nYU8DLknyS+BhwEfa9pcATwVeneSc9murqX4TkiRJM91EhhepqhOBE0e1fWDg8bHAsWM878vAlydZ\noyRJ0tBzRXpJkqQOGLokSZI6YOiSJEnqgKFLkiSpA4YuSZKkDhi6JEmSOmDokiRJ6oChS5IkqQOG\nLkmSpA4YuiRJkjpg6JIkSeqAoUuSJKkDhi5JkqQOGLokSZI6YOiSJEnqgKFLkiSpA4YuSZKkDhi6\nJEmSOmDokiRJ6oChS5IkqQOGLkmSpA4YuiRJkjpg6JIkSeqAoUuSJKkDhi5JkqQOGLokSZI6YOiS\nJEnqwIRCV5Jdk1yS5NIk+4yxf8MkpyQ5L8lpSeYP7HtVkl+1X6+ayuIlSZKGxbihK8mqwGHAc4DN\ngL2SbDbqsEOAL1bVlsCBwEfb564HHAA8EdgeOCDJQ6aufEmSpOEwkZ6u7YFLq+qyqroNOAZ44ahj\nNgNOaR+fOrD/2cB3q+r6qvoj8F1g18mXLUmSNFwmEroeCVw5sL24bRt0LrBH+3h3YO0kcyf4XEmS\npJXeREJXxmirUdvvAXZKcjawE3AVcMcEn0uSNyZZlGTRkiVLJlCSJEnScJlI6FoMbDCwPR+4evCA\nqrq6ql5UVVsD+7VtSyfy3PbYw6tqYVUtnDdv3n18C5IkSTPfRELXGcAmSTZOsjqwJ3DC4AFJ1k8y\ncq59gSPaxycBuyR5SDuBfpe2TZIkaVYZN3RV1R3AW2nC0sXAV6vqwiQHJtmtPexpwCVJfgk8DPhI\n+9zrgQ/TBLczgAPbNkmSpFllzkQOqqoTgRNHtX1g4PGxwLHLee4R3N3zJUmSNCu5Ir0kSVIHDF2S\nJEkdMHRJkiR1wNAlSZLUAUOXJElSBwxdkiRJHTB0SZIkdcDQJUmS1AFDlyRJUgcMXZIkSR0wdEmS\nJHXA0CVJktQBQ5ckSVIHDF2SJEkdMHRJkiR1wNAlSZLUAUOXJElSBwxdkiRJHTB0SZIkdcDQJUmS\n1AFDlyRJUgcMXZIkSR0wdEmSJHXA0CVJktQBQ5ckSVIHDF2SJEkdmNN3AdLKbKN9vtV3CSt0+UHP\n67sESZo1VvrQ5Q89SZI0E0xoeDHJrkkuSXJpkn3G2P+oJKcmOTvJeUme27avluQLSc5PcnGSfaf6\nDUiSJA2DcUNXklWBw4DnAJsBeyXZbNRh+wNfraqtgT2BT7ftfws8oKq2ALYF3pRko6kpXZIkaXhM\npKdre+DSqrqsqm4DjgFeOOqYAtZpH68LXD3Q/qAkc4AHArcBf5p01ZIkSUNmIqHrkcCVA9uL27ZB\nHwRenmQxcCLwtrb9WODPwO+A3wKHVNX1o18gyRuTLEqyaMmSJfftHUiSJA2BiYSujNFWo7b3Ao6s\nqvnAc4EvJVmFppdsGfAIYGPg3Ukefa+TVR1eVQurauG8efPu0xuQJEkaBhMJXYuBDQa253P38OGI\n1wFfBaiqnwBrAOsDLwO+U1W3V9UfgB8BCydbtCRJ0rCZSOg6A9gkycZJVqeZKH/CqGN+C+wMkOTx\nNKFrSdv+jDQeBDwJ+MVUFS9JkjQsxg1dVXUH8FbgJOBimqsUL0xyYJLd2sPeDbwhybnA0cCrq6po\nrnpcC7iAJrx9vqrOm4b3IUmSNKNNaHHUqjqRZoL8YNsHBh5fBOw4xvNuolk2QpIkaVbz3ouSJEkd\nMHRJkiR1wNAlSZLUAUOXJElSBwxdkiRJHTB0SZIkdcDQJUmS1AFDlyRJUgcMXZIkSR0wdEmSJHXA\n0CVJktQBQ5ckSVIHDF2SJEkdMHRJkiR1wNAlSZLUAUOXJElSBwxdkiRJHTB0SZIkdcDQJUmS1AFD\nlyRJUgcMXZIkSR0wdEmSJHXA0CVJktQBQ5ckSVIHDF2SJEkdMHRJkiR1wNAlSZLUgQmFriS7Jrkk\nyaVJ9hlj/6OSnJrk7CTnJXnuwL4tk/wkyYVJzk+yxlS+AUmSpGEwZ7wDkqwKHAY8C1gMnJHkhKq6\naOCw/YGvVtVnkmwGnAhslGQO8GXgFVV1bpK5wO1T/i4kSZJmuIn0dG0PXFpVl1XVbcAxwAtHHVPA\nOu3jdYGr28e7AOdV1bkAVXVdVS2bfNmSJEnDZSKh65HAlQPbi9u2QR8EXp5kMU0v19va9k2BSnJS\nkrOS/MNYL5DkjUkWJVm0ZMmS+/QGJEmShsFEQlfGaKtR23sBR1bVfOC5wJeSrEIzfPkUYO/2++5J\ndr7XyaoOr6qFVbVw3rx59+kNSJIkDYOJhK7FwAYD2/O5e/hwxOuArwJU1U+ANYD12+f+oKquraqb\naXrBtpls0ZIkScNmIqHrDGCTJBsnWR3YEzhh1DG/BXYGSPJ4mtC1BDgJ2DLJmu2k+p2Ai5AkSZpl\nxr16saruSPJWmgC1KnBEVV2Y5EBgUVWdALwb+GySd9EMPb66qgr4Y5KP0wS3Ak6sqm9N15uRJEma\nqcYNXQBVdSLN0OBg2wcGHl8E7Lic536ZZtkISZKkWcsV6SVJkjpg6JIkSeqAoUuSJKkDhi5JkqQO\nGLokSZI6YOiSJEnqgKFLkiSpA4YuSZKkDhi6JEmSOmDokiRJ6oChS5IkqQOGLkmSpA6kqvqu4R6S\nLAGu6LuOFVgfuLbvIoaYn9/k+Pndf352k+PnNzl+fpMzkz+/Datq3kQOnHGha6ZLsqiqFvZdx7Dy\n85scP7/7z89ucvz8JsfPb3JWls/P4UVJkqQOGLokSZI6YOi67w7vu4Ah5+c3OX5+95+f3eT4+U2O\nn9/krBSfn3O6JEmSOmBPlyRJUgcMXZIkSR0wdEmSpBkjyapJ3tV3HdPB0KVpleSBSfZN8u/t9mOT\nPKfvuoZNkjWTLEwyoQX4pMniToIVAAAWxElEQVRK8rcTadPYkmw8kTbdW1UtA17Ydx3TwYn0mlZJ\njgbOB15WVZsnWRP4UVVt3XNpM1qS3YBDgeuB/YHDgGuAjYB/rKov9FfdzJfkAyvYXVX14c6KGVJJ\nzqqqbcZr09iW8/mdWVXb9lXTMEnyEWBd4L+AP4+0V9VZvRU1Beb0XcBMluS85e2i+Y97yy7rGVKb\nVNVeI78hV9XNSdJ3UUPgw8AuNP/pnApsWVWXJXkocApg6FqxP4/RtibwemAuzeerMbQ90c8FHpnk\n0IFd6wB39FPV8EjyOOAJwLpJXjSwax1gjX6qGkpPbr8fONBWwDN6qGXKGLpW7E6aP+SvAN8Abum3\nnKF0W5I1aD7Hke712/otaSjcWVW/BEjym6q6DKCq/pDEH3zjqKp/HXmcZG3gHcBrgWOAf13e8wTA\n1cAiYDfgzIH2G4GVcp7NFPtr4PnAg4EXDLTfCLyhl4qGUFU9ve8apoOhawWqaqv2t5a9aILXRe33\nk6vKH3wTcyDwHWB+ki8AOwGv67ekobBKkofQzLu8s3080kPoXMwJSLIe8PfA3jQ9g9tU1R/7rWrm\nq6pzgXOTfKWqbgdo//5t4Oc3vqr6OvD1JDtU1U/6rmdYJXkY8M/AI6rqOUk2A3aoqs/1XNqkOKfr\nPkjyUpq5Nf9SVR/ru55h0U7+fjJNaPhxVf2h55JmvCSX0/S0jjUUW1X16G4rGi5JPga8iGYV68Oq\n6qaeSxo6SU6j6e2aA5wDLAF+UFV/32ddwyLJwcA/0YyQfAdYALyzqr7ca2FDIsm3gc8D+1XVgiRz\ngLOraoueS5sUQ9c4kjwS2BPYHfgj8FXgOP8Tn7gkewKPqaqPJNkAeGhVnTne86T7K8mdwF9o5iAN\n/ic3Mh9znV4KGyJJzq6qrZO8nqaX64Ak5zmXdWKSnNOOluwO/B+aodlTq2pBz6UNhSRnVNV2I38P\n27ZzqmqrvmubDIcXVyDJD4C1aYLWq2muJANYPcl6VXX98p6rRpJ/A1YDngp8hGaC878D2/VZ10yX\n5FEr2l9Vv+2qlmFUVQ7BTt6cJA8HXgLs13cxQ2i19vtzgaOr6nqvIbpP/pxkLnfPB34SsLTfkibP\n0LViG9L8gb8JeONAe9p2h3jG9+Sq2ibJ2QDtfzyr913UEPgWzd+xwf+lC5gHPBRYtY+iNKscCJxE\ns8TLGUkeDfyq55qGyTeS/IJmePHN7TSLW3uuaZj8PXAC8JgkP6L5v+/F/ZY0eQ4valol+RmwA7Co\nDV9zge+5Ttd9k2Qj4B+BZwKHVtWnei1ohktyI2OH1jnA6lXlL5yadu0FCH+qqmXtGoXrVNXv+65r\nWLTzuP6a5t/xJSMXdgwzu+BXIMlFSd7X/oan++cw4L+BeUk+BPwQ+Jd+SxoeSTZJciTwbZrL9zcz\ncI2vqtauqnXa72sDj6AZ3v498Ml+qxsOSTZNckqSC9rtLZPs33ddwyLJasArgP9KcizNVdvX9VvV\nzJfkGe33F9FcyPHXwKbAC0atezaU7OlagSQLaCbRvwS4Fjga+GpVXd1rYUMmyRNoemhC08t1Qc8l\nzXhJNqeZR/ME4GCaOSHL+q1q+CR5MPBO4JU0y738v6ryB98EtHNa3wv8x8BE5guqavN+KxsOSf6T\nZl7XyELGrwCWVdXr+6tq5kvywar6YJLPj7G7quq1nRc1hexiX4GR9WqAfdtJfC8FfprkUpofgp/t\ntcAZLsmqwFnt1ToX9l3PkDkXuJJmbtf2wPaDk3Cr6u091TUUkqwPvJvm3+wRwNZVNfSTcDu2ZlX9\nfNTkb9cnnLjtRl2p+P0k5/ZWzfAYWQvuc1X1w14rmQaGrgmqqp/SBK6vA/8P+DfA0LUC7TyGi5I8\nsqqu6rueITPUv83NAFfQrCv1eeBm4HWjQuvHe6prmFyb5DHcffXYi4Hf9VvSUFmW5DFV9WuAdpqK\nvdXjew3NFIBDgZXuPp+GrglIsh3NqvR7AJfTLLj4tT5rGiLrAxcn+Qn3vGnp0I/NTydvaD1pH+Pu\n9bnWHrXPORUT8xaa/+sel+Qq4Dc0q/trYt4LnJrkMpqpFRvSBAqt2MXt4tDzRt3/eKW457FzulYg\nyT/TzOe6geaebcdU1eJ+qxouSXYeq72qTum6Fs0eSeYv799qkhdU1Te6rmmYJFkFeHFVfTXJg4BV\nqurGvusaNkkewN1X3/2iqv7Sc0lDIclf0SxXstvofVV1RfcVTR1D1wokORE4qKpOb7dfSdPbdQXw\nQRdHXb4kJ1fVLn3XodkpySXAs6vq8lHtrwH2r6rH9FLYEElyelU9te86hlV79eL/pVkYGuA0mosS\nhn7ZA91/LhmxYn8FjFwu/VTgIOCLNKviHt5jXcNgXt8FaFZ7F/DdJJuMNCTZl2bBxZ16q2q4fDfJ\ne5JskGS9ka++ixoinwG2BT7dfm3btmkFkny1/X5+kvMGvs4fNdw4lOzpWoHB+zwlOQxYUlUfHL1P\n99bOY3jP8vZX1f90WM7QSfIpVjD3yKsXx9cObf8HzX3vXk9z66nnV9UfV/hEAZDkN2M0e7P1CUpy\n7uj7LI7VpntK8vCq+l2SDcfaP+zDi06kX7E5SeZU1R3AztzzVkB+diu2LvB87rki+IgCDF0rtqjv\nAoZdVZ2S5NU0wzo/BnauKm/DMkFVtXHfNQw5r168H6pq5ArZa4FbqurOJJsCj6NZJHqo2dO1Akn2\no7lZ6bXAo4BtqqqSPBb4QlXt2GuBM1iSs6pqpbvcV8Nh1G2AHgDcTvMDb+QKqHV6LG8oOCdpctqe\n1s8D97h6sapO7bWwIZHkTOBvgIcAP6X5RfTmqhrqK2gNXeNoF0V9OHByVf25bdsUWKuqzuq1uBks\nydneX3Hy2pvk/iOwGbDGSHtVPaO3ojQruKL65Hn14v038ot7krcBD6yqg1eGnysOkY2jXRR1dNsv\n+6hlyLyi7wJWEkcB/wU8D/g74FU0i35K080V1e+ndj7Sn6vq2vZG108BNgaO77eyoZIkO9CsDfe6\ntm3oM4tXL2paeH/FKTO3qj4H3F5VP2jvO/akvovSrLCsXZEecE7SRCV5P/B9mjuY/BPwCZpFot+R\n5BO9Fjdc3gnsCxxXVRe2f/+Gfmh26FOjtJIbmT/zuyTPA64G5vdYj2YPV1S/f/YCHg+sCfwW+Kuq\nujnJHOCcXisbIlX1A+AHcNdivdeuDFdtG7qkme2fkqxLc/PmTwHr0KxBJU2r9urPTXBO0n11a1Xd\nBtyW5NdVdTNAVd2R5LaeaxsaSb5CM6ViGXAmsG6Sj1fVx/qtbHIMXZpWSXYEPkjzW/Ic7r56zLV+\nJqCqvtk+XAo8vc9aNDsk+eeqel+7+dSq+m6vBQ2fByd5Ec3/deu0j2m31+2vrKGzWVX9KcnewIk0\nFxSdSXNf1aHl1YuaVkl+QdMzcyYD80Gq6rreihoCSf6hvVpnzEVSV4Zuds1Mg8u9uPTLfZfk8yva\nX1UO0U5AkguBrYCvAP9WVT9YGRaXtadL021pVQ39gnY9uLj97iKp0hAxVE2Z/wAuB84FTm+vCP1T\nrxVNAXu6NK2SHASsSrMC/V3zQVzjTJqZkiwGPk4zHPau9vFdqurjYz1P99Ze/PIE7rnG3oH9VTTc\nBu4QM7Ts6dJ0e2L7feFAWwEu7jkBSRYC+3H3nDgAqmrL3orSyu6zwNpjPNZ9kOTfaa5gfDrwn8CL\ngZ/3WtSQGSu0AkMdWu3pkmawJJfQXLp/PnDnSPuw3/RVWtklOa+qthz4vhbwP1W1S9+1DYPlhdaq\net0KnzjDuTiqplWSdZN8PMmi9utf2yUQNDFLquqEqvpNVV0x8tV3UZLGdUv7/eYkj6BZc8+biE/c\nk6vqlcAfq+pDwA7ABj3XNGkOL2q6HQFcALyk3X4FzU1gX7TcZ2jQAe098E7hnnPi/qe/kiRNwDeT\nPJhmiYOzaKZV/Ge/JQ2V0aH1OlaC0OrwoqZVknOqaqvx2jS2JF8GHgdcyN3Di9XeDkjSEGhvfL1G\nVS3tu5Zh0d5O6VPAzsBhtKG1qt7fa2GTZOjStEryE+C9VfXDdntH4JCq2qHfyoZDkvOraou+69Ds\n0/bSvBLYiHtexOEacSswsBjqmOylvu9WptDq8KKm2/8FvtDO4wpwPfDqXisaLj9NsllVXdR3IZp1\nTgR+yqiLODSuF7TfHwo8mebm19BMCD+NZvkcLceKQmuSoQ+t9nSpE0nWAaiqoV/crktJLgYeA/yG\nZk7XyG2UXDJC08rV6CcnyTeBN1TV79rthwOHVZXzWVdgnBX9h35qhaFL0yLJy6vqy0n+fqz9LrA4\nMe0qzPfiFYyabkneBdwEfJN7XsRxfW9FDZEkF1TV5gPbqwDnV9UTeixLPXN4UdPlQe13F1achKq6\nIskC4G/apv+tqnP7rEmzxm00V97tx933/yzAm9VPzGlJTgKOpvnc9qS5Clkr0P6ivrSqPjeq/W3A\nqlX1iX4qmxr2dEkzWJJ3AG/g7nkguwOHV9Wn+qtKs0GSXwNPrKpr+65lWCXZHXhqu/lH4GFV9ZYe\nS5rxklwAbFNVt41qfwBwxrBPrXBxVE2rJAcnWSfJaklOSXJtkpf3XdcQeR3ND74PVNUHgCfRhDBp\nul0I3Nx3EUPuNzSLou5OM5H+4hUfLpp5W7eN0Tgyp3WoObyo6bZLVf1D+xvfYuBvgVOBL/db1tAI\nsGxgexkrwX88GgrLgHOSnMo953S5ZMQKJNmUZihxL5oFPf+LZlTp6b0WNkSSPKyqrhnd1lc9U8nQ\npem2Wvv9ucDRVXV9Yma4Dz4P/CzJce32/wE+t4LjpalyfPul++YXwP8CL6iqS+GuixI0MR8DvpXk\n3TQr+QNsCxwMHNJbVVPEOV2aVkkOogkKtwDbAw8GvllVT+y1sCGSZFtgR5oertOr6uyeS9IskWR1\nYNN285Kqur3PeoZB26u/J80aXd8BjqFZSX3ob2HTlSTPAfYBNqe5COFC4KCq+navhU0BQ5emXZKH\nAH+qqmVJ1gTWqarf913XsEiyKvAw7rkq+G/7q0izQZKnAV8ALqcJ/BsAr6qq03ssa2gkeRDNL5x7\nAc+g+SyPq6qTey1MvTJ0aVokeUZVfX95qwsP+6rCXWkvkz4AuIa753O5OKqmXZIzgZdV1SXt9qY0\nUwS27bey4ZNkPZr5rC+tqmf0XY/6Y+jStEjyoao6YDmrCw/9qsJdSXIpzdWL1/Vdi2aXJOeNDvdj\ntUmaOEOXNIO1V449q6ru6LsWzS5JjqCZT/OltmlvYE5Vvaa/qqThZujStEryz8DBVXVDu/0Q4N1V\ntX+/lQ2HJJ8D/hr4Fve8bN/bKGlatYtRvgV4Cu1FHMCn2/WSpGm1nFvILQXOrKpzuq5nqhi6NK2S\nnF1VW49q80a6E5TkgLHaq+pDXdciSV1J8hVgIfCNtul5wBnA44CvVdXBfdU2GYYuTask5wHbjfx2\nnOSBwCJv+irNTEnO5+57Ld6Lc7rUhfa+lXtU1U3t9lrAsTSr+59ZVZv1Wd/95eKomm5fBk5pJ9QX\n8FqaS6e1Akk+UVXvTPINxvgBWFW79VCWZofnt99H7hE4OKfL2wKpK4+iuen6iNuBDavqliRDO8Rt\nT5emXZJdgWfSzAs5uapO6rmkGS/JtlV1ZpKdxtpfVT/ouibNLkl+VFU7jtcmTYck76fp1fp62/QC\n4ATgX4HDq2rvvmqbDHu61IWLgTuq6ntJ1kyydlXd2HdRM1lVndk+3KqqPjm4L8k7AEOXptuDkjyl\nqn4IkOTJwIN6rkmzRFV9OMm3uftuHH9XVYva3UMZuMCeLk2zJG8A3gisV1WPSbIJ8O9VtXPPpQ2F\nsS46GOviBGmqtbefOgJYt226AXhtVZ21/GdJU2dlvBuHoUvTKsk5NPdc/NlIUEhyflVt0W9lM1uS\nvYCX0Vyu/78Du9YGllXVM3spTLNOknVoflYs7bsWzR4r6904HF7UdPtLVd2WBIAkc1jBlVG6y4+B\n3wHr08xhGHEjcF4vFWlWadfp2gPYCJgz8m+4qg7ssSzNHu8A/npluxuHoUvT7QdJ3gc8MMmzgDdz\n97orWo6qugK4Atih71o0a32ddjFKBhbmlTpyJc3fv5WKw4uaVklWAV4H7ELTPXwS8J/lX7wVSnIj\nY/cIjnSxr9NxSZplklxQVZv3XYdmp5X1bhz2dGlaVdWdSY4Hjq+qJX3XMyyqau2+a9Cs9+MkW1TV\n+X0Xolnpt+3X6u3XSsGeLk2LNBNADgDeStM7E5rJkJ9yTsjEJXnUWO3DfgWPZr4kFwGPBX5D09Ow\nUkxklvpk6NK0SPIu4LnAG6vqN23bo4HPAN+pqv/XZ33Dor0ly4g1gI2BS7yNkqZbkg3Ham/nG0rT\nYmW/G4ehS9MiydnAs6rq2lHt82hWpXedqfshyTbAm6rqTX3XotkhyUNpAj9gL6um18p+Nw7ndGm6\nrDY6cAFU1ZIkq/VR0Mqgqs5Ksl3fdWjll2Q3muVKHgH8AdiQ5u4S9rJq2ozcjWPYw9XyGLo0XW67\nn/s0IMnfD2yuAmwDeEGCuvBh4EnA96pq6yRPB/bquSbNEu3UitFDcUuBRcA/Dev6XYYuTZcFSf40\nRnsYGKrQuAavYryD5vLp/+6pFs0ut1fVdUlWSbJKVZ2a5F/6LkqzxrdpLr76Sru9J83Pj6XAkTQ3\nwB46hi5Ni6pate8aVgZV9aG+a9CsdUOStYDTgaOS/IEm+Etd2LGqdhzYPj/Jj6pqxyQv762qSTJ0\nSTNQkhNWtH/Yr+DRUHghcAvwLmBvmhtfu9yLurJWkidW1c8AkmwPrNXuG9rw79WL0gyUZAnNbTCO\nBn5G061+l5V1kqlmriSrAntW1VF916KVX3vB0BE0QSvAn4DXAxcCz6uqr/ZY3v1m6JJmoPYH3LNo\nJi5vSTOX6+iqurDXwrTSS7IO8BbgkcAJwHfb7fcC51TVC3ssT7NMknVpssoNfdcyFQxd0gyX5AE0\n4etjwIFV9ameS9JKLMnXgT8CPwF2Bh5CcxuWd1TVOX3Wptlj1JXbI5YCZw7z30NDlzRDtWHreTSB\nayOaXocjquqqPuvSyi3J+VW1Rft4VeBa4FFVdWO/lWk2SfIVYCHwjbbpecAZwOOAr1XVwX3VNhlO\npJdmoCRfADanuWz6Q1V1Qc8lafa4feRBVS1L8hsDl3owF9imqm4CSHIAcCzwVOBMYChDlz1d0gyU\n5E7gz+3m4D/SkZsOr9N9VZoNkizj7r97AR4I3Ix/99ShJBcDC6rqtnb7ATRzCh+f5OxhvZWcPV3S\nDFRVq/Rdg2Yn19jTDPEV4KftHENoFkM9OsmDgIv6K2ty7OmSJEkzTpKFwI40vaw/rKpFPZc0aYYu\nSZI0IyV5KAO3jquq3/ZYzqQ5hCFJkmaUJLsl+RXwG+AH7fdv91vV5Bm6JEnSTPNh4EnAL6tqY+CZ\nwI/6LWnyDF2SJGmmub2qrgNWSbJKVZ0KbNV3UZPl1YuSJGmmuSHJWsDpwFFJ/sAQ3+h6hBPpJUnS\njNIuDXELzYjc3sC6wFFt79fQMnRJkqQZK8n6wHW1EgQW53RJkqQZIcmTkpyW5H+SbJ3kAuAC4Jok\nu/Zd32TZ0yVJkmaEJIuA99EMJx4OPKeqfprkccDRw3r7nxH2dEmSpJliTlWdXFVfA35fVT8FqKpf\n9FzXlDB0SZKkmeLOgce3jNo39ENzDi9KkqQZIcky4M8091t8IHDzyC5gjapara/apoKhS5IkqQMO\nL0qSJHXA0CVJktQBQ5ckSVIHDF2SJEkdMHRJkiR1wNAlSZLUgf8P4fSq8StIYSoAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ee68b57278>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# File:        text_classification.py\n",
    "#\n",
    "# Author:      Rohan Patel\n",
    "#\n",
    "# Date:        05/10/2018\n",
    "#\n",
    "# Description: This script contains the main sms classification code. We first load the processed \n",
    "#              messages, convert each message into a bag of words and then into a tfidf vector \n",
    "#              representation. Then we split the tfidf feature vector into training and test sets, \n",
    "#              build our classifier on the training set, and test it on the test set.  \n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "\n",
    "def Tfidf_Vectorization(messages):\n",
    "    '''\n",
    "    1. Convert word tokens from processed msgs dataframe into a bag of words\n",
    "    2. Convert bag of words representation into tfidf vectorized representation for each message\n",
    "    '''\n",
    "    bow_transformer = CountVectorizer(analyzer=text_process).fit(messages['message'])\n",
    "    bow = bow_transformer.transform(messages['message']) # bag of words\n",
    "\n",
    "    tfidf_transformer = TfidfTransformer().fit(bow)\n",
    "    tfidf_vect = tfidf_transformer.transform(bow) # tfidf vector representation\n",
    "    \n",
    "    pickle.dump(tfidf_vect, open(\"output/tfidf_vector.pickle\", \"wb\")) # stores tfidf vector in a pickle file so it could be used later in future scripts\n",
    "    \n",
    "    return tfidf_vect\n",
    "\n",
    "def TrainTestSplit(feature_vect, messages):\n",
    "    '''\n",
    "    Split dataset into training and test sets. We use a 70/30 split.\n",
    "    '''\n",
    "    X_train, X_test, y_train, y_test = train_test_split(feature_vect, messages['label'], test_size = 0.3, random_state = 101)\n",
    "    \n",
    "    return X_train, X_test, y_train, y_test\n",
    "\n",
    "def train_classifier(classifier, X_train, y_train):\n",
    "    classifier.fit(X_train, y_train)\n",
    "    \n",
    "def predict_labels(classifier, X_test):\n",
    "    return (classifier.predict(X_test))\n",
    "\n",
    "def main():\n",
    "    \n",
    "    messages = pd.read_csv('output/processed_msgs.csv')\n",
    "\n",
    "    tfidf_vect = Tfidf_Vectorization(messages)\n",
    "    \n",
    "    # append our message length feature to the tfidf vector to produce the final feature vector we fit into our classifiers\n",
    "    len_feature = messages['length'].as_matrix()\n",
    "    feat_vect = np.hstack((tfidf_vect.todense(), len_feature[:, None]))\n",
    "    \n",
    "    X_train, X_test, y_train, y_test = TrainTestSplit(feat_vect, messages)\n",
    "    \n",
    "    svm = SVC()\n",
    "    dtree = DecisionTreeClassifier()\n",
    "    mnb = MultinomialNB()\n",
    "    knn = KNeighborsClassifier()\n",
    "    rfc = RandomForestClassifier()\n",
    "    ada_boost = AdaBoostClassifier()\n",
    "    bagging_clf = BaggingClassifier()\n",
    "    \n",
    "    classifiers = {'SVM': svm, 'Decision Tree': dtree, 'Multinomial NB': mnb, 'KNN': knn, 'Random Forest': rfc, \n",
    "                   'AdaBoost': ada_boost, 'Bagging Classifier': bagging_clf}\n",
    "    \n",
    "    X_train2, X_test2 = train_test_split(messages['message'], test_size = 0.3, random_state = 101)\n",
    "    pred_scores = []\n",
    "    pred = dict()\n",
    "    file = open('output/misclassified_msgs.txt', 'a', encoding='utf-8') # misclassified messages will be written in this\n",
    "    for k, v in classifiers.items():\n",
    "        train_classifier(v, X_train, y_train)\n",
    "        pred[k] = predict_labels(v, X_test)\n",
    "        pred_scores.append((k, [accuracy_score(y_test, pred[k])]))\n",
    "        print('\\n############### ' + k + ' ###############\\n')\n",
    "        print(classification_report(y_test, pred[k]))\n",
    "\n",
    "        # write misclassified messages into a new text file\n",
    "        file.write('\\n#################### ' + k + ' ####################\\n')\n",
    "        file.write('\\nMisclassified as Spam:\\n\\n')\n",
    "        for msg in X_test2[y_test < pred[k]]:\n",
    "            file.write(msg)\n",
    "            file.write('\\n')\n",
    "        file.write('\\nMisclassified as Ham:\\n\\n')\n",
    "        for msg in X_test2[y_test > pred[k]]:\n",
    "            file.write(msg)\n",
    "            file.write('\\n')\n",
    "    file.close()\n",
    "\n",
    "    print('\\n############### Accuracy Scores ###############')\n",
    "    accuracy = pd.DataFrame.from_items(pred_scores, orient = 'index', columns = ['Accuracy Rate'])\n",
    "    print('\\n')\n",
    "    print(accuracy)\n",
    "    print('\\n')\n",
    "\n",
    "    #plot accuracy scores in a bar plot\n",
    "    accuracy.plot(kind =  'bar', ylim=(0.85,1.0), figsize=(10,5))\n",
    "    plt.title('Distribution by Classifier')\n",
    "    plt.legend()\n",
    "    \n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Parameter tuning using grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "############### Multinomial NB ###############\n",
      "\n",
      "Fitting 3 folds for each of 20 candidates, totalling 60 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  60 out of  60 | elapsed:   33.5s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best parameter: {'alpha': 0.10000000000000001}\n",
      "\n",
      "Accuracy Score: 0.980263157895\n",
      "\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.99      0.99      0.99      1475\n",
      "       spam       0.93      0.90      0.92       197\n",
      "\n",
      "avg / total       0.98      0.98      0.98      1672\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# File:        parameter_tuning.py\n",
    "#\n",
    "# Author:      Rohan Patel\n",
    "#\n",
    "# Date:        05/12/2018\n",
    "#\n",
    "# Description: This script uses scikit-learn's GridSearchCV to perform an exhaustive grid search.\n",
    "#              Exhaustive grid search is a way to select the best model out of a family of models\n",
    "#              by tuning the model parameters. \n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import classification_report, accuracy_score\n",
    "\n",
    "def SVM_Tuning(X_train, X_test, y_train, y_test):\n",
    "\n",
    "    print('\\n############### SVM ###############\\n')\n",
    "    param_grid = {'kernel':['sigmoid','rbf','linear'],'gamma':[1,0.1,0.01]}\n",
    "\n",
    "    model = GridSearchCV(SVC(), param_grid, verbose = 3)\n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    print('\\nBest parameter:', model.best_params_)\n",
    "\n",
    "    pred = model.predict(X_test)\n",
    "\n",
    "    print('\\nAccuracy Score:', accuracy_score(y_test, pred))\n",
    "    print('\\n')\n",
    "    print(classification_report(y_test, pred))\n",
    "\n",
    "def MNB_Tuning(X_train, X_test, y_train, y_test):\n",
    "\n",
    "    print('\\n############### Multinomial NB ###############\\n')\n",
    "    param_grid = {'alpha': np.arange(0.05, 1.05, 0.05)}\n",
    "\n",
    "    model = GridSearchCV(MultinomialNB(), param_grid, verbose = 1)\n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    print('\\nBest parameter:', model.best_params_)\n",
    "\n",
    "    pred = model.predict(X_test)\n",
    "\n",
    "    print('\\nAccuracy Score:', accuracy_score(y_test, pred))\n",
    "    print('\\n')\n",
    "    print(classification_report(y_test, pred))\n",
    "\n",
    "def DTree_Tuning(X_train, X_test, y_train, y_test):\n",
    "\n",
    "    print('\\n############### Decision Tree ###############\\n')\n",
    "    param_grid = {'min_samples_split': np.arange(2, 21, 1)}\n",
    "    \n",
    "    model = GridSearchCV(DecisionTreeClassifier(), param_grid, verbose = 1)\n",
    "    model.fit(X_train, y_train)\n",
    "\n",
    "    print('\\nBest parameter:', model.best_params_)\n",
    "\n",
    "    pred = model.predict(X_test)\n",
    "\n",
    "    print('\\nAccuracy Score:', accuracy_score(y_test, pred))\n",
    "    print('\\n')\n",
    "    print(classification_report(y_test, pred))\n",
    "\n",
    "def main():\n",
    "\n",
    "    tfidf_vect = pickle.load(open(\"output/tfidf_vector.pickle\", \"rb\"))\n",
    "    messages = pd.read_csv('output/processed_msgs.csv')\n",
    "    \n",
    "    # append our message length feature to the tfidf vector to produce the final feature vector we fit into our classifiers\n",
    "    len_feature = messages['length'].as_matrix()\n",
    "    feat_vect = np.hstack((tfidf_vect.todense(), len_feature[:, None]))\n",
    "\n",
    "    X_train, X_test, y_train, y_test = train_test_split(feat_vect, messages['label'], test_size = 0.3, random_state = 101)\n",
    "\n",
    "    MNB_Tuning(X_train, X_test, y_train, y_test)\n",
    "    #SVM_Tuning(X_train, X_test, y_train, y_test)\n",
    "    #DTree_Tuning(X_train, X_test, y_train, y_test)\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# K-Fold Cross Validation & Learning Curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This may take some time, please wait............................................................................."
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwwAAAHwCAYAAAALsOgwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3Xl4lPW5//HPnZlkJmyGzaIJCrVo\nCasQwF2xShEFV6yoVWyrtR6tbbWKtbX+aD3ltJ7WnmqreIr2tFUE3FBpcVeoCwTFBRBBtBJADMgm\nEMIk9++PLIaQhGQWJnnm/bquXMzzzHeeuWfkusyH7/19vubuAgAAAICGZKW7AAAAAACtF4EBAAAA\nQKMIDAAAAAAaRWAAAAAA0CgCAwAAAIBGERgAAAAANIrAAABoFjP7h5ldmu46AAD7F4EBAFo5M/vI\nzE5Jdx3ufpq7/yUV1zazTmZ2h5l9bGafm9nK6uNuqXg/AEDzERgAADKzcBrfO0fSc5L6SRotqZOk\nYyRtlDQ8juul7bMAQBARGACgDTOzM8xssZltNrNXzGxgnecmmdkHZrbNzJaa2dl1nptoZv8ys9+Z\n2WeSbq0+N9/MbjezTWb2oZmdVuc1L5rZd+q8vqmxvc3s5er3ftbM7jKzvzXyMS6RdIiks919qbtX\nuvun7v4Ld59TfT03s6/Uuf79ZvbL6scnmVmJmd1oZp9Ius/MlpnZGXXGh81sg5kNqT4+qvr72mxm\nb5nZSYn8dwCAICMwAEAbVf3L7zRJ35XUVdI9kmabWaR6yAeSjpd0gKT/J+lvZnZQnUuMkLRK0oGS\nbqtzbrmkbpJ+LenPZmaNlNDU2AckLaiu61ZJ32zio5wi6Z/u/vm+P3WjekjqIulQSVdIelDShDrP\nf13SBnd/w8zyJT0l6ZfVr7le0sNm1j2B9weAwCIwAEDbdbmke9z9dXevqF5fsEvSUZLk7jPdfW31\nv9g/JGmF9mzxWevuf3D3mLvvrD73b3e/190rJP1F0kGSvtTI+zc41swOkTRM0i3uXu7u8yXNbuJz\ndJW0Lq5v4AuVkn7u7ruqP8sDksaZWbvq5y+sPidJF0ua4+5zqr+bZyQVSxqTYA0AEEgEBgBouw6V\ndF11W81mM9ssqaekgyXJzC6p0660WVJ/Vc0G1FjdwDU/qXng7juqH3Zo5P0bG3uwpM/qnGvsvWps\nVFXYSESpu5fVqWelpGWSxlaHhnH6IjAcKml8ve/tuCTUAACBxMIwAGi7Vku6zd1vq/+EmR0q6V5J\nX5P0qrtXmNliSXXbizxFda2T1MXM2tUJDT2bGP+spF+aWXt3397ImB2S2tU57iGppM5xQ5+lpi0p\nS9LS6hAhVX1vf3X3y/fxOQAAYoYBANqKbDOL1vkJqyoQXGlmI6xKezM73cw6Smqvql+iSyXJzC5T\n1QxDyrn7v1XV4nOrmeWY2dGSxjbxkr+q6pf4h83sq2aWZWZdzewnZlbTJrRY0oVmFjKz0ZJObEYp\n0yWNkvQ9fTG7IEl/U9XMw9errxetXjhd0MKPCgAZgcAAAG3DHEk76/zc6u7FqlrHcKekTZJWSpoo\nSe6+VNJ/S3pV0npJAyT9az/We5Gko1XVbvRLSQ+pan3FXtx9l6oWPr8n6RlJW1W1YLqbpNerh12r\nqtCxufraj+2rAHdfp6rPf0z1+9ecXy3pTEk/UVWgWi3px+L/iQDQIHNP1Yw0AABVzOwhSe+5+8/T\nXQsAoGX41xQAQNKZ2TAzO6y6vWi0qv5Ff5+zAgCA1odFzwCAVOgh6RFV3TK1RNL33P3N9JYEAIgH\nLUkAAAAAGkVLEgAAAIBGERgAAAAANKrNrWHo1q2b9+rVK91lAAAAAG3WokWLNrh79+aMbXOBoVev\nXiouLk53GQAAAECbZWb/bu5YWpIAAAAANIrAAAAAAKBRBAYAAAAAjWpzaxgAAAAyye7du1VSUqKy\nsrJ0l4I2KBqNqqCgQNnZ2XFfg8AAAADQipWUlKhjx47q1auXzCzd5aANcXdt3LhRJSUl6t27d9zX\noSUJAACgFSsrK1PXrl0JC2gxM1PXrl0Tnp0iMAAAALRyhAXEKxl/dwgMAAAAaNTGjRs1ePBgDR48\nWD169FB+fn7tcXl5ebOucdlll2n58uVNjrnrrrv097//PRklI8lYwwAAAIBGde3aVYsXL5Yk3Xrr\nrerQoYOuv/76Pca4u9xdWVkN/1v0fffdt8/3+Y//+I/Ei22BWCymcDjc6HFzX5cJmGEAAABAi61c\nuVL9+/fXlVdeqSFDhmjdunW64oorVFRUpH79+mny5Mm1Y4877jgtXrxYsVhMeXl5mjRpkgYNGqSj\njz5an376qSTppz/9qe64447a8ZMmTdLw4cN1xBFH6JVXXpEkbd++Xeeee64GDRqkCRMmqKioqDbM\n1LVw4UKdeOKJGjp0qE477TStX7++9ro333yzTjjhBN155526+OKLdd1112nkyJH6yU9+og0bNmjc\nuHEaOHCgjjnmGL377ru1tX33u9/Vqaeeqssuuyyl32trlFnxCAAAoA37f08s0dK1W5N6zcKDO+nn\nY/vF9dqlS5fqvvvu09133y1JmjJlirp06aJYLKaRI0fqvPPOU2Fh4R6v2bJli0488URNmTJFP/rR\njzRt2jRNmjRpr2u7uxYsWKDZs2dr8uTJ+uc//6k//OEP6tGjhx5++GG99dZbGjJkyF6v27Vrl669\n9lrNnj1b3bp109///nf97Gc/09SpUyVJW7du1csvvyxJuvjii/XBBx/oueeeU1ZWlr73ve9pxIgR\nmj17tp5++mlNnDhRxcXFkqQ333xTL7/8sqLRaFzfVVtGYAAAAEBcDjvsMA0bNqz2+MEHH9Sf//xn\nxWIxrV27VkuXLt0rMOTm5uq0006TJA0dOlTz5s1r8NrnnHNO7ZiPPvpIkjR//nzdeOONkqRBgwap\nX7+9g86yZcu0ZMkSnXLKKZKkiooKFRQU1D5/wQUX7DF+/Pjxta1U8+fP11NPPSVJGjVqlCZOnKjt\n27dLks4888yMDAsSgQEAAKDNiHcmIFXat29f+3jFihX6/e9/rwULFigvL08XX3xxg7fzzMnJqX0c\nCoUUi8UavHYkEtlrjLvvsyZ318CBAxsNInVrrn9c//p1j+u/LpOwhgEAAAAJ27p1qzp27KhOnTpp\n3bp1mjt3btLf47jjjtOMGTMkSe+8846WLl2615jCwkKtWbNGCxYskCSVl5dryZIlzbr+CSecUHun\npmeffVYFBQUZHRRqpHSGwcxGS/q9pJCk/3X3KfWeP0TSXyTlVY+Z5O5zUlkTAAAAkm/IkCEqLCxU\n//799eUvf1nHHnts0t/jmmuu0SWXXKKBAwdqyJAh6t+/vw444IA9xkQiEc2aNUvf//73tW3bNsVi\nMV133XUNti/VN3nyZF122WUaOHCgOnTo0Ky7O2UCa87UTlwXNgtJel/SqZJKJC2UNMHdl9YZM1XS\nm+7+JzMrlDTH3Xs1dd2ioiKvWXwCAAAQdMuWLVPfvn3TXUarEIvFFIvFFI1GtWLFCo0aNUorVqzI\nuNuctlRDf4fMbJG7FzXn9an8dodLWunuq6qLmi7pTEl1545cUqfqxwdIWpvCepJix+4d2l2xWwdE\nD9j3YAAAACTN559/rq997WuKxWJyd91zzz2Ehf0gld9wvqTVdY5LJI2oN+ZWSU+b2TWS2ks6JYX1\nJMV5M85T6Y5SLbx8YbpLAQAAyCh5eXlatGhRusvIOKlc9GwNnKvf/zRB0v3uXiBpjKS/mtleNZnZ\nFWZWbGbFpaWlKSi1+aLhqMpie6/4BwAAAIIolYGhRFLPOscF2rvl6NuSZkiSu78qKSqpW/0LuftU\ndy9y96Lu3bunqNzmiYQj2hXbldYaAAAAgP0llYFhoaQ+ZtbbzHIkXSBpdr0xH0v6miSZWV9VBYb0\nTiHsQyQU0a4KAgMAAAAyQ8oCg7vHJF0taa6kZZJmuPsSM5tsZuOqh10n6XIze0vSg5Imeqpu25Qk\ntCQBAAAgk6R04zZ3n+Puh7v7Ye5+W/W5W9x9dvXjpe5+rLsPcvfB7v50KutJhkiIliQAAJBZPvnk\nE11wwQU67LDDVFhYqDFjxuj9999P+ftOnDhR99xzzx7nHnvsMY0ZM6bJ1/Xq1UsbNmyQJB1zzDGN\nXnvWrFlNXuf+++/X2rVfdNR/5zvfaXCzuKBjp+cWioajtCQBAICM4e46++yzddJJJ+mDDz7Q0qVL\n9Z//+Z9av379HuMqKiqS/t4TJkzQ9OnT9zg3ffp0TZgwodnXeOWVV+J+//qB4X//939VWFgY9/Va\nIhaL7XHc3O/X3VVZWZnUWggMLRQJR1QWK1Mr75wCAABIihdeeEHZ2dm68sora88NHjxYxx9/vF58\n8UWNHDlSF154oQYMGCBJ+u1vf6v+/furf//+uuOOOyRJ27dv1+mnn65Bgwapf//+euihhyRJkyZN\nUmFhoQYOHKjrr79+r/c+5ZRT9N5772ndunWSpB07dujZZ5/VWWedJUk666yzNHToUPXr109Tp05t\nsP4OHTpIqvpF+uqrr1ZhYaFOP/10ffrpp7VjJk+erGHDhql///664oor5O6aNWuWiouLddFFF2nw\n4MHauXOnTjrpJNVsIPzggw9qwIAB6t+/v2688cY93u/mm2/WoEGDdNRRR+0VrGq+j29961saNmyY\njjzySD3++OOSqgLK+PHjNXbsWI0aNarZ3+9HH32kvn376qqrrtKQIUO0evXqvd4zEex00UKRUESS\ntLtyt3JCOWmuBgAAZJIf/PMHWvzJ4qRec3CPwbpj9B2NPv/uu+9q6NChjT6/YMECvfvuu+rdu7cW\nLVqk++67T6+//rrcXSNGjNCJJ56oVatW6eCDD9ZTTz0lSdqyZYs+++wzPfroo3rvvfdkZtq8efNe\n1w6FQjrnnHM0Y8YMXXvttZo9e7ZGjhypjh07SpKmTZumLl26aOfOnRo2bJjOPfdcde3atcE6H330\nUS1fvlzvvPOO1q9fr8LCQn3rW9+SJF199dW65ZZbJEnf/OY39eSTT+q8887TnXfeqdtvv11FRXtu\niLx27VrdeOONWrRokTp37qxRo0bpscce01lnnaXt27frqKOO0m233aYbbrhB9957r37605/u8frb\nbrtNJ598sqZNm6bNmzdr+PDhOuWUqu3IXn31Vb399tvq0qWLXnzxxWZ9v507d9by5ct133336Y9/\n/GOj/63ixQxDC0XDUUli4TMAAICk4cOHq3fv3pKk+fPn6+yzz1b79u3VoUMHnXPOOZo3b54GDBig\nZ599VjfeeKPmzZunAw44QJ06dVI0GtV3vvMdPfLII2rXrl2D16/bllS/Hel//ud/av8lf/Xq1Vqx\nYkWjdb788suaMGGCQqGQDj74YJ188sm1z73wwgsaMWKEBgwYoOeff15Llixp8jMvXLhQJ510krp3\n765wOKyLLrpIL7/8siQpJydHZ5xxhiRp6NCh+uijj/Z6/dNPP60pU6Zo8ODBOumkk1RWVqaPP/5Y\nknTqqaeqS5cuLfp+JenQQw/VUUcd1WTd8WKGoYUi4aoZhl2xXVIkzcUAAICM0tRMQKr069evycXB\n7du3r33cWMv24YcfrkWLFmnOnDm66aabNGrUKN1yyy1asGCBnnvuOU2fPl133nmnnnnmmdrZjHHj\nxmny5Mk69thjtW7dOr311lt65ZVXasPDiy++qGeffVavvvqq2rVrV/uLd1PM9t5XuKysTFdddZWK\ni4vVs2dP3Xrrrfu8TlOt6dnZ2bXvEwqF9lqLUPP6hx9+WEccccQe519//fU9vk+ped9v/XHJxgxD\nC9XMMLDwGQAAZIKTTz5Zu3bt0r333lt7buHChXrppZf2GnvCCSfoscce044dO7R9+3Y9+uijOv74\n47V27Vq1a9dOF198sa6//nq98cYb+vzzz7VlyxaNGTNGd9xxhxYvXqxQKKTFixdr8eLFmjx5sqSq\nX/LPP/98XXrppRozZoyi0arfxbZs2aLOnTurXbt2eu+99/Taa681+TlOOOEETZ8+XRUVFVq3bp1e\neOEFSaoNB926ddPnn3++Rzjq2LGjtm3btte1RowYoZdeekkbNmxQRUWFHnzwQZ144onN/k6//vWv\n6w9/+ENtAHjzzTeb9brGvt9UY4ahhWrWMNCSBAAAMoGZ6dFHH9UPfvADTZkyRdFoVL169dIdd9yh\nNWvW7DF2yJAhmjhxooYPHy6p6jakRx55pObOnasf//jHysrKUnZ2tv70pz9p27ZtOvPMM1VWVnUz\nmd/97neN1jBhwgT95je/0ZQpU2rPjR49WnfffbcGDhyoI444Yp/tOGeffbaef/55DRgwQIcffnjt\nL/h5eXm6/PLLNWDAAPXq1UvDhg2rfc3EiRN15ZVXKjc3V6+++mrt+YMOOki/+tWvNHLkSLm7xowZ\nozPPPLPZ3+nPfvYz/eAHP9DAgQPl7urVq5eefPLJfb6use+3obanZLK2drefoqIir1mdng4zlszQ\nN2Z9Q+9+7131O7Bf2uoAAACZYdmyZerbt2+6y0Ab1tDfITNb5O5FjbxkD7QktRCLngEAAJBJCAwt\nVNOSxBoGAAAAZAICQwvVLnqOERgAAAAQfASGFqq5rSotSQAAYH9pa2tO0Xok4+8OgaGFaEkCAAD7\nUzQa1caNGwkNaDF318aNG2tvRRsvbqvaQix6BgAA+1NBQYFKSkpUWlqa7lLQBkWjURUUFCR0DQJD\nC+2x0zMAAECKZWdnq3fv3ukuAxmMlqQWoiUJAAAAmYTA0EK0JAEAACCTEBhaiJYkAAAAZBICQwsx\nwwAAAIBMQmBooXBWWFmWxRoGAAAAZAQCQxwioQgtSQAAAMgIBIY4RMNRWpIAAACQEQgMcYiEI7Qk\nAQAAICMQGOLADAMAAAAyBYEhDpEQMwwAAADIDASGOETCLHoGAABAZiAwxIGWJAAAAGQKAkMcaEkC\nAABApiAwxIEZBgAAAGQKAkMcWMMAAACATEFgiAMtSQAAAMgUBIY40JIEAACATEFgiEMkREsSAAAA\nMgOBIQ7RcJSWJAAAAGQEAkMcIuEILUkAAADICASGONCSBAAAgExBYIhDNBzV7srdqvTKdJcCAAAA\npBSBIQ6RcESSmGUAAABA4BEY4hANRyWJhc8AAAAIPAJDHCKhqhkGFj4DAAAg6FIaGMxstJktN7OV\nZjapged/Z2aLq3/eN7PNqawnWWhJAgAAQKYIp+rCZhaSdJekUyWVSFpoZrPdfWnNGHf/YZ3x10g6\nMlX1JFNNSxIzDAAAAAi6VM4wDJe00t1XuXu5pOmSzmxi/ARJD6awnqSpaUliDQMAAACCLpWBIV/S\n6jrHJdXn9mJmh0rqLen5FNaTNLWLnmlJAgAAQMClMjBYA+e8kbEXSJrl7hUNXsjsCjMrNrPi0tLS\npBUYr5o1DLQkAQAAIOhSGRhKJPWsc1wgaW0jYy9QE+1I7j7V3Yvcvah79+5JLDE+tCQBAAAgU6Qy\nMCyU1MfMeptZjqpCwez6g8zsCEmdJb2awlqSikXPAAAAyBQpCwzuHpN0taS5kpZJmuHuS8xsspmN\nqzN0gqTp7t5Yu1Krw21VAQAAkClSdltVSXL3OZLm1Dt3S73jW1NZQyqw0zMAAAAyBTs9x4GdngEA\nAJApCAxxoCUJAAAAmYLAEAcWPQMAACBTEBjiwG1VAQAAkCkIDHGgJQkAAACZgsAQhyzLUnZWNi1J\nAAAACDwCQ5wi4QgtSQAAAAg8AkOcouEoMwwAAAAIPAJDnCKhCGsYAAAAEHgEhjjRkgQAAIBMQGCI\nEy1JAAAAyAQEhjhFQswwAAAAIPgIDHFihgEAAACZgMAQp0iYRc8AAAAIPgJDnGhJAgAAQCYgMMSJ\nliQAAABkAgJDnGhJAgAAQCYgMMQpGo7SkgQAAIDAIzDEKRKK0JIEAACAwCMwxCkSoiUJAAAAwUdg\niBOLngEAAJAJCAxxioS5rSoAAACCj8AQp2g4qkqvVKwylu5SAAAAgJQhMMQpEopIEm1JAAAACDQC\nQ5wi4arAwMJnAAAABBmBIU7RcFQSMwwAAAAINgJDnGpaklj4DAAAgCAjMMSpZoaBliQAAAAEGYEh\nTjVrGGhJAgAAQJARGOJESxIAAAAyAYEhTix6BgAAQCYgMMSJ26oCAAAgExAY4lS76JmWJAAAAAQY\ngSFO7PQMAACATEBgiBMtSQAAAMgEBIY4segZAAAAmYDAECduqwoAAIBMQGCIEzs9AwAAIBMQGOLE\nTs8AAADIBASGOGVnZUuiJQkAAADBRmCIk5kpGo4ywwAAAIBAS2lgMLPRZrbczFaa2aRGxpxvZkvN\nbImZPZDKepItEoqwhgEAAACBFk7Vhc0sJOkuSadKKpG00Mxmu/vSOmP6SLpJ0rHuvsnMDkxVPakQ\nDUdpSQIAAECgpXKGYbikle6+yt3LJU2XdGa9MZdLusvdN0mSu3+awnqSLhKO0JIEAACAQEtlYMiX\ntLrOcUn1uboOl3S4mf3LzF4zs9EprCfpIqEIMwwAAAAItJS1JEmyBs55A+/fR9JJkgokzTOz/u6+\neY8LmV0h6QpJOuSQQ5JfaZyi4ShrGAAAABBoqZxhKJHUs85xgaS1DYx53N13u/uHkparKkDswd2n\nunuRuxd17949ZQW3FC1JAAAACLpUBoaFkvqYWW8zy5F0gaTZ9cY8JmmkJJlZN1W1KK1KYU1JRUsS\nAAAAgi5lgcHdY5KuljRX0jJJM9x9iZlNNrNx1cPmStpoZkslvSDpx+6+MVU1JRv7MAAAACDoUrmG\nQe4+R9KceuduqfPYJf2o+qfNiYQj2ly2ed8DAQAAgDaKnZ4TwD4MAAAACDoCQwIiIRY9AwAAINgI\nDAmIhCPcVhUAAACBRmBIQDTEomcAAAAEG4EhAZEwt1UFAABAsBEYEsBOzwAAAAg6AkMCajZuq7o7\nLAAAABA8BIYERMIRSVJ5RXmaKwEAAABSg8CQgGg4KkksfAYAAEBgERgSEAlVzTCw8BkAAABBRWBI\nQM0MAwufAQAAEFQEhgTUrGGgJQkAAABBRWBIAC1JAAAACDoCQwJY9AwAAICgIzAkoKYliTUMAAAA\nCCoCQwJqFz3TkgQAAICAIjAkoGYNAy1JAAAACCoCQwJoSQIAAEDQERgSwKJnAAAABB2BIQHcVhUA\nAABBR2BIADs9AwAAIOgIDAlgp2cAAAAEHYEhAbQkAQAAIOgIDAlghgEAAABBR2BIQDgrrJCFWMMA\nAACAwCIwJCgajtKSBAAAgMAiMCQoEo7QkgQAAIDAIjAkKBKK0JIEAACAwCIwJIiWJAAAAAQZgSFB\ntCQBAAAgyAgMCWKGAQAAAEFGYEhQJMQMAwAAAIKLwJCgSJhFzwAAAAguAkOCaEkCAABAkBEYEkRL\nEgAAAIKMwJCgaDhKSxIAAAACi8CQIG6rCgAAgCAjMCQoEoqwhgEAAACBRWBIEC1JAAAACDICQ4JY\n9AwAAIAgS2lgMLPRZrbczFaa2aQGnp9oZqVmtrj65zuprCcVImFakgAAABBc4VRd2MxCku6SdKqk\nEkkLzWy2uy+tN/Qhd786VXWkWjQcVawyporKCoWyQukuBwAAAEiqVM4wDJe00t1XuXu5pOmSzkzh\n+6VFJBSRJGYZAAAAEEipDAz5klbXOS6pPlffuWb2tpnNMrOeKawnJaLhqCSx8BkAAACBlMrAYA2c\n83rHT0jq5e4DJT0r6S8NXsjsCjMrNrPi0tLSJJeZmEi4aoaBhc8AAAAIolQGhhJJdWcMCiStrTvA\n3Te6e80/zd8raWhDF3L3qe5e5O5F3bt3T0mx8aIlCQAAAEGWysCwUFIfM+ttZjmSLpA0u+4AMzuo\nzuE4SctSWE9K1LQkMcMAAACAIErZXZLcPWZmV0uaKykkaZq7LzGzyZKK3X22pO+b2ThJMUmfSZqY\nqnpSpaYliTUMAAAACKKUBQZJcvc5kubUO3dLncc3SboplTWkWu2iZ1qSAAAAEEDs9JygmjUMtCQB\nAAAgiAgMCaIlCQAAAEFGYEgQi54BAAAQZASGBHFbVQAAAAQZgSFB7PQMAACAICMwJIidngEAABBk\nBIYE0ZIEAACAICMwJIhFzwAAAAgyAkOCuK0qAAAAgozAkCBakgAAABBkBIYEmZlyQjm0JAEAACCQ\nCAxJEAlFaEkCAABAIBEYkiAajtKSBAAAgEAiMCRBJByhJQkAAACBRGBIAmYYAAAAEFQEhiSIhJhh\nAAAAQDARGJIgEmbRMwAAAIKJwJAEtCQBAAAgqAgMSUBLEgAAAIKKwJAE0XCUliQAAAAEUrMCg5nl\nmtkRqS6mreK2qgAAAAiqfQYGMxsrabGkf1YfDzaz2akurC2JhCKsYQAAAEAgNWeG4VZJwyVtliR3\nXyypV+pKantoSQIAAEBQNScwxNx9S8oracNY9AwAAICgCjdjzLtmdqGkkJn1kfR9Sa+ktqy2hduq\nAgAAIKiaM8NwjaR+knZJekDSFknXprKotoZFzwAAAAiq5swwnO7uN0u6ueaEmY2XNDNlVbUxkVDV\nTs/uLjNLdzkAAABA0jRnhuGmZp7LWNFwVC5XrDKW7lIAAACApGp0hsHMTpM0RlK+mf1Pnac6SeI3\n4zoi4YgkqSxWpuxQdpqrAQAAAJKnqZaktZKKJY2TtKjO+W2SfpjKotqaSKgqMOyq2KWO6pjmagAA\nAIDkaTQwuPtbkt4yswfcffd+rKnNiYajksTCZwAAAAROcxY99zKzX0kqlBStOenuX05ZVW1MTUsS\nm7cBAAAgaJqz6Pk+SX9S1bqFkZL+T9JfU1lUW1Mzw8BeDAAAAAia5gSGXHd/TpK5+7/d/VZJJ6e2\nrLalZg0DLUkAAAAImua0JJWZWZakFWZ2taQ1kg5MbVltCy1JAAAACKrmzDD8QFI7Sd+XNFTSNyVd\nmsqi2hoWPQMAACCo9jnD4O4Lqx9+LukySTKzQ1NZVFtT97aqAAAAQJA0OcNgZkeb2XlmdmD18UAz\ne0DS/P1SXRtRu+iZliQAAAAETKOBwcx+I2mapHMlPWVmP5f0jKTXJfXZP+W1DXV3egYAAACCpKmW\npNMlHenuZWbWWVU7Pw909xWM05IrAAAgAElEQVT7p7S2g5YkAAAABFVTLUk73b1Mktx9k6TlhIWG\n0ZIEAACAoGoqMBxmZrNrflS143Pd430ys9FmttzMVprZpCbGnWdmbmZFLf0ArQEtSQAAAAiqplqS\nzqx3/N8tubCZhSTdJelUSSWSFprZbHdfWm9cR1XdsvX1lly/NWGnZwAAAARVo4HB3V9K8NrDJa10\n91WSZGbTVRVCltYb9wtJv5Z0fYLvlzbs9AwAAICgas7GbfHKl7S6znFJ9blaZnakpJ7u/mQK60i5\ncFZYJmMNAwAAAAInlYHBGjjntU+aZUn6naTr9nkhsyvMrNjMiktLS5NYYnKYmaLhKC1JAAAACJx9\nbdwWqt6PIR4lknrWOS5Q1a1Za3SU1F/Si2b2kaSjJM1uaOGzu0919yJ3L+revXuc5aRWJByhJQkA\nAACB02RgcPcKSUPNrKHZgn1ZKKmPmfU2sxxJF0iqvbuSu29x927u3svde0l6TdI4dy+O473SLhqO\n0pIEAACAwGnqLkk13pT0uJnNlLS95qS7P9LUi9w9ZmZXS5orKSRpmrsvMbPJkordvVm3Zm0rIqGI\nyiqYYQAAAECwNCcwdJG0UdLJdc65pCYDgyS5+xxJc+qdu6WRsSc1o5ZWKxKOMMMAAACAwNlnYHD3\ny/ZHIW0di54BAAAQRPu8S5KZFZjZo2b2qZmtN7OHzaxgfxTXlkRCLHoGAABA8DTntqr3qWqx8sGq\n2kfhiepzqINFzwAAAAii5gSG7u5+n7vHqn/ul9Q6722aRtxWFQAAAEHUnMCwwcwurt6TIWRmF6tq\nETTqiIQirGEAAABA4DQnMHxL0vmSPpG0TtJ51edQBy1JAAAACKIm75JkZiFJ57r7uP1UT5tFSxIA\nAACCqDk7PZ+5n2pp06IhbqsKAACA4GnOxm3/MrM7JT2kPXd6fiNlVbVBzDAAAAAgiJoTGI6p/nNy\nnXOuPXd+zniREDs9AwAAIHj2tYYhS9Kf3H3GfqqnzWKnZwAAAATRvtYwVEq6ej/V0qZFwhGVV5Sr\n0ivTXQoAAACQNM25reozZna9mfU0sy41PymvrI2JhqOSpPKK8jRXAgAAACRPc9Yw1Oy58B91zrmk\nLye/nLYrEopIkspiZbXhAQAAAGjr9hkY3L33/iikrYuEqwIDC58BAAAQJI22JJnZDXUej6/33H+m\nsqi2qGZWgYXPAAAACJKm1jBcUOfxTfWeG52CWtq0ui1JAAAAQFA0FRiskccNHWe82hkGWpIAAAAQ\nIE0FBm/kcUPHGa9mDQMzDAAAAAiSphY9DzKzraqaTcitfqzqY24DVE9NSxJrGAAAABAkjQYGdw/t\nz0LaOlqSAAAAEETN2bgNzUBLEgAAAIKIwJAktCQBAAAgiAgMSUJLEgAAAIKIwJAktCQBAAAgiAgM\nScJOzwAAAAgiAkOSsNMzAAAAgojAkCQ1LUmsYQAAAECQEBiShLskAQAAIIgIDEkSygopnBWmJQkA\nAACBQmBooYpK15Yduxt8LhqO0pIEAACAQCEwtNDE+xbo0vsWNPhcJBRhhgEAAACBQmBooYMOiGrN\n5p0NPhcJR1jDAAAAgEAhMLRQfl47lW7bpbLdFXs9Fw1HCQwAAAAIFAJDC+V3zpUkrduyd+sRLUkA\nAAAIGgJDC+XnVQWGNZv2bkti0TMAAACChsDQQgXVMwxrNu/Y67lImBkGAAAABAuBoYV6HBBVljU8\nwxAJsegZAAAAwUJgaKHsUJa+1CmqkgbulERLEgAAAIKGwBCH/LzchmcYaEkCAABAwKQ0MJjZaDNb\nbmYrzWxSA89faWbvmNliM5tvZoWprCdZ8jvnNrgXA7dVBQAAQNCkLDCYWUjSXZJOk1QoaUIDgeAB\ndx/g7oMl/VrSb1NVTzLl5+Xqky1lqqj0Pc5zW1UAAAAETSpnGIZLWunuq9y9XNJ0SWfWHeDuW+sc\ntpe052/grVR+51zFKl3rt+4ZDiKhCGsYAAAAECjhFF47X9LqOsclkkbUH2Rm/yHpR5JyJJ2cwnqS\npnYvhs07dXD1Y4mWJAAAAARPKmcYrIFze80guPtd7n6YpBsl/bTBC5ldYWbFZlZcWlqa5DJbrnYv\nhnoLn1n0DAAAgKBJZWAokdSzznGBpLVNjJ8u6ayGnnD3qe5e5O5F3bt3T2KJ8Tm4zgxDXdxWFQAA\nAEGTysCwUFIfM+ttZjmSLpA0u+4AM+tT5/B0SStSWE/StMsJq2v7HJXUn2EIRVThFYpVxtJUGQAA\nAJBcKVvD4O4xM7ta0lxJIUnT3H2JmU2WVOzusyVdbWanSNotaZOkS1NVT7I1dGvVSDgiSdoV26Vw\nTiqXhwAAAAD7R0p/q3X3OZLm1Dt3S53H16by/VMpPy9X76/ftse5aDgqSdpVsUvt1T4dZQEAAABJ\nxU7PccrPq5phcP9iHXckVDXDwMJnAAAABAWBIU75nXNVtrtSn20vrz1XO8PAwmcAAAAEBIEhTvkN\n3Cmpdg0DezEAAAAgIAgMccpvYC8GWpIAAAAQNASGOBXktZO05wwDLUkAAAAIGgJDnDrlhtUhEt5j\nL4aaliRmGAAAABAUBIY4mVntnZJq1L2tKgAAABAEBIYE5HfO3XOGIfTFxm0AAABAEBAYEpCfl6s1\nm3bUHtOSBAAAgKAhMCQgv3OutpbFtK1styRakgAAABA8BIYE1N+LgduqAgAAIGgIDAmovxdD7cZt\nrGEAAABAQBAYElBQb4aBliQAAAAEDYEhAd06RJQTyvpihoGWJAAAAAQMgSEBWVmmg/OiKqmeYcgJ\n5UiiJQkAAADBQWBIUH7n3NoZBjNTJBRhhgEAAACBQWBIUP3dniPhCGsYAAAAEBgEhgTl57VT6bZd\nKttdIalq4TMtSQAAAAgKAkOCam6tum5LVRtSJBRRWQUtSQAAAAgGAkOCajdv2/TFrVWZYQAAAEBQ\nEBgSVFCzedvmHZKq1jCw6BkAAABBQWBIUI8Dosoy7bEXA4ueAQAAEBQEhgRlh7L0pU5f7MVASxIA\nAACChMCQBPl5X+zFQEsSAAAAgoTAkAQFnb/YiyEajtKSBAAAgMAgMCRBfudcfbKlTBWVzk7PAAAA\nCBQCQxLk57VTrNK1fmtZ1U7PrGEAAABAQBAYkiC/9taqO2lJAgAAQKAQGJKg7uZttCQBAAAgSAgM\nSVAbGGpmGGhJAgAAQEAQGJIgNyekru1zVMIMAwAAAAKGwJAk+Z1zVbJpR9Wi54pdcvd0lwQAAAAk\njMCQJPl5ubUtSZK0u3J3misCAAAAEkdgSJL8vFyt3bxTOaEcSaItCQAAAIFAYEiS/M65KttdqYqK\nbEli4TMAAAACgcCQJDV3StqxyySJvRgAAAAQCASGJKnZvG17dWCgJQkAAABBQGBIkoK8dpKkbTur\nZxhoSQIAAEAAEBiSpFNuWB0iYW3dWXXMDAMAAACCgMCQJGam/LxcbdpedcwaBgAAAAQBgSGJ8jvn\natP2qg3baEkCAABAEKQ0MJjZaDNbbmYrzWxSA8//yMyWmtnbZvacmR2aynpSLT8vVxu3VUiiJQkA\nAADBkLLAYGYhSXdJOk1SoaQJZlZYb9ibkorcfaCkWZJ+nap69of8zrnasSskiZYkAAAABEMqZxiG\nS1rp7qvcvVzSdEln1h3g7i+4+47qw9ckFaSwnpTLz8uVqWrjNmYYAAAAEASpDAz5klbXOS6pPteY\nb0v6RwrrSbmqvRjY6RkAAADBEU7hta2Bc97gQLOLJRVJOrGR56+QdIUkHXLIIcmqL+kK8nJlXh0Y\naEkCAABAAKRyhqFEUs86xwWS1tYfZGanSLpZ0jh3b/C3bHef6u5F7l7UvXv3lBSbDN06RBQJRSTR\nkgQAAIBgSGVgWCipj5n1NrMcSRdIml13gJkdKekeVYWFT1NYy36RlWU6+ICOkmhJAgAAQDCkLDC4\ne0zS1ZLmSlomaYa7LzGzyWY2rnrYbyR1kDTTzBab2exGLtdmFHTuJIkZBgAAAARDKtcwyN3nSJpT\n79wtdR6fksr3T4eenTtKa7NYwwAAAIBAYKfnJMvPayfzbG0v35nuUgAAAICEERiSrKBz1V4Mn+3Y\nnu5SAAAAgIQRGJIsv3OuTDnatGPHvgcDAAAArRyBIcny83Ilz9bmMmYYAAAA0PYRGJKsxwFRZSlb\nW8tYwwAAAIC2j8CQZNmhLGWHIlq3ba02bt+W7nIAAACAhBAYUuAref31aflb+tLt+Tpx6uV6/ePl\n6S4JAAAAiAuBIQXevPoR3TXqcR3afpheXjtNR00r1Jd/c7L++OpsVVZWprs8AAAAoNnM3dNdQ4sU\nFRV5cXFxustotgUfv68b/vkbzV/3kCq0TR2yDtOEwsv1q9FXqWv7jukuDwAAABnIzBa5e1FzxjLD\nkGLDDzlcL15xr0p/vFZXDPgvSdK9707Sl27P1/ce/e80VwcAAAA0jcCwn3Ru10H3nHODttz8vv40\nara6ZH9Fd7/1Y9045+50lwYAAAA0isCwn2VlZenKo8dq+Q9eVtecAfr1gmv0+/mPpLssAAAAoEEE\nhjTp3K6DFl35jDqEeupHz16shxa/lO6SAAAAgL0QGNLo0C4H6uVvPaNs66SLHz9Tz698K90lAQAA\nAHsgMKTZkfmH6YkJcyRJYx44Te+s/TDNFQEAAABfIDC0AqcePkT3jX1Y5ZWbdOyfR2n1lg3pLgkA\nAACQRGBoNS4e8jX9ZuRftK3iQw29a5Q279ye7pIAAAAAAkNrct2J5+uHRXeodPebOvLOM1Qei6W7\nJAAAAGQ4AkMr89szrtYFfX6mj3a8qBF/mqDKysp0lwQAAIAMFk53AdjbgxdO1idTP9GL6+5Vzi86\nK6SIQhZRyHIUtohCWdnKzoooOyuinFBUh3XurVP7DNXxvQerb/e+6hTplO6PAAAAgIAwd093DS1S\nVFTkxcXF6S4j5SorK3XpjF9oSekSlVfsUnnlLpVX7FKscpd2V5YrVrlLMd+l3ZU7tbPyE8m+aF86\nsN1BGtijv/p1L1Tfbn3V78B+GpE/Qtmh7DR+IgAAALQWZrbI3YuaNZbA0PZ9uGGrHnqzWHOWFWtJ\n6VLt0sfy8Brtzlqt3ZU7JUldc7tqfOF4XTjgQh17yLHKMrrRAAAAMhWBIYNt2bFbL77/qZ5eul4v\nLl+vLeWfSNmr1KvnO3qz9Bnt2L1DPTv11IT+EzRhwAQN+tIgmVmj19u0c5OWbVimZaXLtLlss749\n5NvKi+btx08EAACAZCMwQJJUHqvUa6s26u6XPtArH2zUNV/LV48vLdGD7z6ouR/MVawypr7d+urC\nARdq3BHjtHHHRi3bsExLS5fW/vnJ55/scc2enXrqL2f9RSN7j0zTpwIAAECiCAzYQ3msUjfMekuP\nLV6rS44+VD8f20+byjZq1tJZeuCdBzTv43l7jO+Y01GF3QvVt3tfFXYrrH1cur1Ulz52qZZvXK4f\nHfUj3fa12xQNR9P0qQAAABAvAgP2Ulnp+tU/luneeR9qzIAe+u35gxXNDkmSPt7ysZ7/8Hnld8xX\nYfdCHdzx4EbblHbs3qEbnrlBdy28S/2699PfzvmbBvcYvD8/CgAAABJEYECj7n15lW6bs0wjenfR\nvZcWqVM0vjsn/XPlP3XZ45dp446N+sXIX+j6Y65XKCuU5GoBAACQCi0JDNwqJ8NcfsKXdcc3BuuN\njzfp/Ltf1fqtZXFdZ/RXRuud772jcUeM06TnJmnkX0bqo80fJbdYAAAApB2BIQOddWS+pk0cptWf\n7dA5f3xFH5R+Htd1urXrppnjZ+ovZ/1Fiz9ZrIF/Gqipi6Zqe/n2JFcMAACAdKElKYO9U7JFE+9b\noEp3TZs4TEce0jnua320+SNd8uglmvfxPEXDUZ3c+2SNPXyszjj8DBV0Kkhi1QAAAEgUaxjQbB9t\n2K5Lpi1Q6bZdeuKa4/SVAzvEfa1Kr9QLH76gJ95/Qk+8/4RWbVolSTqyx5Eae/hYjT1irIYcNIRN\n4wAAANKMwIAWWb+1TMf/1wu66KhD9POx/ZJyTXfXsg3L9MTyqvDwasmrqvRKHdzxYI0vHK9bT7qV\nDeAAAADShMCAFrvq74v02qrP9NpNX1NOOPkzABt2bNCcFXM0e/lsPfbeYzqo40GaNm6aTj3s1KS/\nFwAAAJrGXZLQYuOH9tRn28v1/HufpuT63dp10yWDLtGs82fp1W+/qg45HTTqb6N01VNX6fPy+BZd\nAwAAIPUIDJAkHd+nmw7sGNGsRatT/l7D8ofpjSve0HVHX6e7i+/WoLsHad6/5+37hQAAANjvCAyQ\nJIVDWTpnSIFeWF6qT7fFtzdDS+Rm5+r2UbfrpYkvSZJOvP9EXf/09SqLpf69AQAA0HwEBtQ6b2iB\nKipdj725Zr+95/GHHq+3rnxLVxZdqf9+9b815J4hWrhm4X57fwAAADSNwIBaXzmwg448JE8zi0u0\nPxfDd8jpoD+e/kfNvXiutu7aqqP/fLQmvzR5v9YAAACAhhEYsIfxQ3tqxaef6+2SLfv9vUcdNkrv\nXvWuLuh/gX7+4s911VNXqdIr93sdAAAA+AKBAXs4Y9BBioSzNHM/LH5uSF40T389+6+64ZgbdPei\nu3XZ45cpVhlLSy0AAABIcWAws9FmttzMVprZpAaeP8HM3jCzmJmdl8pa0Dydotka3b+HZi9eq7Ld\nFWmpwcw05ZQpmnzSZP3fW/+nCx++UOUV5WmpBQAAINOlLDCYWUjSXZJOk1QoaYKZFdYb9rGkiZIe\nSFUdaLnxQ3tqa1lMTy9dn7YazEw/O/Fnuv3U2zVz6UydO+Nc7qAEAACQBqmcYRguaaW7r3L3cknT\nJZ1Zd4C7f+Tub0uiUb0VOeawrsrPy9WsRSXpLkXXHXOd/jjmj3ry/Sc19sGx2l6+Pd0lAQAAZJRU\nBoZ8SXUb4Uuqz6GVy8oynTskX/NWlGrdlp3pLkffG/Y93X/m/Xr+w+f19b99XVvK9v+CbAAAgEyV\nysBgDZyL6z6ZZnaFmRWbWXFpaWmCZaE5zh1aIHfpkTf2354MTbl08KWafu50vb7mdZ3y11P02c7P\n0l0SAABARkhlYCiR1LPOcYGktfFcyN2nunuRuxd17949KcWhaYd2ba/hvbtoZvHqVrMfwvh+4/Xo\nNx7VO+vf0Un3n6T1n6dvjQUAAECmSGVgWCipj5n1NrMcSRdImp3C90OSjR9aoI827tCif29Kdym1\nzjj8DD154ZP6YNMHGvW3USyEBgAASLGUBQZ3j0m6WtJcScskzXD3JWY22czGSZKZDTOzEknjJd1j\nZktSVQ9absyAg9QuJ6SZxelf/FzXKV8+RTPHz9Tb69/WpGf3ulsvAAAAkiil+zC4+xx3P9zdD3P3\n26rP3eLus6sfL3T3Andv7+5d3b1fKutBy7SPhDVmwEF68u212lHeujZPG9NnjK4Zfo1+//rvNXfl\n3HSXAwAAEFjs9IwmjR9aoO3lFfrHO5+ku5S9/Ncp/6V+3ftp4uMTVbqdxfAAAACpQGBAk4b37qJD\nu7bTzEWr9z14P8vNztUD5z6gz3Z+pu888Z1WszgbAAAgSAgMaJKZ6bwhBXpt1Wda/dmOdJezl4Ff\nGqgpX5ui2ctna+qiqekuBwAAIHAIDNinc4YWyEytYufnhlx71LU69cun6odzf6j3NryX7nIAAAAC\nhcCAfcrPy9Wxh3XTrEUlqqxsfW0/WZal+8+6X+2y2+miRy5SeUV5uksCAAAIDAIDmuX8YT21ZvNO\nzXqjdc4yHNzxYN079l69se4N3fLCLekuBwAAIDAIDGiWMwYcpOG9uugXTy7Vui07011Og87ue7Yu\nH3K5fv2vX+vFj15MdzkAAACBQGBAs2RlmX593kDtrqjUpIffabV3JPrd13+nPl376JuPflObdrae\nHaoBAADaKgIDmq1Xt/aaNPqreun90la3+3ON9jnt9fdz/q5PPv9E333yu6022AAAALQVBAa0yCVH\n99KI3lWtSWs3t87WpKKDizT5pMmauXSmfjX/V9q2a1u6SwIAAGizCAxokaws02/OG6RYpWvSI623\nNemGY2/QqMNG6ebnb9aBtx+o8TPH69Flj6osVpbu0gAAANoUAgNa7JCu7TTptK/q5fdL9dDC1rcD\ntCSFskL6x0X/0PzL5uvbR35bL330ks6ZcY6+dPuX9K3Hv6VnPnhGscpYussEAABo9ay1/gtxY4qK\niry4uDjdZWS8ykrXhf/7mt5ds1Vzf3iC8vNy011Sk2KVMT236jk9+O6DemTZI9pWvk0Htj9Q3+j3\nDV017Cp9tdtX010iAADAfmNmi9y9qDljmWFAXGpakyrdNenht1tta1KNcFZYX//K13X/Wfdr/fXr\nNWv8LB1/yPGaumiqBt09SFPmT2HGAQAAoAEEBsStZ5d2uum0r2reig2a3kpbkxqSm52rcwvP1azz\nZ+njH36scUeM003P3aRj/nyMlpYuTXd5AAAArQqBAQm5aMShOvrLXfXLJ5eqZNOOdJfTYge2P1Az\nx8/UQ+c9pA83f6gj7zmS2QYAAIA6CAxISM2GbpJa9YZu+3J+v/O15KolGnv42NrZhiWfLkl3WQAA\nAGlHYEDCenZpp5vG9NX8lRv0wIKP011O3A5sf6BmnT+rdrZhyNQh+tW8XzHbAAAAMhqBAUlx0YhD\ndOxXuuq2p5bppfdL011OQurONvzk+Z/o6D8frcffe1wbd2xMd2kAAAD7HbdVRdKs31qmS6ct0Pvr\nt+knY/rq28f1lpmlu6yEzFwyU1fNuUobdmyQJPXt1lfHH3K8jjvkOB13yHHqlddrv33GisoKrd66\nWqXbSzW4x2Blh7L3y/sCAIDgacltVQkMSKrtu2L60YzFmrtkvcYPLdAvz+6vSDiU7rISUhYr08I1\nCzX/4/ma9/E8/Wv1v7R111ZJUn7HfB13yHE6tuex6tGhh3JCOY3+ZIeyq/7MylZ2KLv2z5pzZqZK\nr9TabWu1YuMKrfhsRe2f7298X6s2rdKuil2SpM7Rzjr7q2fr/H7n6+TeJxMeAABAixAYkFaVla47\nnn1f//P8Sg09tLPuvniouneMpLuspKmorNCS0iWa9+95mr96vub9e57WbFuT8HVDFpKZ7bFmIhKK\n6CtdvqI+XfuoT5eqn06RTnpyxZN6/L3Hta18m7rkdtHZXz1b4wvHEx4AAECzEBjQKjz59lpdP/Mt\ndWmXo6mXFKl//gHpLikl3F1rtq3R5rLNKq8o3+Nnd8Xu2se7KnZpd8Vu7a7cXftnzZiacxVeoUMP\nOLQ2IPQ8oKeyrOGlRmWxMj39wdOasWSGZi+fXRsezvnqOTq38Fwd2/NYdYx03M/fBgAAaAsIDGg1\n3l2zRZf/X7E279it28cP0ukDD0p3SYFUFivT3JVzNXPpTD2+/HF9/v/bu/coOc7yzuPfp7r6orlo\nRtbNN9kWyCAMxjKRDZFzEq/NxWGzeJOF2BCyXtZZNjlmF87JJmuSs0k24Zwlu1mSzXJJiE0CBHC4\nbrxZljtmDxgHycYYG9sg2cYSsi2NJY1memb6UvXsH3Xp6p5paWRrNJr273NOnap6u7r6rXqrq9/n\nfauqm9MEFvDSjS9lx7k7uOK8K9ixaQfnj52/4u8rERERkWdPAYOcVg5MzfHrH72bex4/wr+/+kLe\ncfWFBIEqrUtltjWb3Gvx+Le4c9+d3LXvLqab0wCcNXIWOzbt4IpNV/DTm36abWduoxbWljnHIiIi\ncqopYJDTTqMd8bufu59P372Pq7Zu4DeufD7bz1+j1u5TIIoj7j9wP9/a+y3u3Hsnd+69k0ePPApA\nGIRcvOFiLjv7MrafvZ3LzrmMF69/se6DEBERGXAKGOS05O586FuP8Z4vPUy9GfG89cP88vZN/NLL\nzmHDqFq5T6Unpp7grn13sXP/Tnbt38Wu/bs4PHcYgFpY49IzL2X72du5eMPFnDlyJhuGN7BxZCMb\nhjcwVB5a9Oe4O/VWHYCh8lDf+zFERETk1FLAIKe1eqPN//n+E3xy5152/fgwpcC4ausGrtu+iStf\nuJ6wpErlqebu7Dm8h137d7HzJzvZuX8n9zxxT17ZLxouD7NheEMeRNTCGtPNaaab00w1ppJxMxnX\nm3Uc73rvaHWUkcpIPoxWkvktZ2zhyguuZMemHYxURk7l5ouIiDznKGCQFWP3gWk+tWsvn7lnHxPT\nTTaMVvkXP3Uub7r8PDadsfiWbDn5ojhi39F9HJw5yFPTT3GgfoAD9QM8Ve+enm3NMlodzSv+o9VR\nRsojeWAwWkme1JQFFcWAIhuONo6y5/Ae2nGbMAi57OzLuPKCKxVAiIiILBEFDLLitKKYrz10gE/u\n3MvXHz5ANSxxyw3buWLLuuXOmpwi081p7tx7J3c8dgd3PHYHO/fv7Aogfu78n+PMkTO7HlM77zG2\ncYuNwxvZcsaWfDh79GxdCiUiItJDAYOsaPsOz3Dj3+zi0afrvP9NL+OVF21c7izJMugXQBRl/6Jd\nLVWplCqEQchT9adoRs18mVpY4/lrns+Fay9ky5okiHjhuheydd1WNg5v1I33ixB7zFRjiiNzRzgy\nd4Sp5hQzrRnqzTozrZlkulXvSssCuOL/jrSiFu24TStOxkPlIcaqY6yurmasOsZYbSwfZ2mrq6vz\nYbQ6ShiEz3p73J1G1OBo4yhHG0eZakwx155juDLMaGU07zGrhoPzh5PNqMnk3GRehpONZHpybpJq\nWGWsOsZ4bXxeGSjYFhlcChhkxTsy0+SGD32HB/Yf5T3XbeN1l5y93FmSZTbbmmW2PdsVHCxU2Y/i\niL1H97L70O6u4UeHfsSeQ3toRI182fHaOFvXbWXruq28aN2L8vHmNZuJPeZo4yiTc5N5xXKyMZmn\nTTeniTzC3XGc7FyaTTuOYQyVhxipjDBcGWa4PMxwZTiZT6crpUp+/8dUc2rB8Wx7Nl8n0PWZWZph\nlEtlykGZMAjz6WKamZE20YoAABYESURBVNFoz++ZKfbWTDenOTx3OK9YHp49zGRjktjjRZVTYAFD\n5SGqpeq8PBTzEgYhM60ZJhuT+T6OPDru+ofKQ11BxEhlhMCCrv2fydIij5hqTOXleLRxlFbcOu5n\nlYNyHjyMVkcZq46xfng964fSYXj+uFKqcLB+ML9sr2uYOcDB+kFm27PUwhrVUpVaWFtwiOIoKf/0\n8r3e+4OmGlPEHlMKSoRBSMnScTqfpc20Zjgyd4TZ9uyiyq/XaGWU8do464bW5fcuLTSsXbWWyCPm\n2nP5MNua7ZpvxS1GKiNJ8FcZXXQw6O6043Y+xB7njQWloPSMtmshURzll0hmQ+xx51LL9BLLWlhT\nQ4MMBAUMMhCm5lrc+OFd7HzsEP/lFy/m+svPW+4syQoXe8y+o/t4eOJhHpx4kIcmHsrHT04/mS9n\nWFfFczmFQchQeQjDMDOMpKKSTWfj2OO85T5r0T+erNJVHEYqI4zXxhmvjbOmtqZrnA2rq6sZKg8x\nVB5iuDKcjMvJuFKqPKPKlLvPCyCyAO1Yw1RzKl9Hcd8U0wILGK2mFdTK6q6KajZUwyr1Zn3hoC2d\nPjJ3hIMzBzlYP8jEzMSiAhxIgqhihbsW1mi0GzSiRleFutFO5mfbs4RB2PVQgGKFNUsrBSXacZso\njvLKdORRV+V6qDzUVXbjtfG8NyEry0bUYHJuMt/32bjYEzExM9F179Jce+6Ey/h4VoWrGCoPdW1D\nK2odcz8HFnT1MmZDuVSmZCUCCygF6dhKXdOtuNV1LGX/V3M8JSt1HtpQHWWoPMSqcBWryqtYFa6i\nFtby6Sy9GEBn+auUKl2BtNMdGBX3QVa25aDMqvKqruAy+8xsyNYfBuGCAXu5VFbPkQAKGGSAzDYj\nfv1v7+YbPzzIf/qFi7jxZzYvd5ZkQB2ZO8JDEw/x0MRD7Dm0J79MY3V19bxLZMZqY0mFzUrzKu7Q\nqcw7SSU4e2JUvVWn3qwn8+l0M2rmFY/i5TDZ+NlUwCOPui4DcneqYZVqqdq3h0YWJ/Y4CSDqB/Mg\n4uDMQZpRk/VD67ta389YdcZJbQlfbtnjkos9KRMzE5RL5b49JrWwRhiE1Jv1rmCveFnY0cZR6q16\nXtHNekqKld/suG1Frb73MmVD7DGRR8k4jvL5bLoUlLoue+u9BG51dTVm1reHJ5ufac0w257Ne0Gz\ncdbLMtuenXc55XIrB52yqobdPV3VUpVqWCWwYN6QBeCBBZgZ7k7scVePZ+xxVy/rQmVYnA4s6A6S\nfH7QlPVcDoVJQ8Wq8qq80SIbsvNa75B9dhiEVEvV/L3FIO9kfj/bcTsPQIvn++whH8W0N7/0zZw3\ntnyNoQoYZKA02hFv/8S9fOGBJ/nNV72At121RRUdERFZMWKP856/ZtScN92Mml2Xk3VVeNOKdSko\n0YyaC17yVQxQ+t03VPzMBXu4CvONdiMPBGKPu4YsSIg9zgOHrNEkCyqKjSa9PSXF+5haUYvY465K\n/UJDFEfMtmfze6ay+6ROlkqpkgcQvcFE77gW1phtz3YuVe3pEZ1pzSz6c7/yq1/h6uddfdK240Sd\nSMDw7O8eE1li1bDEe990Kb/96fv471/+IdPNNjdfs1VBg4iIrAiBBUnvHs/uRvoT+ePMQdeO23kP\nTtZb23UpV9yad2lXI2ow25rt6hVacLrQUzQ5N8mT7Se7epCK91KtHVrL5jWbux7gkN1fld2/lt23\n1jtfKVWWezcumgIGWRHCUsCfvOEShqol/vIbj1BvtPnD172EIFDQICIi8lwTBmFy6WZ1FIaXOzeD\nTwGDrBhBYPzRtS9huBryl994hL2HZnnzK87nZ1+wjmo4ONcHi4iIiJxOFDDIimJm3HzNVtaPVHn/\nHXv4Nx/ZxepayGsvPovXbTubl29eS0m9DiIiIiInjW56lhWrFcV8c/cEt9+7ny898CT1ZsSG0Sr/\n7JKzed0lZ/PSc8d0n4OIiIjIAvSUJHnOmW1GfPWhp7j93v3c8fBBmlHMBWuHeMk5YzTbMc0oTsY9\n0412zPhQmQvWDfO8dcNsXjecT48PrZybkUREREROxGkTMJjZNcD/AErALe7+7p7Xq8BHgJ8Cngau\nc/fHjrVOBQxyPJMzLb7wwBP87+89wf4js1TCIBlKwfzpUsDT9SaPTtTZd3iGuPB1GB8qs3ndMJvX\nDnPmWI0zhiusGaok4+EKZwxVWDNcZqTa/3n2Uey0oiRIabVjInfcIXYnipPpKHZid2JPnm9uZpQC\no2RGEFCYTsalkhEGRrkUEAamXhQRERE5YafFY1XNrAS8D3gVsA/YaWa3u/sPCovdCBx29y1mdj3w\nx8B1S5UneW4YGypz3WXncd1lJ/ZnKM12zOOHZnhsos6jE3UefbrOowfrfPuRpzk41aAdLxxcV0oB\n40NlzKAVOa20F6MVxfR5y0kVBkZYMspBQFgywlISCI3WQkaqISO1kNFaEtjkadWQoUqJwAwz0udn\ngxlpWvIXZPnrpH9Glk533pekh1kwEyRDECRpQSEtWQdAzzrT9QHpn/+AQ/rHP5C0aWQ7shNMmdH5\nPMvGUC4F6TD4wZSngWfkThx377/s9c4+TARpGQeFfRjk5Z/8EVMW1MbpuDPfCWyLrxXns7ROvpJx\nFCd5zKfd82OgN09WmO8E1J33RTFd6dn2ec++6ZUdD51jkfSZ8eTp9EkvHv/FPJsVvyfpc+ejQkNB\n5LTT80EznW5HnX2ZNCJk29Yp03acrKcdJ+toRXGynjg5x7RjzxsYinm3ru8XlIKASnpeKJe6p8ul\npOEhv+3Lsr8eJN+ebLqzXwvThT2eHSPtqJP/KI7Tsefj4+ktt35tmv3OM8UGluTc0imvbPuK57Wu\nz+qTp/wctcD5LyjsnK6yoPsY6z23ZduWpxe+T9l3pHhcZMd6J0/zyyb7/Hnvi+cfa174fArfHy98\nnzrrnb9tWPKG+d9z8uk47s7zsWS/O/m5IOj5nhW2r3ff5nnsyfNCZcIC78uSs+nY020o7MOoZ5uO\nt1nF/ZudIz1dd+e87LzjlS/gBRtHF7WPlttS3vR8ObDb3R8BMLPbgGuBYsBwLfAH6fSngfeamflK\nu05KBkIlDNiyYYQtG0bmvebuTDXaHK43OVRvcnimyaF6K5mfaXK4nvyBTF5ZDY1K/qNc+GHOfszy\nk2L3tEGhckR3ZSs7aaU/vq12UnloR8mPciutiLTSS66mG22mG20O1Zs8/vQMR+faTDdazLXiU7xn\nl08l2/dhVllK5oPirwTFH/hOhakYtOAsEMR0Apas0hKWOhWWPFDq8wNV1OqqWKZlGTvNdkw7Tso1\ncieKOhWv7LiQwWZZEBxkFf20dzE9jnsrndB9nLbjNOhoJ8dWMzr13//AIAyCpBI4r5o+X+93ZKGK\nffGcqK/B4hUDXaMTBPUG0N3nwWy6E+hkunrA8+CNPC37rGPJjuE47m18yOY701meihPFIKz4er98\nz39/d6NDto/6NU5lDSzHkzcypPs6C16z/W1m1Bun1z+AH8tSBgznAHsL8/uAl/dbxt3bZjYJrAUm\nljBfIifMzFhdK7O6Vub8tSv7gc+tKKbeaDPTjAotxOkJe4FW494WMCi0htHzw521JHpnOgt6jtW6\nFnt3q1x2gi22EhWDqaylursli7xVsxU5jXbSqttqd1p3s2CqKx/ZjvHuH5b5P6iFE336ljhtYWvH\ncVcwl00323HXZyz0owZJhbBWDgirYR5g5pXDrOcoMEpBkPwYB0E6b10/ZMWW5XktaGnvAXR+jLNW\nzGJLpBV+EIOg04Ke/YgC3Z/X1UPQ2U/ze3+yvHZ+jLN9GKctn3HcyRvp8Vj88c4C6/xyvULQ3TG/\nFTIr02zPFysHvZXs4rHZOS6y8u4cx8U8F+fBqITZJYNBPl1sOOgts5IVKiKF/ZMtezIVezCySyW7\nWpt7tjlLtz77tZi7JHgO8ssmi5XJpdTb09aO464et+xcFaeF1ymvhYKT7oRiz0Aczz//Fc+J2TuK\n89m+LX43F+plzb9n6fFcSlvc80pr0Glp7z2fFI9jnK5jKFtvVvEd9F5XWTpLGTAsdFT2tgMsZhnM\n7K3AWwHOO+/ELjMRkW7lUsD4UIVx/WGoyHOOpT1hYQlq5cH4/5p8m/KUwdgukdNJsITr3gdsKsyf\nC+zvt4yZhcAYcKh3Re7+QXff7u7b169fv0TZFRERERGRXksZMOwELjSzzWZWAa4Hbu9Z5nbghnT6\n9cDXdP+CiIiIiMjpY8kuSUrvSXgb8EWS/sEPufsDZvaHwC53vx24Ffiome0m6Vm4fqnyIyIiIiIi\nJ24p72HA3T8PfL4n7fcK03PAG5YyDyIiIiIi8swt5SVJIiIiIiKywilgEBERERGRvhQwiIiIiIhI\nXwoYRERERESkLwUMIiIiIiLSlwIGERERERHpSwGDiIiIiIj0pYBBRERERET6UsAgIiIiIiJ9KWAQ\nEREREZG+FDCIiIiIiEhfChhERERERKQvBQwiIiIiItKXAgYREREREenL3H2583BCzOwg8ONlzMI6\nYGIZP1+Wjsp2cKlsB5PKdXCpbAeXyvb0cb67r1/MgisuYFhuZrbL3bcvdz7k5FPZDi6V7WBSuQ4u\nle3gUtmuTLokSURERERE+lLAICIiIiIifSlgOHEfXO4MyJJR2Q4ule1gUrkOLpXt4FLZrkC6h0FE\nRERERPpSD4OIiIiIiPSlgGGRzOwaM3vYzHab2c3LnR85PjP7kJkdMLP7C2lnmNmXzexH6XhNmm5m\n9udp+d5nZi8rvOeGdPkfmdkNy7Et0s3MNpnZ183sQTN7wMzenqarfFc4M6uZ2XfM7Htp2f7nNH2z\nmf1jWk5/Z2aVNL2azu9OX7+gsK53pukPm9lrlmeLpMjMSmb2XTP7h3Re5ToAzOwxM/u+md1rZrvS\nNJ2PB4gChkUwsxLwPuDngYuAN5rZRcubK1mEvwGu6Um7Gfiqu18IfDWdh6RsL0yHtwIfgOSEB/w+\n8HLgcuD3s5OeLKs28Jvu/iLgFcBN6XdS5bvyNYCr3P0SYBtwjZm9Avhj4E/Tsj0M3JgufyNw2N23\nAH+aLkd6PFwPvJjkPPD+9Fwuy+vtwIOFeZXr4Pgn7r6t8MhUnY8HiAKGxbkc2O3uj7h7E7gNuHaZ\n8yTH4e7/DzjUk3wt8OF0+sPAPy+kf8QTdwHjZnYW8Brgy+5+yN0PA19mfhAip5i7P+Hu96TTUyQV\nkHNQ+a54aRlNp7PldHDgKuDTaXpv2WZl/mngajOzNP02d2+4+6PAbpJzuSwTMzsX+KfALem8oXId\nZDofDxAFDItzDrC3ML8vTZOVZ6O7PwFJpRPYkKb3K2OV/WkuvVThUuAfUfkOhPSylXuBAySVhj3A\nEXdvp4sUyykvw/T1SWAtKtvT0Z8Bvw3E6fxaVK6DwoEvmdndZvbWNE3n4wESLncGVghbIE2Plxos\n/cpYZX8aM7MR4DPAO9z9aNIAufCiC6SpfE9T7h4B28xsHPgc8KKFFkvHKtsVwMx+ATjg7neb2ZVZ\n8gKLqlxXpivcfb+ZbQC+bGYPHWNZle0KpB6GxdkHbCrMnwvsX6a8yLPzVNr1STo+kKb3K2OV/WnK\nzMokwcLH3P2zabLKd4C4+xHgDpL7VMbNLGvkKpZTXobp62MklyKqbE8vVwCvM7PHSC7rvYqkx0Hl\nOgDcfX86PkAS5F+OzscDRQHD4uwELkyf5lAhueHq9mXOkzwztwPZkxduAP6+kP4v06c3vAKYTLtQ\nvwi82szWpDdfvTpNk2WUXst8K/Cgu7+n8JLKd4Uzs/VpzwJmtgp4Jck9Kl8HXp8u1lu2WZm/Hvia\nJ38wdDtwffq0nc0kN1h+59RshfRy93e6+7nufgHJb+jX3P1XULmueGY2bGaj2TTJefR+dD4eKLok\naRHcvW1mbyM5cEvAh9z9gWXOlhyHmX0CuBJYZ2b7SJ6+8G7gk2Z2I/A48IZ08c8DryW5gW4GeAuA\nux8ysz8iCRoB/tDde2+kllPvCuBXge+n17oD/A4q30FwFvDh9Mk3AfBJd/8HM/sBcJuZvQv4LknA\nSDr+qJntJmmBvh7A3R8ws08CPyB5qtZN6aVOcnr5j6hcV7qNwOfSS0JD4OPu/gUz24nOxwND//Qs\nIiIiIiJ96ZIkERERERHpSwGDiIiIiIj0pYBBRERERET6UsAgIiIiIiJ9KWAQEREREZG+FDCIiKwg\nZrbWzO5NhyfN7CeF+coi1/HXZvbC4yxzk5n9yknK87Vp/r5nZj8ws187zvJXpc9nX+i1s8zs84V1\n3Z6mbzKzvzsZ+RURkW56rKqIyAplZn8ATLv7n/SkG8n5PV6WjHXnpQo8Cmx39/3p/Pnu/sNjvOdd\nwIS7/9kCr90K3OPu70vnX+ru9y1R9kVEBPUwiIgMBDPbYmb3m9lfAPcAZ5nZB81sl5k9YGa/V1j2\nm2a2zcxCMztiZu9OW+y/bWYb0mXeZWbvKCz/bjP7jpk9bGY70vRhM/tM+t5PpJ+1rSdrY4CR/PkW\n7t7IggUz22hmn03f9x0ze4WZPR/4NeC30l6JHT3rOwvYl81kwUK6/fem039d6HWZMLPfTdNvTj/n\nvuL+EBGRY1PAICIyOC4CbnX3S939J8DN7r4duAR4lZldtMB7xoBvuPslwLeBf91n3ebulwO/BWSV\n7X8HPJm+993Apb1vcvcDwBeBH5vZx83sjWaW/fb8OfBf0zz+MnCLu+8BbgH+m7tvc/c7e1b5XpJ/\ngv6amf2OmZ21wGe+xd23Ab8ITAAfMbPXAucBLwe2ATsWCEZERGQBChhERAbHHnffWZh/o5ndQ9Lj\n8CKSgKLXrLv/33T6buCCPuv+7ALL/AxwG4C7fw94YKE3uvu/Al4F7AJuBj6YvvRK4C/SnoH/Bawx\ns1X9Nw/c/fPA84Fb0+35rpmt7V0uXc+ngN9w973Aq4GfB75Lsj+2AC841meJiEgiXO4MiIjISVPP\nJszsQuDtwOXufsTM/haoLfCeZmE6ov/vQmOBZWyxGUsvHbrPzD4OPEhy2ZGl+SvmgeQWjGOu62ng\nY8DHzOwLJIFLb7DyV8Bt7v71Ql7f5e63LjbPIiKSUA+DiMhgWg1MAUfTy3ZeswSf8U2SS4kws4tZ\noAfDzFab2c8WkrYBP06nvwLcVFg2u/9hChhd6APN7OqsF8LMVgObgcd7lnk7UO65GfyLwI1mNpwu\nc66ZrVvkdoqIPKeph0FEZDDdA/wAuB94BPjWEnzG/yS5P+C+9PPuByZ7ljHgnWb2V8AsME3nPomb\ngA+Y2VtIfo++nqb9PfApM/sl4Kae+xguA95rZi2SRq8PuPt3zWxLYZn/AMxkN0ED73X3W8xsK3BX\n2oMxBbyJ5B4HERE5Bj1WVUREnhEzC4HQ3efSS6C+BFzo7u1lzpqIiJxE6mEQEZFnagT4aho4GPBv\nFSyIiAwe9TCIiIiIiEhfuulZRERERET6UsAgIiIiIiJ9KWAQEREREZG+FDCIiIiIiEhfChhERERE\nRKQvBQwiIiIiItLX/wdZSJae8cl4VQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ee68accdd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# File:        learning_curve.py\n",
    "#\n",
    "# Author:      Rohan Patel\n",
    "#\n",
    "# Date:        05/16/2018\n",
    "#\n",
    "# Description: This script plots a learning curve to evaluate our learned classifier. The learning \n",
    "#              curve is generated by plotting the error rate vs the training set size. It is used \n",
    "#              used to diagnose bias and variance and thereby diagnose whether our classifier shows\n",
    "#              any underfitting or overfitting symptoms.\n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import accuracy_score\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "def learning_curve(feature_vect, messages, folds):\n",
    "    training_set_size = []\n",
    "    train_error = []\n",
    "    cv_error = []\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"This may take some time, please wait..\", sep=' ', end='', flush=True)\n",
    "    for training_size in range(10, len(messages), 75):\n",
    "        print('.', sep=' ', end='', flush=True)\n",
    "        X = feature_vect[:training_size]\n",
    "        y = messages['label'][:training_size]\n",
    "        kf = KFold(n_splits=folds, random_state=None, shuffle=False)\n",
    "        err1 = []\n",
    "        err2 = []\n",
    "\n",
    "        # generate K folds and train classifier on every fold\n",
    "        for train_index, test_index in kf.split(X):\n",
    "            #print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n",
    "            X_train, X_test = X[train_index], X[test_index]\n",
    "            y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "            mnb = MultinomialNB(alpha = 0.10000000000000001)\n",
    "            mnb.fit(X_train, y_train)\n",
    "            pred = mnb.predict(X_test)\n",
    "\n",
    "            err = 1.0 - (mnb.score(X_train,y_train))\n",
    "            err1.append(err) # training error\n",
    "            err = 1.0 - (mnb.score(X_test,y_test))\n",
    "            err2.append(err) # cv error\n",
    "\n",
    "        train_error.append(np.average(err1)) # training error =  avg of training error from every fold\n",
    "        cv_error.append(np.average(err2))    # cross-validation error = avg of cv error from every fold\n",
    "        training_set_size.append(training_size)\n",
    "\n",
    "    # Plotting of learning curve using matplotlib\n",
    "    x = training_set_size\n",
    "    y = train_error\n",
    "    z = cv_error\n",
    "\n",
    "    fig = plt.figure(figsize=(13,8))\n",
    "    plt.plot(x,y, label='Training error')\n",
    "    plt.plot(x,z,'g-', label='Cross-Validation error')\n",
    "    plt.xlabel('Training Set Size')\n",
    "    plt.ylabel('Error Rate')\n",
    "    plt.title('Learning Curve')\n",
    "    plt.legend()\n",
    "    plt.show(block = True)\n",
    "\n",
    "def main():\n",
    "\n",
    "    tfidf_vect = pickle.load(open(\"output/tfidf_vector.pickle\", \"rb\"))\n",
    "    messages = pd.read_csv('output/processed_msgs.csv')\n",
    "    \n",
    "    # append our message length feature to the tfidf vector to produce the final feature vector we fit into our classifiers\n",
    "    len_feature = messages['length'].as_matrix()\n",
    "    feat_vect = np.hstack((tfidf_vect.todense(), len_feature[:, None]))\n",
    "\n",
    "    learning_curve(feat_vect, messages, folds = 5)\n",
    "    #learning_curve(feat_vect, messages, folds = 10)\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Test learned classifier on 2nd Dataset to check for bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "################# Multinomial NB #################\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.00      0.00      0.00         0\n",
      "       spam       1.00      0.89      0.94      1353\n",
      "\n",
      "avg / total       1.00      0.89      0.94      1353\n",
      "\n",
      "\n",
      "\n",
      "0.888396156689\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rohan\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1137: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "################# SVM #################\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.00      0.00      0.00         0\n",
      "       spam       1.00      0.84      0.91      1353\n",
      "\n",
      "avg / total       1.00      0.84      0.91      1353\n",
      "\n",
      "\n",
      "\n",
      "0.835181079084\n"
     ]
    }
   ],
   "source": [
    "# File:        check_bias.py\n",
    "#\n",
    "# Author:      Rohan Patel\n",
    "#\n",
    "# Date:        05/17/2018\n",
    "#\n",
    "# Description: This script loads a second message dataset that has been scraped from Dublin Institute\n",
    "#              of Technology's website in order to check the bias of our learned SVM and Multinomial\n",
    "#              NB classifiers. The classfiers are trained on the UCI dataset and tested on the Dublin \n",
    "#              dataset.\n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from xml.dom import minidom\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "\n",
    "def load_messages2():\n",
    "    sms_list = []\n",
    "    \n",
    "    # parse spam.xml containing 2nd dataset\n",
    "    xmldoc = minidom.parse('smsspamcollection/spam.xml')\n",
    "    itemlist = xmldoc.getElementsByTagName('text')\n",
    "    \n",
    "    for msg in itemlist:\n",
    "        sms_list.append(msg.childNodes[0].nodeValue)\n",
    "    \n",
    "    messages2 = pd.DataFrame(data=sms_list, columns=['message'])\n",
    "    messages2['label'] = 'spam'\n",
    "    messages2['length'] = messages2['message'].apply(len)\n",
    "    return messages2\n",
    "\n",
    "def Tfidf_Vectorization(sms_data):\n",
    "    '''\n",
    "    1. Convert word tokens from processed msgs dataframe into a bag of words\n",
    "    2. Convert bag of words representation into tfidf vectorized representation for each message\n",
    "    '''\n",
    "    \n",
    "    bow_transformer = CountVectorizer(analyzer=text_process).fit(sms_data['message'])\n",
    "    bow = bow_transformer.transform(sms_data['message']) # bag of words\n",
    "\n",
    "    tfidf_transformer = TfidfTransformer().fit(bow)\n",
    "    tfidf_vect = tfidf_transformer.transform(bow) # tfidf vector representation\n",
    "    \n",
    "    return tfidf_vect\n",
    "\n",
    "def main():\n",
    "    \n",
    "    messages = pd.read_csv(\"output/processed_msgs.csv\")    \n",
    "    messages2 = load_messages2()\n",
    "    sms_data = pd.concat([messages, messages2]) # concatenate messages1 and messages2 to create a common tfidf feature vector\n",
    "    \n",
    "    tfidf_vect = Tfidf_Vectorization(sms_data) # create a large sparse tfidf feature vector\n",
    "    \n",
    "    # append our message length feature to the tfidf vector to produce the final feature vector we fit into our classifiers\n",
    "    len_feature = sms_data['length'].as_matrix()\n",
    "    feat_vect = np.hstack((tfidf_vect.todense(), len_feature[:, None]))\n",
    "    \n",
    "    X_train = feat_vect[:5572] # training set is comprised of entire UCI message dataset\n",
    "    y_train = messages['label']\n",
    "    X_test = feat_vect[5572:] # test set is comprised of entire Dublin spam message dataset\n",
    "    y_test = messages2['label']\n",
    "    \n",
    "    #Multinomial Naive Bayes\n",
    "    mnb = MultinomialNB(alpha = 0.10000000000000001)\n",
    "    mnb.fit(X_train, y_train)\n",
    "    pred = mnb.predict(X_test)\n",
    "    \n",
    "    print('\\n################# Multinomial NB #################\\n')\n",
    "    print(classification_report(y_test, pred))\n",
    "    print('\\n')\n",
    "    print(accuracy_score(y_test, pred))\n",
    "    \n",
    "    #SVM\n",
    "    svm = SVC(kernel = 'linear', gamma = 1)\n",
    "    svm.fit(X_train, y_train)\n",
    "    pred = svm.predict(X_test)\n",
    "    \n",
    "    print('\\n################# SVM #################\\n')\n",
    "    print(classification_report(y_test, pred))\n",
    "    print('\\n')\n",
    "    print(accuracy_score(y_test, pred))\n",
    "    \n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Address Imbalance of UCI Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "################# Multinomial NB #################\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.94      0.98      0.96       300\n",
      "       spam       0.97      0.92      0.94       225\n",
      "\n",
      "avg / total       0.95      0.95      0.95       525\n",
      "\n",
      "\n",
      "\n",
      "0.950476190476\n",
      "\n",
      "################# SVM #################\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        ham       0.95      0.96      0.95       300\n",
      "       spam       0.95      0.93      0.94       225\n",
      "\n",
      "avg / total       0.95      0.95      0.95       525\n",
      "\n",
      "\n",
      "\n",
      "0.946666666667\n"
     ]
    }
   ],
   "source": [
    "# File:        address_imbalance.py\n",
    "#\n",
    "# Author:      Rohan Patel\n",
    "#\n",
    "# Date:        05/22/2018\n",
    "#\n",
    "# Description: This script cuts down the UCI message dataset to 747 spam messages and 1000 ham messages\n",
    "#              to address the imbalance of the previously used dataset. The classifiers are re-trained \n",
    "#              to check if changing the composition of the dataset affects the final results.\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "\n",
    "def Generate_balanced_dataset(messages):\n",
    "    '''\n",
    "    generate a more balanced dataset containing 1000 ham and 747 spam messages\n",
    "    '''\n",
    "    ham_msg = messages[messages['label'] == 'ham'] # ham messages\n",
    "    spam_msg = messages[messages['label'] == 'spam'] # spam messages\n",
    "    \n",
    "    ham_msg = ham_msg.reset_index(drop=True)[:1000] # pick only top 1000 ham messages\n",
    "    spam_msg = spam_msg.reset_index(drop=True) # pick all 747 spam messages\n",
    "    \n",
    "    balanced_data = pd.concat([ham_msg, spam_msg]).sample(frac=1).reset_index(drop=True) # concatenate spam and ham messages to create a more balanced dataset\n",
    "    \n",
    "    return balanced_data\n",
    "\n",
    "def Tfidf_Vectorization(sms_data):\n",
    "    '''\n",
    "    1. Convert word tokens from processed msgs dataframe into a bag of words\n",
    "    2. Convert bag of words representation into tfidf vectorized representation for each message\n",
    "    '''\n",
    "    \n",
    "    bow_transformer = CountVectorizer(analyzer=text_process).fit(sms_data['message'])\n",
    "    bow = bow_transformer.transform(sms_data['message']) # bag of words\n",
    "\n",
    "    tfidf_transformer = TfidfTransformer().fit(bow)\n",
    "    tfidf_vect = tfidf_transformer.transform(bow) # tfidf vector representation\n",
    "    \n",
    "    return tfidf_vect\n",
    "\n",
    "def main():\n",
    "    \n",
    "    messages = pd.read_csv(\"output/processed_msgs.csv\")\n",
    "    \n",
    "    balanced_data = Generate_balanced_dataset(messages)\n",
    "    tfidf_vect = Tfidf_Vectorization(balanced_data)\n",
    "    \n",
    "    # append our message length feature to the tfidf vector to produce the final feature vector we fit into our classifiers\n",
    "    len_feature = balanced_data['length'].as_matrix()\n",
    "    feat_vect = np.hstack((tfidf_vect.todense(), len_feature[:, None]))\n",
    "    \n",
    "    X_train, X_test, y_train, y_test = train_test_split(feat_vect, balanced_data['label'], test_size=0.3, random_state=101)\n",
    "    \n",
    "    #Multinomial Naive Bayes\n",
    "    mnb = MultinomialNB(alpha = 0.10000000000000001)\n",
    "    mnb.fit(X_train, y_train)\n",
    "    pred = mnb.predict(X_test)\n",
    "    \n",
    "    print('\\n################# Multinomial NB #################\\n')\n",
    "    print(classification_report(y_test, pred))\n",
    "    print('\\n')\n",
    "    print(accuracy_score(y_test, pred))\n",
    "    \n",
    "    #SVM\n",
    "    svm = SVC(kernel = 'linear', gamma = 1)\n",
    "    svm.fit(X_train, y_train)\n",
    "    pred = svm.predict(X_test)\n",
    "    \n",
    "    print('\\n################# SVM #################\\n')\n",
    "    print(classification_report(y_test, pred))\n",
    "    print('\\n')\n",
    "    print(accuracy_score(y_test, pred))\n",
    "    \n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}