NLP/02_basic_text_features.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><ul class=\"toc-item\"><li><span><a href=\"#2.1.2)-Lemmatization\" data-toc-modified-id=\"2.1.2)-Lemmatization-0.1\"><span class=\"toc-item-num\">0.1&nbsp;&nbsp;</span>2.1.2) Lemmatization</a></span></li></ul></li><li><span><a href=\"#Features-for-documents\" data-toc-modified-id=\"Features-for-documents-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Features for documents</a></span><ul class=\"toc-item\"><li><span><a href=\"#From-docs-to-feature-vectors:-Make-your-own-countvectorizer\" data-toc-modified-id=\"From-docs-to-feature-vectors:-Make-your-own-countvectorizer-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>From docs to feature vectors: Make your own countvectorizer</a></span><ul class=\"toc-item\"><li><span><a href=\"#Customising-Vectoriser-classes\" data-toc-modified-id=\"Customising-Vectoriser-classes-1.1.1\"><span class=\"toc-item-num\">1.1.1&nbsp;&nbsp;</span>Customising Vectoriser classes</a></span></li><li><span><a href=\"#Exercise:-Build-a-Simple-countvectorizer\" data-toc-modified-id=\"Exercise:-Build-a-Simple-countvectorizer-1.1.2\"><span class=\"toc-item-num\">1.1.2&nbsp;&nbsp;</span>Exercise: Build a Simple countvectorizer</a></span></li></ul></li><li><span><a href=\"#Training-a-document-classifier-with-SimpleCountVectorizer\" data-toc-modified-id=\"Training-a-document-classifier-with-SimpleCountVectorizer-1.2\"><span class=\"toc-item-num\">1.2&nbsp;&nbsp;</span>Training a document classifier with <code>SimpleCountVectorizer</code></a></span><ul class=\"toc-item\"><li><span><a href=\"#I)-No-Stemmer-and-no-doc_cleaner\" data-toc-modified-id=\"I)-No-Stemmer-and-no-doc_cleaner-1.2.1\"><span class=\"toc-item-num\">1.2.1&nbsp;&nbsp;</span>I) No Stemmer and no doc_cleaner</a></span></li><li><span><a href=\"#II)-No-stemmer-but-doc_cleaner\" data-toc-modified-id=\"II)-No-stemmer-but-doc_cleaner-1.2.2\"><span class=\"toc-item-num\">1.2.2&nbsp;&nbsp;</span>II) No stemmer but doc_cleaner</a></span></li><li><span><a href=\"#III)-Use-a-SnowballStemmer\" data-toc-modified-id=\"III)-Use-a-SnowballStemmer-1.2.3\"><span class=\"toc-item-num\">1.2.3&nbsp;&nbsp;</span>III) Use a SnowballStemmer</a></span></li><li><span><a href=\"#Table-with-results-for-each-pipeline\" data-toc-modified-id=\"Table-with-results-for-each-pipeline-1.2.4\"><span class=\"toc-item-num\">1.2.4&nbsp;&nbsp;</span>Table with results for each pipeline</a></span></li></ul></li><li><span><a href=\"#Ngram-features-with-Sklearn-vectorizer\" data-toc-modified-id=\"Ngram-features-with-Sklearn-vectorizer-1.3\"><span class=\"toc-item-num\">1.3&nbsp;&nbsp;</span>Ngram features with Sklearn vectorizer</a></span><ul class=\"toc-item\"><li><span><a href=\"#Training-a-document-classifier-with-sklearn-CountVectorizer\" data-toc-modified-id=\"Training-a-document-classifier-with-sklearn-CountVectorizer-1.3.1\"><span class=\"toc-item-num\">1.3.1&nbsp;&nbsp;</span>Training a document classifier with sklearn <code>CountVectorizer</code></a></span></li><li><span><a href=\"#V)-Training-a-document-classifier-with-sklearn-CountVectorizer-and-ngrams\" data-toc-modified-id=\"V)-Training-a-document-classifier-with-sklearn-CountVectorizer-and-ngrams-1.3.2\"><span class=\"toc-item-num\">1.3.2&nbsp;&nbsp;</span>V) Training a document classifier with sklearn <code>CountVectorizer</code> and ngrams</a></span></li></ul></li></ul></li><li><span><a href=\"#Feature-selection\" data-toc-modified-id=\"Feature-selection-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Feature selection</a></span><ul class=\"toc-item\"><li><span><a href=\"#SelectKbest\" data-toc-modified-id=\"SelectKbest-2.1\"><span class=\"toc-item-num\">2.1&nbsp;&nbsp;</span>SelectKbest</a></span></li><li><span><a href=\"#Feature-Union\" data-toc-modified-id=\"Feature-Union-2.2\"><span class=\"toc-item-num\">2.2&nbsp;&nbsp;</span>Feature Union</a></span></li></ul></li><li><span><a href=\"#Crossvalidating-results-(Exercise)\" data-toc-modified-id=\"Crossvalidating-results-(Exercise)-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Crossvalidating results (Exercise)</a></span></li><li><span><a href=\"#Hashing-words-(Exercise)\" data-toc-modified-id=\"Hashing-words-(Exercise)-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>Hashing words (Exercise)</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:33:41.174892Z",
     "start_time": "2021-03-01T16:33:40.065410Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in /Users/qijunjin/miniforge3/lib/python3.9/site-packages (3.7)\n",
      "Requirement already satisfied: tqdm in /Users/qijunjin/miniforge3/lib/python3.9/site-packages (from nltk) (4.62.3)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /Users/qijunjin/miniforge3/lib/python3.9/site-packages (from nltk) (2022.1.18)\n",
      "Requirement already satisfied: joblib in /Users/qijunjin/miniforge3/lib/python3.9/site-packages (from nltk) (1.1.0)\n",
      "Requirement already satisfied: click in /Users/qijunjin/miniforge3/lib/python3.9/site-packages (from nltk) (8.0.4)\n"
     ]
    }
   ],
   "source": [
    "!pip install nltk\n",
    "import sys\n",
    "import string\n",
    "import nltk\n",
    "import numpy as np\n",
    "\n",
    "import sklearn\n",
    "from sklearn.feature_selection import SelectKBest, VarianceThreshold\n",
    "from sklearn.feature_selection import chi2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:33:44.142334Z",
     "start_time": "2021-03-01T16:33:44.140271Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sklearn.__version__=1.0.2\n"
     ]
    }
   ],
   "source": [
    "print(f'sklearn.__version__={sklearn.__version__}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:33:44.743707Z",
     "start_time": "2021-03-01T16:33:44.741500Z"
    }
   },
   "outputs": [],
   "source": [
    "from nltk.stem import PorterStemmer\n",
    "from nltk.stem import LancasterStemmer\n",
    "\n",
    "porter    = PorterStemmer()\n",
    "lancaster = LancasterStemmer()\n",
    "# nltk.download()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:34:17.888694Z",
     "start_time": "2021-03-01T16:34:17.884392Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word                PorterStemmer       LancasterStemmer    \n",
      "\n",
      "dogs                dog                 dog                 \n",
      "destabilize         destabil            dest                \n",
      "misunderstanding    misunderstand       misunderstand       \n",
      "railroad            railroad            railroad            \n",
      "moonlight           moonlight           moonlight           \n",
      "football            footbal             footbal             \n",
      "pass                pass                pass                \n",
      "passing             pass                pass                \n",
      "friendship          friendship          friend              \n",
      "friends             friend              friend              \n",
      "friendships         friendship          friend              \n",
      "passed              pass                pass                \n",
      "trouble             troubl              troubl              \n",
      "troubling           troubl              troubl              \n",
      "care                care                car                 \n",
      "believes            believ              believ              \n"
     ]
    }
   ],
   "source": [
    "words = [\"dogs\",\"destabilize\",\"misunderstanding\",\"railroad\",\"moonlight\",\n",
    "         \"football\",\"pass\",\"passing\",\"friendship\", \"friends\", \"friendships\",\n",
    "         \"passed\",\"trouble\",\"troubling\",\"care\", \"believes\"]\n",
    "preprocess = [porter, lancaster]\n",
    "\n",
    "len_bin = 20\n",
    "col_formater = \"{0:len_bin}{1:len_bin}{2:len_bin}\".replace(\"len_bin\",str(len_bin))\n",
    "print(col_formater.format(\"Word\", porter.__class__.__name__, lancaster.__class__.__name__))\n",
    "print(\"\")\n",
    "for w in words:\n",
    "    print( col_formater.format(w, porter.stem(w), lancaster.stem(w)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:35:10.032346Z",
     "start_time": "2021-03-01T16:35:10.029796Z"
    }
   },
   "outputs": [],
   "source": [
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "\n",
    "def stem(sentence):\n",
    "    token_words = word_tokenize(sentence)\n",
    "    sentence_stemmed = []\n",
    "    for word in token_words:\n",
    "        sentence_stemmed.append(porter.stem(word))\n",
    "        sentence_stemmed.append(\" \")\n",
    "    return \"\".join(sentence_stemmed)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:35:15.655313Z",
     "start_time": "2021-03-01T16:35:15.631575Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'j.k. rowl wrote harri potter . she never expect the book to be famou . '"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = \"J.K. Rowling wrote Harry Potter. She never expected the book to be famous.\"\n",
    "stem(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:35:16.898070Z",
     "start_time": "2021-03-01T16:35:16.893744Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['J.K. Rowling wrote Harry Potter.',\n",
       " 'She never expected the book to be famous.']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent_tokenize(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:09.482102Z",
     "start_time": "2021-03-01T12:00:09.479536Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['J',\n",
       " 'K',\n",
       " ' Rowling wrote Harry Potter',\n",
       " ' She never expected the book to be famous',\n",
       " '']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Be carefull separating phrases\n",
    "s.split(\".\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:09.487212Z",
     "start_time": "2021-03-01T12:00:09.484157Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['J.K.',\n",
       " 'Rowling',\n",
       " 'wrote',\n",
       " 'Harry',\n",
       " 'Potter',\n",
       " '.',\n",
       " 'She',\n",
       " 'never',\n",
       " 'expected',\n",
       " 'the',\n",
       " 'book',\n",
       " 'to',\n",
       " 'be',\n",
       " 'famous',\n",
       " '.']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_tokenize(s)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1.2) Lemmatization\n",
    "\n",
    "\n",
    "Lemmatization consist on properly use of a vocabulary and morphological analysis of words, aiming to remove inflectional endings only with the goal of returning any word to a set of base (or dictionary form) words.\n",
    "\n",
    "\n",
    "`Lemmatize(saw) = see`\n",
    "\n",
    "\n",
    "We will use a lemmatizer from WordNet (https://wordnet.princeton.edu) avaliable from nltk.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:37:13.974010Z",
     "start_time": "2021-03-01T16:37:13.972112Z"
    }
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "wordnet_lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:37:14.212960Z",
     "start_time": "2021-03-01T16:37:14.210349Z"
    }
   },
   "outputs": [],
   "source": [
    "sentence = \"I was running and eating. This was a terrible idea.\"\n",
    "punctuations=\"?:!.,;\"\n",
    "sentence_words = nltk.word_tokenize(sentence)\n",
    "for word in sentence_words:\n",
    "    if word in punctuations:\n",
    "        sentence_words.remove(word)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:37:15.497196Z",
     "start_time": "2021-03-01T16:37:14.495792Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word                Lemma               \n",
      "I                   I                   \n",
      "was                 wa                  \n",
      "running             running             \n",
      "and                 and                 \n",
      "eating              eating              \n",
      "This                This                \n",
      "was                 wa                  \n",
      "a                   a                   \n",
      "terrible            terrible            \n",
      "idea                idea                \n"
     ]
    }
   ],
   "source": [
    "sentence_words\n",
    "print(\"{0:20}{1:20}\".format(\"Word\",\"Lemma\"))\n",
    "for word in sentence_words:\n",
    "    print (\"{0:20}{1:20}\".format(word,wordnet_lemmatizer.lemmatize(word)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice that the words did no change!\n",
    "\n",
    "This is because there was no context. If we give a part of speech type then the lemmatizer will do what we would expect."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:37:19.701471Z",
     "start_time": "2021-03-01T16:37:19.698730Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word                Lemma               \n",
      "I                   I                   \n",
      "was                 be                  \n",
      "running             run                 \n",
      "and                 and                 \n",
      "eating              eat                 \n",
      "This                This                \n",
      "was                 be                  \n",
      "a                   a                   \n",
      "terrible            terrible            \n",
      "idea                idea                \n"
     ]
    }
   ],
   "source": [
    "sentence_words\n",
    "print(\"{0:20}{1:20}\".format(\"Word\",\"Lemma\"))\n",
    "for word in sentence_words:\n",
    "    print (\"{0:20}{1:20}\".format(word,wordnet_lemmatizer.lemmatize(word, pos=\"v\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:37:27.522658Z",
     "start_time": "2021-03-01T16:37:27.517302Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word                PorterStemmer       LancasterStemmer    WordNetLemmatizer   \n",
      "\n",
      "dogs                dog                 dog                 dog                 \n",
      "destabilize         destabil            dest                destabilize         \n",
      "misunderstanding    misunderstand       misunderstand       misunderstanding    \n",
      "railroad            railroad            railroad            railroad            \n",
      "moonlight           moonlight           moonlight           moonlight           \n",
      "football            footbal             footbal             football            \n",
      "pass                pass                pass                pas                 \n",
      "passing             pass                pass                passing             \n",
      "friendship          friendship          friend              friendship          \n",
      "friends             friend              friend              friend              \n",
      "friendships         friendship          friend              friendship          \n",
      "passed              pass                pass                passed              \n",
      "trouble             troubl              troubl              trouble             \n",
      "troubling           troubl              troubl              troubling           \n",
      "care                care                car                 care                \n",
      "believes            believ              believ              belief              \n"
     ]
    }
   ],
   "source": [
    "words = [\"dogs\",\"destabilize\",\"misunderstanding\",\"railroad\",\"moonlight\",\n",
    "         \"football\",\"pass\",\"passing\",\"friendship\", \"friends\", \"friendships\",\n",
    "         \"passed\",\"trouble\",\"troubling\",\"care\", \"believes\"]\n",
    "preprocess = [porter, lancaster, wordnet_lemmatizer]\n",
    "\n",
    "len_bin = 20\n",
    "col_formater = \"{0:len_bin}{1:len_bin}{2:len_bin}{3:len_bin}\".replace(\"len_bin\",str(len_bin))\n",
    "print(col_formater.format(\"Word\", porter.__class__.__name__, lancaster.__class__.__name__, wordnet_lemmatizer.__class__.__name__))\n",
    "print(\"\")\n",
    "for w in words:\n",
    "    print( col_formater.format(w, porter.stem(w), lancaster.stem(w), wordnet_lemmatizer.lemmatize(w)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Features for documents\n",
    "\n",
    "\n",
    "### From docs to feature vectors: Make your own countvectorizer\n",
    "\n",
    "\n",
    "Let us build a simple document classifier featurizing each document by word counts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:37:44.041057Z",
     "start_time": "2021-03-01T16:37:44.032200Z"
    }
   },
   "outputs": [],
   "source": [
    "import sklearn.linear_model\n",
    "import sklearn.model_selection\n",
    "import sklearn.pipeline\n",
    "import sklearn.feature_extraction\n",
    "import sklearn.datasets\n",
    "import scipy\n",
    "import scipy.sparse as sp\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:37:45.673369Z",
     "start_time": "2021-03-01T16:37:44.640009Z"
    }
   },
   "outputs": [],
   "source": [
    "X = sklearn.datasets.fetch_20newsgroups()\n",
    "\n",
    "X_train = sklearn.datasets.fetch_20newsgroups(subset=\"train\").data\n",
    "y_train = sklearn.datasets.fetch_20newsgroups(subset=\"train\").target\n",
    "X_test  = sklearn.datasets.fetch_20newsgroups(subset=\"test\").data\n",
    "y_test  = sklearn.datasets.fetch_20newsgroups(subset=\"test\").target\n",
    "\n",
    "#y_train = y_train.reshape(-1,1)\n",
    "#y_test = y_test.reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:37:51.000624Z",
     "start_time": "2021-03-01T16:37:50.998753Z"
    }
   },
   "outputs": [],
   "source": [
    "x = X_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:37:55.724925Z",
     "start_time": "2021-03-01T16:37:55.722195Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"From: lerxst@wam.umd.edu (where's my thing)\\nSubject: WHAT car is this!?\\nNntp-Posting-Host: rac3.wam.umd.edu\\nOrganization: University of Maryland, College Park\\nLines: 15\\n\\n I was wondering if anyone out there could enlighten me on this car I saw\\nthe other day. It was a 2-door sports car, looked to be from the late 60s/\\nearly 70s. It was called a Bricklin. The doors were really small. In addition,\\nthe front bumper was separate from the rest of the body. This is \\nall I know. If anyone can tellme a model name, engine specs, years\\nof production, where this car is made, history, or whatever info you\\nhave on this funky looking car, please e-mail.\\n\\nThanks,\\n- IL\\n   ---- brought to you by your neighborhood Lerxst ----\\n\\n\\n\\n\\n\""
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:11.541032Z",
     "start_time": "2021-03-01T12:00:11.535547Z"
    }
   },
   "outputs": [],
   "source": [
    "#x.split(\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:11.545231Z",
     "start_time": "2021-03-01T12:00:11.542259Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:11.550911Z",
     "start_time": "2021-03-01T12:00:11.546824Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['alt.atheism',\n",
       " 'comp.graphics',\n",
       " 'comp.os.ms-windows.misc',\n",
       " 'comp.sys.ibm.pc.hardware',\n",
       " 'comp.sys.mac.hardware',\n",
       " 'comp.windows.x',\n",
       " 'misc.forsale',\n",
       " 'rec.autos',\n",
       " 'rec.motorcycles',\n",
       " 'rec.sport.baseball',\n",
       " 'rec.sport.hockey',\n",
       " 'sci.crypt',\n",
       " 'sci.electronics',\n",
       " 'sci.med',\n",
       " 'sci.space',\n",
       " 'soc.religion.christian',\n",
       " 'talk.politics.guns',\n",
       " 'talk.politics.mideast',\n",
       " 'talk.politics.misc',\n",
       " 'talk.religion.misc']"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.target_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:11.554243Z",
     "start_time": "2021-03-01T12:00:11.552148Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From: lerxst@wam.umd.edu (where's my thing)\n",
      "Subject: WHAT car is this!?\n",
      "Nntp-Posting-Host: rac3.wam.umd.edu\n",
      "Organization: University of Maryland, College Park\n",
      "Lines: 15\n",
      "\n",
      " I was wondering if anyone out there could enlighten me on this car I saw\n",
      "the other day. It was a 2-door sports car, looked to be from the late 60s/\n",
      "early 70s. It was called a Bricklin. The doors were really small. In addition,\n",
      "the front bumper was separate from the rest of the body. This is \n",
      "all I know. If anyone can tellme a model name, engine specs, years\n",
      "of production, where this car is made, history, or whatever info you\n",
      "have on this funky looking car, please e-mail.\n",
      "\n",
      "Thanks,\n",
      "- IL\n",
      "   ---- brought to you by your neighborhood Lerxst ----\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(X[\"data\"][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Customising Vectoriser classes\n",
    "\n",
    "- **preprocessor**: a callable that takes an entire document as input (as a single string), and returns a possibly transformed version of the document, still as an entire string. This can be used to remove HTML tags, lowercase the entire document, etc.\n",
    "\n",
    "\n",
    "- **tokenizer**: a callable that takes the output from the preprocessor and splits it into tokens, then returns a list of these.\n",
    "\n",
    "\n",
    "- **analyzer**: a callable that replaces the preprocessor and tokenizer. The default analyzers all call the preprocessor and tokenizer, but custom analyzers will skip this. N-gram extraction and stop word filtering take place at the analyzer level, so a custom analyzer may have to reproduce these steps."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Example of how to encode sparse matrix fast\n",
    "\n",
    "\n",
    "Notice that in order to build our data as a matrix we need to use sparse matrices due to the high dimensionality (number of words/features) of the vocabulary.\n",
    "\n",
    "Here there is a little example to illustrate how we can build a csr_matrix (compressed sparse row matrix) fast."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:11.563258Z",
     "start_time": "2021-03-01T12:00:11.555525Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 2],\n",
       "       [2, 1, 0],\n",
       "       [0, 1, 3]])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = np.array([[1,0,2],[2,1,0],[0,1,3]])\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:11.575517Z",
     "start_time": "2021-03-01T12:00:11.569428Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[1, 0, 2],\n",
       "        [2, 1, 0],\n",
       "        [0, 1, 3]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = [1,2,2,1,1,3]\n",
    "row  = [0,0,1,1,2,2]\n",
    "#ind_ptr = [0,3,4,9]\n",
    "col = [0,2,0,1,1,2]\n",
    "sp.csr_matrix( (data,(row,col)), shape=(3,3) ).todense()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exercise: Build a Simple countvectorizer\n",
    "\n",
    "Complete methods `fit` and `transform`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:11.582315Z",
     "start_time": "2021-03-01T12:00:11.579307Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('From: jcm@head-cfa.harvard.edu (Jonathan McDowell)\\nSubject: Re: Shuttle Launch Question\\nOrganization: Smithsonian Astrophysical Observatory, Cambridge, MA,  USA\\nDistribution: sci\\nLines: 23\\n\\nFrom article <C5owCB.n3p@world.std.com>, by tombaker@world.std.com (Tom A Baker):\\n>>In article <C5JLwx.4H9.1@cs.cmu.edu>, ETRAT@ttacs1.ttu.edu (Pack Rat) writes...\\n>>>\"Clear caution & warning memory.  Verify no unexpected\\n>>>errors. ...\".  I am wondering what an \"expected error\" might\\n>>>be.  Sorry if this is a really dumb question, but\\n> \\n> Parity errors in memory or previously known conditions that were waivered.\\n>    \"Yes that is an error, but we already knew about it\"\\n> I\\'d be curious as to what the real meaning of the quote is.\\n> \\n> tom\\n\\n\\nMy understanding is that the \\'expected errors\\' are basically\\nknown bugs in the warning system software - things are checked\\nthat don\\'t have the right values in yet because they aren\\'t\\nset till after launch, and suchlike. Rather than fix the code\\nand possibly introduce new bugs, they just tell the crew\\n\\'ok, if you see a warning no. 213 before liftoff, ignore it\\'.\\n\\n - Jonathan\\n\\n\\n',\n",
       " 14)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[4], y_train[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:11.585220Z",
     "start_time": "2021-03-01T12:00:11.583601Z"
    }
   },
   "outputs": [],
   "source": [
    "# \"David's car\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:51:46.677229Z",
     "start_time": "2021-03-01T16:51:46.582125Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sklearn' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [1]\u001b[0m, in \u001b[0;36m<cell line: 9>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mre\u001b[39;00m\n\u001b[1;32m      6\u001b[0m stemmer \u001b[38;5;241m=\u001b[39m  SnowballStemmer(language\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124menglish\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m----> 9\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mSimpleCountVectorizer\u001b[39;00m(\u001b[43msklearn\u001b[49m\u001b[38;5;241m.\u001b[39mbase\u001b[38;5;241m.\u001b[39mBaseEstimator, sklearn\u001b[38;5;241m.\u001b[39mbase\u001b[38;5;241m.\u001b[39mTransformerMixin):\n\u001b[1;32m     11\u001b[0m     \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m     12\u001b[0m                  min_word_counts\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m,\n\u001b[1;32m     13\u001b[0m                  doc_cleaner_pattern\u001b[38;5;241m=\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m[^a-zA-Z0-9]\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     17\u001b[0m                  tokenizer_func\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m     18\u001b[0m                  word_transformer_func\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m     20\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_retype \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mtype\u001b[39m(re\u001b[38;5;241m.\u001b[39mcompile(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhello, world\u001b[39m\u001b[38;5;124m'\u001b[39m))\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sklearn' is not defined"
     ]
    }
   ],
   "source": [
    "import scipy\n",
    "import scipy.sparse as sp\n",
    "from nltk.stem import WordNetLemmatizer, SnowballStemmer\n",
    "from collections import defaultdict\n",
    "import re\n",
    "stemmer =  SnowballStemmer(language='english')\n",
    "\n",
    "\n",
    "class SimpleCountVectorizer(sklearn.base.BaseEstimator, sklearn.base.TransformerMixin):\n",
    "    \n",
    "    def __init__(self,\n",
    "                 min_word_counts=1,\n",
    "                 doc_cleaner_pattern=r\"[^a-zA-Z]\",\n",
    "                 token_pattern=r\"(?u)\\b\\w\\w+\\b\",\n",
    "                 dtype=np.float32,\n",
    "                 doc_cleaner_func=None,\n",
    "                 tokenizer_func=None,\n",
    "                 word_transformer_func=None):\n",
    "        \n",
    "        self._retype = type(re.compile('hello, world'))\n",
    "\n",
    "        self.min_word_counts     = min_word_counts\n",
    "        self.doc_cleaner_pattern = doc_cleaner_pattern\n",
    "        self.token_pattern       = token_pattern\n",
    "        self.dtype               = dtype\n",
    "        \n",
    "        self.doc_cleaner_func      = doc_cleaner_func\n",
    "        self.tokenizer_func        = tokenizer_func\n",
    "        self.word_transformer_func = word_transformer_func\n",
    "\n",
    "        self.word_to_pos = {}\n",
    "\n",
    "\n",
    "    def build_doc_cleaner(self, lower=True):\n",
    "        \"\"\"\n",
    "        Returns a function that cleans undesirable substrings in a string.\n",
    "        It also lowers the input string if lower=True\n",
    "        \"\"\"\n",
    "        if self.doc_cleaner_func:\n",
    "            return self.doc_cleaner_func\n",
    "        else:\n",
    "            if isinstance(self.doc_cleaner_pattern, self._retype):\n",
    "                #clean_doc_pattern = self.doc_cleaner_pattern.sub(\" \", doc)\n",
    "                clean_doc_pattern = re.compile(self.doc_cleaner_pattern)\n",
    "            else:\n",
    "                clean_doc_pattern = re.compile(self.doc_cleaner_pattern)\n",
    "\n",
    "            if lower:\n",
    "                 return lambda doc: clean_doc_pattern.sub(\" \", doc).lower()\n",
    "            else:\n",
    "                 return lambda doc: clean_doc_pattern.sub(\" \", doc)\n",
    "\n",
    "    def build_tokenizer(self):\n",
    "        \"\"\"Returns a function that splits a string into a sequence of tokens\"\"\"\n",
    "        if self.tokenizer_func:\n",
    "            return self.tokenizer_func\n",
    "        \n",
    "        else:\n",
    "            token_pattern = re.compile(self.token_pattern)\n",
    "            return lambda doc: token_pattern.findall(doc)\n",
    "\n",
    "    def build_word_transformer(self):\n",
    "        \"\"\"Returns a stemmer or lemmaitzer if object has any\"\"\"\n",
    "        \n",
    "        if self.word_transformer_func:\n",
    "            return self.word_transformer_func\n",
    "        else:\n",
    "            return lambda word: word\n",
    "        \n",
    "    def tokenize(self, doc):\n",
    "        doc_cleaner      = self.build_doc_cleaner()\n",
    "        doc_tokenizer    = self.build_tokenizer()\n",
    "        doc     = doc_cleaner(doc)\n",
    "        words = doc_tokenizer(doc)\n",
    "            \n",
    "        return words\n",
    "        \n",
    "    def fit(self, X):\n",
    "\n",
    "        assert isinstance(X,list), \"X is expected to be a list of documents\"\n",
    "        \n",
    "        i = 0\n",
    "        word_to_pos = {}\n",
    "        doc_cleaner      = self.build_doc_cleaner()\n",
    "        doc_tokenizer    = self.build_tokenizer()\n",
    "        word_transformer = self.build_word_transformer()\n",
    "        \n",
    "        for x in X:\n",
    "            x     = doc_cleaner(x)\n",
    "            words = doc_tokenizer(x)\n",
    "            for word in words:\n",
    "                word = word_transformer(word)                  \n",
    "                if word not in word_to_pos:\n",
    "                    word_to_pos[word] = i\n",
    "                    i = i + 1\n",
    "\n",
    "        #self.doc_cleaner = doc_cleaner\n",
    "        #self.doc_tokenizer = doc_tokenizer\n",
    "        #self.word_transformer = word_transformer\n",
    "        \n",
    "        self.word_to_pos = word_to_pos            \n",
    "        self.n_features = len(self.word_to_pos)\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        \n",
    "        doc_cleaner      = self.build_doc_cleaner()\n",
    "        doc_tokenizer    = self.build_tokenizer()\n",
    "        word_transformer = self.build_word_transformer()      \n",
    "        \n",
    "        col_indices = []\n",
    "        row_indices = []\n",
    "        sp_data     = []\n",
    "        \n",
    "        for m, doc in enumerate(X):\n",
    "            doc = doc_cleaner(doc)\n",
    "            word_ind_counter = defaultdict(int)\n",
    "            for word in doc_tokenizer(doc):\n",
    "                word = word_transformer(word)   \n",
    "                if word in self.word_to_pos:\n",
    "                    word_ind_counter[self.word_to_pos[word]] +=1 \n",
    "\n",
    "            sp_data.extend(word_ind_counter.values())\n",
    "            row_indices.extend([m]*len(word_ind_counter))\n",
    "            col_indices.extend(word_ind_counter.keys())\n",
    "\n",
    "        encoded_X = sp.csr_matrix((sp_data,(row_indices,col_indices)),\n",
    "                                   shape=(len(X), self.n_features),\n",
    "                                   dtype=self.dtype)\n",
    "        \n",
    "        return encoded_X\n",
    "    \n",
    "    def transform2(self, X):\n",
    "        \n",
    "        doc_cleaner      = self.build_doc_cleaner()\n",
    "        doc_tokenizer    = self.build_tokenizer()\n",
    "        word_transformer = self.build_word_transformer()                      \n",
    "        data = []\n",
    "        ind_col = []\n",
    "        ind_ptr = [0]\n",
    "        \n",
    "        for x in X:\n",
    "            x     = doc_cleaner(x)\n",
    "            words = doc_tokenizer(x)\n",
    "            for word in words:\n",
    "                word = word_transformer(word)    \n",
    "                if word in self.word_to_pos:\n",
    "                    index = self.word_to_pos[word]\n",
    "                    ind_col.append(index)\n",
    "                    data.append(1)\n",
    "            ind_ptr.append(len(ind_col))\n",
    "                                   \n",
    "        X_transformed = sp.csr_matrix((data, ind_col, ind_ptr),\n",
    "                                       shape=(len(X), self.n_features),\n",
    "                                       dtype=self.dtype)\n",
    "        return X_transformed\n",
    "    \n",
    "    def fit_transform2(self, X, y=None):\n",
    "        self.fit(X)\n",
    "        encoded_X = self.transform(X)\n",
    "        return encoded_X\n",
    "\n",
    "    def fit_transform(self, X, y=None):\n",
    "        word_to_pos = {}\n",
    "        doc_cleaner      = self.build_doc_cleaner()\n",
    "        doc_tokenizer    = self.build_tokenizer()\n",
    "        word_transformer = self.build_word_transformer()\n",
    "        \n",
    "        data = []\n",
    "        ind_col = []\n",
    "        ind_ptr = [0]\n",
    "        \n",
    "        for x in X:\n",
    "            x     = doc_cleaner(x)\n",
    "            words = doc_tokenizer(x)\n",
    "            for word in words:\n",
    "                word = word_transformer(word)                  \n",
    "                index = word_to_pos.setdefault(word, len(word_to_pos))\n",
    "                ind_col.append(index)\n",
    "                data.append(1)\n",
    "            ind_ptr.append(len(ind_col))\n",
    "                           \n",
    "        self.word_to_pos = word_to_pos            \n",
    "        self.n_features = len(self.word_to_pos)\n",
    "        self.word_to_pos = word_to_pos\n",
    "        \n",
    "        #self.doc_cleaner = doc_cleaner\n",
    "        #self.doc_tokenizer = doc_tokenizer\n",
    "        #self.word_transformer = word_transformer\n",
    "        \n",
    "        X_transformed = sp.csr_matrix((data, ind_col, ind_ptr))\n",
    "        return X_transformed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:51:49.811845Z",
     "start_time": "2021-03-01T16:51:47.114567Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SimpleCountVectorizer()"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simple_count_vectorizer = SimpleCountVectorizer()\n",
    "simple_count_vectorizer.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T16:52:29.842654Z",
     "start_time": "2021-03-01T16:52:29.839482Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.0"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_tf2 = simple_count_vectorizer.transform2([\"My house house\"])\n",
    "#np.where(x_tf2.toarray())\n",
    "x_tf2.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:14.111028Z",
     "start_time": "2021-03-01T12:00:14.105467Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 0]), array([   6, 1539]))"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_tf1 = simple_count_vectorizer.transform([\"My house house\"])\n",
    "np.where(x_tf1.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:14.115850Z",
     "start_time": "2021-03-01T12:00:14.112656Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3.0, 3.0)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_tf1.sum(), x_tf2.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:14.119149Z",
     "start_time": "2021-03-01T12:00:14.117057Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n",
      "1539\n"
     ]
    }
   ],
   "source": [
    "print(simple_count_vectorizer.word_to_pos['my'])\n",
    "print(simple_count_vectorizer.word_to_pos['house'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:14.123342Z",
     "start_time": "2021-03-01T12:00:14.120554Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x89012 sparse matrix of type '<class 'numpy.float32'>'\n",
       "\twith 2 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_tf1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:14.127238Z",
     "start_time": "2021-03-01T12:00:14.124718Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x89012 sparse matrix of type '<class 'numpy.float32'>'\n",
       "\twith 3 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_tf2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:14.132798Z",
     "start_time": "2021-03-01T12:00:14.128809Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.allclose(x_tf1.toarray(), x_tf2.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:20.409966Z",
     "start_time": "2021-03-01T12:00:14.134471Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train_ = simple_count_vectorizer.fit_transform2(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:20.413967Z",
     "start_time": "2021-03-01T12:00:20.411233Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11314, 89012)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:23.920912Z",
     "start_time": "2021-03-01T12:00:20.415606Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.41 s, sys: 59.6 ms, total: 3.47 s\n",
      "Wall time: 3.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "X_train_ = simple_count_vectorizer.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:30.000443Z",
     "start_time": "2021-03-01T12:00:23.923302Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.96 s, sys: 67.7 ms, total: 6.03 s\n",
      "Wall time: 6.07 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "X_train_ = simple_count_vectorizer.fit_transform2(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:36.001807Z",
     "start_time": "2021-03-01T12:00:30.002362Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train_ = simple_count_vectorizer.fit_transform2(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:36.005104Z",
     "start_time": "2021-03-01T12:00:36.002977Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#%%time \n",
    "#X_train_matrix = simple_count_vectorizer.transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:36.012833Z",
     "start_time": "2021-03-01T12:00:36.006702Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((11314, 89012), 11314)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_.shape, len(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Should a vector representation have the same number of items as words?\n",
    "\n",
    "There sum of the feature vector should equal to the number of words in a sentence if and only if all words belong to the vocabulary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:38.578688Z",
     "start_time": "2021-03-01T12:00:36.014092Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SimpleCountVectorizer()"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simple_count_vectorizer = SimpleCountVectorizer()\n",
    "simple_count_vectorizer.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:38.585652Z",
     "start_time": "2021-03-01T12:00:38.580902Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x89012 sparse matrix of type '<class 'numpy.float32'>'\n",
       "\twith 6 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = [\"this is a feature vector for this sentence\"]\n",
    "x = simple_count_vectorizer.transform(A)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:38.594035Z",
     "start_time": "2021-03-01T12:00:38.587408Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<2x89012 sparse matrix of type '<class 'numpy.float32'>'\n",
       "\twith 10 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = [\"this is a feature vector for this sentence\",\"This is another sentence\"]\n",
    "x = simple_count_vectorizer.transform(A)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:38.599221Z",
     "start_time": "2021-03-01T12:00:38.595757Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['this', 'is', 'feature', 'vector', 'for', 'this', 'sentence']"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simple_count_vectorizer.tokenize(A[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:38.604218Z",
     "start_time": "2021-03-01T12:00:38.600690Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['this', 'is', 'feature', 'vector', 'for', 'this', 'sentence']"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words_in_vocab = [w for w in simple_count_vectorizer.tokenize(A[0]) if w in simple_count_vectorizer.word_to_pos]\n",
    "words_in_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:38.608760Z",
     "start_time": "2021-03-01T12:00:38.605703Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7, 6)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words_in_vocab), len(set(words_in_vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:38.613833Z",
     "start_time": "2021-03-01T12:00:38.610189Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words_notin_vocab = [w for w in simple_count_vectorizer.tokenize(A[0]) if w not in simple_count_vectorizer.word_to_pos]\n",
    "words_notin_vocab"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Training a document classifier with `SimpleCountVectorizer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:40.019110Z",
     "start_time": "2021-03-01T12:00:38.615255Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SimpleCountVectorizer(doc_cleaner_func=<function <lambda> at 0x7f867276b160>)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vainilla_count_vectorizer = SimpleCountVectorizer( doc_cleaner_func=lambda doc: doc)\n",
    "vainilla_count_vectorizer.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:42.991870Z",
     "start_time": "2021-03-01T12:00:40.020675Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<11314x155448 sparse matrix of type '<class 'numpy.float32'>'\n",
       "\twith 1899366 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vainilla_count_vectorizer.transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:00:45.570892Z",
     "start_time": "2021-03-01T12:00:42.993973Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<11314x155448 sparse matrix of type '<class 'numpy.float32'>'\n",
       "\twith 3369673 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vainilla_count_vectorizer.transform2(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:01:26.948126Z",
     "start_time": "2021-03-01T12:00:45.573090Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davidbuchaca1/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=0.1)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_ = vainilla_count_vectorizer.transform(X_train)\n",
    "logistic = sklearn.linear_model.LogisticRegression(C=0.1)\n",
    "logistic.fit(X_train_, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:01:26.999629Z",
     "start_time": "2021-03-01T12:01:26.949918Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9982322785928938"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(logistic.predict(X_train_) == y_train.flatten())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### I) No Stemmer and no doc_cleaner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:05:12.089240Z",
     "start_time": "2021-03-01T12:05:12.086730Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "vainilla_count_vectorizer = SimpleCountVectorizer( doc_cleaner_func=lambda doc: doc)\n",
    "#vainilla_count_vectorizer = sklearn.feature_extraction.text.CountVectorizer()\n",
    "\n",
    "logistic = sklearn.linear_model.LogisticRegression(C=0.1)\n",
    "\n",
    "model_pipe_0 = sklearn.pipeline.Pipeline([(\"countvectorizer\", vainilla_count_vectorizer),\n",
    "                                         (\"logisticregression\", logistic)],\n",
    "                                         )#memory='/Users/Shared/sklearn_mem/')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:05:13.856064Z",
     "start_time": "2021-03-01T12:05:13.854619Z"
    }
   },
   "outputs": [],
   "source": [
    "#vainilla_count_vectorizer.fit(X_train[0:100])\n",
    "#logistic.fit(vainilla_count_vectorizer.transform(X_train[0:100]),y_train[0:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:05:18.973707Z",
     "start_time": "2021-03-01T12:05:15.527486Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davidbuchaca1/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<11314x7531 sparse matrix of type '<class 'numpy.float32'>'\n",
       "\twith 1283391 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_pipe_0.fit(X_train[0:100],y_train[0:100])\n",
    "model_pipe_0.steps[0][1].transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:06:09.444697Z",
     "start_time": "2021-03-01T12:05:20.635740Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davidbuchaca1/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 28s, sys: 2.67 s, total: 2min 30s\n",
      "Wall time: 48.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model_pipe_0.fit(X_train,y_train)\n",
    "y_test_pred  = model_pipe_0.predict(X_test)\n",
    "y_train_pred = model_pipe_0.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:06:13.725989Z",
     "start_time": "2021-03-01T12:06:11.087461Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<11314x155448 sparse matrix of type '<class 'numpy.float32'>'\n",
       "\twith 1899366 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_pipe_0.steps[0][1].transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:06:15.394958Z",
     "start_time": "2021-03-01T12:06:15.392235Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy train: 0.9982322785928938    Accuracy test: 0.7843866171003717\n"
     ]
    }
   ],
   "source": [
    "acc_train_0 = np.mean(y_train == y_train_pred)\n",
    "acc_test_0 = np.mean(y_test == y_test_pred)\n",
    "print(\"Accuracy train: {}    Accuracy test: {}\".format(acc_train_0, acc_test_0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:06:17.066923Z",
     "start_time": "2021-03-01T12:06:17.063652Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<1x155448 sparse matrix of type '<class 'numpy.float32'>'\n",
       " \twith 94 stored elements in Compressed Sparse Row format>,\n",
       " 155448)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_pipe_0.steps[0][1].transform(X_train[0:1]), model_pipe_0.steps[0][1].n_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:06:18.783093Z",
     "start_time": "2021-03-01T12:06:18.736322Z"
    }
   },
   "outputs": [],
   "source": [
    "vocabulary = list(model_pipe_0.steps[0][1].word_to_pos)\n",
    "vocabulary.sort()\n",
    "#vocabulary[0:500]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### II) No stemmer but doc_cleaner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:06:24.979150Z",
     "start_time": "2021-03-01T12:06:24.976884Z"
    }
   },
   "outputs": [],
   "source": [
    "simple_count_vectorizer = SimpleCountVectorizer(doc_cleaner_pattern=re.compile(\"[^a-zA-Z]\"))\n",
    "logistic = sklearn.linear_model.LogisticRegression(C=0.1)\n",
    "\n",
    "model_pipe_1 = sklearn.pipeline.Pipeline([(\"countvectorizer\", simple_count_vectorizer),\n",
    "                                        (\"logisticregression\", logistic)],\n",
    "                                         )#memory='/Users/Shared/sklearn_mem/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:06:52.680919Z",
     "start_time": "2021-03-01T12:06:26.928292Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 25s, sys: 896 ms, total: 1min 26s\n",
      "Wall time: 25.7 s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davidbuchaca1/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('countvectorizer',\n",
       "                 SimpleCountVectorizer(doc_cleaner_pattern=re.compile('[^a-zA-Z]'))),\n",
       "                ('logisticregression', LogisticRegression(C=0.1))])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "model_pipe_1.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:07:00.535058Z",
     "start_time": "2021-03-01T12:06:54.528964Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy train: 0.9975251900300512    Accuracy test: 0.7885023898035051\n"
     ]
    }
   ],
   "source": [
    "y_test_pred  = model_pipe_1.predict(X_test)\n",
    "y_train_pred = model_pipe_1.predict(X_train)\n",
    "\n",
    "acc_train_1 = np.mean(y_train == y_train_pred)\n",
    "acc_test_1 = np.mean(y_test == y_test_pred)\n",
    "\n",
    "print(\"Accuracy train: {}    Accuracy test: {}\".format(acc_train_1, acc_test_1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:07:02.428372Z",
     "start_time": "2021-03-01T12:07:02.423545Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<1x89012 sparse matrix of type '<class 'numpy.float32'>'\n",
       " \twith 86 stored elements in Compressed Sparse Row format>,\n",
       " 89012)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_pipe_1.steps[0][1].transform(X_train[0:1]), model_pipe_1.steps[0][1].n_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:07:04.402014Z",
     "start_time": "2021-03-01T12:07:04.364820Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['aa',\n",
       " 'aaa',\n",
       " 'aaaa',\n",
       " 'aaaaaaaaaaaa',\n",
       " 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaauuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuugggggggggggggggg',\n",
       " 'aaaaagggghhhh',\n",
       " 'aaaarrgghhhh',\n",
       " 'aaah',\n",
       " 'aaahh',\n",
       " 'aaahhhh']"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocabulary = list(model_pipe_1.steps[0][1].word_to_pos)\n",
    "vocabulary.sort()\n",
    "vocabulary[0:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### III) Use a SnowballStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:07:06.207058Z",
     "start_time": "2021-03-01T12:07:06.204178Z"
    }
   },
   "outputs": [],
   "source": [
    "simple_count_vectorizer_stemmer = SimpleCountVectorizer(word_transformer_func= SnowballStemmer('english').stem,\n",
    "                                                        doc_cleaner_pattern=re.compile(\"[^a-zA-Z]\"))\n",
    "\n",
    "logistic = sklearn.linear_model.LogisticRegression(C=0.1)\n",
    "\n",
    "model_pipe_2 = sklearn.pipeline.Pipeline([(\"countvectorizer\", simple_count_vectorizer_stemmer),\n",
    "                                        (\"logisticregression\", logistic)],\n",
    "                                         )#memory='/Users/Shared/sklearn_mem/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:08:46.522104Z",
     "start_time": "2021-03-01T12:07:08.034092Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davidbuchaca1/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy train: 0.9984090507336044    Accuracy test: 0.7887679235262879\n",
      "CPU times: user 2min 28s, sys: 1.99 s, total: 2min 30s\n",
      "Wall time: 1min 38s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model_pipe_2.fit(X_train,y_train)\n",
    "\n",
    "y_test_pred  = model_pipe_2.predict(X_test)\n",
    "y_train_pred = model_pipe_2.predict(X_train)\n",
    "\n",
    "acc_train_2 = np.mean(y_train == y_train_pred)\n",
    "acc_test_2  = np.mean(y_test == y_test_pred)\n",
    "\n",
    "print(\"Accuracy train: {}    Accuracy test: {}\".format(acc_train_2, acc_test_2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:08:48.542905Z",
     "start_time": "2021-03-01T12:08:48.536155Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<1x69965 sparse matrix of type '<class 'numpy.float32'>'\n",
       " \twith 84 stored elements in Compressed Sparse Row format>,\n",
       " 69965)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_pipe_2.steps[0][1].transform(X_train[0:1]), model_pipe_2.steps[0][1].n_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:08:50.387678Z",
     "start_time": "2021-03-01T12:08:50.360131Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['aa',\n",
       " 'aaa',\n",
       " 'aaaa',\n",
       " 'aaaaaaaaaaaa',\n",
       " 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaauuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuugggggggggggggggg',\n",
       " 'aaaaagggghhhh',\n",
       " 'aaaarrgghhhh',\n",
       " 'aaah',\n",
       " 'aaahh',\n",
       " 'aaahhhh']"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocabulary = list(model_pipe_1.steps[0][1].word_to_pos)\n",
    "vocabulary.sort()\n",
    "vocabulary[0:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Table with results for each pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:08:52.509004Z",
     "start_time": "2021-03-01T12:08:52.222112Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:08:54.275249Z",
     "start_time": "2021-03-01T12:08:54.270254Z"
    }
   },
   "outputs": [],
   "source": [
    "df_results = pd.DataFrame()\n",
    "df_results[\"no clean no stem\"]   = [acc_train_0, acc_test_0]\n",
    "df_results[\"yes clean no stem\"]  = [acc_train_1, acc_test_1]\n",
    "df_results[\"yes clean yes stem\"] = [acc_train_2, acc_test_2]\n",
    "df_results.index=[\"train\",\"test\"] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:08:56.076926Z",
     "start_time": "2021-03-01T12:08:56.065436Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>no clean no stem</th>\n",
       "      <th>yes clean no stem</th>\n",
       "      <th>yes clean yes stem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.998232</td>\n",
       "      <td>0.997525</td>\n",
       "      <td>0.998409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>0.784387</td>\n",
       "      <td>0.788502</td>\n",
       "      <td>0.788768</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       no clean no stem  yes clean no stem  yes clean yes stem\n",
       "train          0.998232           0.997525            0.998409\n",
       "test           0.784387           0.788502            0.788768"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "###   Ngram features with Sklearn vectorizer\n",
    "\n",
    "\n",
    "####  Training a document classifier with sklearn `CountVectorizer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:08:57.897420Z",
     "start_time": "2021-03-01T12:08:57.895297Z"
    }
   },
   "outputs": [],
   "source": [
    "count_vectorizer = sklearn.feature_extraction.text.CountVectorizer()\n",
    "logistic = sklearn.linear_model.LogisticRegression(C=0.1)\n",
    "\n",
    "model_pipe_3 = sklearn.pipeline.Pipeline([(\"countvectorizer\", count_vectorizer),\n",
    "                                          (\"logisticregression\", logistic)],\n",
    "                                         )# memory='/Users/Shared/sklearn_mem/')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:09:41.755666Z",
     "start_time": "2021-03-01T12:08:59.659210Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davidbuchaca1/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy train: 0.9984090507336044    Accuracy test: 0.7908921933085502\n",
      "CPU times: user 2min 15s, sys: 2.24 s, total: 2min 17s\n",
      "Wall time: 42.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model_pipe_3.fit(X_train,y_train)\n",
    "\n",
    "y_test_pred  = model_pipe_3.predict(X_test)\n",
    "y_train_pred = model_pipe_3.predict(X_train)\n",
    "\n",
    "acc_train_3 = np.mean(y_train == y_train_pred)\n",
    "acc_test_3  = np.mean(y_test == y_test_pred)\n",
    "\n",
    "print(\"Accuracy train: {}    Accuracy test: {}\".format(acc_train_3, acc_test_3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:09:43.616597Z",
     "start_time": "2021-03-01T12:09:43.613691Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x130107 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 89 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_pipe_3.steps[0][1].transform(X_train[0:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:09:45.428569Z",
     "start_time": "2021-03-01T12:09:45.426135Z"
    }
   },
   "outputs": [],
   "source": [
    "df_results[\"sklearn countvectorizer\"] = [acc_train_3, acc_test_3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:09:47.234189Z",
     "start_time": "2021-03-01T12:09:47.228044Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>no clean no stem</th>\n",
       "      <th>yes clean no stem</th>\n",
       "      <th>yes clean yes stem</th>\n",
       "      <th>sklearn countvectorizer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.998232</td>\n",
       "      <td>0.997525</td>\n",
       "      <td>0.998409</td>\n",
       "      <td>0.998409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>0.784387</td>\n",
       "      <td>0.788502</td>\n",
       "      <td>0.788768</td>\n",
       "      <td>0.790892</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       no clean no stem  yes clean no stem  yes clean yes stem  \\\n",
       "train          0.998232           0.997525            0.998409   \n",
       "test           0.784387           0.788502            0.788768   \n",
       "\n",
       "       sklearn countvectorizer  \n",
       "train                 0.998409  \n",
       "test                  0.790892  "
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### V) Training a document classifier with sklearn `CountVectorizer` and ngrams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "[The house was nice]\n",
    "\n",
    "0the \n",
    "1house\n",
    "2was \n",
    "3nice\n",
    "\n",
    "4the house\n",
    "5house was\n",
    "6\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:13:51.960008Z",
     "start_time": "2021-03-01T12:13:51.875889Z"
    }
   },
   "outputs": [],
   "source": [
    "count_vectorizer = sklearn.feature_extraction.text.CountVectorizer(ngram_range=(1,2))\n",
    "logistic = sklearn.linear_model.LogisticRegression(C=0.1)\n",
    "\n",
    "model_pipe_4 = sklearn.pipeline.Pipeline([(\"countvectorizer\", count_vectorizer),\n",
    "                                          (\"logisticregression\", logistic)],\n",
    "                                           )# memory='/Users/Shared/sklearn_mem/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:19:03.957095Z",
     "start_time": "2021-03-01T12:13:53.865132Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davidbuchaca1/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy train: 0.999734841788934    Accuracy test: 0.7912904938927243\n",
      "CPU times: user 10min 27s, sys: 14.7 s, total: 10min 42s\n",
      "Wall time: 5min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model_pipe_4.fit(X_train,y_train)\n",
    "\n",
    "y_test_pred  = model_pipe_4.predict(X_test)\n",
    "y_train_pred = model_pipe_4.predict(X_train)\n",
    "\n",
    "acc_train_4 = np.mean(y_train == y_train_pred)\n",
    "acc_test_4  = np.mean(y_test == y_test_pred)\n",
    "\n",
    "print(\"Accuracy train: {}    Accuracy test: {}\".format(acc_train_4, acc_test_4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:20:13.081738Z",
     "start_time": "2021-03-01T13:20:13.078679Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x1181803 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 202 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_pipe_4.steps[0][1].transform(X_train[0:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:20:15.111395Z",
     "start_time": "2021-03-01T13:20:15.108945Z"
    }
   },
   "outputs": [],
   "source": [
    "df_results[\"sklearn countvectorizer 2gram\"] = [acc_train_4, acc_test_4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:20:17.001315Z",
     "start_time": "2021-03-01T13:20:16.995125Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>no clean no stem</th>\n",
       "      <th>yes clean no stem</th>\n",
       "      <th>yes clean yes stem</th>\n",
       "      <th>sklearn countvectorizer</th>\n",
       "      <th>sklearn countvectorizer 2gram</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>0.998232</td>\n",
       "      <td>0.997525</td>\n",
       "      <td>0.998409</td>\n",
       "      <td>0.998409</td>\n",
       "      <td>0.999735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>0.784387</td>\n",
       "      <td>0.788502</td>\n",
       "      <td>0.788768</td>\n",
       "      <td>0.790892</td>\n",
       "      <td>0.791290</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       no clean no stem  yes clean no stem  yes clean yes stem  \\\n",
       "train          0.998232           0.997525            0.998409   \n",
       "test           0.784387           0.788502            0.788768   \n",
       "\n",
       "       sklearn countvectorizer  sklearn countvectorizer 2gram  \n",
       "train                 0.998409                       0.999735  \n",
       "test                  0.790892                       0.791290  "
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:20:19.301620Z",
     "start_time": "2021-03-01T13:20:18.913165Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgsAAAD4CAYAAABixqVQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfA0lEQVR4nO3de5RdZX3/8ffHkMolGFEuDfESL6moIJcMVPwpSuWXotYCAqJVC2JFXa2AXWKxXqooIoptl1YrERGtqPUGolYSRAJUAZmE3FBRC7RV+SkiIhTkknx/f5wn5XA4syckk8xE3q+1zppznv3sZ3/3MxnO5zx7z5CqQpIkaSwPmewCJEnS1GZYkCRJnQwLkiSpk2FBkiR1MixIkqROW0x2AdJYtt9++5ozZ85klyFJm5UlS5b8sqp2mMgxDQuasubMmcPo6OhklyFJm5Uk/znRY3oZQpIkdTIsSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkdfIvOGrKWvnTW5hz4tfXqe/1733BRq5Gkh68XFmQJEmdDAuSJKmTYUGSJHUyLEiSpE6GBUmS1MmwIEmSOhkWJElSp/UKC0muT7L9kPbbNrykqSPJnCR/thHGPT7J1uux3xlJnjLBtbwsyYr2+E6S3SdyfEnS5m9KriwkmTbZNTRzgAkPC8DxwAMKC0mmVdVfVNX3NuTASQb/ENd1wLOr6mnAu4AFGzieJOl3TGdYSLJNkq8nWZ5kVZIjBrZvleT8JK8esu8JSa5sn1jf2dd+bpIlSa5Ockxf+21JTkpyBbBve31yO/blSXYacowZST6RZGU7zqGt/aWtbVWSU/uP0ff8sCRntednJflg+2R9bZLDWrf3As9KsizJG5JckeSpfWMsTjKvzdOZ7XyvSnJQ2z4tyWl99b0+ybHAzsBFSS4ar96BOVmcZCTJn7aaliW5Jsl1rf+8JBe3+V2YZFZfne9JcjFwXP8cVtV3qurm9vJy4FF9x39bkh8kuSDJZ5O8cdh4SV7Y5uaqJN9c+71K8o4kn0yyqK1GvSjJ+9q5np9k+uD3VJI09Yy3snAg8LOq2r2qdgXO79s2A/gq8Jmq+lj/TknmA3OBfYA9gHlJ9mubj66qecAIcGySR7b2bYBVVfWHVfXv7fXlVbU7cAlwv0ACvA24pap2a5+Mv5VkZ+BU4I/asfdOcvA45wkwC3gm8Cf0QgLAicClVbVHVf0D8Dngxe0cZwE7V9US4C3At6pqb2B/4P1JtgGOAR4H7NnqO7uqPgj8DNi/qvYfp97BOQGgqs5rNe0BLAdOa2+8HwIOa/N7JnBy3/k9vKqeXVUf6JiDVwHfaOc3AhwK7Am8iN73q1//eP8OPL2q9mxz9Ka+fk8AXgAcBHwauKiqdgPuaO33keSYJKNJRlfffktHqZKkTWW8sLASOCDJqUmeVVX9//X+CvCJqvrUkP3mt8dVwFJgF3rhAXoBYTm9T7GP7mtfDXypb4y7gK+150voXRIYdADw4bUv2ifkvYHFVXVjVd0DnA3sN2TfQedW1Zq2zH+/VYzm88Dh7fmLgS+05/OBE5MsAxYDWwKPafV9tNVBVf1qyJhd9Q7OyX0keRNwR1V9GHgSsCtwQavjrfStEgD/OtY4baz96YWFv2lNzwS+UlV3VNWt9IJhv/7xHgUsTLISOAF4at+2b1TV3fT+LU3j3sC5kiHf06paUFUjVTUybeuZXSVLkjaRzuvNVfXDJPOA5wOnJFlUVSe1zd8GnpfkM1VVA7sGOKWqTr9PY/Icem+g+1bV7UkW03tjBfhtVa3u635337irx6g1wLBjj3lKfc+3HNh253hjVNVPk9yU5GnAEcBr+vofWlXX3KeQZFh9g7rqHZyT/rGfSy+4rA0WAa6uqn3HGOt/xiygdz5nAM+rqpvWoa7B8T4E/H1Vnde+x+/o23YnQFWtSdL/PV2D/yMzSdosjHfPws7A7VX1aeA0YK++zW8HbgI+MmTXhcDRSWa0cWYn2RGYCdzcgsIuwNM3sP5FwF/11bsdcAXw7CTbp3ej5EuBi1uXnyd5cpKHAIesw/i3AtsOtK1dZp9ZVStb20Lg9S0ckGTPvvpem3YTYJJHDBm3q96hkjyW3ry/uKruaM3XADsk2bf1md5/f0XHWI8Bvgy8oqp+2Lfp34EXJtmyfR+7/reOM4GftudHjndMSdLmZbzLELsB323L2m8B3j2w/XhgyyTv62+sqkXAZ4DL2tL0F+m9OZ4PbJFkBb077y/fwPrfDWzXbgxcTu8+gBuANwMX0buev7SqvtL6n0jv0sa3gBvWYfwVwD3p3WT5htb2ReAl9C5JrPUuYDqwIsmq9hp6n9b/q7Uv597frFgAfCPJRePUO5ajgEcC57SbHP+tqu4CDgNObcdaBjxjHc7x7W2sj7SxRgGq6krgvFbTl4FRYKybCN4BfCHJpcAv1+GYkqTNSO5/BUHqSTKjqm5L729CXAIcU1VLN9XxHzprbs068h/Xqe/17+1a+JCkB48kS6pq8Kb0DeI1Y3VZkN4fgdoS+OSmDAqSpKnDsKAxVdXG+INUkqTNzJT8C46SJGnqMCxIkqROhgVJktTJexY0Ze02eyaj/paDJE06VxYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkdTIsSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkdTIsSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkddpisguQxrLyp7cw58Svj7n9+ve+YBNWI0kPXq4sSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ1+J8JCkuuTbD+k/bbJqGdjSTInyZ9thHGPT7L1eux3RpKnTHQ9kqSp5XciLGxsSaZNdg3NHGDCwwJwPPCAwkKSaVX1F1X1vQ05cBL/1ockTXGbVVhIsk2SrydZnmRVkiMGtm+V5Pwkrx6y7wlJrkyyIsk7+9rPTbIkydVJjulrvy3JSUmuAPZtr09ux748yU5DjjEjySeSrGzHObS1v7S1rUpyav8x+p4fluSs9vysJB9M8p0k1yY5rHV7L/CsJMuSvCHJFUme2jfG4iTz2jyd2c73qiQHte3TkpzWV9/rkxwL7AxclOSi8eodmJPFSUaS/GmraVmSa5Jc1/rPS3Jxm9+FSWb11fmeJBcDx433fZckTa7N7VPdgcDPquoFAElm9m2bAXwO+FRVfap/pyTzgbnAPkCA85LsV1WXAEdX1a+SbAVcmeRLVXUTsA2wqqre3sbYBri8qt6S5H3Aq4F3D9T3NuCWqtqt7bNdkp2BU4F5wM3AoiQHV9W545zrLOCZwC7AecAXgROBN1bVn7TxAV4M/F17I965qpYkeQ/wrao6OsnDge8m+Sbw58DjgD2r6p4kj2jn/tfA/lX1y3HqHZwTAKrqvFYjST4PXJxkOvAh4KCqurEFu5OBo9v5Pbyqnj3OHEiSpoDNamUBWAkckOTUJM+qqlv6tn0F+MRgUGjmt8dVwFJ6b8Bz27ZjkywHLgce3de+GvhS3xh3AV9rz5fQuyQw6ADgw2tfVNXNwN7A4qq6saruAc4G9luHcz23qta0Zf77rWI0nwcOb89fDHyhPZ8PnJhkGbAY2BJ4TKvvo60OqupXQ8bsqndwTu4jyZuAO6rqw8CTgF2BC1odbwUe1df9X8cY45gko0lGV99+y7AukqRNbLNaWaiqHyaZBzwfOCXJoqo6qW3+NvC8JJ+pqhrYNcApVXX6fRqT59B7A923qm5PspjeGyvAb6tqdV/3u/vGXc3wuQsw7NhjnlLf8y0Htt053hhV9dMkNyV5GnAE8Jq+/odW1TX3KaS3FDBY36CuegfnpH/s59ILLmuDRYCrq2rfMcb6n2GNVbUAWADw0Flzx6tVkrQJbFYrC22J/Paq+jRwGrBX3+a3AzcBHxmy60Lg6CQz2jizk+wIzARubkFhF+DpG1jiIuCv+urdDrgCeHaS7dO7UfKlwMWty8+TPDnJQ4BD1mH8W4FtB9o+B7wJmFlVK1vbQuD1LRyQZM+++l6bdlNhkkcMGber3qGSPJbevL+4qu5ozdcAOyTZt/WZ3n9/hSRp87FZhQVgN3rX35cBb+H+9wwcD2zZ7in4X1W1CPgMcFmSlfSu/28LnA9skWQF8C56lyI2xLuB7dqNgcvp3QdwA/Bm4CJgObC0qr7S+p9I79LGt4Ab1mH8FcA96d1k+YbW9kXgJfQuSaz1LmA6sCLJqvYa4Azgv1r7cu79zYoFwDeSXDROvWM5CngkcE67yfHfquou4DDg1HasZcAz1uEcJUlTTO6/Yi9NDQ+dNbdmHfmPY273f1EtSfeXZElVjUzkmJvbyoIkSdrEDAuSJKmTYUGSJHUyLEiSpE6GBUmS1Gmz+qNMenDZbfZMRv2NB0madK4sSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkdTIsSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkdTIsSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkdTIsSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWNiEksxJsmqy65gISf52smuQJG0ahgWtL8OCJD1IGBaAJO9Kclzf65OTHNuen5DkyiQrkryztW2T5OtJlidZleSIIWM+Mck3W5+lSZ4wsH1akvf3jf2a1j4jyYVtn5VJDmrtc5J8P8nHklydZFGSrQbG3DbJdUmmt9cPS3J9kulJnpDk/CRLklyaZJfW5/B2DsuTXDLkPGYluSTJstbvWUneC2zV2s5u/V6e5Lut7fQk01r7bUlObcf9ZpJ9kixOcm2SP92Ab5skaRMxLPR8HDgSIMlDgJcAZyeZD8wF9gH2AOYl2Q84EPhZVe1eVbsC5w8Z82zgw1W1O/AM4IaB7a8CbqmqvYG9gVcneRzwW+CQqtoL2B/4QJK0fea2MZ8K/Bo4tH/AqroVWAy8oDW9BPhSVd0NLABeX1XzgDcCH2l93g78catz2Jv3nwELq2oPYHdgWVWdCNxRVXtU1cuSPBk4Avg/rd9q4GVt/22Axe24twLvBv4vcAhw0uDBkhyTZDTJ6I033jikHEnSprbFZBcwFVTV9UluSrInsBNwVVXd1MLCfOCq1nUGvTfsS4HTkpwKfK2qLu0fL8m2wOyqOqeN/9vW3t9tPvC0JIe11zPb2D8B3tNCyRpgdqsJ4LqqWtaeLwHmDDmdM4A3AecCr6QXQmbQCyxf6Kvhoe3rt4Gzknwe+PKQ8a4EzmyrFef2Hb/fc4F5wJVt/K2AX7Rtd3FvmFoJ3FlVdydZOaz+qlpAL9gwMjJSQ44lSdrEDAv3OgM4Cvh94MzWFuCUqjp9sHOSecDzgVOSLKqq/k/JGew/ROh90l84MO5RwA7AvPamej2wZdt8Z1/X1fTelO+jqr7dLlk8G5hWVauSPAz4dfvUP9j/tUn+kN5qxLIke1TVTX3bL2nB5QXAvyR5f1V9asi5fLKq3jzkPO+uqrVv+mvWnkNVrUnivz9J2gx4GeJe59C7vLA3sPYNfCFwdPtkTpLZSXZMsjNwe1V9GjgN2Kt/oKr6DfCTJAe3/R6aZOuB4y0EXtd3f8EfJNmG3grDL1pQ2B947Hqcy6eAzwKf6KvnuiSHt2Mlye7t+ROq6oqqejvwS+DR/QMleWyr52P0LtesPde719YOXAgclmTHts8j2n6SpN8BfrJrququJBfR+wS+urUtatfjL2vL67cBLweeCLw/yRrgbuB1Q4Z8BXB6kpNan8PpfbJe6wx6y/BL2z0JNwIH07vX4atJRoFlwA/W43TOpndvwGf72l4G/HOStwLTgc8By9t5zKW3OnBha+v3HOCEJHfTO/8/b+0LgBVJlrb7Ft4KLGr3fNwN/CXwn+tRuyRpism9K8QPbu1NbilweFX9aLLr2RDtPoiDquoVk13LhhgZGanR0dHJLkOSNitJllTVyESO6coCkOQpwNeAc34HgsKHgOfRu59CkqQNZlgAqup7wOMnu46JUFWvn+waJEm/W7zBUZIkdTIsSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkdTIsSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkdTIsSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkdTIsSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhYZIlmZNk1WTX8UAlOT7J1pNdhyRp4zMsaH0dDxgWJOlBwLAwhiTvSnJc3+uTkxzbnp+Q5MokK5K8s7Vtk+TrSZYnWZXkiCFjPjHJN1ufpUmeMLB9WpL39439mtY+I8mFbZ+VSQ5q7XOSfD/Jx5JcnWRRkq2GHPesJB9M8p0k1yY5rLWnHW9VG3dYzfc7rzYPOwMXJbmo9Zuf5LJW4xeSzGjt1yd5T9s2mmSvJAuT/EeS167v90eStOkYFsb2ceBIgCQPAV4CnJ1kPjAX2AfYA5iXZD/gQOBnVbV7Ve0KnD9kzLOBD1fV7sAzgBsGtr8KuKWq9gb2Bl6d5HHAb4FDqmovYH/gA0nS9pnbxnwq8Gvg0DHOZxbwTOBPgPe2the1c9gdOAB4f5JZA/vd77yq6oPAz4D9q2r/JNsDbwUOaDWOAn/dN8Z/V9W+wKXAWcBhwNOBk8aoVZI0hWwx2QVMVVV1fZKbkuwJ7ARcVVU3tbAwH7iqdZ1B7w37UuC0JKcCX6uqS/vHS7ItMLuqzmnj/7a193ebDzxt7Sd/YGYb+yfAe1ooWQPMbjUBXFdVy9rzJcCcMU7p3KpaA3wvydp9nwl8tqpWAz9PcjG9kHJe334ru86reTrwFODb7Xx+D7isb/va8VYCM6rqVuDWJL9N8vCq+nXfPB0DHAPwmMc8ZoxTkSRtSoaFbmcARwG/D5zZ2gKcUlWnD3ZOMg94PnBKkkVV1f/JOYP9hwjw+qpaODDuUcAOwLyqujvJ9cCWbfOdfV1XA/e7DDGkXwa+jqmqfjjOea0d54Kqeuk4x14zUMcaBv4NVtUCYAHAyMhIjVefJGnj8zJEt3PoLcPvDax9A18IHN13TX52kh2T7AzcXlWfBk4D9uofqKp+A/wkycFtv4cO+W2ChcDrkkxvff4gyTb0Vhh+0YLC/sBjJ+j8LgGOaPdK7ADsB3y3v0PHed0KbNueXw78nyRPbPtsneQPJqhGSdIkc2WhQ1Xd1W7g+3VbqqeqFiV5MnBZW3K/DXg58ER61/zXAHcDrxsy5CuA05Oc1PocTu/T9Vpn0LuMsLTdk3AjcDC9ex2+mmQUWAb8YIJO8RxgX2A5UMCbqur/DfTZjeHntQD4RpIb2n0LRwGfTfLQtv2twA8nqE5J0iRKlSu9Y2k3Ni4FDq+qH012PQ82IyMjNTo6OtllSNJmJcmSqhqZyDG9DDGGJE8BfgxcaFCQJD2YeRliDFX1PeDxk12HJEmTzZUFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkdTIsSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkdTIsSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkdTIsSJKkToYFSZLUybAgSZI6GRYkSVInw4IkSepkWJAkSZ0MC5IkqZNhQZIkdTIsSJKkToYFSZLUybAwyZLcNtk1PFBJjkqy82TXIUnaNAwLWh9HAYYFSXqQMCysoyRzknw/yceSXJ1kUZKt2rY9klyeZEWSc5JsN2T/ndq25e3xjCF9TkhyZRvnnX3t5yZZ0o57TF/7bUlObuNdnmSnIWO+I8mZSRYnuTbJsX3b/jrJqvY4fsi+05Kc1bavTPKGJIcBI8DZSZYl2SrJvCQXtxoXJpnV9l+c5B+SXNLmbu8kX07yoyTvfsDfBEnSpDAsPDBzgQ9X1VOBXwOHtvZPAX9TVU8DVgJ/N2TfDwIXV9XuwF7A1f0bk8xv4+8D7AHMS7Jf23x0Vc2j9yZ9bJJHtvZtgMvbmJcArx6j7l2AP25j/12S6UnmAa8E/hB4OvDqJHsO7LcHMLuqdq2q3YBPVNUXgVHgZVW1B3AP8CHgsFbjmcDJfWPcVVX7AR8FvgL8JbArcFTfefTPwzFJRpOM3njjjWOcjiRpUzIsPDDXVdWy9nwJMCfJTODhVXVxa/8ksN+Qff8I+GeAqlpdVbcMbJ/fHlcBS+m9wc9t245Nshy4HHh0X/tdwNf66xmj7q9X1Z1V9UvgF8BOwDOBc6rqf6rqNuDLwLMG9rsWeHySDyU5EPjNkLGfRO/N/4Iky4C3Ao/q235e+7oSuLqqbqiqO9vYjx4crKoWVNVIVY3ssMMOY5yOJGlT2mKyC9jM3Nn3fDWw1QSOHeCUqjr9Po3Jc4ADgH2r6vYki4Et2+a7q6r66hnr+zlY9xbteJ2q6uYku9NblfhL4MXA0UPqvrqq9h3n2GsG6ljTUa8kaQpxZWEDtRWCm5Os/VT+CuDiIV0vBF4H/3svwMMGti8Ejk4yo/WZnWRHYCZwcwsKu9C7ZDARLgEOTrJ1km2AQ4BL+zsk2R54SFV9CXgbvcsnALcC27bn1wA7JNm37TM9yVMnqEZJ0hTgJ7uJcSTw0SRb01tef+WQPscBC5K8it6n+9cBl63dWFWLkjwZuCwJwG3Ay4HzgdcmWUHvjfnyiSi4qpYmOQv4bms6o6quGug2G/hEkrWh8s3t61n0zvcOYF/gMOCD7ZLMFsA/MnBPhiRp85V7V7GlqWVkZKRGR0cnuwxJ2qwkWVJVIxM5ppchJElSJ8OCJEnqZFiQJEmdDAuSJKmTYUGSJHUyLEiSpE6GBUmS1MmwIEmSOhkWJElSJ8OCJEnqZFiQJEmdDAuSJKmTYUGSJHUyLEiSpE6GBUmS1MmwIEmSOqWqJrsGaagktwLXTHYd62B74JeTXcQ6sM6JZZ0Ta3Ooc3OoEeBJVbXtRA64xUQOJk2wa6pqZLKLGE+SUeucONY5saxz4mwONUKvzoke08sQkiSpk2FBkiR1MixoKlsw2QWsI+ucWNY5saxz4mwONcJGqNMbHCVJUidXFiRJUifDgiRJ6mRY0CaT5MAk1yT5cZITh2w/Icmy9liVZHWSR7Rtx7W2q5Mc37fPI5JckORH7et2U7TOdyT5ad9+z9/INc5M8tUky1strxxv30may/Wpc0LncgLqPDPJL5KsGthnqs3nWHVOmflM8ugkFyX5fms/rm+fKTOf49Q5JX7Wk2yZ5Lt97e/s2+eBz2VV+fCx0R/ANOA/gMcDvwcsB57S0f+FwLfa812BVcDW9P42yDeBuW3b+4AT2/MTgVOnaJ3vAN64qeYS+Nu1cwHsAPyq9R1z38mYy/Wsc8LmckPrbK/3A/YCVg3sM2Xmc5w6p8x8ArOAvVr7tsAPp+i/z646J2w+N7DGADNa+3TgCuDp6zuXrixoU9kH+HFVXVtVdwGfAw7q6P9S4LPt+ZOBy6vq9qq6B7gYOKRtOwj4ZHv+SeDgKVrnRFqXGgvYNkmAGfT+A3LPOPtOxlyuT50TbUPqpKouaa8HTaX57Kpzoq13nVV1Q1UtbfXeCnwfmN32mTLzOU6dE2lDaqyquq31md4ea3+j4QHPpWFBm8ps4L/7Xv+EMX64kmwNHAh8qTWtAvZL8si27fnAo9u2narqBoD2dccpWifAXyVZ0ZaDN2QJdV1q/Cd64eVnwErguKpaM86+kzGX61MnTNxcbmidXabSfI5nys1nkjnAnvQ+EcMUnc8hdcLU+FknybQky4BfABdU1XrPpWFBm0qGtI31e7svBL5dVb8CqKrvA6cCFwDn01uKu2djFMnGq/OfgScAewA3AB/YyDX+MbAM2Lkd85+SPGwd950oG6vOiZzLDa1zU9pYdU65+Uwyg14IP76qfrOB9YxlY9U5VX7WqarVVbUH8ChgnyS7rm8hhgVtKj/hvp+yH0UvCQ/zEu5d2gegqj5eVXtV1X70ltl+1Db9PMksgPb1F1Oxzqr6efvBXQN8jN7y4sas8ZXAl9tS5I+B64Bdxtl3MubyAdc5wXO5oXV2mUrzOaapNp9JptN7Az67qr7ct8+Ums+x6pxCP+v/q6p+DSymtxIK6zGXhgVtKlcCc5M8Lsnv0XujPW+wU5KZwLOBrwy079i+PgZ4Efe+SZ8HHNmeHzm431Spc+0PZnMIvUsWG7PG/wKe2469E/Ak4Npx9p2MuXzAdU7wXG5onV2m0nyOaSrNZ7vu/nHg+1X19wP7TJn57KpzqvysJ9khycNb+1bAAcAP2j4PfC5rgu6A9eFjvAe9a/g/pHd371ta22uB1/b1OQr43JB9LwW+R29p/7l97Y8ELqT3Cf5C4BFTtM5/oXc9cUX7QZ21MWuktyS5qB1zFfDyrn0nay7Xs84JncsJqPOz9Jab76b3SfBVU3Q+x6pzyswn8Ex6y+wr6C2tLwOeP9Xmc5w6p8TPOvA04KpWxyrg7Rvys+6fe5YkSZ28DCFJkjoZFiRJUifDgiRJ6mRYkCRJnQwLkiSpk2FBkiR1MixIkqRO/x+nm+KMW1RhKQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "df_results.T[\"test\"].plot(kind=\"barh\", xlim=(0.79,0.83))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Feature selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  SelectKbest "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:20:24.297308Z",
     "start_time": "2021-03-01T13:20:24.294423Z"
    }
   },
   "outputs": [],
   "source": [
    "count_vectorizer = sklearn.feature_extraction.text.CountVectorizer(ngram_range=(1,2))\n",
    "feature_selector = SelectKBest(chi2, k = 700000)\n",
    "logistic = sklearn.linear_model.LogisticRegression(C=0.1)\n",
    "\n",
    "model_pipe_5 = sklearn.pipeline.Pipeline([(\"count_vectorizer\", count_vectorizer),\n",
    "                                          (\"feature_selector\", feature_selector),\n",
    "                                          (\"logisticregression\", logistic)],\n",
    "                                         )# memory='/Users/Shared/sklearn_mem/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:23:49.440495Z",
     "start_time": "2021-03-01T13:20:26.279867Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7min 36s, sys: 12.3 s, total: 7min 48s\n",
      "Wall time: 3min 23s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davidbuchaca1/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('count_vectorizer', CountVectorizer(ngram_range=(1, 2))),\n",
       "                ('feature_selector',\n",
       "                 SelectKBest(k=700000,\n",
       "                             score_func=<function chi2 at 0x7f8660f66700>)),\n",
       "                ('logisticregression', LogisticRegression(C=0.1))])"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "model_pipe_5.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:23:59.516522Z",
     "start_time": "2021-03-01T13:23:51.327069Z"
    }
   },
   "outputs": [],
   "source": [
    "acc_train = np.mean(model_pipe_5.predict(X_train) == y_train)\n",
    "acc_test = np.mean(model_pipe_5.predict(X_test) == y_test)\n",
    "df_results[\"sklearn countvectorizer 2gram + selection\"] = [acc_train, acc_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:24:01.492699Z",
     "start_time": "2021-03-01T13:24:01.382054Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkcAAAD4CAYAAADmQEJ+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmZElEQVR4nO3deZRdVZ328e9jiAYIRoZAh4DGIYoIEkiBxGaQlk4jqIAEEIcGsUFcyqBLaHxBGhmEKHb7gtgSkUFFHJmUlgQhAUQCVEImVNCGtC3wSkREkCkkz/vH2SU7l1tVIamkbszzWeuu3LvPPvv8zr43qaf2OVWRbSIiIiKi8ZLBLiAiIiKikyQcRURERFQSjiIiIiIqCUcRERERlYSjiIiIiMo6g11ARDQ22WQTjxkzZrDLiIhYo8yaNesPtkcO5JgJRxEdYsyYMXR3dw92GRERaxRJ/zPQY+ayWkREREQl4SgiIiKiknAUERERUUk4ioiIiKgkHEVERERUEo4iIiIiKglHEREREZX8nqOIDjH/gccYc+K1q2TshWfvs0rGjYj4W5SVo4iIiIhKwlFEREREJeEoIiIiopJwFBEREVFJOIqIiIiorFA4krRQ0iZt2p9Y+ZI6h6Qxkt63CsY9TtJ6K7DfhZK2HuBa3i9pXnn8XNJ2Azl+J5E0Q1LXCuw3TtLe1et3SzpxYKuLiIhO0ZErR5KGDHYNxRhgwMMRcBzwosKRpCG2/8X2L1bmwJJaf33D/cDutt8MnA5MWcnxBoykhatq7BdpHPDXcGT7GttnD145ERGxKvUZjiStL+laSXMlLZB0cMv2dSVdJ+mINvseL+nOsiLx2ar9KkmzJN0t6ciq/QlJp0m6HZhQXp9Zjj1T0mZtjjFc0sWS5pfjHFDaDyltCyRNro9RPZ8k6ZLy/BJJ55aVk/skTSrdzgZ2lTRH0ick3S7pTdUYMySNL/N0UTnfuyTtW7YPkXROVd/Rko4BNgemS5reX70tczJDUldZuZhTHvdIur/0Hy/ppjK/UyWNqur8nKSbgGPrObT9c9uPlpczgS2q439G0q8kXS/pckmfajeepHeVublL0k973itJp0q6VNK0str4HkmfL+d6naShre/piyFplKSbyzwskLRraZ8o6TZJsyV9X9LwNvu27SNpx/I5mCvpDkkjgNOAg8txDpZ0mKQvl/6vknRDeX9vkPTK0t7bZyoiIjpcfytHewEP2t7O9jbAddW24cCPgG/b/lq9k6SJwFhgJ5rvusdL2q1sPtz2eKALOEbSxqV9fWCB7bfY/ll5PdP2dsDNwAsCGPAZ4DHb25aVjxslbQ5MBv6hHHtHSfv1c54Ao4BdgHfShCKAE4FbbI+z/R/Ad4CDyjmOAja3PQs4CbjR9o7AHsAXJK0PHAm8Gti+1HeZ7XOBB4E9bO/RT72tcwL8deVinO1xwFzgnBI0zgMmlfm9CDizOr9X2N7d9hf7mIMPAz8p59cFHABsD7yH5v2q1eP9DNjZ9vZljk6o+r0W2AfYF/gWMN32tsBTpX1lvA+YWuZhO2COmsu9JwN72t4B6AY+We/UWx9JLwW+CxxbPnd7An8BTgG+W+b8uy01fBn4Rs/7C5xbbWv3mVqGpCMldUvqXvLkYys6DxERMYD6uyQyn+YL72Tgx7ZvqbZdDXze9mVt9ptYHneV18NpwtLNNIFo/9K+ZWl/BFgC/LAa41ngx+X5LOAf2xxnT+C9PS9sP1pC2AzbiwAkXQbsBlzVz7leZXsp8Au1WaUqvgdcD/wbTUj6fnW+7+5ZWQGGAa8s9X3V9nOlvj+2GXPHPuptnZNlSDoBeMr2+ZK2AbYBrpcEMAR4qOre+kW9daw9aMLRLqVpF+Bq20+V7T9q2aUebwvguyUwvpTmUl2Pn9heLGl+qaknYM+nuWzZWsdJwIHl5eaS5pTnt9r+WEv3O4GLSjC8yvYcSbsDWwO3lnl4KXBby34799LnDcBDtu8EsP3nUlNrmbUJNOER4JvA56tt/X6mbE+hXMp82aix7utAERGxevQZjmzfK2k8zf0WZ0maZvu0svlW4B2Svm279R91AWfZvmCZRultNIFhgu0nJc2gCRIAT9teUnVfXI27pJdaBbQ7dq+nVD0f1rLtmf7GsP2ApEckvRk4GPhI1f8A2/csU0jzVbW/L3h91ds6J/XYb6cJET0rcgLutj2hl7H+0msBzflcCLzD9iPLUVfreOcB/277mvIen1ptewbA9lJJ9Xu6lDbvqe0zKStekhaWVaG2bN9cwvA+wDclfQF4FLje9iF91K52fco8rGxAqffv9zMVERGdp797jjYHnrT9LeAcYIdq8yk0Kz5fabPrVODw6j6O0ZI2BUYAj5ZgtBXNd/ArYxrw8areDYHbgd0lbaLmxu5DgJtKl99LeqOklwD7v2C0F3oc2KClreey0Qjb80vbVODoEoaQtH1V31EqNy1L2qjNuH3V25akV9HM+0E9KzvAPcBISRNKn6Gq7o/qY6xXAlcAH7R9b7XpZ8C7JA0r72Nfl8BGAA+U54f2d8yBUubh4XJZ9+s0n8+ZwN9Lel3ps56k17fs2lufX9GsVu1Y2jco7127z0GPn/P86uX7aeYtIiLWYP3dc7QtcEe5tHEScEbL9uOAYZLqSwnYngZ8G7itXE75Ac0Xl+uAdSTNo/nJqJkrWf8ZwIblZty5NPfxPAR8GphOcz/ObNtXl/4n0lyqu5FlLzn1Zh7wXLk59xOl7Qc0Xwy/V/U7HRgKzJO0oLyGZjXmt6V9Ls//5NsU4CeSpvdTb28OAzYGriw3Cf+X7WeBScDkcqw5wFuX4xxPKWN9pYzVDVAuLV1TarqC5r6c3m6KORX4vqRbgD8sxzEHytto7jO6i+b+qP9bLk8eBlxePmczga3qnXrrU+bwYOC8MofX06wwTge27rkhu6WGY4APlXE+SMsN7xERsebRC6+IRTQkDbf9hJrfyXQzcKTt2YNd19+ql40a61GHfmmVjL3w7JW99z0iojNJmmX7Rf8Ou76sst9RE38Tpqj5pZPDgEsTjCIiYm2QcBS9sr0qfgFmRERER+vI35AdERERMVgSjiIiIiIquawW0SG2HT2C7tw4HREx6LJyFBEREVFJOIqIiIioJBxFREREVBKOIiIiIioJRxERERGVhKOIiIiISsJRRERERCXhKCIiIqKScBQRERFRSTiKiIiIqCQcRURERFQSjiIiIiIqCUcRERERlYSjiIiIiErCUUREREQl4SgiIiKiknAUERERUVlnsAuIiMb8Bx5jzInXLlffhWfvs4qriYhYe2XlKCIiIqKScBQRERFRSTiKiIiIqCQcRURERFQSjiIiIiIqCUfLQdJCSZu0aX9iMOpZVSSNkfS+VTDucZLWW4H9LpS09QDX8n5J88rj55K2G8jxIyJizZdw1AEkDRnsGooxwICHI+A44EWFI0lDbP+L7V+szIEltf66ivuB3W2/GTgdmLKS40VExN+YhKOKpPUlXStprqQFkg5u2b6upOskHdFm3+Ml3VlWJD5btV8laZakuyUdWbU/Iek0SbcDE8rrM8uxZ0rarM0xhku6WNL8cpwDSvshpW2BpMn1MarnkyRdUp5fIuncsnJyn6RJpdvZwK6S5kj6hKTbJb2pGmOGpPFlni4q53uXpH3L9iGSzqnqO1rSMcDmwHRJ0/urt2VOZkjqkvTuUtMcSfdIur/0Hy/ppjK/UyWNqur8nKSbgGPrObT9c9uPlpczgS2q439G0q8kXS/pckmfajeepHeVublL0k973itJp0q6VNK0str4HkmfL+d6naShre9pRER0noSjZe0FPGh7O9vbANdV24YDPwK+bftr9U6SJgJjgZ2AccB4SbuVzYfbHg90AcdI2ri0rw8ssP0W2z8rr2fa3g64GXhBAAM+Azxme9uy8nGjpM2BycA/lGPvKGm/5TjXUcAuwDtpQhHAicAttsfZ/g/gO8BB5RxHAZvbngWcBNxoe0dgD+ALktYHjgReDWxf6rvM9rnAg8Aetvfop97WOQHA9jWlpnHAXOCcEjTOAyaV+b0IOLM6v1fY3t32F/uYgw8DPynn1wUcAGwPvIfm/arV4/0M2Nn29mWOTqj6vRbYB9gX+BYw3fa2wFOlfRmSjpTULal7yZOP9VFqRESsLglHy5oP7ClpsqRdbddfra4GLrb9jTb7TSyPu4DZwFY0YQmaQDSXZpViy6p9CfDDaoxngR+X57NoLnG12hM4v+dFWQHZEZhhe5Ht54DLgN3a7NvqKttLy2WrF6xSFd8DDizPDwK+X55PBE6UNAeYAQwDXlnq+2qpA9t/bDNmX/W2zskyJJ0APGX7fOANwDbA9aWOk6lWgYDv9jZOGWsPmnD0r6VpF+Bq20/ZfpwmCNfq8bYApkqaDxwPvKna9hPbi2k+S0N4PmDPp817anuK7S7bXUPWG9FXyRERsZrk/omK7XsljQf2Bs6SNM32aWXzrcA7JH3btlt2FXCW7QuWaZTeRhMYJth+UtIMmiAB8LTtJVX3xdW4S2j/3ghod+xeT6l6Pqxl2zP9jWH7AUmPSHozcDDwkar/AbbvWaYQqV19rfqqt3VO6rHfThPUeoKUgLttT+hlrL/0WkBzPhcC77D9yHLU1TreecC/276mvMenVtueAbC9VFL9ni4lf98iItYIWTmqlEs+T9r+FnAOsEO1+RTgEeArbXadChwuaXgZZ7SkTYERwKMlGG0F7LySJU4DPl7VuyFwO7C7pE3U3Nh9CHBT6fJ7SW+U9BJg/+UY/3Fgg5a2nstGI2zPL21TgaNLGELS9lV9R6nctCxpozbj9lVvW5JeRTPvB9l+qjTfA4yUNKH0GVrfH9XHWK8ErgA+aPveatPPgHdJGlbex77+87IRwAPl+aH9HTMiItYsCUfL2ha4o1ymOQk4o2X7ccAwSZ+vG21PA74N3FYutfyAJgxcB6wjaR7NT0bNXMn6zgA2LDcyz6W5j+ch4NPAdJr7cWbbvrr0P5HmUt2NwEPLMf484Dk1N4V/orT9AHgvzSW2HqcDQ4F5khaU19Csxvy2tM/l+Z98mwL8RNL0furtzWHAxsCV5abs/7L9LDAJmFyONQd463Kc4yllrK+UsboBbN8JXFNqugLoBnq7CehU4PuSbgH+sBzHjIiINYheeIUoYu0kabjtJ9T8TqabgSNtz15dx3/ZqLEedeiXlqvvwrP7WtiKiFh7SJplu/WHaFZK7oGIeN4UNb90chhw6eoMRhER0TkSjiIK26viF2BGRMQaJvccRURERFSychTRIbYdPYLu3EsUETHosnIUERERUUk4ioiIiKgkHEVERERUEo4iIiIiKglHEREREZWEo4iIiIhKwlFEREREJeEoIiIiopJwFBEREVFJOIqIiIioJBxFREREVBKOIiIiIioJRxERERGVhKOIiIiISsJRRERERCXhKCIiIqKScBQRERFRWWewC4iIxvwHHmPMidf2un3h2fusxmoiItZeWTmKiIiIqCQcRURERFQSjiIiIiIqCUcRERERlYSjiIiIiErC0RpI0kJJm7Rpf2Iw6llVJI2R9L5VMO5xktZbgf0ulLT1QNcTERGdJeEoXkDSkMGuoRgDDHg4Ao4DXlQ4kjTE9r/Y/sXKHFhSfn1GRESHSzjqYJLWl3StpLmSFkg6uGX7upKuk3REm32Pl3SnpHmSPlu1XyVplqS7JR1ZtT8h6TRJtwMTyuszy7FnStqszTGGS7pY0vxynANK+yGlbYGkyfUxqueTJF1Snl8i6VxJP5d0n6RJpdvZwK6S5kj6hKTbJb2pGmOGpPFlni4q53uXpH3L9iGSzqnqO1rSMcDmwHRJ0/urt2VOZkjqkvTuUtMcSfdIur/0Hy/ppjK/UyWNqur8nKSbgGP7e98jImJw5bvYzrYX8KDtfQAkjai2DQe+A3zD9jfqnSRNBMYCOwECrpG0m+2bgcNt/1HSusCdkn5o+xFgfWCB7VPKGOsDM22fJOnzwBHAGS31fQZ4zPa2ZZ8NJW0OTAbGA48C0yTtZ/uqfs51FLALsBVwDfAD4ETgU7bfWcYHOAj4txI8Nrc9S9LngBttHy7pFcAdkn4K/DPwamB7289J2qic+yeBPWz/oZ96W+cEANvXlBqR9D3gJklDgfOAfW0vKkH2TODwcn6vsL1760mXgHokwJCXj+xniiIiYnXIylFnmw/sKWmypF1tP1Ztuxq4uDUYFRPL4y5gNk3gGFu2HSNpLjAT2LJqXwL8sBrjWeDH5fksmktcrfYEzu95YftRYEdghu1Ftp8DLgN2W45zvcr20nLZ6gWrVMX3gAPL84OA75fnE4ETJc0BZgDDgFeW+r5a6sD2H9uM2Ve9rXOyDEknAE/ZPh94A7ANcH2p42Rgi6r7d9uNYXuK7S7bXUPWG9GuS0RErGZZOepgtu+VNB7YGzhL0jTbp5XNtwLvkPRt227ZVcBZti9YplF6G01gmGD7SUkzaIIEwNO2l1TdF1fjLqH9Z0VAu2P3ekrV82Et257pbwzbD0h6RNKbgYOBj1T9D7B9zzKFNEs9rfW16qve1jmpx347TVDrCVIC7rY9oZex/tJPHRER0SGyctTByiWfJ21/CzgH2KHafArwCPCVNrtOBQ6XNLyMM1rSpsAI4NESjLYCdl7JEqcBH6/q3RC4Hdhd0iZqbuw+BLipdPm9pDdKegmw/3KM/ziwQUvbd4ATgBG255e2qcDRJQwhafuqvqNUboKWtFGbcfuqty1Jr6KZ94NsP1Wa7wFGSppQ+gyt74+KiIg1R8JRZ9uW5v6ZOcBJvPCen+OAYeWeoL+yPQ34NnCbpPk09+9sAFwHrCNpHnA6zaW1lXEGsGG5kXkuzX08DwGfBqYDc4HZtq8u/U+kuVR3I/DQcow/D3hOzU3hnyhtPwDeS3OJrcfpwFBgnqQF5TXAhcBvS/tcnv/JtynATyRN76fe3hwGbAxcWW7K/i/bzwKTgMnlWHOAty7HOUZERIfRC6/IRMRgeNmosR516Jd63b7w7H1WXzEREWsISbNsdw3kmFk5ioiIiKgkHEVERERUEo4iIiIiKvlR/ogOse3oEXTnvqKIiEGXlaOIiIiISsJRRERERCXhKCIiIqKScBQRERFRSTiKiIiIqCQcRURERFQSjiIiIiIqCUcRERERlYSjiIiIiErCUUREREQl4SgiIiKiknAUERERUUk4ioiIiKgkHEVERERUEo4iIiIiKglHEREREZWEo4iIiIhKwlFEREREJeEoIiIiopJwFBEREVFJOIqIiIioJBxFREREVBKOIiIiIioJR7FKSBojacFg1zEQJP2fwa4hIiJWn4SjiP4lHEVErEUSjtYykk6XdGz1+kxJx5Tnx0u6U9I8SZ8tbetLulbSXEkLJB3cZszXSfpp6TNb0mtbtg+R9IVq7I+U9uGSbij7zJe0b2kfI+mXkr4m6W5J0ySt2zLmBpLulzS0vH65pIWShkp6raTrJM2SdIukrUqfA8s5zJV0c5vzGCXpZklzSr9dJZ0NrFvaLiv9PiDpjtJ2gaQhpf0JSZPLcX8qaSdJMyTdJ+ndK/G2RUTEapRwtPb5OnAogKSXAO8FLpM0ERgL7ASMA8ZL2g3YC3jQ9na2twGuazPmZcD5trcD3go81LL9w8BjtncEdgSOkPRq4Glgf9s7AHsAX5Skss/YMuabgD8BB9QD2n4cmAHsU5reC/zQ9mJgCnC07fHAp4CvlD6nAP9U6mwXVt4HTLU9DtgOmGP7ROAp2+Nsv1/SG4GDgb8v/ZYA7y/7rw/MKMd9HDgD+Edgf+C0NsdD0pGSuiV1L1q0qF2XiIhYzdYZ7AJi9bK9UNIjkrYHNgPusv1ICUcTgbtK1+E0AeUW4BxJk4Ef276lHk/SBsBo21eW8Z8u7XW3icCbJU0qr0eUsX8HfK6EsKXA6FITwP2255Tns4AxbU7nQuAE4CrgQzShazhNQPt+VcPLyp+3ApdI+h5wRZvx7gQuKqtRV1XHr70dGA/cWcZfF3i4bHuW58PjfOAZ24slze+lfmxPoQlzdHV1uV2fiIhYvRKO1k4XAocBfwdcVNoEnGX7gtbOksYDewNnSZpmu14FUWv/NkSzkjO1ZdzDgJHA+BIiFgLDyuZnqq5LaELIMmzfWi7B7Q4Msb1A0suBP5VVndb+R0l6C81q0xxJ42w/Um2/uQS1fYBvSvqC7W+0OZdLbX+6zXkutt0TcJb2nIPtpZLydy0iYg2Ry2prpytpLpftCPQElqnA4WXlBUmjJW0qaXPgSdvfAs4BdqgHsv1n4HeS9iv7vUzSei3Hmwp8tLo/6PWS1qdZQXq4BKM9gFetwLl8A7gcuLiq535JB5ZjSdJ25flrbd9u+xTgD8CW9UCSXlXq+RrN5ceec13cUztwAzBJ0qZln43KfhER8Tci382uhWw/K2k6zQrLktI2rdxPc1u5XPQE8AHgdcAXJC0FFgMfbTPkB4ELJJ1W+hxIs3LS40Kay0qzyz1Fi4D9aO5V+pGkbmAO8KsVOJ3LaO7tubxqez/wn5JOBoYC3wHmlvMYS7P6c0Npq70NOF7SYprz/+fSPgWYJ2l2ue/oZGBauWdrMfAx4H9WoPaIiOhAev4qQKwtyhf12cCBtn892PWsjHIf0762PzjYtaysrq4ud3d3D3YZERFrFEmzbHcN5JhZOVrLSNoa+DFw5d9AMDoPeAfN/VAREREDIuFoLWP7F8BrBruOgWD76MGuISIi/vbkhuyIiIiISsJRRERERCXhKCIiIqKScBQRERFRSTiKiIiIqCQcRURERFQSjiIiIiIqCUcRERERlYSjiIiIiErCUUREREQl4SgiIiKiknAUERERUUk4ioiIiKgkHEVERERUEo4iIiIiKglHEREREZWEo4iIiIhKwlFEREREJeEoIiIiopJwFBEREVFJOIqIiIioJBxFREREVBKOIiIiIioJRxERERGVhKOIiIiISsJRrDaSxkhaMNh1vFiSjpO03mDXERERq0fCUUT/jgMSjiIi1hIJR4Gk0yUdW70+U9Ix5fnxku6UNE/SZ0vb+pKulTRX0gJJB7cZ83WSflr6zJb02pbtQyR9oRr7I6V9uKQbyj7zJe1b2sdI+qWkr0m6W9I0Seu2Oe4lks6V9HNJ90maVNpVjregjNuu5hecV5mHzYHpkqaXfhMl3VZq/L6k4aV9oaTPlW3dknaQNFXSf0s6akXfn4iIWL0SjgLg68ChAJJeArwXuEzSRGAssBMwDhgvaTdgL+BB29vZ3ga4rs2YlwHn294OeCvwUMv2DwOP2d4R2BE4QtKrgaeB/W3vAOwBfFGSyj5jy5hvAv4EHNDL+YwCdgHeCZxd2t5TzmE7YE/gC5JGtez3gvOyfS7wILCH7T0kbQKcDOxZauwGPlmN8b+2JwC3AJcAk4CdgdPaFSrpyBKkuhctWtTL6URExOqUcBTYXgg8Iml7YCJwl+1HyvOJwF3AbGArmoAyH9hT0mRJu9p+rB5P0gbAaNtXlvGftv1ky2EnAv8saQ5wO7BxGVvA5yTNA34KjAY2K/vcb3tOeT4LGNPLKV1le6ntX1T77gJcbnuJ7d8DN9GEslqf51XsDGwN3FpqPxR4VbX9mmqs220/bnsR8LSkV7QOZnuK7S7bXSNHjuzldCIiYnVaZ7ALiI5xIXAY8HfARaVNwFm2L2jtLGk8sDdwlqRptuuVEbX2b0PA0bantox7GDASGG97saSFwLCy+Zmq6xLgBZfV2vRTy5+9sn1vP+fVM871tg/p59hLW+pYSv6+RUSsEbJyFD2upLmstCPQE1imAodX99SMlrSppM2BJ21/CzgH2KEeyPafgd9J2q/s97I2P+01FfiopKGlz+slrQ+MAB4uwWgPll2VWRk3AweXe51GArsBd9Qd+jivx4ENyvOZwN9Lel3ZZz1Jrx+gGiMiogPkO9kAwPaz5YbjP9leUtqmSXojcFu57ecJ4APA62ju2VkKLAY+2mbIDwIXSDqt9DmQZvWkx4U0l8Vml3uKFgH70dyr9CNJ3cAc4FcDdIpXAhOAuYCBE2z/v5Y+29L+vKYAP5H0ULnv6DDgckkvK9tPBu4doDojImKQyfZg1xAdoNyIPRs40PavB7uetVFXV5e7u7sHu4yIiDWKpFm2uwZyzFxWCyRtDfwGuCHBKCIi1na5rBaUn+p6zWDXERER0QmychQRERFRSTiKiIiIqCQcRURERFQSjiIiIiIqCUcRERERlYSjiIiIiErCUUREREQl4SgiIiKiknAUERERUUk4ioiIiKgkHEVERERUEo4iIiIiKglHEREREZWEo4iIiIhKwlFEREREJeEoIiIiopJwFBEREVFJOIqIiIioJBxFREREVBKOIiIiIioJRxERERGVhKOIiIiISsJRRERERCXhKCIiIqKScBQRERFRSTiKiIiIqCQcxWol6YnBruHFknSYpM0Hu46IiFg9Eo4i+ncYkHAUEbGWSDiKtiSNkfRLSV+TdLekaZLWLdvGSZopaZ6kKyVt2Gb/zcq2ueXx1jZ9jpd0Zxnns1X7VZJmleMeWbU/IenMMt5MSZu1GfNUSRdJmiHpPknHVNs+KWlBeRzXZt8hki4p2+dL+oSkSUAXcJmkOZLWlTRe0k2lxqmSRpX9Z0j6D0k3l7nbUdIVkn4t6YwX/SZERMSgSDiKvowFzrf9JuBPwAGl/RvAv9p+MzAf+Lc2+54L3GR7O2AH4O56o6SJZfydgHHAeEm7lc2H2x5PE0qOkbRxaV8fmFnGvBk4ope6twL+qYz9b5KGShoPfAh4C7AzcISk7Vv2GweMtr2N7W2Bi23/AOgG3m97HPAccB4wqdR4EXBmNcaztncDvgpcDXwM2AY4rDqPeh6OlNQtqXvRokW9nE5ERKxOCUfRl/ttzynPZwFjJI0AXmH7ptJ+KbBbm33/AfhPANtLbD/Wsn1iedwFzKYJNGPLtmMkzQVmAltW7c8CP67r6aXua20/Y/sPwMPAZsAuwJW2/2L7CeAKYNeW/e4DXiPpPEl7AX9uM/YbaMLO9ZLmACcDW1Tbryl/zgfutv2Q7WfK2Fu2DmZ7iu0u210jR47s5XQiImJ1WmewC4iO9kz1fAmw7gCOLeAs2xcs0yi9DdgTmGD7SUkzgGFl82Lbrurp7fPbWvc65Xh9sv2opO1oVp0+BhwEHN6m7rttT+jn2Etb6ljaR70REdFBsnIUL0pZAXpUUs+qyweBm9p0vQH4KPz1Xp6Xt2yfChwuaXjpM1rSpsAI4NESjLaiuQQ2EG4G9pO0nqT1gf2BW+oOkjYBXmL7h8BnaC4HAjwObFCe3wOMlDSh7DNU0psGqMaIiOgA+U42VsShwFclrUdzuehDbfocC0yR9GGa1ZuPArf1bLQ9TdIbgdskATwBfAC4DjhK0jyaIDJzIAq2PVvSJcAdpelC23e1dBsNXCyp55uGT5c/L6E536eACcAk4NxyiXEd4Eu03FMVERFrLj1/lSIiBlNXV5e7u7sHu4yIiDWKpFm2uwZyzFxWi4iIiKgkHEVERERUEo4iIiIiKglHEREREZWEo4iIiIhKwlFEREREJeEoIiIiopJwFBEREVFJOIqIiIioJBxFREREVBKOIiIiIioJRxERERGV/MezER1C0uPAPYNdx3LYBPjDYBexHFLnwFkTaoTUOdDWlDrfYHuDgRxwnYEcLCJWyj0D/T9LrwqSulPnwFkT6lwTaoTUOdDWpDoHesxcVouIiIioJBxFREREVBKOIjrHlMEuYDmlzoG1JtS5JtQIqXOgrbV15obsiIiIiEpWjiIiIiIqCUcRERERlYSjiFVA0l6S7pH0G0knttl+vKQ55bFA0hJJG5Vtx5a2uyUdV+2zkaTrJf26/Llhh9Z5qqQHqv32Xg11jpD0I0lzSz0f6m/fQZrPFamz0+bzIkkPS1rQsk+nzWdvdQ7ofK5ojZK2lDRd0i9L+7HVPh0zl/3U2TGfTUnDJN1RtX+22ufFz6ftPPLIYwAfwBDgv4HXAC8F5gJb99H/XcCN5fk2wAJgPZrfQ/ZTYGzZ9nngxPL8RGByh9Z5KvCp1TmfwP/pmQ9gJPDH0rfXfQdjPlewzo6Zz/J6N2AHYEHLPh0zn/3UOWDzuZLv+Shgh9K+AXBvh342+6qzYz6bgIDhpX0ocDuw84rOZ1aOIgbeTsBvbN9n+1ngO8C+ffQ/BLi8PH8jMNP2k7afA24C9i/b9gUuLc8vBfbr0DoH2vLUaWADSQKG0/yD+Vw/+w7GfK5InQNtZerE9s3ldatOms++6hxIK1yj7Ydszy61Pg78Ehhd9umYueynzoG2MnXa9hOlz9Dy6PmJsxc9nwlHEQNvNPC/1evf0cs/JpLWA/YCfliaFgC7Sdq4bNsb2LJs28z2QwDlz007tE6Aj0uaVy5trOwlgeWp88s0ge1BYD5wrO2l/ew7GPO5InVC58xnXzppPvszUPM5IDVKGgNsT7PaAR06l23qhA76bEoaImkO8DBwve0Vns+Eo4iBpzZtvf3OjHcBt9r+I4DtXwKTgeuB62iWlZ9bFUWy6ur8T+C1wDjgIeCLq6HOfwLmAJuX435Z0suXc9+Bsqrq7KT5XJ1WVZ0DOZ8rXaOk4TTfdBxn+88rUUtfVlWdHfXZtL3E9jhgC2AnSdusaCEJRxED73csu4qyBc13Oe28l+cvVQFg++u2d7C9G82S8a/Lpt9LGgVQ/ny4E+u0/fvyj9RS4Gs0S+Wrus4PAVeUpfXfAPcDW/Wz72DM54uus8Pmsy+dNJ+9GuD5XKkaJQ2lCRyX2b6i2qej5rK3Ojv1s2n7T8AMmtVuWIH5TDiKGHh3AmMlvVrSS2mCxTWtnSSNAHYHrm5p37T8+UrgPTwfSq4BDi3PD23dr1Pq7PlHqNif5hLcqq7zt8Dby/E3A94A3NfPvoMxny+6zg6bz7500nz2aoDnc4VrLPfMfB34pe1/b9mnY+ayrzo76bMpaaSkV5T2dYE9gV+VfV78fHqA7jLPI488nn/Q3INzL81PXpxU2o4Cjqr6HAZ8p82+twC/oLlU9faqfWPgBpoVmhuAjTq0zm/S3Aswr/yjNGpV10mzxD6tHHcB8IG+9h2s+VzBOjttPi+nuYSymOY7/Q936Hz2VueAzueK1gjsQnPJaB7NZaI5wN6dNpf91Nkxn03gzcBdpZYFwCkr83c9/31IRERERCWX1SIiIiIqCUcRERERlYSjiIiIiErCUUREREQl4SgiIiKiknAUERERUUk4ioiIiKj8f4eHKYw7OyMpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_results.T[\"test\"].plot(kind=\"barh\", xlim=(0.79,0.83))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Feature Union"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:24:03.354145Z",
     "start_time": "2021-03-01T13:24:03.351799Z"
    }
   },
   "outputs": [],
   "source": [
    "simple_count_vectorizer_stemmer = SimpleCountVectorizer(word_transformer_func= SnowballStemmer('english').stem,\n",
    "                                                        doc_cleaner_pattern=re.compile(\"[^a-zA-Z]\"))\n",
    "\n",
    "count_vectorizer = sklearn.feature_extraction.text.CountVectorizer(ngram_range=(1,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:24:05.228472Z",
     "start_time": "2021-03-01T13:24:05.226390Z"
    }
   },
   "outputs": [],
   "source": [
    "union = sklearn.pipeline.FeatureUnion([(\"simple_count_vectorizer_stemmer\", simple_count_vectorizer_stemmer),\n",
    "                                       (\"count_vectorizer\", count_vectorizer)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:24:07.133052Z",
     "start_time": "2021-03-01T13:24:07.130039Z"
    }
   },
   "outputs": [],
   "source": [
    "logistic = sklearn.linear_model.LogisticRegression(C=0.1)\n",
    "feature_selector = SelectKBest(chi2, k = 700000)\n",
    "model_pipe_6 = sklearn.pipeline.Pipeline([(\"union_vectorizers\", union),\n",
    "                                          (\"feature_selector\", feature_selector),\n",
    "                                          (\"logisticregression\", logistic)],\n",
    "                                         )# memory='/Users/Shared/sklearn_mem/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:28:00.134193Z",
     "start_time": "2021-03-01T13:24:09.065420Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8min 18s, sys: 9.91 s, total: 8min 28s\n",
      "Wall time: 3min 51s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davidbuchaca1/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('union_vectorizers',\n",
       "                 FeatureUnion(transformer_list=[('simple_count_vectorizer_stemmer',\n",
       "                                                 SimpleCountVectorizer(doc_cleaner_pattern=re.compile('[^a-zA-Z]'),\n",
       "                                                                       word_transformer_func=<bound method EnglishStemmer.stem of <EnglishStemmer>>)),\n",
       "                                                ('count_vectorizer',\n",
       "                                                 CountVectorizer(ngram_range=(1,\n",
       "                                                                              2)))])),\n",
       "                ('feature_selector',\n",
       "                 SelectKBest(k=700000,\n",
       "                             score_func=<function chi2 at 0x7f8660f66700>)),\n",
       "                ('logisticregression', LogisticRegression(C=0.1))])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "model_pipe_6.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:29:01.299014Z",
     "start_time": "2021-03-01T13:28:02.436371Z"
    }
   },
   "outputs": [],
   "source": [
    "acc_train = np.mean(model_pipe_6.predict(X_train) == y_train)\n",
    "acc_test = np.mean(model_pipe_6.predict(X_test) == y_test)\n",
    "df_results[\"Feature union + selection\"] = [acc_train, acc_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T13:29:03.333324Z",
     "start_time": "2021-03-01T13:29:03.209539Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkcAAAD4CAYAAADmQEJ+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqeklEQVR4nO3deZRdVZ328e9jiAQSjCKRN0QwtkYRGQJJlNgMonQaRQUkgIg2ERvU1c1gL6FjozSiCBHs9gW1JSLigBPKpCgJQgYEglTmoILdkO4WeDUgIggyJM/7x9klO5dbVRkqqRvyfNa6q87d0/mdfW9Sv9r73CrZJiIiIiIazxvoACIiIiI6SZKjiIiIiEqSo4iIiIhKkqOIiIiISpKjiIiIiMoWAx1ARDS22247jx49eqDDiIjYpMyfP/8B2yP6c8wkRxEdYvTo0XR1dQ10GBERmxRJ/93fY2ZbLSIiIqKS5CgiIiKikuQoIiIiopLkKCIiIqKS5CgiIiKikuQoIiIiopKP8kd0iKX3PszoqdcOdBjPacvPPXigQ4iITUBWjiIiIiIqSY4iIiIiKkmOIiIiIipJjiIiIiIqSY4GgKSVkhZVj9HrMMahknbZAOGtE0nvkDR1oOPojaTRkpatY98pknaonl/cSfMfERH9J59WGxiP2x67nmMcCvwI+MWadpC0he2n1/O8bdm+BrimP8eU9EZgiu0p/TnuOpoCLAPuA7D99wMaTUREbDBZOeoQksZJmiNpvqQZkkaW8uMl3S5psaQfSNpa0huAdwDnlZWnV0iaLWl86bOdpOXleIqkyyX9EJgpaaikS8qYCyUd0iaWN0r6UfX885KmlOPlkj4haYGkpZJ2rs7z+XL8Mkk3SFpSvu5Uyi+VdIGkWyTdLWlyP8zbEZKWlfmZW8oGSTqvXOMSSR9o06/HNpJOK9e2WNK5Jc7xwGVlvrdqme+jS/tlkqZV4zwq6ewyzjxJ26/v9UZExIaX5GhgbFVtqV0paTBwITDZ9jjgEuDs0vYK2xNs7wH8Eni/7VtoVmlOtT3W9n/1cb6JwLG23wScDtxoewJwAE2CNXQt43/A9l7AfwAfaVP/eeDrtncHLgMuqOpGAvsAbwPOXcvztnMG8Ldlft5Ryt4PPFyucQJwvKSXt/Rr20bSW2hW5V5fxvyM7e8DXcAxZb4f7x6kbLVNA94EjAUmSDq0VA8F5pVx5gLHtwYv6QRJXZK6Vj728PrORURE9INsqw2M1bbVJO0K7ApcLwlgEHB/qd5V0qeAFwLDgBnrcL7rbf++HE8C3iGpO6kZAuxEk3itqSvK1/nAO9vUT6zKvwF8pqq7yvYq4Bc9raRIug3YkuZ6t5W0qFT9s+3W678ZuFTS96q4JgG7VytTw4ExwF1Vv57aHAh81fZjANW89WQCMNv2ihL7ZcB+wFXAkzRbn9DM1d+0drY9HZgOsOXIMe7jXBERsREkOeoMAu6wPbFN3aXAobYXl62tN/YwxtM8sxI4pKXuTy3nOtz2nb3EU4/VbrwnyteVrNl7qP6m/0R1rLaN7dfDmt1zZPuDkl4PHAwskjS2jHtiayLVcuN7T20Oaom3L22voXjKdvdYazpXERExwLKt1hnuBEZImgggabCk15a6bYD7y9bbMVWfR0pdt+XAuHLc2708M4ATVZaoJO3Zps1/A7tI2lLScODNa3k9twDvKsfHAD9by/5rTNIrbN9m+wzgAWBHmmv8UJkzJL2qzdZhT21mAsdJ2rqUb1vat853t9uA/ct9XoOAo4E5/XuVERGxMeUn2Q5g+8myvXNBSUa2AD4H3AF8nOYb8H8DS3nmG/R3gC9LOokmGTof+J6k9wI39nK6T5axl5QEaTnN/T91PP9btqmWAL8GFq7lJZ0EXCLpVGAF8L617L82zpM0hmYF5wZgMU3co4EF5RpX0NxHVLu4XRvb15XVpy5JTwI/Bv6FZgXvS5Iep9k2BMD2/ZI+CswqMfzY9tUb5EojImKj0DOr/hExkLYcOcYjj/3cQIfxnJY/PBvx3CNpvu3x/TlmttUiIiIiKkmOIiIiIiq55yiiQ+w2ajhd2faJiBhwWTmKiIiIqCQ5ioiIiKgkOYqIiIioJDmKiIiIqCQ5ioiIiKgkOYqIiIioJDmKiIiIqCQ5ioiIiKgkOYqIiIioJDmKiIiIqCQ5ioiIiKgkOYqIiIioJDmKiIiIqCQ5ioiIiKhsMdABRERj6b0PM3rqtRtk7OXnHrxBxo2IeC7KylFEREREJclRRERERCXJUUREREQlyVFEREREZZ2SI0nLJW3XpvzR9Q+pc0gaLendG2DcUyRtvQ79Lpa0Sz/HcoykJeVxi6Q9+nP8TiJptqTx69BvrKS3Vs/fIWlq/0YXERGdoiNXjiQNGugYitFAvydHwCnAWiVHkgbZ/nvbv1ifE0tq/YTiPcD+tncHPglMX8/x+o2k5Rtq7LU0FvhLcmT7GtvnDlw4ERGxIfWaHEkaKulaSYslLZN0VEv9VpKuk3R8m76nSrq9rEh8oiq/StJ8SXdIOqEqf1TSWZJuAyaW52eXc8+TtH2bcwyT9FVJS8t5Di/lR5eyZZKm1eeojidLurQcXyrpgrJycrekyaXZucC+khZJ+rCk2yS9thpjtqRxZZ4uKde7UNIhpX6QpPOr+E6UdBKwAzBL0qy+4m2Zk9mSxpeVi0Xlcaeke0r7cZLmlPmdIWlkFeenJc0BTq7n0PYtth8qT+cBL63O/3FJv5J0vaRvS/pIu/Ekvb3MzUJJP+1+rSSdKelrkmaW1cZ3SvpMudbrJA1ufU3XhqSRkuaWeVgmad9SPknSrZIWSLpc0rA2fdu2kTShvA8WS/q5pOHAWcBR5TxHSZoi6fOl/csk3VBe3xsk7VTKe3pPRUREh+tr5egg4D7be9jeFbiuqhsG/BD4lu0v150kTQLGAK+j+al7nKT9SvVxtscB44GTJL24lA8Fltl+ve2flefzbO8BzAWelYABHwcetr1bWfm4UdIOwDTgTeXcEyQd2sd1AowE9gHeRpMUAUwFbrI91va/A98BjizXOBLYwfZ84HTgRtsTgAOA8yQNBU4AXg7sWeK7zPYFwH3AAbYP6CPe1jkB/rJyMdb2WGAxcH5JNC4EJpf5vQQ4u7q+F9re3/Zne5mD9wM/Kdc3Hjgc2BN4J83rVavH+xmwt+09yxydVrV7BXAwcAjwTWCW7d2Ax0v5+ng3MKPMwx7AIjXbvR8DDrS9F9AF/FPdqac2kp4PfBc4ubzvDgT+BJwBfLfM+XdbYvg88PXu1xe4oKpr955ajaQTJHVJ6lr52MPrOg8REdGP+toSWUrzjXca8CPbN1V1VwOfsX1Zm36TymNheT6MJlmaS5MQHVbKdyzlDwIrgR9UYzwJ/Kgczwf+ps15DgTe1f3E9kMlCZttewWApMuA/YCr+rjWq2yvAn6hNqtUxfeA64F/pUmSLq+u9x3dKyvAEGCnEt+XbD9d4vt9mzEn9BJv65ysRtJpwOO2vyBpV2BX4HpJAIOA+6vmrd/UW8c6gCY52qcU7QNcbfvxUv/Dli71eC8FvlsSxufTbNV1+4ntpyQtLTF1J9hLabYtW+M4HTiiPN1B0qJyfLPtf2hpfjtwSUkMr7K9SNL+wC7AzWUeng/c2tJv7x7avBq43/btALb/WGJqDbM2kSZ5BPgG8Jmqrs/3lO3plK3MLUeOcW8nioiIjaPX5Mj2XZLG0dxvcY6kmbbPKtU3A2+R9C3brf+pCzjH9kWrFUpvpEkYJtp+TNJsmkQC4M+2V1bNn6rGXdlDrALanbvHS6qOh7TUPdHXGLbvlfSgpN2Bo4APVO0Pt33naoE031X7+obXW7ytc1KP/WaaJKJ7RU7AHbYn9jDWn3oMoLmei4G32H5wDeJqHe9C4N9sX1Ne4zOruicAbK+SVL+mq2jzmto+m7LiJWl5WRVqy/bckgwfDHxD0nnAQ8D1to/uJXa1a1PmYX0TlLp/n++piIjoPH3dc7QD8JjtbwLnA3tV1WfQrPh8sU3XGcBx1X0coyS9BBgOPFQSo51pfoJfHzOBf6zifRFwG7C/pO3U3Nh9NDCnNPmtpNdIeh5w2LNGe7ZHgG1ayrq3jYbbXlrKZgAnlmQISXtW8X1Q5aZlSdu2Gbe3eNuS9DKaeT+ye2UHuBMYIWliaTNY1f1RvYy1E3AF8F7bd1VVPwPeLmlIeR172wIbDtxbjo/t65z9pczD78q27ldo3p/zgL+W9MrSZmtJr2rp2lObX9GsVk0o5duU167d+6DbLTyzenkMzbxFRMQmrK97jnYDfl62Nk4HPtVSfwowRFK9lYDtmcC3gFvLdsr3ab65XAdsIWkJzSej5q1n/J8CXlRuxl1Mcx/P/cBHgVk09+MssH11aT+VZqvuRlbfcurJEuDpcnPuh0vZ92m+GX6vavdJYDCwRNKy8hya1Zj/KeWLeeaTb9OBn0ia1Ue8PZkCvBi4stwk/GPbTwKTgWnlXIuAN6zBNZ5RxvpiGasLoGwtXVNiuoLmvpyeboo5E7hc0k3AA2twzv7yRpr7jBbS3B/1f8v25BTg2+V9Ng/Yue7UU5syh0cBF5Y5vJ5mhXEWsEv3DdktMZwEvK+M815abniPiIhNj569IxbRkDTM9qNqfifTXOAE2wsGOq7nqi1HjvHIYz+3QcbOH56NiOcqSfNtr/XvsOvNBvsdNfGcMF3NL50cAnwtiVFERGwOkhxFj2xviF+AGRER0dGSHEV0iN1GDacr218REQOuI/98SERERMRASXIUERERUUlyFBEREVFJchQRERFRSXIUERERUUlyFBEREVFJchQRERFRSXIUERERUUlyFBEREVFJchQRERFRSXIUERERUUlyFBEREVFJchQRERFRSXIUERERUdlioAOIiMbSex9m9NRr16jt8nMP3sDRRERsvrJyFBEREVFJchQRERFRSXIUERERUUlyFBEREVFJcrQGJC2XtF2b8kcHIp4NRdJoSe/eAOOeImnrdeh3saRd+jmWYyQtKY9bJO3Rn+NHRMSmL8lRB5A0aKBjKEYD/Z4cAacAa5UcSRpk++9t/2J9Tiyp9ROZ9wD7294d+CQwfT3Hi4iI55gkRxVJQyVdK2mxpGWSjmqp30rSdZKOb9P3VEm3lxWJT1TlV0maL+kOSSdU5Y9KOkvSbcDE8vzscu55krZvc45hkr4qaWk5z+Gl/OhStkzStPoc1fFkSZeW40slXVBWTu6WNLk0OxfYV9IiSR+WdJuk11ZjzJY0rszTJeV6F0o6pNQPknR+Fd+Jkk4CdgBmSZrVV7wtczJb0nhJ7ygxLZJ0p6R7SvtxkuaU+Z0haWQV56clzQFOrufQ9i22HypP5wEvrc7/cUm/knS9pG9L+ki78SS9vczNQkk/7X6tJJ0p6WuSZpbVxndK+ky51uskDW59TSMiovMkOVrdQcB9tvewvStwXVU3DPgh8C3bX647SZoEjAFeB4wFxknar1QfZ3scMB44SdKLS/lQYJnt19v+WXk+z/YewFzgWQkY8HHgYdu7lZWPGyXtAEwD3lTOPUHSoWtwrSOBfYC30SRFAFOBm2yPtf3vwHeAI8s1jgR2sD0fOB240fYE4ADgPElDgROAlwN7lvgus30BcB9wgO0D+oi3dU4AsH1NiWkssBg4vyQaFwKTy/xeApxdXd8Lbe9v+7O9zMH7gZ+U6xsPHA7sCbyT5vWq1eP9DNjb9p5ljk6r2r0COBg4BPgmMMv2bsDjpXw1kk6Q1CWpa+VjD/cSakREbCxJjla3FDhQ0jRJ+9quv1tdDXzV9tfb9JtUHguBBcDONMkSNAnRYppVih2r8pXAD6oxngR+VI7n02xxtToQ+EL3k7ICMgGYbXuF7aeBy4D92vRtdZXtVWXb6lmrVMX3gCPK8ZHA5eV4EjBV0iJgNjAE2KnE96USB7Z/32bM3uJtnZPVSDoNeNz2F4BXA7sC15c4Pka1CgR8t6dxylgH0CRH/1yK9gGutv247UdoEuFaPd5LgRmSlgKnAq+t6n5i+yma99Ignkmwl9LmNbU93fZ42+MHbT28t5AjImIjyf0TFdt3SRoHvBU4R9JM22eV6puBt0j6lm23dBVwju2LViuU3kiTMEy0/Zik2TSJBMCfba+smj9VjbuS9q+NgHbn7vGSquMhLXVP9DWG7XslPShpd+Ao4ANV+8Nt37laIFK7+Fr1Fm/rnNRjv5kmUetOpATcYXtiD2P9qccAmuu5GHiL7QfXIK7W8S4E/s32NeU1PrOqewLA9ipJ9Wu6ivx7i4jYJGTlqFK2fB6z/U3gfGCvqvoM4EHgi226zgCOkzSsjDNK0kuA4cBDJTHaGdh7PUOcCfxjFe+LgNuA/SVtp+bG7qOBOaXJbyW9RtLzgMPWYPxHgG1ayrq3jYbbXlrKZgAnlmQISXtW8X1Q5aZlSdu2Gbe3eNuS9DKaeT/S9uOl+E5ghKSJpc3g+v6oXsbaCbgCeK/tu6qqnwFvlzSkvI69/X2O4cC95fjYvs4ZERGbliRHq9sN+HnZpjkd+FRL/SnAEEmfqQttzwS+Bdxatlq+T5MMXAdsIWkJzSej5q1nfJ8CXlRuZF5Mcx/P/cBHgVk09+MssH11aT+VZqvuRuD+NRh/CfC0mpvCP1zKvg+8i2aLrdsngcHAEknLynNoVmP+p5Qv5plPvk0HfiJpVh/x9mQK8GLgynJT9o9tPwlMBqaVcy0C3rAG13hGGeuLZawuANu3A9eUmK4AuoCebgI6E7hc0k3AA2twzoiI2ITo2TtEEZsnScNsP6rmdzLNBU6wvWBjnX/LkWM88tjPrVHb/OHZiIiGpPm2Wz9Es15yD0TEM6ar+aWTQ4CvbczEKCIiOkeSo4jC9ob4BZgREbGJSXIU0SF2GzWcrmyXRUQMuNyQHREREVFJchQRERFRSXIUERERUUlyFBEREVFJchQRERFRSXIUERERUUlyFBEREVFJchQRERFRSXIUERERUUlyFBEREVFJchQRERFRSXIUERERUUlyFBEREVFJchQRERFR2WKgA4iIxtJ7H2b01Gt7rF9+7sEbMZqIiM1XVo4iIiIiKkmOIiIiIipJjiIiIiIqSY4iIiIiKkmONkGSlkvark35owMRz4YiabSkd2+AcU+RtPU69LtY0i79HU9ERHSWJEfxLJIGDXQMxWig35Mj4BRgrZIjSYNs/73tX6zPiSXlE6IRER0uyVEHkzRU0rWSFktaJumolvqtJF0n6fg2fU+VdLukJZI+UZVfJWm+pDsknVCVPyrpLEm3ARPL87PLuedJ2r7NOYZJ+qqkpeU8h5fyo0vZMknT6nNUx5MlXVqOL5V0gaRbJN0taXJpdi6wr6RFkj4s6TZJr63GmC1pXJmnS8r1LpR0SKkfJOn8Kr4TJZ0E7ADMkjSrr3hb5mS2pPGS3lFiWiTpTkn3lPbjJM0p8ztD0sgqzk9LmgOc3NfrHhERAys/xXa2g4D7bB8MIGl4VTcM+A7wddtfrztJmgSMAV4HCLhG0n625wLH2f69pK2A2yX9wPaDwFBgme0zyhhDgXm2T5f0GeB44FMt8X0ceNj2bqXPiyTtAEwDxgEPATMlHWr7qj6udSSwD7AzcA3wfWAq8BHbbyvjAxwJ/GtJPHawPV/Sp4EbbR8n6YXAzyX9FPg74OXAnraflrRtufZ/Ag6w/UAf8bbOCQC2rykxIul7wBxJg4ELgUNsryiJ7NnAceX6Xmh7/9aLLgnqCQCDXjCijymKiIiNIStHnW0pcKCkaZL2tf1wVXc18NXWxKiYVB4LgQU0CceYUneSpMXAPGDHqnwl8INqjCeBH5Xj+TRbXK0OBL7Q/cT2Q8AEYLbtFbafBi4D9luDa73K9qqybfWsVarie8AR5fhI4PJyPAmYKmkRMBsYAuxU4vtSiQPbv28zZm/xts7JaiSdBjxu+wvAq4FdgetLHB8DXlo1/267MWxPtz3e9vhBWw9v1yQiIjayrBx1MNt3SRoHvBU4R9JM22eV6puBt0j6lm23dBVwju2LViuU3kiTMEy0/Zik2TSJBMCfba+smj9VjbuS9u8VAe3O3eMlVcdDWuqe6GsM2/dKelDS7sBRwAeq9ofbvnO1QJqlntb4WvUWb+uc1GO/mSZR606kBNxhe2IPY/2pjzgiIqJDZOWog5Utn8dsfxM4H9irqj4DeBD4YpuuM4DjJA0r44yS9BJgOPBQSYx2BvZezxBnAv9Yxfsi4DZgf0nbqbmx+2hgTmnyW0mvkfQ84LA1GP8RYJuWsu8ApwHDbS8tZTOAE0syhKQ9q/g+qHITtKRt24zbW7xtSXoZzbwfafvxUnwnMELSxNJmcH1/VEREbDqSHHW23Wjun1kEnM6z7/k5BRhS7gn6C9szgW8Bt0paSnP/zjbAdcAWkpYAn6TZWlsfnwJeVG5kXkxzH8/9wEeBWcBiYIHtq0v7qTRbdTcC96/B+EuAp9XcFP7hUvZ94F00W2zdPgkMBpZIWlaeA1wM/E8pX8wzn3ybDvxE0qw+4u3JFODFwJXlpuwf234SmAxMK+daBLxhDa4xIiI6jJ69IxMRA2HLkWM88tjP9VifPzwbEfFskubbHt+fY2blKCIiIqKS5CgiIiKikk+rRXSI3UYNpytbZxERAy4rRxERERGVJEcRERERlSRHEREREZUkRxERERGVJEcRERERlSRHEREREZUkRxERERGVJEcRERERlSRHEREREZUkRxERERGVJEcRERERlSRHEREREZUkRxERERGVJEcRERERlSRHEREREZUkRxERERGVJEcRERERlSRHEREREZUkRxERERGVJEexQUgaLWnZQMfRHyT9y0DHEBERG0+So4i+JTmKiNiMJDnazEj6pKSTq+dnSzqpHJ8q6XZJSyR9opQNlXStpMWSlkk6qs2Yr5T009JmgaRXtNQPknReNfYHSvkwSTeUPkslHVLKR0v6paQvS7pD0kxJW7WMuY2keyQNLs9fIGm5pMGSXiHpOknzJd0kaefS5ohyDYslzW1zHSMlzZW0qLTbV9K5wFal7LLS7j2Sfl7KLpI0qJQ/KmlaOe9PJb1O0mxJd0t6x3q8bBERsRElOdr8fAU4FkDS84B3AZdJmgSMAV4HjAXGSdoPOAi4z/YetncFrmsz5mXAF2zvAbwBuL+l/v3Aw7YnABOA4yW9HPgzcJjtvYADgM9KUukzpoz5WuAPwOH1gLYfAWYDB5eidwE/sP0UMB040fY44CPAF0ubM4C/LXG2S1beDcywPRbYA1hkeyrwuO2xto+R9BrgKOCvS7uVwDGl/1BgdjnvI8CngL8BDgPOanM+JJ0gqUtS14oVK9o1iYiIjWyLgQ4gNi7byyU9KGlPYHtgoe0HS3I0CVhYmg6jSVBuAs6XNA34ke2b6vEkbQOMsn1lGf/PpbxuNgnYXdLk8nx4Gfs3wKdLErYKGFViArjH9qJyPB8Y3eZyLgZOA64C3keTdA2jSdAur2LYsny9GbhU0veAK9qMdztwSVmNuqo6f+3NwDjg9jL+VsDvSt2TPJM8LgWesP2UpKU9xI/t6TTJHOPHj3e7NhERsXElOdo8XQxMAf4PcEkpE3CO7YtaG0saB7wVOEfSTNv1Koha27chmpWcGS3jTgFGAONKErEcGFKqn6iarqRJQlZj++ayBbc/MMj2MkkvAP5QVnVa239Q0utpVpsWSRpr+8Gqfm5J1A4GviHpPNtfb3MtX7P90TbX+ZTt7gRnVfc12F4lKf/WIiI2EdlW2zxdSbNdNgHoTlhmAMeVlRckjZL0Ekk7AI/Z/iZwPrBXPZDtPwK/kXRo6belpK1bzjcD+FB1f9CrJA2lWUH6XUmMDgBetg7X8nXg28BXq3jukXREOZck7VGOX2H7NttnAA8AO9YDSXpZiefLNNuP3df6VHfswA3AZEkvKX22Lf0iIuI5Ij/NboZsPylpFs0Ky8pSNrPcT3Nr2S56FHgP8ErgPEmrgKeAD7UZ8r3ARZLOKm2OoFk56XYxzbbSgnJP0QrgUJp7lX4oqQtYBPxqHS7nMpp7e75dlR0D/IekjwGDge8Ai8t1jKFZ/bmhlNXeCJwq6Sma6/+7Uj4dWCJpQbnv6GPAzHLP1lPAPwD/vQ6xR0REB9IzuwCxuSjf1BcAR9j+9UDHsz7KfUyH2H7vQMeyvsaPH++urq6BDiMiYpMiab7t8f05ZlaONjOSdgF+BFz5HEiMLgTeQnM/VERERL9IcrSZsf0L4K8GOo7+YPvEgY4hIiKee3JDdkREREQlyVFEREREJclRRERERCXJUUREREQlyVFEREREJclRRERERCXJUUREREQlyVFEREREJclRRERERCXJUUREREQlyVFEREREJclRRERERCXJUUREREQlyVFEREREJclRRERERCXJUUREREQlyVFEREREJclRRERERCXJUUREREQlyVFEREREJclRbDSSRktaNtBxrC1Jp0jaeqDjiIiIjSPJUUTfTgGSHEVEbCaSHAWSPinp5Or52ZJOKsenSrpd0hJJnyhlQyVdK2mxpGWSjmoz5isl/bS0WSDpFS31gySdV439gVI+TNINpc9SSYeU8tGSfinpy5LukDRT0lZtznuppAsk3SLpbkmTS7nK+ZaVcdvF/KzrKvOwAzBL0qzSbpKkW0uMl0saVsqXS/p0qeuStJekGZL+S9IH1/X1iYiIjSvJUQB8BTgWQNLzgHcBl0maBIwBXgeMBcZJ2g84CLjP9h62dwWuazPmZcAXbO8BvAG4v6X+/cDDticAE4DjJb0c+DNwmO29gAOAz0pS6TOmjPla4A/A4T1cz0hgH+BtwLml7J3lGvYADgTOkzSypd+zrsv2BcB9wAG2D5C0HfAx4MASYxfwT9UY/2t7InATcCkwGdgbOKtdoJJOKIlU14oVK3q4nIiI2JiSHAW2lwMPStoTmAQstP1gOZ4ELAQWADvTJChLgQMlTZO0r+2H6/EkbQOMsn1lGf/Pth9rOe0k4O8kLQJuA15cxhbwaUlLgJ8Co4DtS597bC8qx/OB0T1c0lW2V9n+RdV3H+Dbtlfa/i0whyYpq/V6XcXewC7AzSX2Y4GXVfXXVGPdZvsR2yuAP0t6YetgtqfbHm97/IgRI3q4nIiI2Ji2GOgAomNcDEwB/g9wSSkTcI7ti1obSxoHvBU4R9JM2/XKiFrbtyHgRNszWsadAowAxtl+StJyYEipfqJquhJ41rZam3Zq+doj23f1cV3d41xv++g+zr2qJY5V5N9bRMQmIStH0e1Kmm2lCUB3wjIDOK66p2aUpJdI2gF4zPY3gfOBveqBbP8R+I2kQ0u/Ldt82msG8CFJg0ubV0kaCgwHflcSowNYfVVmfcwFjir3Oo0A9gN+Xjfo5boeAbYpx/OAv5b0ytJna0mv6qcYIyKiA+Qn2QDA9pPlhuM/2F5ZymZKeg1wa7nt51HgPcArae7ZWQU8BXyozZDvBS6SdFZpcwTN6km3i2m2xRaUe4pWAIfS3Kv0Q0ldwCLgV/10iVcCE4HFgIHTbP+/lja70f66pgM/kXR/ue9oCvBtSVuW+o8Bd/VTnBERMcBke6BjiA5QbsReABxh+9cDHc/maPz48e7q6hroMCIiNimS5tse359jZlstkLQL8J/ADUmMIiJic5dttaB8quuvBjqOiIiITpCVo4iIiIhKkqOIiIiISpKjiIiIiEqSo4iIiIhKkqOIiIiISpKjiIiIiEqSo4iIiIhKkqOIiIiISpKjiIiIiEqSo4iIiIhKkqOIiIiISpKjiIiIiEqSo4iIiIhKkqOIiIiISpKjiIiIiEqSo4iIiIhKkqOIiIiISpKjiIiIiEqSo4iIiIhKkqOIiIiISpKj2KgkPTrQMawtSVMk7TDQcURExMaR5Ciib1OAJEcREZuJJEfRlqTRkn4p6cuS7pA0U9JWpW6spHmSlki6UtKL2vTfvtQtLo83tGlzqqTbyzifqMqvkjS/nPeEqvxRSWeX8eZJ2r7NmGdKukTSbEl3SzqpqvsnScvK45Q2fQdJurTUL5X0YUmTgfHAZZIWSdpK0jhJc0qMMySNLP1nS/p3SXPL3E2QdIWkX0v61Fq/CBERMSCSHEVvxgBfsP1a4A/A4aX868A/294dWAr8a5u+FwBzbO8B7AXcUVdKmlTGfx0wFhgnab9SfZztcTRJyUmSXlzKhwLzyphzgeN7iHtn4G/L2P8qabCkccD7gNcDewPHS9qzpd9YYJTtXW3vBnzV9veBLuAY22OBp4ELgcklxkuAs6sxnrS9H/Al4GrgH4BdgSnVddTzcIKkLkldK1as6OFyIiJiY0pyFL25x/aicjwfGC1pOPBC23NK+deA/dr0fRPwHwC2V9p+uKV+UnksBBbQJDRjSt1JkhYD84Adq/IngR/V8fQQ97W2n7D9APA7YHtgH+BK23+y/ShwBbBvS7+7gb+SdKGkg4A/thn71TTJzvWSFgEfA15a1V9Tvi4F7rB9v+0nytg7tg5me7rt8bbHjxgxoofLiYiIjWmLgQ4gOtoT1fFKYKt+HFvAObYvWq1QeiNwIDDR9mOSZgNDSvVTtl3F09P7tzXuLcr5emX7IUl70Kw6/QNwJHBcm7jvsD2xj3OvaoljVS/xRkREB8nKUayVsgL0kKTuVZf3AnPaNL0B+BD85V6eF7TUzwCOkzSstBkl6SXAcOChkhjtTLMF1h/mAodK2lrSUOAw4Ka6gaTtgOfZ/gHwcZrtQIBHgG3K8Z3ACEkTS5/Bkl7bTzFGREQHyE+ysS6OBb4kaWua7aL3tWlzMjBd0vtpVm8+BNzaXWl7pqTXALdKAngUeA9wHfBBSUtoEpF5/RGw7QWSLgV+Xooutr2wpdko4KuSun9o+Gj5einN9T4OTAQmAxeULcYtgM/Rck9VRERsuvTMLkVEDKTx48e7q6troMOIiNikSJpve3x/jplttYiIiIhKkqOIiIiISpKjiIiIiEqSo4iIiIhKkqOIiIiISpKjiIiIiEqSo4iIiIhKkqOIiIiISpKjiIiIiEqSo4iIiIhK/nxIRIeQ9AjN35PrdNsBDwx0EGsgcfafTSFGSJz9bVOJ89W2t+m72ZrLH56N6Bx39vffB9oQJHUlzv6zKcS5KcQIibO/bUpx9veY2VaLiIiIqCQ5ioiIiKgkOYroHNMHOoA1lDj716YQ56YQIyTO/rbZxpkbsiMiIiIqWTmKiIiIqCQ5ioiIiKgkOYrYACQdJOlOSf8paWqb+lMlLSqPZZJWStq21J1cyu6QdErVZ1tJ10v6dfn6og6N80xJ91b93roR4hwu6YeSFpd43tdX3wGaz3WJs9Pm8xJJv5O0rKVPp81nT3H263yua4ySdpQ0S9IvS/nJVZ+Omcs+4uyY96akIZJ+XpV/ouqz9vNpO4888ujHBzAI+C/gr4DnA4uBXXpp/3bgxnK8K7AM2Jrm95D9FBhT6j4DTC3HU4FpHRrnmcBHNuZ8Av/SPR/ACOD3pW2PfQdiPtcxzo6Zz/J8P2AvYFlLn46Zzz7i7Lf5XM/XfCSwVynfBrirQ9+bvcXZMe9NQMCwUj4YuA3Ye13nMytHEf3vdcB/2r7b9pPAd4BDeml/NPDtcvwaYJ7tx2w/DcwBDit1hwBfK8dfAw7t0Dj725rEaWAbSQKG0fyH+XQffQdiPtclzv62PnFie2553qqT5rO3OPvTOsdo+37bC0qsjwC/BEaVPh0zl33E2d/WJ07bfrS0GVwe3Z84W+v5THIU0f9GAf9bPf8NPfxnImlr4CDgB6VoGbCfpBeXurcCO5a67W3fD1C+vqRD4wT4R0lLytbG+m4JrEmcn6dJ2O4DlgIn217VR9+BmM91iRM6Zz5700nz2Zf+ms9+iVHSaGBPmtUO6NC5bBMndNB7U9IgSYuA3wHX217n+UxyFNH/1Kasp9+Z8XbgZtu/B7D9S2AacD1wHc2y8tMbIkg2XJz/AbwCGAvcD3x2I8T5t8AiYIdy3s9LesEa9u0vGyrOTprPjWlDxdmf87neMUoaRvNDxym2/7gesfRmQ8XZUe9N2yttjwVeCrxO0q7rGkiSo4j+9xtWX0V5Kc1POe28i2e2qgCw/RXbe9nej2bJ+Nel6reSRgKUr7/rxDht/7b8J7UK+DLNUvmGjvN9wBVlaf0/gXuAnfvoOxDzudZxdth89qaT5rNH/Tyf6xWjpME0Ccdltq+o+nTUXPYUZ6e+N23/AZhNs9oN6zCfSY4i+t/twBhJL5f0fJrE4prWRpKGA/sDV7eUv6R83Ql4J88kJdcAx5bjY1v7dUqc3f8JFYfRbMFt6Dj/B3hzOf/2wKuBu/voOxDzudZxdth89qaT5rNH/Tyf6xxjuWfmK8Avbf9bS5+Omcve4uyk96akEZJeWMq3Ag4EflX6rP18up/uMs8jjzyeedDcg3MXzScvTi9lHwQ+WLWZAnynTd+bgF/QbFW9uSp/MXADzQrNDcC2HRrnN2juBVhS/lMauaHjpFlin1nOuwx4T299B2o+1zHOTpvPb9NsoTxF85P++zt0PnuKs1/nc11jBPah2TJaQrNNtAh4a6fNZR9xdsx7E9gdWFhiWQacsT7/1vPnQyIiIiIq2VaLiIiIqCQ5ioiIiKgkOYqIiIioJDmKiIiIqCQ5ioiIiKgkOYqIiIioJDmKiIiIqPx/FVmfToJaR5gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_results.T[\"test\"].plot(kind=\"barh\", xlim=(0.79,0.83))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Crossvalidating results (Exercise)\n",
    "\n",
    "It is very (VERY) important you do crossvalidation (or at least a train/test split)\n",
    "\n",
    "\n",
    "If you have never use Bayesian Optimization for selecting hyperparameters I recommend you to try: https://scikit-optimize.github.io"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:02:12.037840Z",
     "start_time": "2021-03-01T12:00:08.818Z"
    }
   },
   "outputs": [],
   "source": [
    "# Can't deepcopy objects with compiled regular expressions in python 3.6\n",
    "# Therefore can't use GridSearch with our doc_cleaner with a compiled regular expression\n",
    "# In python 3.7 you can!!\n",
    "# Minimal working example\n",
    "#import re,copy\n",
    "#class MyClass():\n",
    "#    def __init__(self):\n",
    "#        self.regex=re.compile('\\d+')\n",
    "#\n",
    "#myobj = MyClass()    \n",
    "#copy.deepcopy(myobj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:02:12.038899Z",
     "start_time": "2021-03-01T12:00:08.819Z"
    }
   },
   "outputs": [],
   "source": [
    "#X_train_vec = union.fit_transform(X_train)\n",
    "#n_features = X_train_vec.shape[1]\n",
    "#min_k = n_features//2\n",
    "#max_k = n_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:02:12.039919Z",
     "start_time": "2021-03-01T12:00:08.821Z"
    }
   },
   "outputs": [],
   "source": [
    "simple_count_vectorizer_stemmer = SimpleCountVectorizer(lemmatizer= None,\n",
    "                                                        stemmer= SnowballStemmer('english'),\n",
    "                                                        doc_cleaner=r\"[^a-zA-Z]\")\n",
    "\n",
    "count_vectorizer = sklearn.feature_extraction.text.CountVectorizer(ngram_range=(1,3))\n",
    "\n",
    "union = sklearn.pipeline.FeatureUnion([(\"simple_count_vectorizer_stemmer\", simple_count_vectorizer_stemmer),\n",
    "                                       (\"count_vectorizer\", count_vectorizer)])\n",
    "\n",
    "logistic = sklearn.linear_model.LogisticRegression(C=0.1)\n",
    "feature_selector = SelectKBest(chi2)\n",
    "\n",
    "\n",
    "# Hint\n",
    "#model_pipe_6 = sklearn.pipeline.Pipeline([(\"union_vectorizers\", union),\n",
    "#                                          (\"feature_selector\", feature_selector),\n",
    "#                                          (\"logisticregression\", logistic)],\n",
    "#                                         )# memory='/Users/Shared/sklearn_mem/')\n",
    "\n",
    "#possible_K = [int(x)-1 for x in np.linspace(min_k, max_k,10)]\n",
    "#parameteres = {'feature_selector__k':possible_K, 'logisticregression__C':[0.1,0.01,0.001]}\n",
    "\n",
    "#grid_6 = sklearn.model_selection.RandomizedSearchCV(model_pipe_6,\n",
    "#                                                    param_distributions=parameteres, \n",
    "#                                                    cv=3,\n",
    "#                                                    n_iter=2,\n",
    "#                                                    n_jobs=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:02:12.041030Z",
     "start_time": "2021-03-01T12:00:08.823Z"
    }
   },
   "outputs": [],
   "source": [
    "#%time\n",
    "#grid_6.fit(X_train, y_train)\n",
    "# Returns error? Welcome to your custom sklearn pipelines\n",
    "# Question: Why is there an error?\n",
    "# ValueError: k should be >=0, <= n_features; got 3238254.Use k='all' to return all features.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:02:12.042173Z",
     "start_time": "2021-03-01T12:00:08.824Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#acc_train = np.mean(grid_6.predict(X_train) == y_train)\n",
    "#acc_test = np.mean(grid_6.predict(X_test) == y_test)\n",
    "#print(\"acc_train={} acc_test={}\".format(acc_train, acc_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-01T12:02:12.043133Z",
     "start_time": "2021-03-01T12:00:08.826Z"
    }
   },
   "outputs": [],
   "source": [
    "#df_results[\"Feature union + selection + CV\"] = [acc_train, acc_test]\n",
    "#df_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Hashing words (Exercise)\n",
    "\n",
    "Explore the results you get by hashing the input words. Add `df_results[\"Feature hash + selection + CV\"]` in the table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}