FULL_PIPELINE.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8702979d",
   "metadata": {},
   "source": [
    "# FIRST FULL PIPELINE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14326cb2",
   "metadata": {},
   "source": [
    "Step 1: Import venue reviews file (OUTPUT: df with venue reviews)\n",
    "\n",
    "Step 2: Sentence Tokenization (OUTPUT: list of sentences)\n",
    "\n",
    "Step 3: Clean + Pre-Process Sentences (OUTPUT: list of cleaned sentences)\n",
    "\n",
    "Step 4: Binary Text Classification (OUTPUT: list of filtered sentences only discussing accesisbility)\n",
    "\n",
    "Step 5: Text Classification (OUTPUT: sentences categorized into class/aspect)\n",
    "\n",
    "Step 6: Summarization (OUTPUT: per class, summary of sentences)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8ceb749",
   "metadata": {},
   "source": [
    "## Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d5bad7fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# General\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# Data loading\n",
    "import os\n",
    "import glob\n",
    "\n",
    "# Cleaning and Pre-processing\n",
    "import re\n",
    "\n",
    "import spacy\n",
    "from spacy.language import Language\n",
    "from spacy_langdetect import LanguageDetector\n",
    "\n",
    "# NLTK\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tokenize import sent_tokenize\n",
    "\n",
    "# Spacy\n",
    "import spacy\n",
    "from spacy.language import Language\n",
    "from spacy_langdetect import LanguageDetector\n",
    "\n",
    "# TextBlob\n",
    "from textblob import TextBlob\n",
    "\n",
    "# From pre_processing.py\n",
    "from pre_processing import remove_nan\n",
    "from pre_processing import clean_translation\n",
    "\n",
    "from pre_processing import remove_emoji\n",
    "from pre_processing import clean_string\n",
    "\n",
    "from pre_processing import get_lang_detector\n",
    "from pre_processing import remove_non_ENG\n",
    "\n",
    "from pre_processing import remove_stopwords\n",
    "from pre_processing import lemmatize_string\n",
    "\n",
    "#from pre_processing import sent_tok\n",
    "from pre_processing import word_tok\n",
    "\n",
    "#from pre_processing import rating_to_sent\n",
    "from pre_processing import abs_date"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1dbfb803",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "19a34f18",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All_files: 1948\n",
      "File_Count: 1949\n"
     ]
    }
   ],
   "source": [
    "path = \"./Data/GoogleReviews\"\n",
    "all_files = glob.glob(path + \"/*.csv\")\n",
    "\n",
    "# Count number of venues in path\n",
    "file_count = 0\n",
    "dir = path\n",
    "for path in os.listdir(dir):\n",
    "    if os.path.isfile(os.path.join(dir, path)):\n",
    "        file_count += 1\n",
    "\n",
    "print(\"All_files:\", len(all_files))\n",
    "print(\"File_Count:\", file_count)\n",
    "\n",
    "# Create a list with all the data names\n",
    "data_names = []\n",
    "\n",
    "for i in range(file_count):\n",
    "    data_names.append(\"data\"+str(i))\n",
    "\n",
    "# Create empty list\n",
    "dataframes_list = []\n",
    "  \n",
    "# append datasets to the list\n",
    "for i in range(file_count-1):\n",
    "    temp_df = pd.read_csv(all_files[i])\n",
    "    temp_df = temp_df.drop(columns=\"Unnamed: 0\")\n",
    "    dataframes_list.append(temp_df)\n",
    "      \n",
    "# display datasets\n",
    "#for dataset in dataframes_list:\n",
    "#    display(dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b086560e",
   "metadata": {},
   "source": [
    "# Cleaning and Pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "527e18ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "df0 = dataframes_list[0]\n",
    "df1 = dataframes_list[1]\n",
    "df2 = dataframes_list[2]\n",
    "df3 = dataframes_list[3]\n",
    "df4 = dataframes_list[4]\n",
    "df5 = dataframes_list[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0778c5d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 15 µs, sys: 3 µs, total: 18 µs\n",
      "Wall time: 21.2 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_test_list = [df0, df1, df2, df3, df4, df5]\n",
    "\n",
    "def clean_preprocess(df):\n",
    "    # Drop columns we don't need and change names of columns\n",
    "    df = df.drop(columns=\"Review Rate\")\n",
    "    df = df.rename(columns={\"Review Time\": \"Date\", \"Review Text\": \"Text\"})\n",
    "    df[\"Cleaned_Sentence\"] = 0\n",
    "\n",
    "    # Drop NaN\n",
    "    df = remove_nan(df, \"Text\")\n",
    "\n",
    "    # Remove original language and keep translation\n",
    "    df[\"Text\"] = df[\"Text\"].apply(clean_translation)\n",
    "\n",
    "    # Remove emoji's\n",
    "    df[\"Text\"] = df[\"Text\"].apply(remove_emoji)\n",
    "\n",
    "    # Tokenize into sentences\n",
    "    punctuations = '\\.\\!\\?'\n",
    "    df = (df.drop('Text',axis=1).merge(df.Text.str\n",
    "                               .extractall(f'(?P<Sentence>[^{punctuations}]+[{punctuations}])\\s?')\n",
    "                               .reset_index('match'),left_index=True, right_index=True, how='left'))\n",
    "\n",
    "    # Remove NaN\n",
    "    df = remove_nan(df, \"Sentence\")\n",
    "\n",
    "    # Remove non-English sentences\n",
    "    df['Sentence'] = df['Sentence'].apply(remove_non_ENG)\n",
    "\n",
    "    # Remove NaN\n",
    "    df = remove_nan(df, \"Sentence\")\n",
    "\n",
    "    # Clean Sentences\n",
    "    df[\"Cleaned_Sentence\"] = df[\"Sentence\"].apply(clean_string)\n",
    "\n",
    "    # Change Absolute Date to Relative Date\n",
    "    df[\"Date\"] = df[\"Date\"].apply(abs_date)\n",
    "\n",
    "    # Remove Stopwords\n",
    "    df[\"Cleaned_Sentence\"] = df[\"Cleaned_Sentence\"].apply(remove_stopwords)\n",
    "\n",
    "    # Lemmatize\n",
    "    df[\"Cleaned_Sentence\"] = df[\"Cleaned_Sentence\"].apply(lemmatize_string)\n",
    "\n",
    "    df[\"Tokens\"] = df[\"Cleaned_Sentence\"].apply(word_tok)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fde8626",
   "metadata": {},
   "outputs": [],
   "source": [
    "df0 = clean_preprocess(df0)\n",
    "df1 = clean_preprocess(df1)\n",
    "df2 = clean_preprocess(df2)\n",
    "df3 = clean_preprocess(df3)\n",
    "df4 = clean_preprocess(df4)\n",
    "df5 = clean_preprocess(df5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd3a8ea0",
   "metadata": {},
   "source": [
    "# EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 474,
   "id": "c137b813",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_distributions(df):\n",
    "    print(\"Total number of reviews in:\", df.shape[0])\n",
    "    print(\"Class = 0:\", df[df[\"Class\"]==0].shape[0], \"-->\", df[df[\"Class\"]==0].shape[0]/df.shape[0]*100,\"%\")\n",
    "    print(\"Class = 1:\", df[df[\"Class\"]==1].shape[0], \"-->\", df[df[\"Class\"]==1].shape[0]/df.shape[0]*100,\"%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 475,
   "id": "d2b5c3b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of reviews in: 166\n",
      "Class = 0: 133 --> 80.12048192771084 %\n",
      "Class = 1: 33 --> 19.879518072289155 %\n",
      "Total number of reviews in: 125\n",
      "Class = 0: 124 --> 99.2 %\n",
      "Class = 1: 1 --> 0.8 %\n",
      "Total number of reviews in: 234\n",
      "Class = 0: 190 --> 81.19658119658119 %\n",
      "Class = 1: 44 --> 18.803418803418804 %\n",
      "Total number of reviews in: 109\n",
      "Class = 0: 90 --> 82.56880733944955 %\n",
      "Class = 1: 19 --> 17.431192660550458 %\n",
      "Total number of reviews in: 465\n",
      "Class = 0: 391 --> 84.08602150537634 %\n",
      "Class = 1: 74 --> 15.913978494623656 %\n",
      "Total number of reviews in: 466\n",
      "Class = 0: 368 --> 78.96995708154506 %\n",
      "Class = 1: 98 --> 21.030042918454935 %\n"
     ]
    }
   ],
   "source": [
    "get_distributions(df0)\n",
    "get_distributions(df1)\n",
    "get_distributions(df2)\n",
    "get_distributions(df3)\n",
    "get_distributions(df4)\n",
    "get_distributions(df5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 476,
   "id": "1f7f83a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(166, 8)\n",
      "(125, 8)\n",
      "(234, 8)\n",
      "(109, 8)\n",
      "(465, 8)\n",
      "(466, 8)\n"
     ]
    }
   ],
   "source": [
    "# Shape of the dataset\n",
    "print(df0.shape)\n",
    "print(df1.shape)\n",
    "print(df2.shape)\n",
    "print(df3.shape)\n",
    "print(df4.shape)\n",
    "print(df5.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0dd0f7d5",
   "metadata": {},
   "source": [
    "# Binary Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e78f888e",
   "metadata": {},
   "source": [
    "## Rule-Based Binary Classification (to filter reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "713a7e09",
   "metadata": {},
   "outputs": [],
   "source": [
    "from binary_classifier import rule_based_classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "id": "1cf8bdbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14.4 ms, sys: 1.87 ms, total: 16.3 ms\n",
      "Wall time: 17.1 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df0[\"Class\"] = df0[\"Cleaned_Sentence\"].apply(rule_based_classification)\n",
    "df1[\"Class\"] = df1[\"Cleaned_Sentence\"].apply(rule_based_classification)\n",
    "df2[\"Class\"] = df2[\"Cleaned_Sentence\"].apply(rule_based_classification)\n",
    "df3[\"Class\"] = df3[\"Cleaned_Sentence\"].apply(rule_based_classification)\n",
    "df4[\"Class\"] = df4[\"Cleaned_Sentence\"].apply(rule_based_classification)\n",
    "df5[\"Class\"] = df5[\"Cleaned_Sentence\"].apply(rule_based_classification)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "id": "b50266a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Name</th>\n",
       "      <th>Date</th>\n",
       "      <th>Cleaned_Sentence</th>\n",
       "      <th>match</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>Tokens</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Ellis</td>\n",
       "      <td>2019</td>\n",
       "      <td>bit quite went let fool ever case</td>\n",
       "      <td>0.0</td>\n",
       "      <td>It was a bit quite when we went in, but don’t ...</td>\n",
       "      <td>[bit, quite, went, let, fool, ever, case]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Ellis</td>\n",
       "      <td>2019</td>\n",
       "      <td>little place cozily busy</td>\n",
       "      <td>1.0</td>\n",
       "      <td>After a little while the place was cozily busy.</td>\n",
       "      <td>[little, place, cozily, busy]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>Ellis</td>\n",
       "      <td>2019</td>\n",
       "      <td>rightfully</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Rightfully so!</td>\n",
       "      <td>[rightfully]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>Ellis</td>\n",
       "      <td>2019</td>\n",
       "      <td>burger nacho lovely staff</td>\n",
       "      <td>3.0</td>\n",
       "      <td>The burgers (and nachos) were lovely, as was t...</td>\n",
       "      <td>[burger, nacho, lovely, staff]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>Ellis</td>\n",
       "      <td>2019</td>\n",
       "      <td>would definitely recommend place youre around ...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>I would definitely recommend this place if you...</td>\n",
       "      <td>[would, definitely, recommend, place, youre, a...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index   Name  Date                                   Cleaned_Sentence  \\\n",
       "0      0  Ellis  2019                  bit quite went let fool ever case   \n",
       "1      0  Ellis  2019                           little place cozily busy   \n",
       "2      0  Ellis  2019                                         rightfully   \n",
       "3      0  Ellis  2019                          burger nacho lovely staff   \n",
       "4      0  Ellis  2019  would definitely recommend place youre around ...   \n",
       "\n",
       "   match                                           Sentence  \\\n",
       "0    0.0  It was a bit quite when we went in, but don’t ...   \n",
       "1    1.0    After a little while the place was cozily busy.   \n",
       "2    2.0                                     Rightfully so!   \n",
       "3    3.0  The burgers (and nachos) were lovely, as was t...   \n",
       "4    4.0  I would definitely recommend this place if you...   \n",
       "\n",
       "                                              Tokens  Class  \n",
       "0          [bit, quite, went, let, fool, ever, case]      0  \n",
       "1                      [little, place, cozily, busy]      0  \n",
       "2                                       [rightfully]      0  \n",
       "3                     [burger, nacho, lovely, staff]      1  \n",
       "4  [would, definitely, recommend, place, youre, a...      0  "
      ]
     },
     "execution_count": 394,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# View the dataset\n",
    "# Drop extra index\n",
    "#df0 = df0.drop(columns=\"level_0\")\n",
    "df1 = df1.drop(columns=\"level_0\")\n",
    "df2 = df2.drop(columns=\"level_0\")\n",
    "df3 = df3.drop(columns=\"level_0\")\n",
    "df4 = df4.drop(columns=\"level_0\")\n",
    "df5 = df5.drop(columns=\"level_0\")\n",
    "\n",
    "df0.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 694,
   "id": "e3cd21f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop match column\n",
    "df0 = df0.drop(columns=\"match\")\n",
    "df1 = df1.drop(columns=\"match\")\n",
    "df2 = df2.drop(columns=\"match\")\n",
    "df3 = df3.drop(columns=\"match\")\n",
    "df4 = df4.drop(columns=\"match\")\n",
    "df5 = df5.drop(columns=\"match\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d75a412",
   "metadata": {},
   "source": [
    "## Machine Learning Binary Classification (to filter reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "id": "69d0b7e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5f2056a",
   "metadata": {},
   "source": [
    "### Train and Test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 695,
   "id": "14471c96",
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_data(df):\n",
    "    \n",
    "    # Determine X and y variables\n",
    "    X = df[\"Cleaned_Sentence\"]\n",
    "    y = df[\"Class\"]\n",
    "    \n",
    "    # Split dataset into train and test set\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=42)\n",
    "    \n",
    "    return X_train, X_test, y_train, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 696,
   "id": "ea67dc87",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 738,
   "id": "8a36f8b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Determine X and y variables\n",
    "X0 = df0[\"Cleaned_Sentence\"]   # Predictor\n",
    "y0 = df0[\"Class\"]              # Target\n",
    "\n",
    "# Split dataset into train and test set\n",
    "X_train0, X_test0, y_train0, y_test0 = train_test_split(X0, y0, test_size=0.10, random_state=42)\n",
    "\n",
    "X1 = df1[\"Cleaned_Sentence\"]   # Predictor\n",
    "y1 = df1[\"Class\"]              # Target\n",
    "\n",
    "# Split dataset into train and test set\n",
    "X_train1, X_test1, y_train1, y_test1 = train_test_split(X1, y1, test_size=0.10, random_state=42)\n",
    "\n",
    "X2 = df2[\"Cleaned_Sentence\"]   # Predictor\n",
    "y2 = df2[\"Class\"]              # Target\n",
    "\n",
    "# Split dataset into train and test set\n",
    "X_train2, X_test2, y_train2, y_tes2t = train_test_split(X2, y2, test_size=0.10, random_state=42)\n",
    "\n",
    "X3 = df3[\"Cleaned_Sentence\"]   # Predictor\n",
    "y3 = df3[\"Class\"]              # Target\n",
    "\n",
    "# Split dataset into train and test set\n",
    "X_train3, X_test3, y_train3, y_test3 = train_test_split(X3, y3, test_size=0.10, random_state=42)\n",
    "\n",
    "X4 = df4[\"Cleaned_Sentence\"]   # Predictor\n",
    "y4 = df4[\"Class\"]              # Target\n",
    "\n",
    "# Split dataset into train and test set\n",
    "X_train4, X_test4, y_train4, y_test4 = train_test_split(X4, y4, test_size=0.10, random_state=42)\n",
    "\n",
    "X5 = df5[\"Cleaned_Sentence\"]   # Predictor\n",
    "y5 = df5[\"Class\"]              # Target\n",
    "\n",
    "# Split dataset into train and test set\n",
    "X_train5, X_test5, y_train5, y_test5 = train_test_split(X5, y5, test_size=0.10, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "id": "a8922c77",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check if y_test Class labels are correct\n",
    "#y_test\n",
    "\n",
    "#y_test_ind = [113, 164, 169, 101, 100, 15, 177, 35, 119, 152, 24, 76, 156, 118, 68, 16, 122, 30, 136]\n",
    "#for i in y_test_ind:\n",
    "#    print(i, df1[\"Sentence\"].iloc[i], df1[\"Class\"].iloc[i])\n",
    "\n",
    "# Adjust Class labels manually\n",
    "# Not necessary"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f168dda",
   "metadata": {},
   "source": [
    "## Method 1: from sklearn (Multinomial Naive Bayes, Support Vector Machinec)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48cbac16",
   "metadata": {},
   "source": [
    "### Extract Features from Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 740,
   "id": "986e4b79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(419, 728)"
      ]
     },
     "execution_count": 740,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "#df5\n",
    "\n",
    "count_vect = CountVectorizer()\n",
    "X_train_counts5 = count_vect.fit_transform(X_train5)\n",
    "X_train_counts5.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 741,
   "id": "96dd0e40",
   "metadata": {},
   "outputs": [],
   "source": [
    "count_vect.vocabulary_.get(u'algorithm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 742,
   "id": "2f7b8c91",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(419, 728)"
      ]
     },
     "execution_count": 742,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "tf_transformer = TfidfTransformer(use_idf=False).fit(X_train_counts5)\n",
    "X_train_tf5 = tf_transformer.transform(X_train_counts5)\n",
    "X_train_tf5.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 743,
   "id": "9ed4c7d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(419, 728)"
      ]
     },
     "execution_count": 743,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_transformer = TfidfTransformer()\n",
    "X_train_tfidf5 = tfidf_transformer.fit_transform(X_train_counts5)\n",
    "X_train_tfidf5.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "827d9b5a",
   "metadata": {},
   "source": [
    "### Training Classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 744,
   "id": "434b4cab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'The staff is friendly.' => 1\n",
      "'The toilet is not accessible.' => 0\n",
      "'The entrance has a ramp.' => 0\n",
      "'The food is delicious.' => 0\n",
      "'The toilet is downstairs.' => 0\n"
     ]
    }
   ],
   "source": [
    "# MULTINOMIAL NAIVE BAYES: df5\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "clf = MultinomialNB().fit(X_train_tfidf5, y_train5)\n",
    "\n",
    "docs_new = ['The staff is friendly.', 'The toilet is not accessible.',\n",
    "            'The entrance has a ramp.', 'The food is delicious.',\n",
    "            'The toilet is downstairs.']\n",
    "X_new_counts5 = count_vect.transform(docs_new)\n",
    "X_new_tfidf5 = tfidf_transformer.transform(X_new_counts5)\n",
    "\n",
    "predicted = clf.predict(X_new_tfidf5)\n",
    "\n",
    "for doc, category in zip(docs_new, predicted):\n",
    "    print('%r => %s' % (doc, category))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38b62b18",
   "metadata": {},
   "source": [
    "### Building Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 749,
   "id": "c0a1c215",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.851063829787234"
      ]
     },
     "execution_count": 749,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MULTINOMIAL NAIVE BAYES\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "text_clf = Pipeline([\n",
    "     ('vect', CountVectorizer()),\n",
    "     ('tfidf', TfidfTransformer()),\n",
    "     ('clf', MultinomialNB()) ])\n",
    "\n",
    "text_clf.fit(X_train5, y_train5)\n",
    "\n",
    "# Eval\n",
    "docs_test = X_test5\n",
    "predicted = text_clf.predict(docs_test)\n",
    "np.mean(predicted == y_test5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 746,
   "id": "c5dac78f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9787234042553191"
      ]
     },
     "execution_count": 746,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# SUPPOR VECTOR MACHINES \n",
    "\n",
    "# We can change the learner by simply plugging a different classifier object into our pipeline\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "text_clf = Pipeline([\n",
    "     ('vect', CountVectorizer()),\n",
    "     ('tfidf', TfidfTransformer()),\n",
    "     ('clf', SGDClassifier(loss='hinge', penalty='l2',\n",
    "                           alpha=1e-3, random_state=42,\n",
    "                           max_iter=5, tol=None)) ])\n",
    "\n",
    "text_clf.fit(X_train5, y_train5)\n",
    "\n",
    "# Eval\n",
    "docs_test = X_test5\n",
    "predicted = text_clf.predict(docs_test)\n",
    "np.mean(predicted == y_test5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 750,
   "id": "42506c24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      1.00      0.91        35\n",
      "           1       1.00      0.42      0.59        12\n",
      "\n",
      "    accuracy                           0.85        47\n",
      "   macro avg       0.92      0.71      0.75        47\n",
      "weighted avg       0.88      0.85      0.83        47\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# More detailed performance analysis of the results\n",
    "from sklearn import metrics\n",
    "print(metrics.classification_report(y_test5, predicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bd79e3d",
   "metadata": {},
   "source": [
    "## Method 2: from Medium (SVM and Naive Bayes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 751,
   "id": "002c046a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk import pos_tag\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from collections import defaultdict\n",
    "from nltk.corpus import wordnet as wn\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn import model_selection, naive_bayes, svm\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb8b46cd",
   "metadata": {},
   "source": [
    "### Extra pre-process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 752,
   "id": "9afc5d9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# WordNetLemmatizer requires Pos tags to understand if the word is noun or verb or adjective etc. By default it is set to Noun\n",
    "tag_map = defaultdict(lambda : wn.NOUN)\n",
    "tag_map['J'] = wn.ADJ\n",
    "tag_map['V'] = wn.VERB\n",
    "tag_map['R'] = wn.ADV\n",
    "for index,entry in enumerate(df5['Tokens']):\n",
    "    # Declaring Empty List to store the words that follow the rules for this step\n",
    "    Final_words = []\n",
    "    # Initializing WordNetLemmatizer()\n",
    "    word_Lemmatized = WordNetLemmatizer()\n",
    "    # pos_tag function below will provide the 'tag' i.e if the word is Noun(N) or Verb(V) or something else.\n",
    "    for word, tag in pos_tag(entry):\n",
    "        # Below condition is to check for Stop words and consider only alphabets\n",
    "        if word not in stopwords.words('english') and word.isalpha():\n",
    "            word_Final = word_Lemmatized.lemmatize(word,tag_map[tag[0]])\n",
    "            Final_words.append(word_Final)\n",
    "    # The final processed set of words for each iteration will be stored in 'text_final'\n",
    "    df5.loc[index,'text_final'] = str(Final_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 753,
   "id": "b699c314",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Name</th>\n",
       "      <th>Date</th>\n",
       "      <th>Cleaned_Sentence</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>Tokens</th>\n",
       "      <th>Class</th>\n",
       "      <th>text_final</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2022</td>\n",
       "      <td>absolutely top restaurant special fantastic dish</td>\n",
       "      <td>An absolutely top restaurant with special and ...</td>\n",
       "      <td>[absolutely, top, restaurant, special, fantast...</td>\n",
       "      <td>0</td>\n",
       "      <td>['absolutely', 'top', 'restaurant', 'special',...</td>\n",
       "      <td>Label 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2022</td>\n",
       "      <td>lovely vibe good friendly staff</td>\n",
       "      <td>Lovely vibe and very good and friendly staff.</td>\n",
       "      <td>[lovely, vibe, good, friendly, staff]</td>\n",
       "      <td>1</td>\n",
       "      <td>['lovely', 'vibe', 'good', 'friendly', 'staff']</td>\n",
       "      <td>Label 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2022</td>\n",
       "      <td>place go amsterdam</td>\n",
       "      <td>The place to go in Amsterdam.</td>\n",
       "      <td>[place, go, amsterdam]</td>\n",
       "      <td>0</td>\n",
       "      <td>['place', 'go', 'amsterdam']</td>\n",
       "      <td>Label 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2021</td>\n",
       "      <td>great place try everything order dish wine enjoy</td>\n",
       "      <td>Great place to try everything, order all the d...</td>\n",
       "      <td>[great, place, try, everything, order, dish, w...</td>\n",
       "      <td>0</td>\n",
       "      <td>['great', 'place', 'try', 'everything', 'order...</td>\n",
       "      <td>Label 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2019</td>\n",
       "      <td>favourite restaurant neighborhood moment</td>\n",
       "      <td>My favourite restaurant in the neighborhood at...</td>\n",
       "      <td>[favourite, restaurant, neighborhood, moment]</td>\n",
       "      <td>0</td>\n",
       "      <td>['favourite', 'restaurant', 'neighborhood', 'm...</td>\n",
       "      <td>Label 0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index               Name  Date  \\\n",
       "0      0  Firma Pekelharing  2022   \n",
       "1      0  Firma Pekelharing  2022   \n",
       "2      0  Firma Pekelharing  2022   \n",
       "3      1  Firma Pekelharing  2021   \n",
       "4      2  Firma Pekelharing  2019   \n",
       "\n",
       "                                   Cleaned_Sentence  \\\n",
       "0  absolutely top restaurant special fantastic dish   \n",
       "1                   lovely vibe good friendly staff   \n",
       "2                                place go amsterdam   \n",
       "3  great place try everything order dish wine enjoy   \n",
       "4          favourite restaurant neighborhood moment   \n",
       "\n",
       "                                            Sentence  \\\n",
       "0  An absolutely top restaurant with special and ...   \n",
       "1      Lovely vibe and very good and friendly staff.   \n",
       "2                      The place to go in Amsterdam.   \n",
       "3  Great place to try everything, order all the d...   \n",
       "4  My favourite restaurant in the neighborhood at...   \n",
       "\n",
       "                                              Tokens  Class  \\\n",
       "0  [absolutely, top, restaurant, special, fantast...      0   \n",
       "1              [lovely, vibe, good, friendly, staff]      1   \n",
       "2                             [place, go, amsterdam]      0   \n",
       "3  [great, place, try, everything, order, dish, w...      0   \n",
       "4      [favourite, restaurant, neighborhood, moment]      0   \n",
       "\n",
       "                                          text_final    label  \n",
       "0  ['absolutely', 'top', 'restaurant', 'special',...  Label 0  \n",
       "1    ['lovely', 'vibe', 'good', 'friendly', 'staff']  Label 1  \n",
       "2                       ['place', 'go', 'amsterdam']  Label 0  \n",
       "3  ['great', 'place', 'try', 'everything', 'order...  Label 0  \n",
       "4  ['favourite', 'restaurant', 'neighborhood', 'm...  Label 0  "
      ]
     },
     "execution_count": 753,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels=[]\n",
    "for c in df5[\"Class\"]:\n",
    "    if c == 1:\n",
    "        labels.append(\"Label 1\")\n",
    "    else:\n",
    "        labels.append(\"Label 0\")\n",
    "df5['label'] = labels\n",
    "df5.head()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 758,
   "id": "08231a2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "Train_X, Test_X, Train_Y, Test_Y = model_selection.train_test_split(df5['text_final'],df5['label'],test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 755,
   "id": "afde8072",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "355    Label 0\n",
       "389    Label 0\n",
       "451    Label 0\n",
       "127    Label 0\n",
       "121    Label 0\n",
       "        ...   \n",
       "332    Label 0\n",
       "51     Label 0\n",
       "90     Label 0\n",
       "221    Label 0\n",
       "370    Label 0\n",
       "Name: label, Length: 140, dtype: object"
      ]
     },
     "execution_count": 755,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Test_Y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51ece132",
   "metadata": {},
   "source": [
    "### Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 759,
   "id": "e617d04e",
   "metadata": {},
   "outputs": [],
   "source": [
    "Encoder = LabelEncoder()\n",
    "Train_Y = Encoder.fit_transform(Train_Y)\n",
    "Test_Y = Encoder.fit_transform(Test_Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6a0424c",
   "metadata": {},
   "source": [
    "### Word Vectorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 760,
   "id": "7b82f774",
   "metadata": {},
   "outputs": [],
   "source": [
    "Tfidf_vect = TfidfVectorizer(max_features=5000)\n",
    "Tfidf_vect.fit(df5['text_final'])\n",
    "Train_X_Tfidf = Tfidf_vect.transform(Train_X)\n",
    "Test_X_Tfidf = Tfidf_vect.transform(Test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 714,
   "id": "cd4c5799",
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(Tfidf_vect.vocabulary_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 715,
   "id": "27c31e17",
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(Train_X_Tfidf)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a15681e9",
   "metadata": {},
   "source": [
    "### Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 761,
   "id": "ff9f95ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Bayes Accuracy Score ->  84.28571428571429\n"
     ]
    }
   ],
   "source": [
    "# fit the training dataset on the NB classifier\n",
    "Naive = naive_bayes.MultinomialNB()\n",
    "Naive.fit(Train_X_Tfidf,Train_Y)\n",
    "# predict the labels on validation dataset\n",
    "predictions_NB = Naive.predict(Test_X_Tfidf)\n",
    "# Use accuracy_score function to get the accuracy\n",
    "print(\"Naive Bayes Accuracy Score -> \",accuracy_score(predictions_NB, Test_Y)*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ac65c9e",
   "metadata": {},
   "source": [
    "### Support Vector Machines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 764,
   "id": "5f7ff399",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVM Accuracy Score ->  95.71428571428572\n"
     ]
    }
   ],
   "source": [
    "# Classifier - Algorithm - SVM\n",
    "# fit the training dataset on the classifier\n",
    "SVM = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')\n",
    "SVM.fit(Train_X_Tfidf,Train_Y)\n",
    "# predict the labels on validation dataset\n",
    "predictions_SVM = SVM.predict(Test_X_Tfidf)\n",
    "# Use accuracy_score function to get the accuracy\n",
    "print(\"SVM Accuracy Score -> \",accuracy_score(predictions_SVM, Test_Y)*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b58f67a",
   "metadata": {},
   "source": [
    "## Re-Sampling Reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 731,
   "id": "bb87b0ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "import imblearn\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.over_sampling import RandomOverSampler \n",
    "\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61d8f5d8",
   "metadata": {},
   "source": [
    "### Undersampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 727,
   "id": "8408c961",
   "metadata": {},
   "outputs": [],
   "source": [
    "def undersample(df):\n",
    "    X = df\n",
    "    y = df[\"Class\"]\n",
    "    \n",
    "    # define the undersampling method\n",
    "    us = RandomUnderSampler(sampling_strategy='majority')\n",
    "\n",
    "    # define undersample strategy\n",
    "    us = RandomUnderSampler(sampling_strategy=0.95)\n",
    "\n",
    "    # summarize the new class distribution\n",
    "    print('Original dataset shape %s' % Counter(y))\n",
    "\n",
    "    # fit and apply the transform\n",
    "    X_under, y_under = us.fit_resample(X, y)\n",
    "    print('Resampled dataset shape %s' % Counter(y_under))\n",
    "    \n",
    "    return X_under, y_under"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 765,
   "id": "7813d395",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original dataset shape Counter({0: 133, 1: 33})\n",
      "Resampled dataset shape Counter({0: 34, 1: 33})\n",
      "Original dataset shape Counter({0: 124, 1: 1})\n",
      "Resampled dataset shape Counter({0: 1, 1: 1})\n",
      "Original dataset shape Counter({0: 190, 1: 44})\n",
      "Resampled dataset shape Counter({0: 46, 1: 44})\n",
      "Original dataset shape Counter({0: 90, 1: 19})\n",
      "Resampled dataset shape Counter({0: 20, 1: 19})\n",
      "Original dataset shape Counter({0: 391, 1: 74})\n",
      "Resampled dataset shape Counter({0: 77, 1: 74})\n",
      "Original dataset shape Counter({0: 368, 1: 98})\n",
      "Resampled dataset shape Counter({0: 103, 1: 98})\n"
     ]
    }
   ],
   "source": [
    "X_under0, y_under0 = undersample(df0)\n",
    "X_under1, y_under1 = undersample(df1)\n",
    "X_under2, y_under2 = undersample(df2)\n",
    "X_under3, y_under3 = undersample(df3)\n",
    "X_under4, y_under4 = undersample(df4)\n",
    "X_under5, y_under5 = undersample(df5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97f1c2f2",
   "metadata": {},
   "source": [
    "### Oversampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 732,
   "id": "1c53e42b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def oversample(df):\n",
    "    X = df\n",
    "    y = df[\"Class\"]\n",
    "    \n",
    "    # define the undersampling method\n",
    "    os = RandomOverSampler(sampling_strategy='majority')\n",
    "\n",
    "    # define undersample strategy\n",
    "    os = RandomOverSampler(sampling_strategy=0.95)\n",
    "\n",
    "    # summarize the new class distribution\n",
    "    print('Original dataset shape %s' % Counter(y))\n",
    "\n",
    "    # fit and apply the transform\n",
    "    X_over, y_over = os.fit_resample(X, y)\n",
    "    print('Resampled dataset shape %s' % Counter(y_over))\n",
    "    \n",
    "    return X_over, y_over"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 733,
   "id": "17508de4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original dataset shape Counter({0: 133, 1: 33})\n",
      "Resampled dataset shape Counter({0: 133, 1: 126})\n",
      "Original dataset shape Counter({0: 124, 1: 1})\n",
      "Resampled dataset shape Counter({0: 124, 1: 117})\n",
      "Original dataset shape Counter({0: 190, 1: 44})\n",
      "Resampled dataset shape Counter({0: 190, 1: 180})\n",
      "Original dataset shape Counter({0: 90, 1: 19})\n",
      "Resampled dataset shape Counter({0: 90, 1: 85})\n",
      "Original dataset shape Counter({0: 391, 1: 74})\n",
      "Resampled dataset shape Counter({0: 391, 1: 371})\n",
      "Original dataset shape Counter({0: 368, 1: 98})\n",
      "Resampled dataset shape Counter({0: 368, 1: 349})\n"
     ]
    }
   ],
   "source": [
    "X_over0, y_over0 = oversample(df0)\n",
    "X_over1, y_over1 = oversample(df1)\n",
    "X_over2, y_over2 = oversample(df2)\n",
    "X_over3, y_over3 = oversample(df3)\n",
    "X_over4, y_over4 = oversample(df4)\n",
    "X_over5, y_over5 = oversample(df5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8d5747f",
   "metadata": {},
   "source": [
    "### Data Augmentation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9dcc68ad",
   "metadata": {},
   "source": [
    "For text, data augmentation can be done by tokenizing document into a sentence, shuffling and rejoining them to generate new texts, or replacing adjectives, verbs etc by its a synonym to generate different text with the same meaning. Any pre-trained word embedding or NLTK’s wordnet can be used to find the synonym of a word."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 810,
   "id": "7f51649d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Name</th>\n",
       "      <th>Date</th>\n",
       "      <th>Cleaned_Sentence</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>Tokens</th>\n",
       "      <th>Class</th>\n",
       "      <th>text_final</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2022</td>\n",
       "      <td>absolutely top restaurant special fantastic dish</td>\n",
       "      <td>An absolutely top restaurant with special and ...</td>\n",
       "      <td>[fantastic, dish, special, top, restaurant, ab...</td>\n",
       "      <td>0</td>\n",
       "      <td>['absolutely', 'top', 'restaurant', 'special',...</td>\n",
       "      <td>Label 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2022</td>\n",
       "      <td>lovely vibe good friendly staff</td>\n",
       "      <td>Lovely vibe and very good and friendly staff.</td>\n",
       "      <td>[good, staff, friendly, vibe, lovely]</td>\n",
       "      <td>1</td>\n",
       "      <td>['lovely', 'vibe', 'good', 'friendly', 'staff']</td>\n",
       "      <td>Label 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2022</td>\n",
       "      <td>place go amsterdam</td>\n",
       "      <td>The place to go in Amsterdam.</td>\n",
       "      <td>[place, go, amsterdam]</td>\n",
       "      <td>0</td>\n",
       "      <td>['place', 'go', 'amsterdam']</td>\n",
       "      <td>Label 0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index               Name  Date  \\\n",
       "0      0  Firma Pekelharing  2022   \n",
       "1      0  Firma Pekelharing  2022   \n",
       "2      0  Firma Pekelharing  2022   \n",
       "\n",
       "                                   Cleaned_Sentence  \\\n",
       "0  absolutely top restaurant special fantastic dish   \n",
       "1                   lovely vibe good friendly staff   \n",
       "2                                place go amsterdam   \n",
       "\n",
       "                                            Sentence  \\\n",
       "0  An absolutely top restaurant with special and ...   \n",
       "1      Lovely vibe and very good and friendly staff.   \n",
       "2                      The place to go in Amsterdam.   \n",
       "\n",
       "                                              Tokens  Class  \\\n",
       "0  [fantastic, dish, special, top, restaurant, ab...      0   \n",
       "1              [good, staff, friendly, vibe, lovely]      1   \n",
       "2                             [place, go, amsterdam]      0   \n",
       "\n",
       "                                          text_final    label  \n",
       "0  ['absolutely', 'top', 'restaurant', 'special',...  Label 0  \n",
       "1    ['lovely', 'vibe', 'good', 'friendly', 'staff']  Label 1  \n",
       "2                       ['place', 'go', 'amsterdam']  Label 0  "
      ]
     },
     "execution_count": 810,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df5.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 829,
   "id": "5708ab07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Cleaned_Sentence</th>\n",
       "      <th>Tokens</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>lovely vibe good friendly staff</td>\n",
       "      <td>[lovely, friendly, vibe, staff, good]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>delicious food friendly staff</td>\n",
       "      <td>[delicious, food, friendly, staff]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>friendly sightly cheeky staff</td>\n",
       "      <td>[sightly, cheeky, staff, friendly]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>restaurant nice atmosphere good service</td>\n",
       "      <td>[good, service, nice, atmosphere, restaurant]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>waiter always available explained whole menu</td>\n",
       "      <td>[explained, menu, available, waiter, always, w...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>443</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>pleasant service</td>\n",
       "      <td>[pleasant, service]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>great food super nice service</td>\n",
       "      <td>[great, food, service, nice, super]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>446</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>great food good service</td>\n",
       "      <td>[good, food, great, service]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>good food nice service</td>\n",
       "      <td>[service, good, nice, food]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>450</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>good cuisine good service</td>\n",
       "      <td>[cuisine, service, good, good]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>98 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Name                              Cleaned_Sentence  \\\n",
       "1    Firma Pekelharing               lovely vibe good friendly staff   \n",
       "5    Firma Pekelharing                 delicious food friendly staff   \n",
       "7    Firma Pekelharing                 friendly sightly cheeky staff   \n",
       "8    Firma Pekelharing       restaurant nice atmosphere good service   \n",
       "9    Firma Pekelharing  waiter always available explained whole menu   \n",
       "..                 ...                                           ...   \n",
       "443  Firma Pekelharing                              pleasant service   \n",
       "444  Firma Pekelharing                 great food super nice service   \n",
       "446  Firma Pekelharing                       great food good service   \n",
       "449  Firma Pekelharing                        good food nice service   \n",
       "450  Firma Pekelharing                     good cuisine good service   \n",
       "\n",
       "                                                Tokens  Class  \n",
       "1                [lovely, friendly, vibe, staff, good]      1  \n",
       "5                   [delicious, food, friendly, staff]      1  \n",
       "7                   [sightly, cheeky, staff, friendly]      1  \n",
       "8        [good, service, nice, atmosphere, restaurant]      1  \n",
       "9    [explained, menu, available, waiter, always, w...      1  \n",
       "..                                                 ...    ...  \n",
       "443                                [pleasant, service]      1  \n",
       "444                [great, food, service, nice, super]      1  \n",
       "446                       [good, food, great, service]      1  \n",
       "449                        [service, good, nice, food]      1  \n",
       "450                     [cuisine, service, good, good]      1  \n",
       "\n",
       "[98 rows x 4 columns]"
      ]
     },
     "execution_count": 829,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# New DF for DF incl. augmented data\n",
    "df5_aug = df5[df5['Class']==1]\n",
    "df5_aug = df5_aug[['Name', 'Cleaned_Sentence', 'Tokens', 'Class']]\n",
    "df5_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 830,
   "id": "05a85266",
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_aug(data):\n",
    "    data_aug_list=[]\n",
    "    for token_list in data['Tokens']:\n",
    "        random.shuffle(token_list)\n",
    "        new_row = ' '.join(token_list)\n",
    "        data_aug_list.append(new_row)\n",
    "\n",
    "    return data_aug_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 831,
   "id": "83df979d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_aug_list = data_aug(df5_aug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 834,
   "id": "7b200191",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Cleaned_Sentence</th>\n",
       "      <th>Tokens</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>lovely vibe good friendly staff</td>\n",
       "      <td>[staff, good, vibe, friendly, lovely]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>delicious food friendly staff</td>\n",
       "      <td>[food, friendly, delicious, staff]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>friendly sightly cheeky staff</td>\n",
       "      <td>[cheeky, sightly, staff, friendly]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>restaurant nice atmosphere good service</td>\n",
       "      <td>[atmosphere, nice, restaurant, good, service]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>waiter always available explained whole menu</td>\n",
       "      <td>[menu, available, waiter, always, whole, expla...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>pleasant service</td>\n",
       "      <td>[pleasant, service]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>service food great nice super</td>\n",
       "      <td>[service, food, great, nice, super]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>good food great service</td>\n",
       "      <td>[good, food, great, service]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>nice food good service</td>\n",
       "      <td>[nice, food, good, service]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>good service cuisine good</td>\n",
       "      <td>[good, service, cuisine, good]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>196 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Name                              Cleaned_Sentence  \\\n",
       "0    Firma Pekelharing               lovely vibe good friendly staff   \n",
       "1    Firma Pekelharing                 delicious food friendly staff   \n",
       "2    Firma Pekelharing                 friendly sightly cheeky staff   \n",
       "3    Firma Pekelharing       restaurant nice atmosphere good service   \n",
       "4    Firma Pekelharing  waiter always available explained whole menu   \n",
       "..                 ...                                           ...   \n",
       "191  Firma Pekelharing                              pleasant service   \n",
       "192  Firma Pekelharing                 service food great nice super   \n",
       "193  Firma Pekelharing                       good food great service   \n",
       "194  Firma Pekelharing                        nice food good service   \n",
       "195  Firma Pekelharing                     good service cuisine good   \n",
       "\n",
       "                                                Tokens Class  \n",
       "0                [staff, good, vibe, friendly, lovely]     1  \n",
       "1                   [food, friendly, delicious, staff]     1  \n",
       "2                   [cheeky, sightly, staff, friendly]     1  \n",
       "3        [atmosphere, nice, restaurant, good, service]     1  \n",
       "4    [menu, available, waiter, always, whole, expla...     1  \n",
       "..                                                 ...   ...  \n",
       "191                                [pleasant, service]     1  \n",
       "192                [service, food, great, nice, super]     1  \n",
       "193                       [good, food, great, service]     1  \n",
       "194                        [nice, food, good, service]     1  \n",
       "195                     [good, service, cuisine, good]     1  \n",
       "\n",
       "[196 rows x 4 columns]"
      ]
     },
     "execution_count": 834,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df = pd.DataFrame(columns=[\"Name\", \"Cleaned_Sentence\", \"Tokens\", \"Class\"])\n",
    "new_df[\"Name\"] = df5_aug[\"Name\"]\n",
    "new_df[\"Cleaned_Sentence\"] = data_aug_list\n",
    "new_df[\"Tokens\"] = new_df[\"Cleaned_Sentence\"].apply(word_tok)\n",
    "new_df[\"Class\"] = 1\n",
    "df5_aug = pd.concat([df5_aug, new_df], ignore_index=True)\n",
    "df5_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 835,
   "id": "65c8d96b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Cleaned_Sentence</th>\n",
       "      <th>Tokens</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>lovely vibe good friendly staff</td>\n",
       "      <td>[staff, good, vibe, friendly, lovely]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>delicious food friendly staff</td>\n",
       "      <td>[food, friendly, delicious, staff]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>friendly sightly cheeky staff</td>\n",
       "      <td>[cheeky, sightly, staff, friendly]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>restaurant nice atmosphere good service</td>\n",
       "      <td>[atmosphere, nice, restaurant, good, service]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>waiter always available explained whole menu</td>\n",
       "      <td>[menu, available, waiter, always, whole, expla...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>fresh product well cooked nothing say</td>\n",
       "      <td>[fresh, product, well, cooked, nothing, say]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>delicious eaten</td>\n",
       "      <td>[delicious, eaten]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>561</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>nice drink regain strength</td>\n",
       "      <td>[nice, drink, regain, strength]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>okay special</td>\n",
       "      <td>[okay, special]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>cozy good</td>\n",
       "      <td>[cozy, good]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>564 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Name                              Cleaned_Sentence  \\\n",
       "0    Firma Pekelharing               lovely vibe good friendly staff   \n",
       "1    Firma Pekelharing                 delicious food friendly staff   \n",
       "2    Firma Pekelharing                 friendly sightly cheeky staff   \n",
       "3    Firma Pekelharing       restaurant nice atmosphere good service   \n",
       "4    Firma Pekelharing  waiter always available explained whole menu   \n",
       "..                 ...                                           ...   \n",
       "559  Firma Pekelharing         fresh product well cooked nothing say   \n",
       "560  Firma Pekelharing                               delicious eaten   \n",
       "561  Firma Pekelharing                    nice drink regain strength   \n",
       "562  Firma Pekelharing                                  okay special   \n",
       "563  Firma Pekelharing                                     cozy good   \n",
       "\n",
       "                                                Tokens Class  \n",
       "0                [staff, good, vibe, friendly, lovely]     1  \n",
       "1                   [food, friendly, delicious, staff]     1  \n",
       "2                   [cheeky, sightly, staff, friendly]     1  \n",
       "3        [atmosphere, nice, restaurant, good, service]     1  \n",
       "4    [menu, available, waiter, always, whole, expla...     1  \n",
       "..                                                 ...   ...  \n",
       "559       [fresh, product, well, cooked, nothing, say]     0  \n",
       "560                                 [delicious, eaten]     0  \n",
       "561                    [nice, drink, regain, strength]     0  \n",
       "562                                    [okay, special]     0  \n",
       "563                                       [cozy, good]     0  \n",
       "\n",
       "[564 rows x 4 columns]"
      ]
     },
     "execution_count": 835,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Re-join with df5\n",
    "df5_class_0 = df5[df5[\"Class\"]==0]\n",
    "df5_class_0 = df5_class_0[['Name', 'Cleaned_Sentence', 'Tokens', 'Class']]\n",
    "df5_aug = pd.concat([df5_aug, df5_class_0], ignore_index=True)\n",
    "df5_aug"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f32c1f66",
   "metadata": {},
   "source": [
    "## (AFTER RESAMPLING) Machine Learning Binary Classification (to filter reviews)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2abb809",
   "metadata": {},
   "source": [
    "### Method 1 & Undersample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 766,
   "id": "41949ac6",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train0, X_test0, y_train0, y_test0 = train_test_split(X_under0[\"Cleaned_Sentence\"], y_under0, test_size=0.10, random_state=42)\n",
    "X_train1, X_test1, y_train1, y_test1 = train_test_split(X_under1[\"Cleaned_Sentence\"], y_under1, test_size=0.10, random_state=42)\n",
    "X_train2, X_test2, y_train2, y_test2 = train_test_split(X_under2[\"Cleaned_Sentence\"], y_under2, test_size=0.10, random_state=42)\n",
    "X_train3, X_test3, y_train3, y_test3 = train_test_split(X_under3[\"Cleaned_Sentence\"], y_under3, test_size=0.10, random_state=42)\n",
    "X_train4, X_test4, y_train4, y_test4 = train_test_split(X_under4[\"Cleaned_Sentence\"], y_under4, test_size=0.10, random_state=42)\n",
    "X_train5, X_test5, y_train5, y_test5 = train_test_split(X_under5[\"Cleaned_Sentence\"], y_under5, test_size=0.10, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 767,
   "id": "b97d2d55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(180, 401)\n",
      "(180, 401)\n",
      "(180, 401)\n"
     ]
    }
   ],
   "source": [
    "count_vect = CountVectorizer()\n",
    "X_train_counts5 = count_vect.fit_transform(X_train5)\n",
    "print(X_train_counts5.shape)\n",
    "\n",
    "count_vect.vocabulary_.get(u'algorithm')\n",
    "\n",
    "tf_transformer = TfidfTransformer(use_idf=False).fit(X_train_counts5)\n",
    "X_train_tf5 = tf_transformer.transform(X_train_counts5)\n",
    "print(X_train_tf5.shape)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer()\n",
    "X_train_tfidf5 = tfidf_transformer.fit_transform(X_train_counts5)\n",
    "print(X_train_tfidf5.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 768,
   "id": "cdbc91f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7619047619047619"
      ]
     },
     "execution_count": 768,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MULTINOMIAL NAIVE BAYES\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "text_clf = Pipeline([\n",
    "     ('vect', CountVectorizer()),\n",
    "     ('tfidf', TfidfTransformer()),\n",
    "     ('clf', MultinomialNB()) ])\n",
    "\n",
    "text_clf.fit(X_train5, y_train5)\n",
    "\n",
    "# Evaluation\n",
    "docs_test = X_test5\n",
    "predicted = text_clf.predict(docs_test)\n",
    "np.mean(predicted == y_test5)\n",
    "# 0.7619047619047619"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 769,
   "id": "f67aeef5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9047619047619048"
      ]
     },
     "execution_count": 769,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# SUPPOR VECTOR MACHINES \n",
    "\n",
    "# We can change the learner by simply plugging a different classifier object into our pipeline\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "text_clf = Pipeline([\n",
    "     ('vect', CountVectorizer()),\n",
    "     ('tfidf', TfidfTransformer()),\n",
    "     ('clf', SGDClassifier(loss='hinge', penalty='l2',\n",
    "                           alpha=1e-3, random_state=42,\n",
    "                           max_iter=5, tol=None)) ])\n",
    "\n",
    "text_clf.fit(X_train5, y_train5)\n",
    "\n",
    "# Evaluation\n",
    "predicted = text_clf.predict(X_test5)\n",
    "np.mean(predicted == y_test5)\n",
    "\n",
    "# 0.9047619047619048"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d78ffc97",
   "metadata": {},
   "source": [
    "### Method 1 & Oversample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 838,
   "id": "ed1cf0ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train0, X_test0, y_train0, y_test0 = train_test_split(X_over0[\"Cleaned_Sentence\"], y_over0, test_size=0.10, random_state=42)\n",
    "X_train1, X_test1, y_train1, y_test1 = train_test_split(X_over1[\"Cleaned_Sentence\"], y_over1, test_size=0.10, random_state=42)\n",
    "X_train2, X_test2, y_train2, y_test2 = train_test_split(X_over2[\"Cleaned_Sentence\"], y_over2, test_size=0.10, random_state=42)\n",
    "X_train3, X_test3, y_train3, y_test3 = train_test_split(X_over3[\"Cleaned_Sentence\"], y_over3, test_size=0.10, random_state=42)\n",
    "X_train4, X_test4, y_train4, y_test4 = train_test_split(X_over4[\"Cleaned_Sentence\"], y_over4, test_size=0.10, random_state=42)\n",
    "X_train5, X_test5, y_train5, y_test5 = train_test_split(X_over5[\"Cleaned_Sentence\"], y_over5, test_size=0.10, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 848,
   "id": "39ed13c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "132    0\n",
       "227    1\n",
       "237    1\n",
       "143    0\n",
       "19     0\n",
       "      ..\n",
       "20     0\n",
       "188    1\n",
       "71     0\n",
       "106    1\n",
       "102    0\n",
       "Name: Class, Length: 233, dtype: int64"
      ]
     },
     "execution_count": 848,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 839,
   "id": "5310a5e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9305555555555556"
      ]
     },
     "execution_count": 839,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MULTINOMIAL NAIVE BAYES\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "text_clf = Pipeline([\n",
    "     ('vect', CountVectorizer()),\n",
    "     ('tfidf', TfidfTransformer()),\n",
    "     ('clf', MultinomialNB()) ])\n",
    "\n",
    "text_clf.fit(X_train5, y_train5)\n",
    "\n",
    "# Evaluation\n",
    "docs_test = X_test5\n",
    "predicted = text_clf.predict(docs_test)\n",
    "np.mean(predicted == y_test5)\n",
    "# 0.9305555555555556"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 772,
   "id": "06c9c3e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 772,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# SUPPOR VECTOR MACHINES \n",
    "\n",
    "# We can change the learner by simply plugging a different classifier object into our pipeline\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "text_clf = Pipeline([\n",
    "     ('vect', CountVectorizer()),\n",
    "     ('tfidf', TfidfTransformer()),\n",
    "     ('clf', SGDClassifier(loss='hinge', penalty='l2',\n",
    "                           alpha=1e-3, random_state=42,\n",
    "                           max_iter=5, tol=None)) ])\n",
    "\n",
    "text_clf.fit(X_train5, y_train5)\n",
    "\n",
    "# Evaluation\n",
    "predicted = text_clf.predict(X_test5)\n",
    "np.mean(predicted == y_test5)\n",
    "\n",
    "# 1.0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6260e8c6",
   "metadata": {},
   "source": [
    "### Method 1 & Data Augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 849,
   "id": "b11dc0e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df5_aug[\"Cleaned_Sentence\"]\n",
    "y = df5_aug[\"Class\"]\n",
    "y = y.astype('int')\n",
    "\n",
    "X_train5, X_test5, y_train5, y_test5 = train_test_split(X, y, test_size=0.10, random_state=42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 850,
   "id": "c7d2cdbb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9473684210526315"
      ]
     },
     "execution_count": 850,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MULTINOMIAL NAIVE BAYES\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "text_clf = Pipeline([\n",
    "     ('vect', CountVectorizer()),\n",
    "     ('tfidf', TfidfTransformer()),\n",
    "     ('clf', MultinomialNB()) ])\n",
    "\n",
    "text_clf.fit(X_train5, y_train5)\n",
    "\n",
    "# Evaluation\n",
    "docs_test = X_test5\n",
    "predicted = text_clf.predict(docs_test)\n",
    "np.mean(predicted == y_test5)\n",
    "# 0.9473684210526315"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 851,
   "id": "33d2c712",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9824561403508771"
      ]
     },
     "execution_count": 851,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# SUPPOR VECTOR MACHINES \n",
    "\n",
    "# We can change the learner by simply plugging a different classifier object into our pipeline\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "text_clf = Pipeline([\n",
    "     ('vect', CountVectorizer()),\n",
    "     ('tfidf', TfidfTransformer()),\n",
    "     ('clf', SGDClassifier(loss='hinge', penalty='l2',\n",
    "                           alpha=1e-3, random_state=42,\n",
    "                           max_iter=5, tol=None)) ])\n",
    "\n",
    "text_clf.fit(X_train5, y_train5)\n",
    "\n",
    "# Evaluation\n",
    "predicted = text_clf.predict(X_test5)\n",
    "np.mean(predicted == y_test5)\n",
    "\n",
    "# 0.9824561403508771"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94c1f4b7",
   "metadata": {},
   "source": [
    "### Method 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 853,
   "id": "7026d1b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Bayes Accuracy Score ->  94.73684210526315\n",
      "SVM Accuracy Score ->  98.24561403508771\n"
     ]
    }
   ],
   "source": [
    "# WordNetLemmatizer requires Pos tags to understand if the word is noun or verb or adjective etc. By default it is set to Noun\n",
    "tag_map = defaultdict(lambda : wn.NOUN)\n",
    "tag_map['J'] = wn.ADJ\n",
    "tag_map['V'] = wn.VERB\n",
    "tag_map['R'] = wn.ADV\n",
    "\n",
    "#X_over5, y_over5\n",
    "\n",
    "Train_X, Test_X, Train_Y, Test_Y = model_selection.train_test_split(X_over5,y_over5,test_size=0.3)\n",
    "\n",
    "Encoder = LabelEncoder()\n",
    "Train_Y = Encoder.fit_transform(y_train5)\n",
    "Test_Y = Encoder.fit_transform(y_test5)\n",
    "\n",
    "Tfidf_vect = TfidfVectorizer(max_features=5000)\n",
    "Tfidf_vect.fit(X_over5['text_final'])\n",
    "Train_X_Tfidf = Tfidf_vect.transform(X_train5)\n",
    "Test_X_Tfidf = Tfidf_vect.transform(X_test5)\n",
    "\n",
    "# fit the training dataset on the NB classifier\n",
    "Naive = naive_bayes.MultinomialNB()\n",
    "Naive.fit(Train_X_Tfidf,Train_Y)\n",
    "# predict the labels on validation dataset\n",
    "predictions_NB = Naive.predict(Test_X_Tfidf)\n",
    "# Use accuracy_score function to get the accuracy\n",
    "print(\"Naive Bayes Accuracy Score -> \",accuracy_score(predictions_NB, Test_Y)*100)\n",
    "\n",
    "# Classifier - Algorithm - SVM\n",
    "# fit the training dataset on the classifier\n",
    "SVM = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')\n",
    "SVM.fit(Train_X_Tfidf,Train_Y)\n",
    "# predict the labels on validation dataset\n",
    "predictions_SVM = SVM.predict(Test_X_Tfidf)\n",
    "# Use accuracy_score function to get the accuracy\n",
    "print(\"SVM Accuracy Score -> \",accuracy_score(predictions_SVM, Test_Y)*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fba8ff41",
   "metadata": {},
   "source": [
    "## Filter Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 854,
   "id": "0b47bfd9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Name</th>\n",
       "      <th>Date</th>\n",
       "      <th>Cleaned_Sentence</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>Tokens</th>\n",
       "      <th>Class</th>\n",
       "      <th>text_final</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2022</td>\n",
       "      <td>lovely vibe good friendly staff</td>\n",
       "      <td>Lovely vibe and very good and friendly staff.</td>\n",
       "      <td>[staff, good, vibe, friendly, lovely]</td>\n",
       "      <td>1</td>\n",
       "      <td>['lovely', 'vibe', 'good', 'friendly', 'staff']</td>\n",
       "      <td>Label 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2019</td>\n",
       "      <td>delicious food friendly staff</td>\n",
       "      <td>Delicious food and friendly staff.</td>\n",
       "      <td>[food, friendly, delicious, staff]</td>\n",
       "      <td>1</td>\n",
       "      <td>['delicious', 'food', 'friendly', 'staff']</td>\n",
       "      <td>Label 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2018</td>\n",
       "      <td>friendly sightly cheeky staff</td>\n",
       "      <td>Very friendly - and sightly cheeky - staff.</td>\n",
       "      <td>[cheeky, sightly, staff, friendly]</td>\n",
       "      <td>1</td>\n",
       "      <td>['friendly', 'sightly', 'cheeky', 'staff']</td>\n",
       "      <td>Label 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>4</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2020</td>\n",
       "      <td>restaurant nice atmosphere good service</td>\n",
       "      <td>The restaurant had a nice atmosphere and very ...</td>\n",
       "      <td>[atmosphere, nice, restaurant, good, service]</td>\n",
       "      <td>1</td>\n",
       "      <td>['restaurant', 'nice', 'atmosphere', 'good', '...</td>\n",
       "      <td>Label 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>4</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2020</td>\n",
       "      <td>waiter always available explained whole menu</td>\n",
       "      <td>The waiters were always available and explaine...</td>\n",
       "      <td>[menu, available, waiter, always, whole, expla...</td>\n",
       "      <td>1</td>\n",
       "      <td>['waiter', 'always', 'available', 'explained',...</td>\n",
       "      <td>Label 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>443</th>\n",
       "      <td>241</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2019</td>\n",
       "      <td>pleasant service</td>\n",
       "      <td>Pleasant service.</td>\n",
       "      <td>[pleasant, service]</td>\n",
       "      <td>1</td>\n",
       "      <td>['pleasant', 'service']</td>\n",
       "      <td>Label 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444</th>\n",
       "      <td>243</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2019</td>\n",
       "      <td>great food super nice service</td>\n",
       "      <td>Great food and super nice service.</td>\n",
       "      <td>[service, food, great, nice, super]</td>\n",
       "      <td>1</td>\n",
       "      <td>['great', 'food', 'super', 'nice', 'service']</td>\n",
       "      <td>Label 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>446</th>\n",
       "      <td>248</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2018</td>\n",
       "      <td>great food good service</td>\n",
       "      <td>Great food, good service.</td>\n",
       "      <td>[good, food, great, service]</td>\n",
       "      <td>1</td>\n",
       "      <td>['great', 'food', 'good', 'service']</td>\n",
       "      <td>Label 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449</th>\n",
       "      <td>252</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2019</td>\n",
       "      <td>good food nice service</td>\n",
       "      <td>Good food and very nice service.</td>\n",
       "      <td>[nice, food, good, service]</td>\n",
       "      <td>1</td>\n",
       "      <td>['good', 'food', 'nice', 'service']</td>\n",
       "      <td>Label 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>450</th>\n",
       "      <td>255</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2018</td>\n",
       "      <td>good cuisine good service</td>\n",
       "      <td>Good cuisine, good service.</td>\n",
       "      <td>[good, service, cuisine, good]</td>\n",
       "      <td>1</td>\n",
       "      <td>['good', 'cuisine', 'good', 'service']</td>\n",
       "      <td>Label 1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>98 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index               Name  Date  \\\n",
       "1        0  Firma Pekelharing  2022   \n",
       "5        2  Firma Pekelharing  2019   \n",
       "7        3  Firma Pekelharing  2018   \n",
       "8        4  Firma Pekelharing  2020   \n",
       "9        4  Firma Pekelharing  2020   \n",
       "..     ...                ...   ...   \n",
       "443    241  Firma Pekelharing  2019   \n",
       "444    243  Firma Pekelharing  2019   \n",
       "446    248  Firma Pekelharing  2018   \n",
       "449    252  Firma Pekelharing  2019   \n",
       "450    255  Firma Pekelharing  2018   \n",
       "\n",
       "                                 Cleaned_Sentence  \\\n",
       "1                 lovely vibe good friendly staff   \n",
       "5                   delicious food friendly staff   \n",
       "7                   friendly sightly cheeky staff   \n",
       "8         restaurant nice atmosphere good service   \n",
       "9    waiter always available explained whole menu   \n",
       "..                                            ...   \n",
       "443                              pleasant service   \n",
       "444                 great food super nice service   \n",
       "446                       great food good service   \n",
       "449                        good food nice service   \n",
       "450                     good cuisine good service   \n",
       "\n",
       "                                              Sentence  \\\n",
       "1        Lovely vibe and very good and friendly staff.   \n",
       "5                   Delicious food and friendly staff.   \n",
       "7          Very friendly - and sightly cheeky - staff.   \n",
       "8    The restaurant had a nice atmosphere and very ...   \n",
       "9    The waiters were always available and explaine...   \n",
       "..                                                 ...   \n",
       "443                                  Pleasant service.   \n",
       "444                 Great food and super nice service.   \n",
       "446                          Great food, good service.   \n",
       "449                   Good food and very nice service.   \n",
       "450                        Good cuisine, good service.   \n",
       "\n",
       "                                                Tokens  Class  \\\n",
       "1                [staff, good, vibe, friendly, lovely]      1   \n",
       "5                   [food, friendly, delicious, staff]      1   \n",
       "7                   [cheeky, sightly, staff, friendly]      1   \n",
       "8        [atmosphere, nice, restaurant, good, service]      1   \n",
       "9    [menu, available, waiter, always, whole, expla...      1   \n",
       "..                                                 ...    ...   \n",
       "443                                [pleasant, service]      1   \n",
       "444                [service, food, great, nice, super]      1   \n",
       "446                       [good, food, great, service]      1   \n",
       "449                        [nice, food, good, service]      1   \n",
       "450                     [good, service, cuisine, good]      1   \n",
       "\n",
       "                                            text_final    label  \n",
       "1      ['lovely', 'vibe', 'good', 'friendly', 'staff']  Label 1  \n",
       "5           ['delicious', 'food', 'friendly', 'staff']  Label 1  \n",
       "7           ['friendly', 'sightly', 'cheeky', 'staff']  Label 1  \n",
       "8    ['restaurant', 'nice', 'atmosphere', 'good', '...  Label 1  \n",
       "9    ['waiter', 'always', 'available', 'explained',...  Label 1  \n",
       "..                                                 ...      ...  \n",
       "443                            ['pleasant', 'service']  Label 1  \n",
       "444      ['great', 'food', 'super', 'nice', 'service']  Label 1  \n",
       "446               ['great', 'food', 'good', 'service']  Label 1  \n",
       "449                ['good', 'food', 'nice', 'service']  Label 1  \n",
       "450             ['good', 'cuisine', 'good', 'service']  Label 1  \n",
       "\n",
       "[98 rows x 9 columns]"
      ]
     },
     "execution_count": 854,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter df on Sentences discussing Accessibility\n",
    "df5_filtered = df5[df5[\"Class\"]==1]\n",
    "df5_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 692,
   "id": "9f5c1446",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'lovely vibe good friendly staff'"
      ]
     },
     "execution_count": 692,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df5_filtered.iloc[0][3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34466b6e",
   "metadata": {},
   "source": [
    "# Topic Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2e9f0d5",
   "metadata": {},
   "source": [
    "## Rule-Based Topic Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 866,
   "id": "e3a618fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "from topic_classification import topic_classification\n",
    "#from topic_classification import topic_bool\n",
    "\n",
    "def topic_classification(text):\n",
    "\n",
    "    general = [\"wheelchair\", \"disability\", \"disabled\", \"handicap\", \"handicapped\", \"mobility\"]\n",
    "    entrance = [\"entrance\", \"door\", \"ramp\", \"narrow\"]\n",
    "    bathroom = [\"toilet\", \"bathroom\", \"restroom\"]\n",
    "    transport = [\"parking\"]\n",
    "    stairs = [\"elevator\", \"steps\", \"stair\", \"steep\", \"narrow\"]\n",
    "    space = [\"cramped\", \"spacious\"]\n",
    "    staff = [\"service\", \"staff\", \"waiter\", \"waitress\"]\n",
    "\n",
    "    category = str()\n",
    "\n",
    "    if any(word in text for word in general):\n",
    "        category = category + \"general\"\n",
    "    elif any(word in text for word in entrance):\n",
    "        category = category + \"entrance\"\n",
    "    elif any(word in text for word in bathroom):\n",
    "        category = category + \"bathroom\"\n",
    "    elif any(word in text for word in transport):\n",
    "        category = category + \"transport\"\n",
    "    elif any(word in text for word in stairs):\n",
    "        category = category + \"stairs\"\n",
    "    elif any(word in text for word in space):\n",
    "        category = category + \"space\"\n",
    "    elif any(word in text for word in staff):\n",
    "        category = category + \"staff\"\n",
    "\n",
    "    return category\n",
    "\n",
    "def topic_bool(text):\n",
    "    \n",
    "    if text == \"general\":\n",
    "        category_id = 1\n",
    "    elif text == \"entrance\":\n",
    "        category_id = 2\n",
    "    elif text == \"bathroom\":\n",
    "        category_id = 3\n",
    "    elif text == \"transport\":\n",
    "        category_id = 4\n",
    "    elif text == \"stairs\":\n",
    "        category_id = 5\n",
    "    elif text == \"space\":\n",
    "        category_id = 6\n",
    "    elif text == \"staff\":\n",
    "        category_id = 7\n",
    "    \n",
    "    return category_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 864,
   "id": "51b9e592",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.38 ms, sys: 21.7 ms, total: 26.1 ms\n",
      "Wall time: 35.9 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df5_filtered[\"Category\"] = df5_filtered[\"Cleaned_Sentence\"].apply(topic_classification)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 867,
   "id": "18798ddb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.89 ms, sys: 8.37 ms, total: 11.3 ms\n",
      "Wall time: 14.9 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df5_filtered[\"Category_ID\"] = df5_filtered[\"Category\"].apply(topic_bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 868,
   "id": "94aeefd6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Cleaned_Sentence</th>\n",
       "      <th>Tokens</th>\n",
       "      <th>Class</th>\n",
       "      <th>Category</th>\n",
       "      <th>Category_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>lovely vibe good friendly staff</td>\n",
       "      <td>[staff, good, vibe, friendly, lovely]</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>delicious food friendly staff</td>\n",
       "      <td>[food, friendly, delicious, staff]</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>friendly sightly cheeky staff</td>\n",
       "      <td>[cheeky, sightly, staff, friendly]</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>restaurant nice atmosphere good service</td>\n",
       "      <td>[atmosphere, nice, restaurant, good, service]</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>waiter always available explained whole menu</td>\n",
       "      <td>[menu, available, waiter, always, whole, expla...</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Name                              Cleaned_Sentence  \\\n",
       "1  Firma Pekelharing               lovely vibe good friendly staff   \n",
       "5  Firma Pekelharing                 delicious food friendly staff   \n",
       "7  Firma Pekelharing                 friendly sightly cheeky staff   \n",
       "8  Firma Pekelharing       restaurant nice atmosphere good service   \n",
       "9  Firma Pekelharing  waiter always available explained whole menu   \n",
       "\n",
       "                                              Tokens  Class Category  \\\n",
       "1              [staff, good, vibe, friendly, lovely]      1    staff   \n",
       "5                 [food, friendly, delicious, staff]      1    staff   \n",
       "7                 [cheeky, sightly, staff, friendly]      1    staff   \n",
       "8      [atmosphere, nice, restaurant, good, service]      1    staff   \n",
       "9  [menu, available, waiter, always, whole, expla...      1    staff   \n",
       "\n",
       "   Category_ID  \n",
       "1            7  \n",
       "5            7  \n",
       "7            7  \n",
       "8            7  \n",
       "9            7  "
      ]
     },
     "execution_count": 868,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df5_filtered.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 873,
   "id": "1adadc66",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(98, 6)"
      ]
     },
     "execution_count": 873,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df5_filtered.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83f6dc93",
   "metadata": {},
   "source": [
    "## Topic Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 879,
   "id": "31e9f422",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name                1\n",
      "Cleaned_Sentence    1\n",
      "Tokens              1\n",
      "Class               1\n",
      "Category            1\n",
      "Category_ID         1\n",
      "dtype: int64\n",
      "Name                1\n",
      "Cleaned_Sentence    1\n",
      "Tokens              1\n",
      "Class               1\n",
      "Category            1\n",
      "Category_ID         1\n",
      "dtype: int64\n",
      "Name                1\n",
      "Cleaned_Sentence    1\n",
      "Tokens              1\n",
      "Class               1\n",
      "Category            1\n",
      "Category_ID         1\n",
      "dtype: int64\n",
      "Name                95\n",
      "Cleaned_Sentence    95\n",
      "Tokens              95\n",
      "Class               95\n",
      "Category            95\n",
      "Category_ID         95\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#print(df5_filtered[df5_filtered[\"Category_ID\"]==1].count()) ==> 0\n",
    "print(df5_filtered[df5_filtered[\"Category_ID\"]==2].count())# ==> 1\n",
    "print(df5_filtered[df5_filtered[\"Category_ID\"]==3].count())# ==> 1\n",
    "#print(df5_filtered[df5_filtered[\"Category_ID\"]==4].count()) ==> 0\n",
    "#print(df5_filtered[df5_filtered[\"Category_ID\"]==5].count()) ==> 0\n",
    "print(df5_filtered[df5_filtered[\"Category_ID\"]==6].count())# ==> 1\n",
    "print(df5_filtered[df5_filtered[\"Category_ID\"]==7].count())# ==> 95"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f3fe654",
   "metadata": {},
   "source": [
    "## Re-Sample Categories"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49273809",
   "metadata": {},
   "source": [
    "## Method 1: Machine Learning Topic Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cf98c12",
   "metadata": {},
   "source": [
    "https://medium.com/@bedigunjit/simple-guide-to-text-classification-nlp-using-svm-and-naive-bayes-with-python-421db3a72d34"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 857,
   "id": "ea75ae86",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.stats import randint\n",
    "import seaborn as sns # used for plot interactive graph.\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from io import StringIO\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import chi2\n",
    "from IPython.display import display\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ffde3f17",
   "metadata": {},
   "source": [
    "### Pre-Process / EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 861,
   "id": "eee79633",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(98, 5)"
      ]
     },
     "execution_count": 861,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df5_filtered = df5_filtered[[\"Name\", \"Cleaned_Sentence\", \"Tokens\", \"Class\", \"Category\"]]\n",
    "df5_filtered.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 862,
   "id": "94b4a914",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Cleaned_Sentence</th>\n",
       "      <th>Tokens</th>\n",
       "      <th>Class</th>\n",
       "      <th>Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>lovely vibe good friendly staff</td>\n",
       "      <td>[staff, good, vibe, friendly, lovely]</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>delicious food friendly staff</td>\n",
       "      <td>[food, friendly, delicious, staff]</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>friendly sightly cheeky staff</td>\n",
       "      <td>[cheeky, sightly, staff, friendly]</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>restaurant nice atmosphere good service</td>\n",
       "      <td>[atmosphere, nice, restaurant, good, service]</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>waiter always available explained whole menu</td>\n",
       "      <td>[menu, available, waiter, always, whole, expla...</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Name                              Cleaned_Sentence  \\\n",
       "1  Firma Pekelharing               lovely vibe good friendly staff   \n",
       "5  Firma Pekelharing                 delicious food friendly staff   \n",
       "7  Firma Pekelharing                 friendly sightly cheeky staff   \n",
       "8  Firma Pekelharing       restaurant nice atmosphere good service   \n",
       "9  Firma Pekelharing  waiter always available explained whole menu   \n",
       "\n",
       "                                              Tokens  Class Category  \n",
       "1              [staff, good, vibe, friendly, lovely]      1    staff  \n",
       "5                 [food, friendly, delicious, staff]      1    staff  \n",
       "7                 [cheeky, sightly, staff, friendly]      1    staff  \n",
       "8      [atmosphere, nice, restaurant, good, service]      1    staff  \n",
       "9  [menu, available, waiter, always, whole, expla...      1    staff  "
      ]
     },
     "execution_count": 862,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df5_filtered.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 863,
   "id": "531441e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Cleaned_Sentence</th>\n",
       "      <th>Tokens</th>\n",
       "      <th>Class</th>\n",
       "      <th>Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>unfortunately one waitress quite rude toilet c...</td>\n",
       "      <td>[hence, one, waitress, star, 3, rude, quite, t...</td>\n",
       "      <td>1</td>\n",
       "      <td>bathroom</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Name                                   Cleaned_Sentence  \\\n",
       "240  Firma Pekelharing  unfortunately one waitress quite rude toilet c...   \n",
       "\n",
       "                                                Tokens  Class  Category  \n",
       "240  [hence, one, waitress, star, 3, rude, quite, t...      1  bathroom  "
      ]
     },
     "execution_count": 863,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df5_filtered[df5_filtered[\"Category\"]==\"bathroom\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 632,
   "id": "aa613268",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/rn/k7vf4gp97fb11dzjbcr7w8bc0000gn/T/ipykernel_9062/1932043138.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df5_filtered['category_id'] = df5_filtered['Category'].factorize()[0]\n"
     ]
    }
   ],
   "source": [
    "# Create a new column 'category_id' with encoded categories \n",
    "df5_filtered['category_id'] = df5_filtered['Category'].factorize()[0]\n",
    "category_id_df = df5_filtered[['Category', 'category_id']].drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 634,
   "id": "e6a7a390",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Name</th>\n",
       "      <th>Date</th>\n",
       "      <th>Cleaned_Sentence</th>\n",
       "      <th>match</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>Tokens</th>\n",
       "      <th>Class</th>\n",
       "      <th>text_final</th>\n",
       "      <th>label</th>\n",
       "      <th>Category</th>\n",
       "      <th>category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2022</td>\n",
       "      <td>lovely vibe good friendly staff</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Lovely vibe and very good and friendly staff.</td>\n",
       "      <td>[lovely, vibe, good, friendly, staff]</td>\n",
       "      <td>1</td>\n",
       "      <td>['lovely', 'vibe', 'good', 'friendly', 'staff']</td>\n",
       "      <td>Label 1</td>\n",
       "      <td>staff</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2019</td>\n",
       "      <td>delicious food friendly staff</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Delicious food and friendly staff.</td>\n",
       "      <td>[delicious, food, friendly, staff]</td>\n",
       "      <td>1</td>\n",
       "      <td>['delicious', 'food', 'friendly', 'staff']</td>\n",
       "      <td>Label 1</td>\n",
       "      <td>staff</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2018</td>\n",
       "      <td>friendly sightly cheeky staff</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Very friendly - and sightly cheeky - staff.</td>\n",
       "      <td>[friendly, sightly, cheeky, staff]</td>\n",
       "      <td>1</td>\n",
       "      <td>['friendly', 'sightly', 'cheeky', 'staff']</td>\n",
       "      <td>Label 1</td>\n",
       "      <td>staff</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>4</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2020</td>\n",
       "      <td>restaurant nice atmosphere good service</td>\n",
       "      <td>0.0</td>\n",
       "      <td>The restaurant had a nice atmosphere and very ...</td>\n",
       "      <td>[restaurant, nice, atmosphere, good, service]</td>\n",
       "      <td>1</td>\n",
       "      <td>['restaurant', 'nice', 'atmosphere', 'good', '...</td>\n",
       "      <td>Label 1</td>\n",
       "      <td>staff</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>4</td>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>2020</td>\n",
       "      <td>waiter always available explained whole menu</td>\n",
       "      <td>1.0</td>\n",
       "      <td>The waiters were always available and explaine...</td>\n",
       "      <td>[waiter, always, available, explained, whole, ...</td>\n",
       "      <td>1</td>\n",
       "      <td>['waiter', 'always', 'available', 'explained',...</td>\n",
       "      <td>Label 1</td>\n",
       "      <td>staff</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index               Name  Date  \\\n",
       "1      0  Firma Pekelharing  2022   \n",
       "5      2  Firma Pekelharing  2019   \n",
       "7      3  Firma Pekelharing  2018   \n",
       "8      4  Firma Pekelharing  2020   \n",
       "9      4  Firma Pekelharing  2020   \n",
       "\n",
       "                               Cleaned_Sentence  match  \\\n",
       "1               lovely vibe good friendly staff    1.0   \n",
       "5                 delicious food friendly staff    1.0   \n",
       "7                 friendly sightly cheeky staff    1.0   \n",
       "8       restaurant nice atmosphere good service    0.0   \n",
       "9  waiter always available explained whole menu    1.0   \n",
       "\n",
       "                                            Sentence  \\\n",
       "1      Lovely vibe and very good and friendly staff.   \n",
       "5                 Delicious food and friendly staff.   \n",
       "7        Very friendly - and sightly cheeky - staff.   \n",
       "8  The restaurant had a nice atmosphere and very ...   \n",
       "9  The waiters were always available and explaine...   \n",
       "\n",
       "                                              Tokens  Class  \\\n",
       "1              [lovely, vibe, good, friendly, staff]      1   \n",
       "5                 [delicious, food, friendly, staff]      1   \n",
       "7                 [friendly, sightly, cheeky, staff]      1   \n",
       "8      [restaurant, nice, atmosphere, good, service]      1   \n",
       "9  [waiter, always, available, explained, whole, ...      1   \n",
       "\n",
       "                                          text_final    label Category  \\\n",
       "1    ['lovely', 'vibe', 'good', 'friendly', 'staff']  Label 1    staff   \n",
       "5         ['delicious', 'food', 'friendly', 'staff']  Label 1    staff   \n",
       "7         ['friendly', 'sightly', 'cheeky', 'staff']  Label 1    staff   \n",
       "8  ['restaurant', 'nice', 'atmosphere', 'good', '...  Label 1    staff   \n",
       "9  ['waiter', 'always', 'available', 'explained',...  Label 1    staff   \n",
       "\n",
       "   category_id  \n",
       "1            0  \n",
       "5            0  \n",
       "7            0  \n",
       "8            0  \n",
       "9            0  "
      ]
     },
     "execution_count": 634,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dictionaries for future use\n",
    "category_to_id = dict(category_id_df.values)\n",
    "id_to_category = dict(category_id_df[['category_id', 'Category']].values)\n",
    "# New dataframe\n",
    "df5_filtered.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 635,
   "id": "a6816a34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhEAAAGDCAYAAACV2ELHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAj6UlEQVR4nO3deZgkVZm28fsRcGRRQFq5bBlpxRUVUFtHEBUU9w1X5HMBZxSVEXBBXHBs2mUUUQZ3BUcBwQUQFVdQRFRk60Y2ZRAHGJcGEVF2AeX9/ogozU6rurKPVV1Vzf27rro688SJiDdOVmc+dSIyM1WFJEnSyrrdTBcgSZLmJkOEJElqYoiQJElNDBGSJKmJIUKSJDUxREiSpCaGCEmS1MQQoRmX5NIkv02y7kDby5N8v7+9IEklWXNovUOTvKu/vWvf58ChPjv27YcObeu6/ue3ST6WZK2hem4c6HNdko8M7Ocvfds1Sc5J8vRJjm+DJB9PcnmSG5Kcl+Rl44zB8D7nT7C9tya5pO/z6yRfHFj2/SR/GtrO1/pl2/XH/tGh7f2oP663Dqzzp4HjvC7JT/u+leT6oe3v0y/br1/+/IFtr9m3LRhoe0SSbyb5Y5KrkpwxNh59jbcObf+6JFv3yx+Y5IQkf+jXX5rkqROM0+Bjtdzj2C/fJsn3klyb5OokX0uy+cDywVquTXLh8OM2tL/h362xn52G+o2N0yPG2cZkY/Prcdb5fpKXr6CuuyX57ySX9cfxP0kWZ/n/b0lycZKfDbR9a+AYbkly88D9T0z2WPXbeEKSk/r9/j7J2UnelOQOA302T3Jc/xhc2/ffZgXjemmSN/fLXpHkgiT/NNB/oyRXJHnyRGOiqWOI0GyxJrDXP7iN/wV2yvJh46XAz8fpu0FVrQc8GNga+Peh5c+oqvUGfl4zsOzUft0NgI8BX0iywXgFJbk98F1g034/6wNvBN6b5PWT7HPZONvbBXgJsENfw0LgxKFurxnazjMGll0PvHTwRX1MVf3n2DrAq8aOs/954EDXLYe2/76BZVcB70iyxgTjsTXwPeBk4N7ARsCrgacMdFs2tP31qurUftnXgO8AGwN3BfYErhlvX71Tx3sc+zpOAL4KzAfuCZwDnJLkXsO1AHcCXgcckuR+K9gf9L9bAz+DIS90j99VwC4NY7NSktwZOBVYG9i6qu4IPIHud3ezga6PoRvPeyV5OEBVPWXg9+FI4H0Dx/Sqfr0JH6t0YfIY4HPAplW1EbATsAnwz32fzYBTgPPoHoP5wJeBEwbDSG/s/+zzgP9I8oSqOgT4NfD2gX4HAd+sqm+3jptGZ4jQbHEAsPdEL8YjupzuyehJ8Ncn0G2A4yZaoaquoHtR2nyiPitY91bgs8C6wH0m6PYS4B7A86vqkqq6pX9y25PuxfZOK7nbhwPHV9X/9jVcXlUHr8T6fwQOBRat5H5H9W3gZuDFEyw/ADisqvavqiurs7SqXjDZhpPMo3uhOaSqbu5/TqmqHzXU+T7g8Kr6YFVdW1VXVdXbgNOA/YY793V+k+7Ff4uG/Y15NN0L5V7AC/uQOaZ5bFbg9cC1wIur6lKAqvpVVe1VVecO9NuFLlB9k6Fw06IPSwcC76iqQ6rqqn7fF1bVHlV1Ud91P7qgt2//GFxbVR+i+3+1/3jbrqolwE+BrfqmVwC7J9kqyROBx9MFPq0ChgjNFkuA7wN7/4PbOZxu9gHghXRPjDdN1DndKYMn0b14rJT+r+2XAbcA/zdBtycA36qq64favwTcgW52YmWcRjeT8MYkCyf6i38S7waeO8Jf1C0K+A9gUQZOEQEkWYfueI9p3PbvgV8AR6Q7TbVxy0b6OrYBjh5n8VF0j9nwOrdL8kxgXl9Dq13oZlPGZieePlDTPzI2E9kBOLYPvOPq9/08utmGI/n7cNPifnQzDl+apN8TmPhxeFRf23KSPBJ4EP3j0IejtwOfBj4J7F5Vf2iuXCvFEKHZ5O3AHknu8g9s48vAdknWpwsTh0/Q78okfwR+QzfFP/zk/ZX+vPTYzysGlj2yX/dPwPvp/sq7YoL9zAMuG26sqj8DV/bLx9vnV8bbWFUdAexBF3xOBq4YOz884ENDtb9zaBuXA58A3jFBzZM5a2j7Txra/nHA74Dh8/Qb0j3n/N14DJk/tP0/Jlm3ui/62R64FPgAcFmSHySZaBYI+sdq4OeRwJ1XUMdlLP+YzO8f6xvpfrdeX1U/maT+K4f2+QD464v184HPVdUtdL9zY3/1N48NsO0K+m80wjafQxe0TwC+Tndq8WmTrDNhPemutRgbw8vHOib5Qr/8hiQv6ZvH/f/Rt92OblzGXJnkRrrTMx8DvjKw7CN0Yf7sqhps1zQzRGjWqKrz6Z7Ehl8U/9z/u9ZQ+1p0TxyD27gR+AbwNmBeVZ0ywe7mVdUGwDp052SHz5/uWFUbDPwcMrDstH7dDelOlTx6BYd1JXC34cb+uo15/fLx9rnjRBusqiOrage689qvojstMvhCvudQ7f8xzmb2B56UZMsV1D6Rhw5t//hx+rwN2JdutmXMH4BbGWc8hiwb2v4GYzM5VfXrqnpNVW1Gd53J9UwcFKF/rAZ+Tpukjrux/GOyrH+s7wR8CHjcJLVD/7s18HNB3/5sut/lb/b3jwSe0ofm5rEBVnQ65/cjbHMX4Kiq+nNV3QQcy+inNCZ6rH7fL//rvqvqhX29ZwFjM2jj/v/o226lG5cx84D16GYrt2Pg+aAPmBfQnebQKmSI0GyziO4c590H2i6jCwsLhvrek/FPIxwOvIHuvOoK9aHjUGDr/pz7yKrqOmB34CVJHjJBt+/SvVCsO9T+XLq//lb6NMrA/m+pqqOBc+mmd1dm3d/TXYD2zkm6Nqmq79BNN+8+0HYD3V+Rz52iffwK+Cgrf+zX93U8f5zFL+DvL1Slf3F9E/DgJDuudLGdXeheBH+Z5HK6afy1gJ2nemwGfBd4dpJxn+uTbEIXjF6c7t1Dl9Od2njqyv5/GPI/dLN8zxmhvokeh1P7cfmrqvpLVX2AbhZw93HW0ypmiNCsUlW/oDtfvOdA21/ozq2+u3/71lpJdqa7GPJb42zmZLpzrR+ebH/9W8NeQjft+vtJuo9X7++BT7H81eGDPkt39fjR6d6qtlY/a/AhYL+qunpl9pfubYtPS3LH/jz9U4AHAqevbO10F75tAzygYd1R7AvsM9S2D7Brf03HRgBJtkzyhck2lmTDdG9NvHd/7POAf6UtiL0Z2CXJnv1Ybpju7cJbA4vHW6GqbqY7jTLRY72i2u9Od8Hf0+kuCNwK2JJuRmjsr/7msVmBA+lmUQ5LsulYLUkOTLIF3e/+z+muYRir6750v7M7t+60nxl4A921Ma/oxzf9qafBa1kWA9skeXeSO/ePxR50pyLftIJdvBfYJwNvFdXMMERoNnoH3TseBu1Od2X8ucAVwGuAp1XVb4dX7q9qP3HsivAJ/DHJdcBv6V44ntk/8Y35WpZ/7/uXV7Ctg+j+cvu7q/b7v2B3AH5F90J/Dd0T+75VdcAKtjmRa4C3Ar+ke6fF+4BX1/LvUPjIUO1Lx9tQVV3Tr3/nlazhnKHtHzTB9k8Bzhhq+zHdX76PAy5OchVwMH+b4ofuPPvwZw88l+5dHwvo/nq9BjifbjZn15Wsn368nkT3l/JldDNaDwG2HXjnwHg+DdwjyTNW0OePQ7W/nu7F+uyqOqG6d9Rc3l+b8iFgiyQPGnFsVvY4r6ILircApye5lm6m5Wq6maJdgI8N1jRwzcwopzQmeqyo7q2tL6B7p86v6E5dHNUf09F9n4vorunYku5al8voZmOetIJTkdCdsvwD3aylZlCWf96UJEkajTMRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpqsOXkXDZo3b14tWLBgpsuQJGmVWbp06ZVV9XdfSWCIWEkLFixgyZIlM12GJEmrTJJxv2TQ0xmSJKmJIUKSJDUxREiSpCaGCEmS1MQQIUmSmhgiJElSE0OEJElqYoiQJElNDBGSJKmJIUKSJDUxREiSpCaGCEmS1MQQIUmSmhgiJElSE0OEJElqYoiQJElNDBGSJKmJIUKSJDVZc6YLmGuWLVvG4sWLZ7oMSZL+zqJFi1bp/pyJkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJarLahYgkr02yzgj9Hp3kp0nOTrJ2kgP6+wesijolSZrr1pzpAqbBa4EjgBsm6fci4P1V9RmAJK8E7lJVN01veZIkrR7mdIhIsi5wFLAJsAZwNDAfOCnJlVW1fZKPAw8H1gaOqapFSV4OvAB4UpIdgDsC6wKnJ3lPVX1xJo5HkqS5ZE6HCODJwLKqehpAkvWBlwHbV9WVfZ99q+qqJGsAJybZoqo+lWRb4OtVdUy/7nVVtdV4O0myG7AbwPrrrz+9RyRJ0hwx16+JOA/YIcn+SR5dVVeP0+cFSc4CfgI8ENh8ZXdSVQdX1cKqWrjOOpNebiFJ0m3CnJ6JqKqfJ3kY8FTgPUlOGFye5J7A3sDDq+oPSQ4F7rDqK5UkafUzp2cikswHbqiqI4D3Aw8FrqW7xgHgTsD1wNVJNgaeMiOFSpK0GprTMxHAg4EDktwK3AK8Gtga+FaSy/oLK38C/BS4GDhl5kqVJGn1MqdDRFUdDxw/1LwE+PBAn10nWHfXofvrTXF5kiSt1ub06QxJkjRzDBGSJKmJIUKSJDUxREiSpCaGCEmS1MQQIUmSmhgiJElSE0OEJElqYoiQJElNDBGSJKmJIUKSJDUxREiSpCaGCEmS1MQQIUmSmhgiJElSk1TVTNcwpyxcuLCWLFky02VIkrTKJFlaVQuH252JkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJarLmTBcw1yxbtozFixcv17Zo0aIZqkaSpJnjTIQkSWpiiJAkSU0MEZIkqYkhQpIkNTFESJKkJoYISZLUxBAhSZKaGCIkSVITQ4QkSWpiiJAkSU0MEZIkqYkhQpIkNTFESJKkJoYISZLUxBAhSZKaGCIkSVITQ4QkSWpiiJAkSU3mRIhIsm6SbyQ5J8n5SXZKcmmS/ZOc0f/cu+/7jCSnJ/lJku8m2bhvXy/JZ5Kcl+TcJM/t25+Y5NQkZyU5Osl6M3mskiTNFXMiRABPBpZV1ZZV9SDg2337NVX1COAjwEF924+AR1bVQ4AvAPv07f8BXF1VD66qLYDvJZkHvA3YoaoeCiwBXj+88yS7JVmSZMkNN9wwTYcoSdLcsuZMFzCi84D3J9kf+HpV/TAJwOf75Z8H/qu/vQnwxSR3A24PXNK37wC8cGyDVfWHJE8HNgdO6bd3e+DU4Z1X1cHAwQDz58+vqT00SZLmpjkRIqrq50keBjwVeE+SE8YWDXbr//0wcGBVHZdkO2C/vj1D/cfavlNVO09H3ZIkrc7mxOmMJPOBG6rqCOD9wEP7RTsN/Ds2g7A+8Jv+9i4DmzkBeM3ANjcETgMeNXA9xTpJ7jstByFJ0mpmToQI4MHAGUnOBvYF3tW3/1OS04G9gNf1bfsBRyf5IXDlwDbeBWzYX5h5DrB9Vf0O2BX4fJJz6ULF/af5WCRJWi3MldMZxwPHD7b11zB8tKoWD/X9KvDVcbZxHcvPTIy1fw94+FTWK0nSbcFcmYmQJEmzzJyYiRhPVS2Y6RokSbotcyZCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpqkqibvlKxRVX9ZBfXMegsXLqwlS5bMdBmSJK0ySZZW1cLh9lFnIn6R5IAkm09xXZIkaY4aNURsAfwc+FSS05LsluRO01iXJEma5UYKEVV1bVUdUlXbAPsAi4DLkhyW5N7TWqEkSZqVRgoRSdZI8swkXwY+CHwAuBfwNeCb01ifJEmapdYcsd9FwEnAAVX144H2Y5I8ZurLkiRJs92kISLJGsChVfWO8ZZX1Z5TXpUkSZr1Jj2d0b+1c/tVUIskSZpDRj2d8eMkHwG+CFw/1lhVZ01LVZIkadYbNURs0/87eEqjgMdNbTmSJGmuGClEVJWnMyRJ0nJGfYvn+kkOTLKk//lAkvWnuzhJkjR7jfqJlZ8GrgVe0P9cA3xmuoqSJEmz36jXRGxWVc8duL84ydnTUI8kSZojRp2JuDHJtmN3kjwKuHF6SpIkSXPBqDMRrwYO66+DCHAVsOt0FSVJkma/Ud+dcTaw5dg3d1bVNdNZlCRJmv1GChFJXj90H+BqYGkfMCRJ0m3MqNdELAReBdy9/9kN2A44JMk+01OaJEmazUa9JmIj4KFVdR1AkkXAMcBjgKXA+6anPEmSNFuNOhNxD+Dmgfu3AJtW1Y3ATVNelSRJmvVGnYn4HHBakq/2958BfD7JusDPpqUySZI0q4367ox3JvkmsC3dWzxfVVVL+sUvmq7iJEnS7DXq6QyAtYFrquog4P+S3HN6SpIkSXPBqF/AtQh4E/CWvmkt4IjpKkqSJM1+o85EPBt4JnA9QFUtA+44XUVJkqTZb9QQcXNVFVAA/QWVkiTpNmzUEHFUkk8CGyR5BfBd4FPTV5YkSZrtRn13xvuTPAG4Brgf8Paq+s60ViZJkma1Ub87Y/+qehPwnXHaJEnSbdCopzOeME7bU6ayEEmSNLescCYiyauB3YF7JTl3YNEdgVOmszBJkjS7TXY643PAt4D3AG8eaL+2qq6atqokSdKst8IQUVVXA1cDOwMkuStwB2C9JOtV1S+nv0RJkjQbjfqJlc9IchFwCXAycCndDIUkSbqNGvXCyncBjwR+XlX3BB6P10RIknSbNmqIuKWqfg/cLsntquokYKvpK0uSJM12I31OBPDHJOsBPwCOTHIF8OfpK0uSJM12k73F897AxsCzgBuB1wEvAjYF9pj26iRJ0qw12emMg+jeznl9Vd1aVX+uqsOAbwL7TXdxkiRp9posRCyoqnOHG6tqCbBgWiqSJElzwmTXRNxhBcvWnspC5oply5axePHi5doWLVo0Q9VIkjRzJpuJOLP/6u/lJPk3YOn0lCRJkuaCyWYiXgt8OcmL+FtoWAjcHnj2NNYlSZJmuck+9vq3wDZJtgce1Dd/o6q+N+2VSZKkWW2kz4noP1zqpGmuRZIkzSGjfmKlJEnScgwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1MURIkqQmszJEJNkxyeYzXYckSZrYrAwRwI7AuCEiyUjfPCpJkqbXKgsRSV6c5IwkZyf5ZJI1klyX5N1JzklyWpKNk2wDPBM4oO+7WZLvJ/nPJCcDeyV5RpLTk/wkyXeTbNzvY78kn+77X5xkz4H9vzTJuf2+Ptu33SXJl5Kc2f88alWNhyRJc90q+as+yQOAnYBHVdUtST4GvAhYFzitqvZN8j7gFVX1riTHAV+vqmP69QE2qKrH9vc3BB5ZVZXk5cA+wBv63d0f2B64I3Bhko8D9wX27fd/ZZI7930/CPxXVf0oyT2A44EHjFP/bsBuAOuvv/7UDo4kSXPUqjo18HjgYcCZfSBYG7gCuBn4et9nKfCEFWzjiwO3NwG+mORuwO2BSwaWfaOqbgJuSnIFsDHwOOCYqroSoKqu6vvuAGze1wRwpyR3rKprB3dcVQcDBwPMnz+/Rj1oSZJWZ6sqRAQ4rKreslxjsndVjb0o/2WSeq4fuP1h4MCqOi7JdsB+A8tuGrg9ts0A47343w7YuqpuHOEYJEnSgFV1TcSJwPOS3BUgyZ2TbLqC/tfSnY6YyPrAb/rbu4y4/xck2Whs/337CcBrxjol2WqEbUmSJFZRiKiqnwFvA05Ici7wHeBuK1jlC8Ab+wsnNxtn+X7A0Ul+CFw5wv5/CrwbODnJOcCB/aI9gYX9BZc/A1416jFJknRbl7+dTdAo5s+fX6985SuXa1u0aNEMVSNJ0vRLsrSqFg63z9bPiZAkSbOcIUKSJDUxREiSpCaGCEmS1MQQIUmSmhgiJElSE0OEJElqYoiQJElNDBGSJKmJIUKSJDUxREiSpCaGCEmS1MQQIUmSmhgiJElSE0OEJElqkqqa6RrmlIULF9aSJUtmugxJklaZJEurauFwuzMRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTdac6QLmmmXLlrF48eLl2hYtWjRD1UiSNHOciZAkSU0MEZIkqYkhQpIkNTFESJKkJoYISZLUxBAhSZKaGCIkSVITQ4QkSWpiiJAkSU0MEZIkqYkhQpIkNTFESJKkJoYISZLUxBAhSZKaGCIkSVITQ4QkSWoybSEiyYIk569E/12TzB+4f2mSedNTnSRJ+kfNppmIXYH5k3UalGTN6SlFkiRNZrpDxJpJDktybpJjkqyT5O1JzkxyfpKD03kesBA4MsnZSdbu198jyVlJzktyf4Ak+/XrnQAcnmTTJCf2+zgxyT36fhO1H5rk40lOSnJxkscm+XSSC5IcOs3jIUnSamO6Q8T9gIOragvgGmB34CNV9fCqehCwNvD0qjoGWAK8qKq2qqob+/WvrKqHAh8H9h7Y7sOAZ1XV/wM+Ahze7+NI4EN9n4naATYEHge8Dvga8F/AA4EHJ9lq+CCS7JZkSZIlN9xwwz84JJIkrR6mO0T8qqpO6W8fAWwLbJ/k9CTn0b2QP3AF6x/b/7sUWDDQftxA0Nga+Fx/+7P9PlbUDvC1qirgPOC3VXVeVd0K/HRoPwBU1cFVtbCqFq6zzjorOl5Jkm4zpvuaghrn/seAhVX1qyT7AXdYwfo39f/+heVrvX4l9jle+9h2bx24PXbf6ywkSRrBdM9E3CPJ1v3tnYEf9bevTLIe8LyBvtcCd2zYx4+BF/a3XzSwj4naJUnSFJjuv7ovAHZJ8kngIrprGzakO41wKXDmQN9DgU8kuZHuVMSo9gQ+neSNwO+Al03SLkmSpkC6SwM0qvnz59crX/nK5doWLVo0Q9VIkjT9kiytqoXD7bPpcyIkSdIcYoiQJElNDBGSJKmJIUKSJDUxREiSpCaGCEmS1MQQIUmSmhgiJElSE0OEJElqYoiQJElNDBGSJKmJIUKSJDUxREiSpCaGCEmS1MSvAl9JCxcurCVLlsx0GZIkrTJ+FbgkSZpShghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1MURIkqQmhghJktTEECFJkpoYIiRJUpNU1UzXMKckuRa4cKbruA2YB1w500Ws5hzjVcNxnn6O8fTbtKruMty45kxUMsddWFULZ7qI1V2SJY7z9HKMVw3Hefo5xjPH0xmSJKmJIUKSJDUxRKy8g2e6gNsIx3n6OcarhuM8/RzjGeKFlZIkqYkzEZIkqYkhYiUkeXKSC5P8IsmbZ7qe1UGSf05yUpILkvw0yV59+52TfCfJRf2/G850rXNdkjWS/CTJ1/v7jvEUS7JBkmOS/E//O7214zz1kryuf744P8nnk9zBcZ4ZhogRJVkD+CjwFGBzYOckm89sVauFPwNvqKoHAI8E/r0f1zcDJ1bVfYAT+/v6x+wFXDBw3zGeeh8Evl1V9we2pBtvx3kKJbk7sCewsKoeBKwBvBDHeUYYIkb3COAXVXVxVd0MfAF41gzXNOdV1WVVdVZ/+1q6J927043tYX23w4AdZ6TA1USSTYCnAZ8aaHaMp1CSOwGPAf4boKpurqo/4jhPhzWBtZOsCawDLMNxnhGGiNHdHfjVwP1f922aIkkWAA8BTgc2rqrLoAsawF1nsLTVwUHAPsCtA22O8dS6F/A74DP9aaNPJVkXx3lKVdVvgPcDvwQuA66uqhNwnGeEIWJ0GafNt7ZMkSTrAV8CXltV18x0PauTJE8HrqiqpTNdy2puTeChwMer6iHA9TilPuX6ax2eBdwTmA+sm+TFM1vVbZchYnS/Bv554P4mdFNo+gclWYsuQBxZVcf2zb9Ncrd++d2AK2aqvtXAo4BnJrmU7jTc45IcgWM81X4N/LqqTu/vH0MXKhznqbUDcElV/a6qbgGOBbbBcZ4RhojRnQncJ8k9k9ye7kKe42a4pjkvSejOIV9QVQcOLDoO2KW/vQvw1VVd2+qiqt5SVZtU1QK639vvVdWLcYynVFVdDvwqyf36pscDP8Nxnmq/BB6ZZJ3++ePxdNdSOc4zwA+bWglJnkp3bnkN4NNV9e6ZrWjuS7It8EPgPP52vv6tdNdFHAXcg+5J4/lVddWMFLkaSbIdsHdVPT3JRjjGUyrJVnQXr94euBh4Gd0fa47zFEqyGNiJ7t1dPwFeDqyH47zKGSIkSVITT2dIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkFYzSSrJBwbu751kvyna9qFJnjcV25pkP8/vvwXzpOnel6R2hghp9XMT8Jwk82a6kEH9N+GO6t+A3atq++mqZ0z/JU4T3pc0MUOEtPr5M3Aw8LrhBcMzCUmu6//dLsnJSY5K8vMk703yoiRnJDkvyWYDm9khyQ/7fk/v118jyQFJzkxybpJXDmz3pCSfo/tAseF6du63f36S/fu2twPbAp9IcsBQ//T7Ob9fb6eBZfv0beckeW/f9v0kC/vb8/qP/ibJrkmOTvI14IRx7q+b5NP98fwkybMG1js2ybeTXJTkfQP7f3KSs/r9n9i3TbSdB/Zje3Y/XvcZ6ZGVZhkTt7R6+ihw7uCL3Ai2BB4AXEX3aYufqqpHJNkL2AN4bd9vAfBYYDPgpCT3Bl5K922KD0/yT8ApSU7o+z8CeFBVXTK4syTzgf2BhwF/oHvx3rGq3pHkcXSfrLlkqMbnAFv1tc4Dzkzyg75tR+BfquqGJHce4Xi3BraoqquS7Dp0/z/pPh78X5NsAJyR5Lv9elvRfdvsTcCFST4M/Ak4BHhMVV0ysP99J9jOq4APVtWR/cfor8wsjTRrGCKk1VBVXZPkcGBP4MYRVztz7KuUk/wvMBYCzgMGTyscVVW3AhcluRi4P/BEYIuBWY71gfsANwNnDAeI3sOB71fV7/p9Hgk8BvjKCmrcFvh8Vf2F7guXTu6381jgM1V1Q3/8o3zc8XeG+g3efyLdl5bt3d+/A93HKQOcWFVX9zX/DNgU2BD4wdhxjrCdU4F9k2wCHFtVF41QrzTrGCKk1ddBwFnAZwba/kx/GrP/8qLbDyy7aeD2rQP3b2X554rhz8ovIMAeVXX84IL+uzqun6C+TFL/yqyTceqCgeOlewEfNFzX4P0Az62qC5fbSfIvLD9Of6Ebm4n2P+52gAuSnA48DTg+ycur6nvjrC/Nal4TIa2m+r+Gj6K7SHHMpXSnDwCeBazVsOnnJ7ldf53EvYALgeOBV6f7WneS3DfJupNs53Tgsf21CmsAOwMnT7LOD4Cd+msw7kI3c3EG3azJvyZZp9//2OmES/nb8a7Mu0qOB/bogxZJHjJJ/1P7Y7nn0P7H3U6SewEXV9WH6L59couVqE2aNQwR0urtA3TXDow5hO7F7gzgX5h4lmBFLqR7sf8W8Kqq+hPdN1f+DDgryfnAJ5lkprM/dfIW4CTgHOCsqprs65u/DJzb9/8esE9VXV5V36Z7MV6S5Gxg7PTB++nCzY9Zfhwm8066gHVufzzvnORYfgfsBhyb5Bzgi5NsZyfg/L7W+wOHr0Rt0qzht3hKkqQmzkRIkqQmhghJktTEECFJkpoYIiRJUhNDhCRJamKIkCRJTQwRkiSpiSFCkiQ1+f9gEyPJtxmXRwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(8,6))\n",
    "colors = ['grey','grey','grey','grey','grey','grey','grey','grey','grey',\n",
    "    'grey','darkblue','darkblue','darkblue']\n",
    "df5_filtered.groupby('Category').Sentence.count().sort_values().plot.barh(\n",
    "    ylim=0, color=colors, title= 'NUMBER OF SENTENCES FOR EACH CATEGORY')\n",
    "plt.xlabel('Number of ocurrences', fontsize = 10);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "817be6eb",
   "metadata": {},
   "source": [
    "### Text Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 643,
   "id": "1e556ed1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Each of the 98 complaints is represented by 28 features (TF-IDF score of unigrams and bigrams)\n"
     ]
    }
   ],
   "source": [
    "tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5,\n",
    "                        ngram_range=(1, 2), \n",
    "                        stop_words='english')\n",
    "# We transform each complaint into a vector\n",
    "features = tfidf.fit_transform(df5_filtered.Sentence).toarray()\n",
    "labels = df5_filtered.category_id\n",
    "print(\"Each of the %d complaints is represented by %d features (TF-IDF score of unigrams and bigrams)\" %(features.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 644,
   "id": "650a6b8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n==> bathroom:\n",
      "  * Most Correlated Unigrams are: friendly, nice, good, food, service\n",
      "  * Most Correlated Bigrams are: great food, friendly staff, good food, friendly service, good service\n",
      "n==> entrance:\n",
      "  * Most Correlated Unigrams are: friendly, nice, good, food, service\n",
      "  * Most Correlated Bigrams are: great food, friendly staff, good food, friendly service, good service\n",
      "n==> space:\n",
      "  * Most Correlated Unigrams are: nice, friendly, good, wine, restaurant\n",
      "  * Most Correlated Bigrams are: great food, friendly staff, good service, friendly service, good food\n",
      "n==> staff:\n",
      "  * Most Correlated Unigrams are: great, staff, service, wine, restaurant\n",
      "  * Most Correlated Bigrams are: great food, friendly staff, good service, friendly service, good food\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Applications/anaconda3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "/Applications/anaconda3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "/Applications/anaconda3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "/Applications/anaconda3/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# Finding the three most correlated terms with each of the product categories\n",
    "N = 5\n",
    "for Sentence, category_id in sorted(category_to_id.items()):\n",
    "    features_chi2 = chi2(features, labels == category_id)\n",
    "    indices = np.argsort(features_chi2[0])\n",
    "    feature_names = np.array(tfidf.get_feature_names())[indices]\n",
    "    unigrams = [v for v in feature_names if len(v.split(' ')) == 1]\n",
    "    bigrams = [v for v in feature_names if len(v.split(' ')) == 2]\n",
    "    print(\"n==> %s:\" %(Sentence))\n",
    "    print(\"  * Most Correlated Unigrams are: %s\" %(', '.join(unigrams[-N:])))\n",
    "    print(\"  * Most Correlated Bigrams are: %s\" %(', '.join(bigrams[-N:])))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d755086",
   "metadata": {},
   "source": [
    "### Multi-Class models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 645,
   "id": "a93afa44",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df5_filtered['Sentence'] # Collection of documents\n",
    "y = df5_filtered['Category'] # Target or the labels we want to predict (i.e., the 13 different complaints of products)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, \n",
    "                                                    test_size=0.25,\n",
    "                                                    random_state = 0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 646,
   "id": "ee827c07",
   "metadata": {},
   "outputs": [],
   "source": [
    "models = [\n",
    "    RandomForestClassifier(n_estimators=100, max_depth=5, random_state=0),\n",
    "    LinearSVC(),\n",
    "    MultinomialNB(),\n",
    "    LogisticRegression(random_state=0),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 647,
   "id": "4f87e68a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5 Cross-validation\n",
    "CV = 5\n",
    "cv_df = pd.DataFrame(index=range(CV * len(models)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 649,
   "id": "0dee8e7f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Applications/anaconda3/lib/python3.9/site-packages/sklearn/model_selection/_split.py:676: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/Applications/anaconda3/lib/python3.9/site-packages/sklearn/model_selection/_split.py:676: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/Applications/anaconda3/lib/python3.9/site-packages/sklearn/model_selection/_split.py:676: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/Applications/anaconda3/lib/python3.9/site-packages/sklearn/model_selection/_split.py:676: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "entries = []\n",
    "for model in models:\n",
    "    model_name = model.__class__.__name__\n",
    "    accuracies = cross_val_score(model, features, labels, scoring='accuracy', cv=CV)\n",
    "    for fold_idx, accuracy in enumerate(accuracies):\n",
    "        entries.append((model_name, fold_idx, accuracy))\n",
    "cv_df = pd.DataFrame(entries, columns=['model_name', 'fold_idx', 'accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31014051",
   "metadata": {},
   "source": [
    "### Compare Text Classification Model Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 650,
   "id": "1d2180ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Mean Accuracy</th>\n",
       "      <th>Standard deviation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>LinearSVC</th>\n",
       "      <td>0.97</td>\n",
       "      <td>0.027386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LogisticRegression</th>\n",
       "      <td>0.97</td>\n",
       "      <td>0.027386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MultinomialNB</th>\n",
       "      <td>0.97</td>\n",
       "      <td>0.027386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForestClassifier</th>\n",
       "      <td>0.97</td>\n",
       "      <td>0.027386</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Mean Accuracy  Standard deviation\n",
       "model_name                                               \n",
       "LinearSVC                        0.97            0.027386\n",
       "LogisticRegression               0.97            0.027386\n",
       "MultinomialNB                    0.97            0.027386\n",
       "RandomForestClassifier           0.97            0.027386"
      ]
     },
     "execution_count": 650,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_accuracy = cv_df.groupby('model_name').accuracy.mean()\n",
    "std_accuracy = cv_df.groupby('model_name').accuracy.std()\n",
    "\n",
    "acc = pd.concat([mean_accuracy, std_accuracy], axis= 1, \n",
    "          ignore_index=True)\n",
    "acc.columns = ['Mean Accuracy', 'Standard deviation']\n",
    "acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 651,
   "id": "a0d922ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfgAAAFPCAYAAACh59yyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAp6ElEQVR4nO3deZwcVbn/8c+XhLAEJSEELgJhUKMSI0aIcVcQF1CUxQUQL7JGvOJ69ScoIvLzXnC/IEhAiIgLyKph0YBR4KpsgQRIQpAYtkjUEIEQtjDJc/84Z0il6ZnpZKaY9Mn3/Xr1q7uqTlU93aernjqnqrsUEZiZmVlZ1hvoAMzMzKz/OcGbmZkVyAnezMysQE7wZmZmBXKCNzMzK5ATvJmZWYGc4M3MzArkBG9mbU/S8ZImD3QcfSHpVZL+JmnoQMdiZXCCt2JIOkdSSDqrybRv5WmXV8Ydn8c1Pv7eZP59JS2X9PMm0zryfIslbdow7RpJp7YY/5S8jnd2M/0lks6W9ICkpyXdK+kiSW9sKLeLpMslPSTpSUlzJf1AUkeefrCkpd2sIyR9sOF9dT0elXSDpPd1M+8pOf4jupm+paSTJf01x/83Sb+R9B5Jr5T0lKT9GuaRpGsl/aaHz20L4PPAN7orszbI9dX4XTupa3pE3AHcQHovZn3mBG+leQDYr9oKkjQY+Hfg/ibl7wK2ani8qkm5w4FvAXtLGt7NujcGjl6ToCVtBewGfD+vq3H6eOBW4JXAfwBjgPcDtwA/qJT7ODANWAx8CNgBOIy0rR+7JrEBu5M+l9cBNwEXSxrbEN8GwIHASd3E35HjfzdwDLAj8A7gCmBSRMwGvgKcJunfKrN+BhgLHNpDfIcDN0XE/DV5c8+zE1j1u9Z4UPJj4BP5O2vWNxHhhx9FPIBzgMtJieSQyvi9gHuAnwCXV8YfD8xqYbnbAE8CI0jJ86iG6R1AAN8EngC2rky7Bji1hXUcA1wMjOpaV2WagFnADGBQk3mHVeJ8Gjilm3V0lTsYWNpNmQA+2PC+xlemvyCP+1TDfAeQDjY2BpYAYxumXwk8CGzSZJ3D8/N6wLXAlDz8svx5friXz24W8OmGcQL+E7g7fyYLgBPztOuB7zaUf2H+3Pep8ft5L/CFXsoMAZ4C3tEw37HAGfmzXQB88fnYpvxo74db8Fais1m1xXcoqWW0pjdeOAS4KiIWAz+lSQs1uxC4g9RKa5kk5Rh/FhH3AzeSehy6jCO13L8dEcsb54+IR/LLD5ESxEmNZRrKrRFJ6wNd3e/PNEw+nBT/E8AlVD4jSZuRegFOjYjnnBqIiIfz8wrgY8Aukg4nHZBdGhEX9BDTZqTejOkNk/4b+CpwIumz+xCpdwfgZ8D+kqr7vw+QEvwV3axnlKSlvTwmdRdnxRfyqZyZkr4iaUjDZ7EMmAm8rWG+z5G+WzuRDiS/JekNLazP1mUDfYThhx/99WBlC344aWc9Gvg3UgtuVNf0SvnjgeXA0obHeZUyAuazslW7CfA4sHOlTAe5pUvaMXcCr8zTrqGXFjywK6lLfUgePhS4ozL9w3n5r+llOT8EHm3hczqY1WvBP5E/l+V5eD6wWWWeFwPLgH/Lw28HHgI2yMMT8nwttY7z+19OaqkO66XsuLzs7SvjNiG1go/sZp4ROd7dKuN+B5zRw3oGAy/t5bFFL7F+Ptf1jqQDoIeAs5qUuwT4aWX43up3Mo+7Gzh2oLc5P9buh8/zWHEi4mFJl5ISxSPANRFxf2ooP8dfgfc0jKu2MncjHTBclpe9VNKvSDvoW5qs+1pJU0ktx/e3GPJhwAWRWm8AFwGnSnpdRNxIOshohVjzXoqefASYTeoy/z4wMSL+VZl+KDAtIrouTryGdFCwN/BLWo8fgIiYLOkE0oHRI70U3yg/P1UZNwbYgHQ6pdnyF+c6OhCYlq9/2BX4eg8xdQLzWnsH3S7je5XB2yUtAX4p6UuReoe6PMnK9/Vs+YbhB4Et+hKPlc9d9FaqycBBpOTT08+nlkXEvIZH9Sr6w4FhwOOSOiV1AvsDB0jauJtlfgl4r6S39BakpGGk7uGJleX/i7SD7+rm/kt+3qGXxf0F2FTSi3optwTYKHe5N8YC8GhD+QURcXdEXEHqor9A0uZ5nkGkHoF3V+JfRroeoCv+u0kHHr3FX9WZH715KD9XL3xs5YDiZ8AHJG1Iun7gAeCP3RXuxy76qhvz80sbxm8GLGoY13hKJPD+23rhL4iVahop0WwO/GpNFpDP7+5NOi88rvJ4Nanb/4PN5ouIWcC5pKvue3MgaWf+6oZ1TGTlrwFmAnOAL+aE2hjnsPzyItJ7bnolf6XcXaRt/zUNRXaqTG8qIq7NsRyXR+1O6vIe3xD/nsBukjpya38qcJSkTXqIa038lXTAMqYybg6pfnbrYb5f5+c9SXXw84joqffjQVZ9f80exz1nrp6Ny88LG8aPJV0oatYn7qK3IkVESNoRUEQ83UPRwQ0/y+qa/++kC90eI+38V7m4TVLXhWTndrPc41jZ8p7Vw/oPAy7KBwXV5f8F+DawX+6yPoR0nvhPkr4B3Em6Yn0P0jn68RHxgKTPkbr3NyVdWHgP8CJSN/uGwBERMVvSVcBZkj5PSpIvA04mnSpo9nPCqu8CF0r6dv4MfhMRjQlplqS7SD0ox5F+2vdnYLqkr5K6nEXqGj+GdI3EaouIFZJ+B7yZdIBDRDwm6WTgRElPA9eRDkJ2jojTc5mnch0eSzq4+mgv6+lTF32+IO71wB9IPSSvJZ3umFL9vPPPCbcGrlrTdZl1cQveihURj0XEkl6KvZzUglrlkX+HfBjpKu7nXLlOumL+LZJe1s26HwBOISXVpiTtRGpFX9Rk/mXAFHI3d0TcBOxMSuyT8vMVpAvYjqrM90PgncBI0s/u7iJdXAir/uZ6P1KymUQ6v/4DUqv2sO7irbicdOHXCaQW8HPizy4EDpG0XkTcQ+ohuJp0FfjtwO9J1yl8vIV19uRMUm9HtXfjmLyer5I+q4tJpw2qfkpK7rdGxJ19jKE3T5M+82tIPQwnAD8inR6oOoD0i437ao7H1gHquVfKzGztJ+l64IcR8dOBjmVN5T8Luhs4ICL+NNDxWPtzC97MSvBx2n9/th3wX07u1l/cgjczMytQux/xmpmZWRNO8GZmZgUq6mdym2++eXR0dAx0GGZmZs+LW2655aGIGNlsWlEJvqOjg+nTG+85YWZmViZJ3f6k0l30ZmZmBXKCNzMzK5ATvJmZWYGc4M3MzArkBG9mZlYgJ3gzM7MCOcGbmZkVqLYEL2mypH9KanovbCWnSJon6fZ868yuabtLuitPO7quGM3MzEpVZwv+HGD3HqbvAYzOj4nA6QD5ns6n5eljgAMkjakxTjMzs+LU9k92EXGdpI4eiuwFnBvpdnY3SBomaSugA5gXEfMBJJ2fy87pr9guv/xyFi5c2F+Le9bixYtZtmxZvy+3bkOGDGHEiBH9vtytttqKPffcs1+XWVfdQXvWX111B+1Vf+1Yd9Be2x64/qraYdsbyL+q3Rp4oDK8II9rNv513S1E0kRSDwCjRo1qacULFy7k/gV/Y+jw/q2cp57pZPnyFf26zOfDimc6Wfz4U/26zMcfXtyvy+tSV91Be9ZfHXUH7Vd/7Vh30F7bHrj+qtph2xvIBK8m46KH8U1FxJnAmQDjx49v+eb2Q4eP4FXvfF+rxW013XH1ZbUt23VXP9df+6qz7sD1V7f+rL+BTPALgG0rw9sADwJDuhlvZmZmLRrIn8lNAQ7KV9O/Hng0IhYCNwOjJW0vaQiwfy5rZmZmLaqtBS/pPGAXYHNJC4CvAesDRMQk4ErgPcA84AngkDytU9JRwFRgEDA5ImbXFaeZmVmJ6ryK/oBepgfwyW6mXUk6ADAzM7M14H+yMzMzK5ATvJmZWYGc4M3MzArkBG9mZlYgJ3gzM7MCOcGbmZkVyAnezMysQE7wZmZmBXKCNzMzK5ATvJmZWYGc4M3MzArkBG9mZlYgJ3gzM7MCOcGbmZkVyAnezMysQE7wZmZmBXKCNzMzK5ATvJmZWYGc4M3MzArkBG9mZlYgJ3gzM7MCOcGbmZkVyAnezMysQE7wZmZmBXKCNzMzK5ATvJmZWYGc4M3MzArkBG9mZlYgJ3gzM7MCOcGbmZkVyAnezMysQE7wZmZmBXKCNzMzK5ATvJmZWYGc4M3MzArkBG9mZlYgJ3gzM7MCOcGbmZkVyAnezMysQE7wZmZmBao1wUvaXdJdkuZJOrrJ9OGSLpV0u6SbJI2tTPuMpFmSZkv6bJ1xmpmZlaa2BC9pEHAasAcwBjhA0piGYl8GZkbEjsBBwMl53rHAEcAE4NXAnpJG1xWrmZlZaepswU8A5kXE/IhYBpwP7NVQZgwwDSAi5gIdkrYEdgBuiIgnIqITuBbYp8ZYzczMilJngt8aeKAyvCCPq7oN2BdA0gRgO2AbYBbwVkkjJG0MvAfYtsZYzczMijK4xmWrybhoGD4JOFnSTOAOYAbQGRF3SvomcDWwlHQg0Nl0JdJEYCLAqFGj+idyMzOzNldnC34Bq7a6twEerBaIiCURcUhEjCOdgx8J3JOnnR0RO0XEW4F/AXc3W0lEnBkR4yNi/MiRI2t4G2ZmZu2nzgR/MzBa0vaShgD7A1OqBSQNy9MADgeui4gledoW+XkUqRv/vBpjNTMzK0ptXfQR0SnpKGAqMAiYHBGzJR2Zp08iXUx3rqTlwBzgsMoiLpY0AngG+GREPFxXrGZmZqWp8xw8EXElcGXDuEmV19cDTX/+FhFvqTM2MzOzkvmf7MzMzArkBG9mZlYgJ3gzM7MCOcGbmZkVyAnezMysQE7wZmZmBXKCNzMzK5ATvJmZWYGc4M3MzArkBG9mZlYgJ3gzM7MCOcGbmZkVyAnezMysQE7wZmZmBXKCNzMzK5ATvJmZWYGc4M3MzArkBG9mZlYgJ3gzM7MCOcGbmZkVyAnezMysQE7wZmZmBXKCNzMzK5ATvJmZWYGc4M3MzArkBG9mZlYgJ3gzM7MCOcGbmZkVyAnezMysQE7wZmZmBXKCNzMzK5ATvJmZWYGc4M3MzArkBG9mZlYgJ3gzM7MCOcGbmZkVyAnezMysQE7wZmZmBXKCNzMzK5ATvJmZWYFqTfCSdpd0l6R5ko5uMn24pEsl3S7pJkljK9M+J2m2pFmSzpO0YZ2xmpmZlaS2BC9pEHAasAcwBjhA0piGYl8GZkbEjsBBwMl53q2BTwPjI2IsMAjYv65YzczMSlNnC34CMC8i5kfEMuB8YK+GMmOAaQARMRfokLRlnjYY2EjSYGBj4MEaYzUzMytKnQl+a+CByvCCPK7qNmBfAEkTgO2AbSLib8B3gPuBhcCjEXFVjbGamZkVpc4ErybjomH4JGC4pJnAp4AZQKek4aTW/vbAi4Chkj7adCXSREnTJU1ftGhRvwVvZmbWzupM8AuAbSvD29DQzR4RSyLikIgYRzoHPxK4B3gHcE9ELIqIZ4BLgDc2W0lEnBkR4yNi/MiRI2t4G2ZmZu2nzgR/MzBa0vaShpAukptSLSBpWJ4GcDhwXUQsIXXNv17SxpIE7AbcWWOsZmZmRRlc14IjolPSUcBU0lXwkyNitqQj8/RJwA7AuZKWA3OAw/K0GyVdBNwKdJK67s+sK1YzM7PS1JbgASLiSuDKhnGTKq+vB0Z3M+/XgK/VGZ+ZmVmpWuqil3SxpPdK8j/fmZmZtYFWE/bpwEeAuyWdJOkVNcZkZmZmfdRSgo+I30XEgcBOwL3A1ZL+LOkQSevXGaCZmZmtvpa73CWNAA4mXe0+g/S3sjsBV9cSmZmZma2xli6yk3QJ8Argp8D7ImJhnvRLSdPrCs7MzMzWTKtX0Z8aEb9vNiEixvdjPGZmZtYPWu2i30HSsK6BfJvX/6gnJDMzM+urVhP8ERHxSNdARDwMHFFLRGZmZtZnrSb49fJfxgLP3ut9SA/lzczMbAC1eg5+KnCBpEmkO8IdCfy2tqjMzMysT1pN8F8CPg58gnQb2KuAs+oKyszMzPqmpQQfEStI/2Z3er3hmJmZWX9o9Xfwo4ETgTHAhl3jI+LFNcVlZmZmfdDqRXY/JrXeO4FdgXNJf3pjZmZma6FWE/xGETENUETcFxHHA2+vLywzMzPri1Yvsnsq3yr2bklHAX8DtqgvLDMzM+uLVlvwnwU2Bj4N7Ax8FPhYTTGZmZlZH/Xags9/avPhiPgisBQ4pPaozMzMrE96bcFHxHJg5+o/2ZmZmdnardVz8DOAX0u6EHi8a2REXFJLVGZmZtYnrSb4zYDFrHrlfABO8GZmZmuhVv/JzufdzczM2kir/2T3Y1KLfRURcWi/R2RmZmZ91moX/eWV1xsC+wAP9n84ZmZm1h9a7aK/uDos6Tzgd7VEZGZmZn3W6h/dNBoNjOrPQMzMzKz/tHoO/jFWPQf/d9I94s3MzGwt1GoX/QvqDsTMzMz6T0td9JL2kbRpZXiYpL1ri8rMzMz6pNVz8F+LiEe7BiLiEeBrtURkZmZmfdZqgm9WrtWf2JmZmdnzrNUEP13S9yS9RNKLJX0fuKXOwMzMzGzNtZrgPwUsA34JXAA8CXyyrqDMzMysb1q9iv5x4OiaYzEzM7N+0upV9FdLGlYZHi5pam1RmZmZWZ+02kW/eb5yHoCIeBjYopaIzMzMrM9aTfArJD3717SSOmhydzkzMzNbO7T6U7evAH+UdG0efiswsZ6QzMzMrK9avcjut5LGk5L6TODXpCvpzczMbC3U6s1mDgc+A2xDSvCvB64H3l5bZGZmZrbGWj0H/xngtcB9EbEr8BpgUW1RmZmZWZ+0muCfioinACRtEBFzgZf3NpOk3SXdJWmepOf8jj7/3O5SSbdLuknS2Dz+5ZJmVh5LJH12Nd6XmZnZOq3VBL8g/w7+V8DVkn4NPNjTDJIGAacBewBjgAMkjWko9mVgZkTsCBwEnAwQEXdFxLiIGAfsDDwBXNpirMVZsuxfnDH3aB575uGBDsXWgOuvvbn+2te6XnctJfiI2CciHomI44GvAmcDe/cy2wRgXkTMj4hlwPnAXg1lxgDT8jrmAh2Stmwosxvw14i4r5VYSzRt4fncu3QO0x48f6BDsTXg+mtvrr/2ta7XXast+GdFxLURMSUn7Z5sDTxQGV6Qx1XdBuwLIGkCsB3pQr6q/YHzVjfOUixZ9i9ueWgaQTD9od+ts0ei7cr1195cf+3LdbcGCX41qMm4xj/HOQkYLmkm6YY2M4DOZxcgDQHeD1zY7UqkiZKmS5q+aFF51/1NW3g+wQoAghXr7JFou3L9tTfXX/ty3dWb4BcA21aGt6HhvH1ELImIQ/K59oOAkcA9lSJ7ALdGxD+6W0lEnBkR4yNi/MiRI/st+LVB1xHo8kjHPMujc509Em1Hrr/25vprX667pM4EfzMwWtL2uSW+PzClWkDSsDwN4HDguohYUilyAOtw93z1CLTLunok2o5cf+3N9de+XHdJq39Vu9oiolPSUcBUYBAwOSJmSzoyT58E7ACcK2k5MAc4rGt+SRsD7wQ+XleMa7v7l8599gi0y/Lo5L6ldw5QRLY6XH/tzfXXvlx3SW0JHiAirgSubBg3qfL6emB0N/M+AYyoM7613WdeecpAh2B94Pprb66/9uW6S+rsojczM7MB4gRvZmZWICd4MzOzAjnBm5mZFcgJ3szMrEBO8GZmZgVygjczMyuQE7yZmVmBnODNzMwK5ARvZmZWICd4MzOzAjnBm5mZFcgJ3szMrEBO8GZmZgVygjczMyuQE7yZmVmBnODNzMwK5ARvZmZWICd4MzOzAjnBm5mZFcgJ3szMrEBO8GZmZgVygjczMyuQE7yZmVmBnODNzMwK5ARvZmZWICd4MzOzAjnBm5mZFcgJ3szMrEBO8GZmZgVygjczMyuQE7yZmVmBnODNzMwK5ARvZmZWICd4MzOzAjnBm5mZFcgJ3szMrEBO8GZmZgVygjczMyuQE7yZmVmBak3wknaXdJekeZKObjJ9uKRLJd0u6SZJYyvThkm6SNJcSXdKekOdsZqZmZWktgQvaRBwGrAHMAY4QNKYhmJfBmZGxI7AQcDJlWknA7+NiFcArwburCtWMzOz0tTZgp8AzIuI+RGxDDgf2KuhzBhgGkBEzAU6JG0p6YXAW4Gz87RlEfFIjbGamZkVpc4EvzXwQGV4QR5XdRuwL4CkCcB2wDbAi4FFwI8lzZB0lqShNcZqZmZWlDoTvJqMi4bhk4DhkmYCnwJmAJ3AYGAn4PSIeA3wOPCcc/gAkiZKmi5p+qJFi/ordjMzs7ZWZ4JfAGxbGd4GeLBaICKWRMQhETGOdA5+JHBPnndBRNyYi15ESvjPERFnRsT4iBg/cuTIfn4LZmZm7anOBH8zMFrS9pKGAPsDU6oF8pXyQ/Lg4cB1Oen/HXhA0svztN2AOTXGamZmVpTBdS04IjolHQVMBQYBkyNitqQj8/RJwA7AuZKWkxL4YZVFfAr4eT4AmA8cUlesZmZmpaktwQNExJXAlQ3jJlVeXw+M7mbemcD4OuMzMzMrlf/JzszMrEBO8GZmZgVygjczMyuQE7yZmVmBnODNzMwK5ARvZmZWICd4MzOzAjnBm5mZFcgJ3szMrEBO8GZmZgVygjczMyuQE7yZmVmBnODNzMwK5ARvZmZWICd4MzOzAjnBm5mZFcgJ3szMrEBO8GZmZgVygjczMyuQE7yZmVmBnODNzMwK5ARvZmZWICd4MzOzAjnBm5mZFcgJ3szMrEBO8GZmZgVygjczMyuQE7yZmVmBnODNzMwK5ARvZmZWICd4MzOzAjnBm5mZFcgJ3szMrEBO8GZmZgVygjczMyuQE7yZmVmBnODNzMwK5ARvZmZWICd4MzOzAjnBm5mZFcgJ3szMrEC1JnhJu0u6S9I8SUc3mT5c0qWSbpd0k6SxlWn3SrpD0kxJ0+uM08zMrDSD61qwpEHAacA7gQXAzZKmRMScSrEvAzMjYh9Jr8jld6tM3zUiHqorRjMzs1LV2YKfAMyLiPkRsQw4H9irocwYYBpARMwFOiRtWWNMZmZm64TaWvDA1sADleEFwOsaytwG7Av8UdIEYDtgG+AfQABXSQrgjIg4s78CW7x4MUsee4wbLjinvxYJwIrlnUREvy7z+SCJ9Qb171dheecz8NQL+nWZUF/dQXvWXx11B+1Xf+1Yd9Be2x64/qraYdurM8GrybjGGjwJOFnSTOAOYAbQmae9KSIelLQFcLWkuRFx3XNWIk0EJgKMGjWqpcCGDh3KsmXLWiq7Op5ZIVa02ZcUYD2J9Qf1b2fO+oM2YOjQof26TKiv7qA966+OuoP2q792rDtor20PXH9V7bDtqa6jJklvAI6PiHfn4WMAIuLEbsoLuAfYMSKWNEw7HlgaEd/paZ3jx4+P6dN9PZ6Zma0bJN0SEeObTavzHPzNwGhJ20saAuwPTGkIbFieBnA4cF1ELJE0VNILcpmhwLuAWTXGamZmVpTauugjolPSUcBUYBAwOSJmSzoyT58E7ACcK2k5MAc4LM++JXBpatQzGPhFRPy2rljNzMxKU1sX/UBwF72Zma1LBqqL3szMzAaIE7yZmVmBnODNzMwK5ARvZmZWICd4MzOzAjnBm5mZFcgJ3szMrEBF/Q5e0iLgvoGOoyabA751bvty/bU311/7Kr3utouIkc0mFJXgSyZpend/ZmBrP9dfe3P9ta91ue7cRW9mZlYgJ3gzM7MCOcG3jzMHOgDrE9dfe3P9ta91tu58Dt7MzKxAbsGbmZkVqIgEL2m5pJmSZkm6TNKwflruwZJO7adl3SvpjhznTElv7I/lNlnPOEnvaRi3h6Tpku6UNFfSd/L44yV9oR/X/efK629Lmp2fj5R0UH+tp79JWtpk3PMSs6RD8/fi9vz93St/785rKLe5pEWSNpC0vqSTJN2d57lJ0h51x/p8khSSfloZHpzf/+UtzLs0P3dI+khl/HhJp9QT8bPreL+ko3sp8+x+JW+DT0jaojJ9aeV1177tNkm31rXfaFWzbWUNltFjPaxJvVX2r7dLulbSdn2Ns78M5P5v8ECstAZPRsQ4AEk/AT4J/NeARtTcrhGxWr/HlDQ4IjpXY5ZxwHjgyjz/WOBU4L0RMVfSYGDi6sTQqoio7nw+DoyMiKdXdzlr8J77XURMqnP5kgRsC3wF2CkiHpW0CTASWAx8R9LGEfFEnuWDwJSIeFrSScBWwNg8vCXwtjrjHQCPA2MlbRQRTwLvBP62msvoAD4C/AIgIqYD0/szyEYRMQWYspqzPQT8J/ClJtOq+7Z3AyfS5nXdQj10sGb1tmtEPCTp68CxwBF9iTNvo4qIFX1ZTt37kp4U0YJvcD2wNYCkCZL+LGlGfn55Hn+wpEsk/Ta3gr7VNbOkQyT9RdK1wJsq47eTNC0fIU6TNCqPP0fS6ZL+IGm+pLdJmpxby+f0FGgvy/yepD8A35T0khzrLZL+V9IrcrkP5RbcbZKukzQEOAHYLx/17wf8P+C/ImIuQER0RsQPm8RyhKSb87IulrRxs3Xkca/MrcaZOfbReXxXy2kKMBS4UdJ+qvQU9PBeVnnPq1HftWiI+RpJ38zv+S+S3pLHD1Lqobg5fw4fz+M3yfV5a25V7JXHd+TvxQ+BW4HtgceApQARsTQi7omIJcB1wPsqIe0PnJfr5QjgU10HTxHxj4i44Pn4XJ5nvwHem18fADzbq6GG3qf8He1omP8k4C35e/o5Sbso9wDk+Sfnup0v6dOVZX0+L2+WpM/mcR1KvV9n5fE/l/QOSX/K+5AJuVy1df4+STfm/c/vlA7EmplM2mY36+XzeCHwcC9lnndKvYY35G3gUknD8/jX5nHX5+1kVh5frYe3aWWv5gxJL6DnettE0o+1srX+gSYhVXPAyLw/uzk/3lQZf3XeRs+QdJ9SL1njNrqtpC9WtvGv5/mHSroi7xdnKe1rUepZm5PLPqentIfPquk+ps8iou0fwNL8PAi4ENg9D78QGJxfvwO4OL8+GJgPbApsSPr3u21JraL7Sa2oIcCfgFPzPJcBH8uvDwV+lV+fA5wPCNgLWAK8inTwdAswLpe7F7gDmAnc2MIyLwcG5eFpwOj8+nXA7/PrO4Ct8+thlfd2auWzuRV4dTef2/HAF/LrEZXx3yAlkO7W8QPgwPx6CLBRtR6avK6up7v3ssp7HojvTw+fzTXAd/Pr9wC/y68nAsfm1xuQWhnbk3rGXpjHbw7My9+PDmAF8PrK93Uq6Tv3Y+B9lfV/CLg0v34R8GAuvyMwY6C3ueejTvJ7vYi0jc4EdgEub6yfPDwL6KjWZ7V843Ce/8+53jYn9ZqsD+ycv/NDgU2A2cBrct11suq2PZmV233XtnswK/cZw1l5IfPhle9QtczxwBeA44CvN9l2luf3Phd4FNh5oOulybjbgbfl1ycA/1Opkzfm1ycBs5rUw2XAm/LrTfK201O9fbNr+V2fcX6+F9g8v/4fYGJ+/Qvgzfn1KODO/PpU4Jj8encg8vegg1W30XeRrsJXrvfLgbcCHwB+VIljU2Az4K5KnQ9r/K728FldQ5N9TF8fpXTRbyRpJqlybgGuzuM3BX6i1MIM0gbcZVpEPAogaQ6wHamCr4mIRXn8L4GX5fJvAPbNr38KfKuyrMsiIiTdAfwjIu7I88/OMc3M5Rq76Hta5oURsVyp2/aNwIWSuqZtkJ//BJwj6QLgkh4+n1aMlfQNYBhpQ5vawzquB74iaRvgkoi4u5UV9PJeIL/nPr2L+nS991tIdQpp499R0gfz8KbAaGAB8N+S3kraWWwNdLXe7ouIGwBy/e4OvBbYDfi+pJ0j4njSjuSHkl4IfBi4KJev8S2uXSLi9twqP4B8yqmfXRGpF+RpSf8k1dGbSQdWjwNIugR4C6nb/Z6GbXtaZbvvaLL8bYBfStqKdCB8Tw+xnALMlPTdhvHVLvo3AOdKGhs5Eww0SZuSEtm1edRPSNv3MOAFEdF1Xc4vgD2bLOJPwPck/Zy0L1nQy3f8HaTeLAAiotqj8YfcS/JPUhd9V/kxlWW+MPcSvBnYJy/jt5Kqy3l2GyVt4+8CZuThTUjb+P+STqN9k3Tw8b9Kpz+fAs6SdAVpG35Wd59VpUizfUyflNJF37URbEfakD6Zx/9/4A8RMZbU3blhZZ7queHlrLweodUNp1qua1krGpa7gtW7zqG6zMfz83rAIxExrvLYASAijiR9kbcl7RxGNFnmbFKrpDfnAEdFxKuAr5M/q2briIhfAO8HngSmSnp7i++v2/fS8J7XRl31Wv2uiNTT0fVeto+Iq4ADSb1AO+fv5T9Y+d1b5T1GclNEnEjacX0gj38S+C1pJ7Q/K7un5wGj8k5qXTAF+A6V7vmsk1X3Xxuy+prtA3rKLo3bdnW7b7ad/4DUUn8V6ZqUbmOMiEdISfA/eihzPakR0vR/x9cyLR2JRsRJpN6NjYAblE/Z9bLc7vbRu5JywGxS6xjSd+QNlW1064h4rJf4qtuogBMr8780Is6OiL+wsrfnREnHRbpuaAJwMbA3aftdHc32MX1SSoIHILfIPw18QdL6pBZV14U5B7ewiBuBXSSNyPN/qDLtz6w8cjwQ+GM/hNzrMiOdj71H0ocgXfgh6dX59Usi4saIOI50oc62pHO61Z3/t4EvS3pZnmc9SZ9vEssLgIX5fR/YNbLZOiS9GJgfEaeQdsA7tvJme3ovbWoq8In8mSHpZZKGkr53/4yIZyR17XSeQ9KLJO1UGTWOVW+WdB7weVLLsqvV/wRwNnCK0jUXSNpK0kf79Z2tPSYDJ3S1nCvuBXYCyJ/h9k3mbdwWWnEdsLekjXNd7kNqra2J6v7nYy2U/x7pQKDpzj0nv0Gk0wlrhbzPfbhyzvjfgWtzy/oxSa/P4/dvNn/ev9wREd8kneJ6BT3X21XAUZX5hzfE8yTwWeAgpWsaGsuPyy//SOoZQ9K7SKdTmpkKHJp7H5G0taQtJL0IeCIifkY6AN0pl9k0Iq7MMYyrLqi7z6qb9faLUrronxURMyTdRvpCfYvURf954PctzLtQ0vGkLuiFpPPXg/LkTwOTJX0RWAQc0g/htrrMA4HTJR1LOs1wPnAb8O18+kGkc9u3kc7nHp1PWZwYEb9UulCo6wKtAK5oso6vkg5w7iMdlXZtYM3WcTTwUUnPAH9n5dFyK7p7LwNpY0kLKsPfa3G+s0hdabcq9QEuIh25/xy4TNJ0Vp4/bWZ9Ujffi0hde4uAIyvTryJ1453d0CV7LOk6iTmSniK1OI5rMea2EhELgJObTLqYtBOfCdwM/KVJmduBzrw/OIeV3aw9re9WpYtjb8qjzsr7lI7VDj6de71Q0t9IB2jNDkKq635I0qXA5yqju04/QtoGPzbAp7GabSsfAybl/ct8Vu7HDgN+JOlx0jnmR5ss77P5IHg5MId0YeUKuq+3bwCnKV2wt5zU27jK6cm8Hz+P1JP76Vz+dlK+u460jX2dtE/cj5RkF5IOLDZpWNZVknYArs/d/EuBjwIvJe0bVwDPAJ8g7TN/LWlDUl1V67FLd59VLfxPdmZm1u8kbRIRXb+sORrYKiI+M8BhASBpA2B5RHTmaxtO77rWoSTFteDNzGyt8F5Jx5DyzH20dpr0+TIKuEDSesAy+vib+bWVW/BmZmYFKuoiOzMzM0uc4M3MzArkBG9mZlYgJ3gzM7MCOcGb2XMo3X5z876WMbOB4wRvZmZWICd4s0KohVuaStpM0q+Ubld5g6Qd87wjJF2ldMvOM6j8V7ekj2rl7YHPkDSo2yBWjeVOST+SNDsve6M8rbtbE7d062VJ71K6Bemtki7s+htRM1uVE7xZWV5K+mvXHUn/6/0R0p2zvgB8mfQXnTMiYsc8fG6e72vAHyPiNaT7C4wCyH/TuR/plp7jSH8P+uy9CnoxGjgtIl4JPEK+kQ7prmGvjYhXA3eS/tK0y3Dg7aS/+bwM+D7wSuBVSvfS3pz0V73viIidSP9f3uzeCmbrPP+TnVlZerul6XasvGPd73PLfVPSPa73zeOv0MrbZ+5GumvWzfm/uDci3Y6z1Vhm5tfVW2B2d2ti6P3Wy9sAY4A/5XiGkO4dYWYNnODNytLbLU07m8wTDc9VAn4SEcf0MZblpIMDSDcQ2TsibpN0MLBLk3m6u/XycuDqiDhgDeIxW6e4i95s3XIduYtd0i7AQ/k2vtXxe7Dy9pnTgA9K2iJP20xS09vfroamtyZu0Q3AmyS9NMezsfKtkM1sVW7Bm61bjgd+nG+f+QQr71PedfvMW0m3z7wfICLm5Fv7XpVvzPEM6Tac9zUueDV0d2viXkXEotzqPy/fEQzSOflmt4s1W6f5ZjNmZmYFche9mZlZgdxFb2ZrTNII0nn6RrtFxOLnOx4zW8ld9GZmZgVyF72ZmVmBnODNzMwK5ARvZmZWICd4MzOzAjnBm5mZFej/AKd5oJ/B/9FSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8,5))\n",
    "sns.boxplot(x='model_name', y='accuracy', \n",
    "            data=cv_df, \n",
    "            color='lightblue', \n",
    "            showmeans=True)\n",
    "plt.title(\"MEAN ACCURACY (cv = 5)n\", size=14);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39d2d898",
   "metadata": {},
   "source": [
    "### Evaluation of Text Classification Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 653,
   "id": "3fcc0ef4",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test,indices_train,indices_test = train_test_split(features, \n",
    "                                                               labels, \n",
    "                                                               df5_filtered.index, test_size=0.25, \n",
    "                                                               random_state=1)\n",
    "model = LinearSVC()\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 655,
   "id": "de42dd0e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ttttCLASSIFICATIION METRICSn\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Number of classes, 2, does not match size of target_names, 4. Try specifying the labels parameter",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Input \u001b[0;32mIn [655]\u001b[0m, in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# Classification report\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mttttCLASSIFICATIION METRICSn\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[43mmetrics\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclassification_report\u001b[49m\u001b[43m(\u001b[49m\u001b[43my_test\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_pred\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[43m                                    \u001b[49m\u001b[43mtarget_names\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mdf5_filtered\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mCategory\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43munique\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m)\n",
      "File \u001b[0;32m/Applications/anaconda3/lib/python3.9/site-packages/sklearn/metrics/_classification.py:2132\u001b[0m, in \u001b[0;36mclassification_report\u001b[0;34m(y_true, y_pred, labels, target_names, sample_weight, digits, output_dict, zero_division)\u001b[0m\n\u001b[1;32m   2126\u001b[0m         warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[1;32m   2127\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlabels size, \u001b[39m\u001b[38;5;132;01m{0}\u001b[39;00m\u001b[38;5;124m, does not match size of target_names, \u001b[39m\u001b[38;5;132;01m{1}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\n\u001b[1;32m   2128\u001b[0m                 \u001b[38;5;28mlen\u001b[39m(labels), \u001b[38;5;28mlen\u001b[39m(target_names)\n\u001b[1;32m   2129\u001b[0m             )\n\u001b[1;32m   2130\u001b[0m         )\n\u001b[1;32m   2131\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 2132\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   2133\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNumber of classes, \u001b[39m\u001b[38;5;132;01m{0}\u001b[39;00m\u001b[38;5;124m, does not match size of \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   2134\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtarget_names, \u001b[39m\u001b[38;5;132;01m{1}\u001b[39;00m\u001b[38;5;124m. Try specifying the labels \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   2135\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mparameter\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\u001b[38;5;28mlen\u001b[39m(labels), \u001b[38;5;28mlen\u001b[39m(target_names))\n\u001b[1;32m   2136\u001b[0m         )\n\u001b[1;32m   2137\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m target_names \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   2138\u001b[0m     target_names \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m l \u001b[38;5;28;01mfor\u001b[39;00m l \u001b[38;5;129;01min\u001b[39;00m labels]\n",
      "\u001b[0;31mValueError\u001b[0m: Number of classes, 2, does not match size of target_names, 4. Try specifying the labels parameter"
     ]
    }
   ],
   "source": [
    "# Classification report\n",
    "print('ttttCLASSIFICATIION METRICSn')\n",
    "print(metrics.classification_report(y_test, y_pred, \n",
    "                                    target_names= df5_filtered['Category'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 657,
   "id": "3d3a2dd9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdIAAAHyCAYAAABfzE8AAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAw10lEQVR4nO3dd5gtVZn3/e/vgEhQonBEQYIwKiYYcRQjiBkTgmkMOIoYxviqY3wEfdQxO46O4lEUTIwBHVEcRJEgiAFEARUdB3mQrAiKiAiH+/2jqmHTdDpnd3V1nf5++qqr965du9ba8d73WqtWpaqQJEmrZ1nfFZAkacgMpJIkjcFAKknSGAykkiSNwUAqSdIYDKSSJI3BQNqBJLsl+UKSC5P8LcllSb6VZL8ka03a9t5JjkhySZJrkpyb5MNJbj/Ffo9PUkk+NcVt+7e3bTuy7tB23VTL2iPbnD/N49i93fahI+s2TPLmJD9PclWSy5OcmeSjSbaYVPbN9ptkkyT/muSXSf6a5A9JvpnkEVNs++y2/CuSbDLptrXb2w6aqu4j22078pgPmOL2DZJc2d7+1mn28cb29i9Ps9+ZluNHno/R9X9J8uMkz5mivIMmvUZbts/z4VNse68k101X99XRvs9OmmWbWZ/7viW5ZZJXJPlp+xr/KcnZSQ5LsmO7zU+T/GyGfdxxqsea5FFJvp7k0iTXtp/fI5Ps3fHD0iJkIJ1nSV4OnAxsCrwGeCjwHOBXwEeAx4xs+0zgFGAz4GXAw4B/BR4BnJ7kHtMU8/QkO82xSr8Ddpu8VNV1q/TAmvquBXwbeCFwCPA4YD/gcOB+wO1muf/WwI+A5wIrgEfSPDd/Bo5O8vpp7roRzXM5jiuBZ06xfh9gtoOpn9X+3yvJZu3li7j58wpw6KR1LxrZz+hr8TTgCuCQJPvOVHhVXQS8GnhqksdNrE9yC+ATNO+t/zvLY5hvuwEfX+AyV9XhNM/LETSv81No3nd/B0x8fg4Ddkpyr2n2MfHa3/DjNcl7gW8AVwMvBvZs/18BfCHJPef1UWjxqyqXeVqABwHXA/8+ze13BO7RXr4T8FfgS8CySdttBvya5gvyFiPrjwdOBy4Bjph0n/1pAsK2I+sOBc6fpc7TbgPs3u7zoe31h7TXHz/N9stm2m9b/8uA7aa47/vb5273kXXPbsv7JnAVcNuR29Zubztolse3bbvdoe3+t5t0+7eBT7bbvHWK+9+vve2o9v+LZyhryn3M8HzcCrgcOHrS+oPafa09af2xwAXARu31NwEraX4Yzef7+HjgpIX4zHSxALcEtm+fw5fN9F4FlgPXAh+YZrv/BU4cuf6Mdr+vnGb7ewF36Ps5cFnYxYx0fr0W+APwL1PdWFX/W1VntFdfDqwFvKSqrp+03WXA64EdgSdO2s1VwNuBJybZdf6qPiebtv8vnurGyY9jVJL7AA8G3lFVv5lik9fRBJWpMs+JZss3zL2qN3MScA7NF+FEnbYC9mAk25jCfjTB6nnAb7kxQxlbVf2Z5sfSHeZ4l+cBGwPvTXJXmufjg1V1ynzVaa4mN3eONEfvmOSoJH9O8v+SvCnJskn3vU2SjyS5IE13xtmTm92TbN52F/yqbQb/bZLPZVKXx0i5d2u7CP4MfIE5vler6hKaH2pPm2hKH9n3A2kC8uj74/XAWVX13mn2e1pVnbeqz4mGzRdznrTNnrsDx1TVX+dwlz2BU6tptpvKUTQZ1EOmuO1g4DxuDDCz1W3tScvqvu4/Bq4DPppk70zqt5zFnu3/I6e6sX3OvgU8KJP6kWmaUT8EHJBkm1Ws86jPcNPm3WcA59NkYDeTZF3gycC3qurC9v73TnKXMeowuv+1gK1psp5ZVdU5wP+haRr/GnAh4/246MJXgO8ATwD+C3gzzY8RoOljp+n62Ism896L5rF8JMlLRvazKU2LzetougBeTfPD8uT2dZnsq8AJNN0N7wfOBv4EvCPJM5Isn6HOhwGb03SpjHomTfPtF9u63w64S1vfVTHjc6LhM5DOn9sA6wH/b47bbw2cO92NVXUVTZ/a1lPcdg3wFuARSR40Szm3p2m6Gl3eMsc6Ti73HJr+0R2ALwOXJflZkne3XzIzmXgc586wzbnA+jRN25O9k+ZL7cBVqfMknwJ2THLf9vozgc9U1XR9pI+nyQAnMpLD2v+r/SU48mPmdsAHaPp/V2Wg0L/RNO9uB7y0fZ8sJu+tqvdW1ber6mXAWTT9wRNeBmwD7FlVH2u3ezVNX++BE1lhVf2yql5WVUdU1Yk0weyJ7X0fNUW5/15Vb6+q71TV8W22/wya5vNPAxcn+d8kH0py50n3PZKmNeSG1oYktwSeBPxXVf2xXT3xHp7rZ3yuz4kGzkC6uGWG2w6laRZ82yz7uBS496Tlw6tboar6OM0XyjNoBm4sA14F/KxtbpzOTI9l1m2q6g/Ae4FnJbnT3Gt8k32cQ5MNPbNtFt+J2Zt1/0STRVBVvwR+ADxjNbP60R81F9AMRHpOVf1gFfbxhHY/xY1Z/oySLJvUIjE5459PR026fhY3bbp+JM1z+JvROtE0r27GjYOASPLCdlTtn2laQs5rb5rq9f/K5BVV9TWaPvInAh+kGQz0IpqBfA8d2e4a4PPA45Js1K6e/CNqHLM9Jxo4A+n8uYwmY5pr0+P5NB/yKSXZgCbL/e1Ut1fVSprBJg9IMtUv9AnXVtWpk5YLR26/jqavdiprjWwzWvblVfXZqnpBVd2F5st9Q5omq+lMPI5tZ9hmG5rn8LJpbn8/TR/0amXUrU/RjN7cH/hhGxxvJsltgYfTfAneMsnGSTamGQF6e+YYxCaZ+FFzH+Afgd8An5giQ5pS25T+H8B/0zwHL0nyD3O465u4aYvEsate9Tn7w6Tr1wCjTbFb0AzKm9xK8sX29s0A2mbeD9MMBnsi8A/AREvCVE27U3aRVNVVVfWVqnppVd2LZvDYSuAdkzY9rN3vk9rrz2r3+a2RbSbew6vavTDbc6KBM5DOk2oOJzkeeFjbLDSbY4Fdk2w5ze170bw+35lhH18AfkLTNDiXjG8qlwK3SbLOFLdNNNdeMtMOquqrwE8ZySamMPHl/bipbmz7vR4GnND+SJiqnD/THB70JGDnmeo0gy8AG9AM3Jkp23gGzQ+Jp9E0+00s72pvX53m3YkfNT+sqsOBR9OMPp5y4MoU3kfTVPkCmgFnZwMfbw+DmckKbtoi8fzVqPt8uQz4HjdvJZlYTm23eypwbFW9sqqOqaof0bxXpzOn80FW1feBY5j0Xm3X/5KmtWKiv/Qzo+/F9gfoL4DHzqUsLR0G0vn1Dppf1O+e6sYk240cG/oBmsFEH5xiVOOmNF+Uv6bpi5xS27f3RuDvaY6TWx3H0XyZTxXg9qH5Vf7Ltl63mWqgR5s9b800WUFb1+8D3wVem2S7KTb5V5oBJlM+dyM+TNMsuloTEFTVFW1ZRwL/OcOmz6LpC9tjiuVoYO8kt16dOozU5Zc0Geajk9x7pm2TPIzmcKDXVdV5VfU3mqz6rjSjxWcq58JJLRJTZuEL5GjgzsB5U7SUnFpVV7bbrU+TqY76p7kWkuTW7cCmyevXohm0NNV79VPAA2lG5q7N1D+03g7cLcn/N025uySx2XaJWXv2TTRXVXVi+wF7Xzuy81Cafp1NaJoC96dp0jujqn6R5Pk0B7Ufm+Rgmg/3nWkOn9kYeFhVTf4ymVzmUUlO5uYjDufq2zTNV4e2TYw/AG5NkxE8HvinkcNadqcZXXkoTVC8gqaZ6yU0QfB9s5T1TJrA/f0k76LJPjamCVpPBN5UVTNl4FTVNUneQpNlrZaqmrFpOMnfA3enOUb1+CluX5emr29fmmNQx/EO4ACa5tcpM532h8oKmkzuhv7tqjolyYeBNyT5YlWdPWZdRm2WqSeKOKOqfjXGft9P07T+3STvp/mRtgHN+/6BVfX4drujgdekmaTjhzSj12ecuGKSOwHHJfk0TWvIpcCWNJ/Bu3HTiTImfJpmAoeXAT+uqrMmb1BVn2nfH+9NshtNC8fFNE3We9G8x3flxv5cLQWrewCqy/QLTT/MF2kC47U0fSTH0DQXTp584b40AyV+B/yNJgs6GNh6iv0ezxQHytMcn1msxoQM7Xbr0mR4v6Lpv7mSJlA+ftJ2W9Fkcz+g+WK6tq33N4CHTNp2yrJpAu4727L+StNcegzwqCm2fXb7mHaYtH7t9v6rMiHD/rNsd8NkCtzYWrDNNNsuo/miPH66fUxxn2lfC5osp4Bd2usHMTIhQ1ufa4C7THHfW7Xvme8Cmaf37/Ej76fJy6tGHutBI/e5SZ0nPe5zJ63bhCag/qZ9z1/a1v/lI9usRzMT2O/a9+PXaUYqz7XcjWl+nJzIjZ/Dy2l+yO07w2P/NjNM5DCy3aNp+s9/1+77EppDcB67Os+Jy7CXtC+sJElaDfaRSpI0BgOpJEljMJBKkjQGA6kkSWMwkEqSNIZFexzperu82OHEi9DlP/pQ31XQFNZdtJ9krUFWd/a0WXXxfX/16R/qrL6TmZFKkjQGf8dKkvo18POcG0glSf3KgrXCdmLYPwMkSeqZGakkqV8Db9oddu0lSeqZGakkqV8D7yM1kEqS+mXTriRJS5cZqSSpXwNv2jUjlSRpDGakkqR+2UcqSdLSZUYqSerXwPtIDaSSpH7ZtCtJ0tJlRipJ6tfAm3bNSCVJGoMZqSSpXwPvIzWQSpL6ZdOuJElLlxmpJKlfA2/aHXbtJUnqmRmpJKlfA89IDaSSpH4tc7CRJElLlhmpJKlfA2/aHXbtJUnqmRmpJKlfA5+QwUAqSeqXTbuSJC1dZqSSpH4NvGnXjFSSpDGYkUqS+mUfqSRJS5cZqSSpXwPvIzWQSpL6ZdOuJElLlxmpJKlfA2/aNSOVJGkMZqSSpH4NvI/UQCpJ6pdNu5IkLV1mpJKkfg28aXfYtZckqWdmpJKkfg08IzWQSpL65WAjSZKWLjNSSVK/Bt60O+zaS5LUMzNSSVK/7COVJGnp6iSQJnln+/9JXexfkrQGybL5XxZQV6U9OsktgNd1tH9J0poimf9lAXXVR3o08HtggyR/GlkfoKpqw47KlSRpQXWVkb6xqjYCjqqqDUeWWxtEJUmjksz7spC6CqSntP//NONWa6Ctlm/M0SteyulHvJHTvvQG/vlpu9/k9pc/c0+uPv1DbLbxBv1UUACc/N0Tedxej+Axj3wYh3xsRd/VkTRgXTXtrpNkP+B+SZ44+caq+nJH5fbuupXX89r3fZmfnH0+t1r/lnzvc6/h2B+czdnnXMxWyzfmIfe9M+dd9Ie+q7mkrVy5kre/7S189GOfZPny5fzjU/Zl9z0ewh132KHvqklL0kJnkPOtq4z0BcB9gY2Bx05aHtNRmYvCxb//Ez85+3wA/vyXazj7Nxdzu803BuBdr9qHN3zgv6iqHmuos848g6233oattt6aW6yzDo989F4cf9yxfVdLWrrSwbKAOslIq+ok4KQkp1bVIV2UMQR32HJTdr7TVvzorHPZ68F358JLr+DMX13Qd7WWvEsvuYTbbnnbG65vsXw5Z55xRo81kjRknc5sVFWHJLkbsBOw7sj6T3VZ7mKwwXrrcPh79ufV7zmC61au5DXPfQSPedGH+q6WgOLmLQJDb1qShmzon79Oj1pNciDwwXbZA3gX8LgZtj8gyalJTr3u9z/rsmqdWnvtZRz+nufx+f8+la9+56dsv9XmbHP7zfjh51/H2Ue9mdtvsTGnfO41LN/s1n1XdUlavvy2XHzRxTdcv/SSS9hiiy16rJGkIet6+od9gT2Bi6vqn4B7ArecbuOqWlFVu1bVrmvf5q4dV607Bx/4dH75m4v59898B4Cf/fpCttnzddx5rwO5814HcsGlV7DbP76TSy67sueaLk13vdvdOe+8czn//N9y7d/+xtHfOIoH7/GQvqslLVlDP/yl60nrr66q65Ncl2RD4FJg+47L7NX9dt6epz/mPpz5qwv4/n++FoADP3Qk3zzp5z3XTBPWXnttXveGN/HCA/bn+utX8oS992GHHXbsu1rSkjX0pt2uA+mpSTYGPgacBvwZ+GHHZfbqez85h/V2efGM29x5rwMXqDaazgMf9GAe+KAH910NSWuArgcbvai9eHCSo4ENq8rhkZKkGww9I+16sNENB+dV1blVdcboOkmShq6TjDTJusD6wG2SbMKNh8duCNyuizIlSQM17IS0s6bd5wMvpwmap9Ge9QW4EvBgSklSr5JsDXwKuC1wPbCiqj6QZFPg88C2wLnAk6vq8pn21UnTblV9oKq2A94G7Nxe/iRwDjdOaC9JUl+Hv1wHvLKq7kIzpe0/J9kJeC1wbFXtCBzbXp9R58eRVtWfkjwAeBhwKPCRjsuUJA1IH4G0qi6qqh+3l68EfgHcHng8cFi72WHAE2bbV9eBdGX7fy/g4Kr6KrBOx2VKkpa40Zny2uWAGbbdFtgF+AGwvKougibYArNOe9b1caQXJPko8FDgnUluSffBW5I0IF0c/lJVK4BZTzac5FbAEcDL2xbUVS6r66D2ZOCbwCOr6gpgU+DVHZcpSdKsktyCJoh+duQ82Zck2bK9fUuaGflm1PWEDH8Bvjxy/SLgoi7LlCQNSx8TMqQp9BDgF1X1vpGbjgT2A97R/v/qbPvqumlXkqSZ9XMc6f2BZwJnJvlJu+71NAH0C0meC5wHPGm2HRlIJUlLTlWdxPQhfM9V2ZeBVJLUK+falSRpCTMjlST1augZqYFUktSroQdSm3YlSRqDGakkqV/DTkjNSCVJGocZqSSpV/aRSpK0hJmRSpJ6NfSM1EAqSerV0AOpTbuSJI3BjFSS1CszUkmSljAzUklSv4adkBpIJUn9smlXkqQlzIxUktQrM1JJkpYwM1JJUq+GnpEaSCVJ/Rp2HLVpV5KkcZiRSpJ6NfSmXTNSSZLGYEYqSeqVGakkSUuYGakkqVdDz0gNpJKkXg09kNq0K0nSGMxIJUn9GnZCakYqSdI4zEglSb0aeh+pgVSS1KuhB1KbdiVJGoMZqSSpVwNPSM1IJUkahxmpJKlXQ+8jNZBKkno18Dhq064kSeMwI5Uk9WroTbtmpJIkjcGMVJLUq4EnpGakkiSNw4xUktSrZcuGnZIaSCVJvbJpV5KkJWzRZqQXfe8DfVdBkrQAPPxFkqQlbNFmpJKkpWHgCamBVJLUL5t2JUlawsxIJUm9MiOVJGkJMyOVJPVq4AmpgVSS1C+bdiVJWsLMSCVJvRp4QmpGKknSOMxIJUm9so9UkqQlzIxUktSrgSekBlJJUr9s2pUkaQkzI5Uk9WrgCakZqSRJ4zAjlST1auh9pAZSSVKvBh5HbdqVJGkcZqSSpF4NvWnXjFSSpDGYkUqSejXwhNRAKknql027kiQtYWakkqReDTwhNSOVJGkcZqSSpF7ZRypJ0hJmRipJ6tXQM1IDqSSpVwOPozbtSpI0DjNSSVKvht60a0YqSVqSknwiyaVJzhpZd1CSC5L8pF0ePdt+DKSSpF4l87/M0aHAI6dY//6q2rldvjHbTmzalST1qq+m3ao6Mcm24+7HjFSSpJt6cZIz2qbfTWbb2EAqSepVF027SQ5IcurIcsAcq/MR4I7AzsBFwHtnu4NNu5KkNU5VrQBWrMb9Lpm4nORjwNdnu4+BVJLUq2WL6PCXJFtW1UXt1b2Bs2baHhYokCbZoKquWoiyJEnD0lccTXI4sDtwmyTnAwcCuyfZGSjgXOD5s+2n00Ca5H7Ax4FbAXdIck/g+VX1oi7LlSRpNlX1tClWH7Kq++l6sNH7gUcAlwFU1U+BB3VcpiRpQJLM+7KQOh+1W1W/nbRqZddlSpK0ULruI/1t27xbSdYBXgr8ouMyJUkDsmzxjDVaLV0H0hcAHwBuD5wPHAP8c8dlSpIGZOiT1ncaSKvq98DTuyxDkqQ+ddpHmuSwJBuPXN8kySe6LFOSNCw9Tlo/L7oebHSPqrpi4kpVXQ7s0nGZkiQtmK77SJcl2aQNoCTZdAHKlCQNSLCPdCbvBb6X5Evt9ScBb+u4TEmSFkzXg40+leQ0YA8gwBOr6uddlilJGhYPf5nd2cDlE2UluUNVnbcA5UqSBsDDX2aQ5CU0kwBfQjOjUWgmAr5Hl+VKkrRQus5IXwbcqaou67gcSdJADTwh7fzwl98Cf+y4DEmSetN1RnoOcHySo4BrJlZW1fs6LndR+L8HvoGTTzyBTTbdlMOPOLLv6mjEyd89kXe+421cv/J69t7nSTz3eQf0XSVpyVpMJ/ZeHV1npOcB3wLWAW49siwJj3nc3vzbh1f0XQ1NsnLlSt7+trfw4YM/zleOPIqjv/F1/vfXv+67WtKSNfSZjbo+/OXNXe5/sdvlXrty4QUX9F0NTXLWmWew9dbbsNXWWwPwyEfvxfHHHcsdd9ih55pJGqKuR+1uDvwLcFdg3Yn1VfWQLsuVZnLpJZdw2y1ve8P1LZYv58wzzuixRtLSNvTDX7pu2v0szXGk2wFvBs4FfjTdxkkOSHJqklMPPeRjHVdNS1VRN1s39A+ypP50Pdhos6o6JMnLquoE4IQkJ0y3cVWtAFYAXHH1ypt/20nzYPny23LxRRffcP3SSy5hiy226LFG0tI29N+xXWek17b/L0qyV5JdgK06LlOa0V3vdnfOO+9czj//t1z7t79x9DeO4sF72Nsg9WVZMu/LQuo6I31rko2AVwIfBDYEXtFxmYvGG1/7Kn586g+54ooreMzD9+CAF76Yx+29T9/VWvLWXnttXveGN/HCA/bn+utX8oS992GHHXbsu1qSBipV3bSgJlkLeGlVvX917m/T7uK07i3W6rsKmsK6npxQ3esszXvqYafP+/f9f+63y4KlpZ017VbVSuBxXe1fkqTFoOvfsd9L8iHg88BVEyur6scdlytJGoihj5rvOpDer/3/lpF1BTiyQ5K0Rug6kD63qs4ZXZFk+47LlCQNyNBP7N314S9fmmLdFzsuU5I0IEnmfVlInWSkSe5MMy3gRkmeOHLThoxMFShJ0tB11bR7J+AxwMbAY0fWXwk8r6MyJUkDNPCxRt0E0qr6KvDVJLtV1SldlCFJ0mLQ9WCjXyd5PbDtaFlV9ZyOy5UkDYSHv8zsq8B3gW8DKzsuS5I0QEMftdt1IF2/ql7TcRmSJPWm68Nfvp7k0R2XIUkasKEf/tJ1IH0Z8LUkVyf5U5Irk/yp4zIlSVow0zbtJvkgzXR+U6qql85h/xsBTwe2q6q3JLkDsOUq11KStMYaeBfpjH2kp87D/v8DuJ5mbt230BxHegRw73nYtyRpDbDQJ+Keb9MG0qo6bB72f5+q+vskp7f7vDzJOvOwX0mSFoVZR+0m2Rx4DbATI9P7VdVczuBybXuC7xrZ1/WrV1VJ0ppo4AnpnAYbfRb4BbAd8GbgXOBHc9z/vwNfAbZI8jbgJODtq15NSZIWp7kcR7pZVR2S5GVVdQJwQpIT5rLzqvpsktOAPWn6k59QVb8Yo76SpDXMUpjZ6Nr2/0VJ9gIuBLaaawFVdTZw9mrUTZKkRW8ugfStSTYCXgl8kOZUaK/otFaSpCVj4Anp7IG0qr7eXvwjsEe31ZEkLTVr7OEvE5J8kikmZvAMLpIkza1p9+sjl9cF9qbpJ5UkaWwDT0jn1LR7xOj1JIfTnBZNkqQlb3VOo7YjcIf5rogkaWla4w9/SXIlN+0jvZhmpiNJksbW9WnIujaXpt1bL0RFJEkaoll/CCQ5di7rJElaHUM/sfdM5yNdF1gfuE2STbjxlHEbArdbgLpJkrTozdS0+3zg5TRB8zRuDKR/ojnPqCRJY1s27LFGM56P9APAB5K8pKo+uIB1kiQtIUMPpHMZLHV9ko0nriTZJMmLuquSJEnDMZdA+ryqumLiSlVdDjyvsxpJkpaUoQ82mksgXZaRWiVZC1inuypJkjQcc5nZ6JvAF5IcTDMxwwuA/+60VpKkJWPofaRzCaSvAQ4AXkgzcvd0YMsuKyVJ0lDMZWaj65N8H9geeAqwKXDEzPeSJGluBj7V7owTMvwd8FTgacBlwOcBqsqTe0uS5s2afGLvs4HvAo+tql8DJHnFgtRKkqSBmGnU7j40Z3o5LsnHkuzJjbMbSZI0L5Z1sCykacurqq9U1VOAOwPHA68Alif5SJKHL1D9JEla1GYN3FV1VVV9tqoeA2wF/AR4bdcVkyQtDcn8LwtpLoe/3KCq/gB8tF0kSRrb0AcbDf3E5JIk9WqVMlJJkubbwBNSM1JJksZhRipJ6tVSmGtXkqTOONhIkqQlzIxUktSrgSekZqSSJI3DjFSS1KuhDzYyI5UkaQxmpJKkXmXgJxYzkEqSemXTriRJS9iizUg3Xm+tvqsgSVoAZqSSJC1hBlJJUq+SzPsyx3I/keTSJGeNrNs0ybeS/E/7f5PZ9mMglST1alnmf5mjQ4FHTlr3WuDYqtoROLa9PnP9V+GxSpK0xqiqE4E/TFr9eOCw9vJhwBNm28+iHWwkSVoaFtlcu8ur6iKAqrooyRaz3cGMVJK0xklyQJJTR5YDuirLjFSS1KsuzkdaVSuAFatx10uSbNlmo1sCl852BzNSSVKvehxsNJUjgf3ay/sBX521/mMVJ0nSQCU5HDgFuFOS85M8F3gH8LAk/wM8rL0+I5t2JUm96muwUVU9bZqb9lyV/ZiRSpI0BjNSSVKvlg38NGpmpJIkjcGMVJLUq0U2IcMqM5BKknrladQkSVrCzEglSb3qYmajhWRGKknSGMxIJUm9GnhCaiCVJPXLpl1JkpYwM1JJUq8GnpCakUqSNA4zUklSr4ae0RlIJUm9ysDbdof+Q0CSpF6ZkUqSejXsfNSMVJKksZiRSpJ65YQMkiQtYWakkqReDTsfNZBKkno28JZdm3YlSRqHGakkqVdOyCBJ0hJmRipJ6tXQMzoDqSSpVzbtSpK0hJmRSpJ6Nex81IxUkqSxdJ6RJrkHsO1oWVX15a7LlSQNw9D7SDsNpEk+AdwD+Blwfbu6AAOpJAkYftNo1xnpfatqp47LkCSpN10H0lOS7FRVP++4HEnSQNm0O7PDaILpxcA1NIOzqqru0XG5kiQtiK4D6SeAZwJncmMfqSRJNxh2Ptp9ID2vqo7suAxJknrTdSA9O8nngK/RNO0CHv4iSbrRwLtIOw+k69EE0IePrPPwF0nSDZYNvHG300BaVf/U5f4lSepbp8fBJtkqyVeSXJrkkiRHJNmqyzIlScOSzP+ykLqeUOKTwJHA7YDb0/SVfrLjMiVJWjBdB9LNq+qTVXVduxwKbN5xmZKkAUkHfwup60D6+yTPSLJWuzwDuKzjMiVJA2LT7syeAzwZuLhd9m3XSZK0Ruh61O55wOO6LEOSNGxDP/zFUbuSJI3BUbuSpF7ZRzozR+1KkmZkIJ2Zo3YlSWu0hRy1exGO2pUkTTL040g7G7WbZC3g7VXlqF1J0hqrs4y0qlYCmydZZ673SXJAklOTnLpixYquqiZJWkSWZf6XhdT1adTOBU5OciRw1cTKqnrfVBtX1QpgIoJWx3WTJC0CC90UO9+6DqQXtssy4NYdlyVJ0oLremajN3e5f0nS8C304SrzrdNAmuTvgFcB246WVVUP6bJcSZIWStdNu18EDgY+DqzsuCxJ0gDZRzqz66rqIx2XIUlSbzoJpEk2bS9+LcmLgK8A10zcXlV/6KJcSdLwLPThKvOtq4z0NJrDVyaenleP3FbA9h2VK0kaGJt2p1BV2wEkWbeq/jp6W5J1uyhTkqQ+dD3X7vfmuE6StEQN/ewvXfWR3pbm/KPrJdmFG5t4NwTW76JMSZL60FUf6SOAZwNbAaPTAV4JvL6jMiVJAzTsHtLu+kgPAw5Lsk9VHdFFGZKkNcOygU9t1PUUgUck2Qu4K7DuyPq3dFmuJEkLpespAg+m6RPdg2Z2o32BH3ZZpiRpWIadj3Y/avd+VfUs4PJ2AvvdgK07LlOSpAXT9RSBV7f//5LkdsBlwHYdlylJGpKBp6RdB9KvJ9kYeBfNbEfQNPFKkgQ4s9Fs3gO8EHggcArwXcBJ7CVJa4yuA+lhNMeO/nt7/WnAp4And1yuJGkgBn70S+eB9E5Vdc+R68cl+WnHZUqStGC6HrV7epL7TlxJch/g5I7LlCQNSDpYFlJXc+2eSXO6tFsAz0pyXnt9G+DnXZQpSVIfumrafUxH+5UkrWnsI725qvp/XexXkrTmGfrhL133kUqStEbretSuJEkzGvrhL2akkiSNwYxUktSrvhLSJOfSTBq0EriuqnZdnf0YSCVJ/eq3aXePqvr9ODuwaVeSpDEYSCVJvUoHf3NUwDFJTktywOrW36ZdSdIapw2Mo8FxRVWtmLTZ/avqwiRbAN9KcnZVnbjKZVXVOHXt0qKtmCQtQZ31ZP7kvCvn/ft+5zvcepXqm+Qg4M9V9Z5VLcumXUlSr/qYtD7JBkluPXEZeDhw1urU36ZdSdJStBz4SprZINYGPldVR6/OjgykkqR+9XD4S1WdA9xz1g3nwKZdSZLGYEYqSeqVZ3+RJGkJMyOVJPVq6Gd/MZBKkno18Dhq064kSeMwI5Uk9WvgKakZqSRJYzAjlST1auiHvxhIJUm9GvqoXZt2JUkagxmpJKlXA09IzUglSRqHGakkqV8DT0kNpJKkXg191K5Nu5IkjcGMVJLUKw9/kSRpCTMjlST1auAJqRmpJEnjMCOVJPVr4CmpgVSS1CsPf5EkaQkzI5Uk9crDXyRJWsLMSCVJvRp4QmoglST1bOCR1KZdSZLGYEYqSeqVh79IkrSEmZFKkno19MNfDKSSpF4NPI7atCtJ0jjMSCVJ/Rp4SmpGKknSGMxIJUm98vAXSZKWMDNSSVKvPPxFkqQxDDyO2rQrSdI4zEglSb0aetOuGakkSWMwI5Uk9WzYKamBVJLUK5t2JUlawsxIJUm9GnhCakYqSdI4zEglSb0aeh+pgVSS1CsnrZckaQkzI5Uk9WvYCakZqSRJ4zAjlST1auAJqRmpJEnjMCOVJPXKw18kSRqDh79IkrSEmZFKkvo17ITUjFSSpHGYkUqSejXwhNRAKknq19BH7dq0K0nSGMxIJUm98vAXSZKWMDNSSVKv7COVJGkJM5BKkjQGm3YlSb2yaVeSpCXMjFSS1CsPf5EkaQkzI5Uk9WrofaQGUklSrwYeR23alSRpHJ0G0iTrJ/k/ST7WXt8xyWO6LFOSNDDpYFlAXWeknwSuAXZrr58PvLXjMiVJWjBdB9I7VtW7gGsBqupqht8cLkmaR+ngbyF1Pdjob0nWAwogyR1pMlRJkgBH7c7mQOBoYOsknwXuDzy74zIlSVowqapuC0g2A+5L06T7/ar6/Rzv2m3FJEmrorO88S9/m/9AtP46C5fndj1qd2/guqo6qqq+DlyX5AldlilJ0kLqNCNN8pOq2nnSutOrapc53N2MVJIWj+4y0ms7yEhvsYZkpNPs39mUJEk36GvUbpJHJvllkl8nee3q1r/rQHpqkvcluWOS7ZO8Hzit4zIlSZpRkrWA/wAeBewEPC3JTquzr64D6UuAvwGfB74I/BX4547LlCQNSDL/yxz8A/Drqjqnqv4G/Cfw+NWpf6fNrFV1FbDa6bIkSR25PfDbkevnA/dZnR11GkiTbA78C3BXYN2J9VX1kGm2PwA4oL36/Kpa0WX9FkqSA9aUx7Im8XVZnHxdFqcuX5d1157/gUyT4gnAikn1n6rM1Rr01PWo3WNomnVfBbwA2A/4XVW9prNCF6Ekp1bVrn3XQzfl67I4+bosTmva65JkN+CgqnpEe/11AFX1r6u6r677SDerqkOAa6vqhKp6Ds3kDJIk9elHwI5JtkuyDvBU4MjV2VHXh6Jc2/6/KMlewIXAVh2XKUnSjKrquiQvBr4JrAV8oqp+tjr76jqQvjXJRsArgQ8CGwKv6LjMxcj+nsXJ12Vx8nVZnNa416WqvgF8Y9z9dD7XriRJa7Ku59rdPsnXkvw+yaVJvppk+y7LlCRpIXU92OhzwBeA2wK3o5mU4fCOy+xVkpcnWX8O2z0wyc+S/CTJekne3V5/90LUc02W5AmrO0OJ5ibJtknOWoXtn53kdiPXz01ym25qJy2srgNpqurTVXVdu3yGNX8y+pcDswZS4OnAe6pq56q6Gng+8PdV9eouK7dEPIFmyq+bSeJcz/14Ns2P6TnztdJQdB1Ij0vy2vbX6zZJ/gU4KsmmSTbtuOzOJdkgyVFJfprkrCQH0nxZHJfkuHabjyQ5tc0239yu2x94MvCmJJ9NciSwAfCDJE/p6/EsZkmekeSHbQb/0SRrJflzkre1z//3kyxPcj/gccC7223vmOT4JG9PcgLwsiSPTfKDJKcn+XaS5W0ZByX5RLv9OUleOlL+s5Kc0Zb16Xbd5kmOSPKjdrl/L09Of9ZOclj7vHwpyfpJ3tQ+F2clWZHGvsCuwGcnWmDa+78kyY+TnJnkznDDa7CiPQb9U+33xrFtGccmuUO73XTrD20/c8e1r+GD29f0F0kO7eE5WnSm+N56SttC8M72M/bDJDu02073WblVkk+2r90ZSfZp1z88ySnt6/rFJLfq87EumKrqbAF+M7Kc0y43XO+y7IVYgH2Aj41c3wg4F7jNyLpN2/9rAccD92ivHwrsO7Ldn/t+PIt1Ae4CfA24RXv9w8CzaFo3Htuuexfwxmme2+OBD49c34QbB9rtD7y3vXwQ8D3glsBtgMuAW9DMzPXLidd15DX9HPCA9vIdgF/0/Vwt4Guybfv837+9/gmaiVc2Hdnm0yOvz/HAriO3nQu8pL38IuDjI6/BacB67fWvAfu1l58D/Ncs6w+lmTM1NPOm/gm4O03ScBqwc9/PXd/LDN9bb2ivPwv4ent5us/KO4F/G9nHJu1n5kRgg3bda4A39f14F2LpOiN9DXDPqtoO+CTwU2CfqtquqtaEQUdnAg9tf8k9sKr+OMU2T07yY+B0mi9k++5W3Z7AvYAfJflJe317mhMifL3d5jSaL/fpfH7k8lbAN5OcCbya5nWZcFRVXVNVvwcuBZYDDwG+1K6jqv7QbvtQ4ENtnY4ENkxy69V8jEP026o6ub38GeABwB5tBnMmzfN212nvDV9u/09+7Y6sprsDYDeaHyzQBOYHzLIe4GvVfJOfCVxSVWdW1fXAz5j5PbJUTPe9dfjI/93ay9N9Vh5Kc+YUAKrqcprJdnYCTm4/E/sB23T5QBaLrvsg3lhVX0jyAOBhwHuBj7CaEwMvNlX1qyT3Ah4N/GvbHHWDJNvR/Eq/d1Vd3jYtrXvzPWkWAQ6rqtfdZGXyqvYLE2AlM7+frxq5/EHgfVV1ZJLdabKgCdeMXJ7YZ5i6b38ZsNvIl/5SM/k5KZrWgl2r6rdJDmLm9/vEcz35tbtqim2nK3Oq9RP7vZ6bvp7X4/mQZ/reGn0OJy5P91mZ6jMR4FtV9bQu6r2YdZ2Rrmz/7wUcXFVfBdbpuMwFk2YU4l+qGUT1HuDvgSuBiaxkQ5ovhT+2fQuP6qWiw3cssG+SLQDS9LHP9Et39DWYykbABe3l/eZY/pOTbDZRfrv+GODFExsl2XkO+1qT3CHNfKUATwNOai//vu0b23dk29lek+l8j2bqNmgG6J00y3rNYprvLYCnjPw/pb083Wdl8nt/E+D7wP1H+lfXT/J3nTyIRabrX2cXJPkoTTPAO5Pcku6D90K6O82glutppkN8IU2TyH8nuaiq9khyOk2T0jnAydPvStOpqp8neSNwTJJlNM/1TOe1/U/gY+1goX2nuP0g4ItJLqD58G83S/k/S/I24IQkK2ma6Z8NvBT4jyRn0HyWTqQ5OcNS8Qtgv/Yz/j80rU2b0DQdnkszl+mEQ4GDk1zNjc2Gc/FS4BNJXg38DvinWdZrdlN9b30JuGWSH9B8R09klQcx9WflrTTv/bNoEqY3V9WXkzwbOLz9rgd4I/Cr7h9Sv7o++8v6wCOBM6vqf5JsCdy9qo6Z5a6SpAWS5FyaJvnf912XIXKKQEla4gyk4zGQSpI0hjWpv1KSpAVnIJUkaQwGUkmSxmAglYAkK9t5YM9q5widy4kHptvXoWnmlyXJxzPDmWiS7J5mfuBVLcOzp0iLhIFUalxdzZl47kYz9eBNjgdNstbq7LSq9q+qn8+wye7AKgdSSYuHgVS6ue8CO7TZ4nFJPgecmeaMM+9Oc3aTM5I8HyCNDyX5eZKjgC0mdpTmTDK7tpcf2Z4V46dpzliyLU3AfkWbDT8w05xRJslmSY5JcxaOj9JMxyZpEVjy805Ko9KcA/NRwNHtqn8A7lZVv0lyAPDHqrp3O3PLye08pbsAd6KZMWY58HOas6GM7ndz4GPAg9p9bVpVf0hyMM2Zf97Tbvc54P1VdVKaU4N9k+bsNwcCJ1XVW5LsBRzQ6RMhac4MpFJjvfaMFdBkpIfQNLn+sKp+065/OHCPif5PmnlIdwQeBBxeVSuBC5N8Z4r93xc4cWJfI2eQmeyhwE7JDQnnxBllHgQ8sb3vUUkuX72HKWm+GUilxtVVtfPoijaYjZ6JJDTn0PzmpO0ezfRnJRm971xmP5nyjDJtXZw9RVqE7COV5u6bwAuT3AIgyd8l2YBmsvqntn2oWwJ7THHfU4AHt6fWGz2DzOSzokx3RpkTac5yQpJH0UwOL2kRMJBKc/dxmv7PH7dnvfgoTavOV2jOfnImzRlQTph8x6r6HU2/5peT/JQbTzT+NWDvicFGNGc12bUdzPRzbhw9/GbgQWlOEv9w4LyOHqOkVeRcu5IkjcGMVJKkMRhIJUkag4FUkqQxGEglSRqDgVSSpDEYSCVJGoOBVJKkMRhIJUkaw/8PXdhpt+cE3ssAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "conf_mat = confusion_matrix(y_test, y_pred)\n",
    "fig, ax = plt.subplots(figsize=(8,8))\n",
    "sns.heatmap(conf_mat, annot=True, cmap=\"Blues\", fmt='d',\n",
    "            xticklabels=category_id_df.Category.values, \n",
    "            yticklabels=category_id_df.Category.values)\n",
    "plt.ylabel('Actual')\n",
    "plt.xlabel('Predicted')\n",
    "plt.title(\"CONFUSION MATRIX - LinearSVCn\", size=16);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 659,
   "id": "ffbaba8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, \n",
    "                                                    test_size=0.25,\n",
    "                                                    random_state = 0)\n",
    "\n",
    "tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5,\n",
    "                        ngram_range=(1, 2), \n",
    "                        stop_words='english')\n",
    "\n",
    "fitted_vectorizer = tfidf.fit(X_train)\n",
    "tfidf_vectorizer_vectors = fitted_vectorizer.transform(X_train)\n",
    "\n",
    "model = LinearSVC().fit(tfidf_vectorizer_vectors, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 660,
   "id": "a5f77073",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['staff']\n"
     ]
    }
   ],
   "source": [
    "complaint = \"\"\"The toilet was upstairs.\"\"\"\n",
    "print(model.predict(fitted_vectorizer.transform([complaint])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 662,
   "id": "4814a9c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['staff']\n"
     ]
    }
   ],
   "source": [
    "complaint = \"\"\"The entrance was upstairs.\"\"\"\n",
    "print(model.predict(fitted_vectorizer.transform([complaint])))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "170e9d8a",
   "metadata": {},
   "source": [
    "## Method 2: Machine Learning Topic Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe874d5f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "dd02c120",
   "metadata": {},
   "source": [
    "# Sentiment Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 881,
   "id": "2c5a3687",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Cleaned_Sentence</th>\n",
       "      <th>Tokens</th>\n",
       "      <th>Class</th>\n",
       "      <th>Category</th>\n",
       "      <th>Category_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>lovely vibe good friendly staff</td>\n",
       "      <td>[staff, good, vibe, friendly, lovely]</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>delicious food friendly staff</td>\n",
       "      <td>[food, friendly, delicious, staff]</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>friendly sightly cheeky staff</td>\n",
       "      <td>[cheeky, sightly, staff, friendly]</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>restaurant nice atmosphere good service</td>\n",
       "      <td>[atmosphere, nice, restaurant, good, service]</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Firma Pekelharing</td>\n",
       "      <td>waiter always available explained whole menu</td>\n",
       "      <td>[menu, available, waiter, always, whole, expla...</td>\n",
       "      <td>1</td>\n",
       "      <td>staff</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Name                              Cleaned_Sentence  \\\n",
       "1  Firma Pekelharing               lovely vibe good friendly staff   \n",
       "5  Firma Pekelharing                 delicious food friendly staff   \n",
       "7  Firma Pekelharing                 friendly sightly cheeky staff   \n",
       "8  Firma Pekelharing       restaurant nice atmosphere good service   \n",
       "9  Firma Pekelharing  waiter always available explained whole menu   \n",
       "\n",
       "                                              Tokens  Class Category  \\\n",
       "1              [staff, good, vibe, friendly, lovely]      1    staff   \n",
       "5                 [food, friendly, delicious, staff]      1    staff   \n",
       "7                 [cheeky, sightly, staff, friendly]      1    staff   \n",
       "8      [atmosphere, nice, restaurant, good, service]      1    staff   \n",
       "9  [menu, available, waiter, always, whole, expla...      1    staff   \n",
       "\n",
       "   Category_ID  \n",
       "1            7  \n",
       "5            7  \n",
       "7            7  \n",
       "8            7  \n",
       "9            7  "
      ]
     },
     "execution_count": 881,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df5_filtered.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfd20186",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "28f8c966",
   "metadata": {},
   "source": [
    "# Topic Summarization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ed689e4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from summarization import ex_summarization\n",
    "from summarization import conc_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "11a291b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.summarization.summarizer import summarize\n",
    "from gensim.summarization import keywords\n",
    "\n",
    "def conc_sentences(df):\n",
    "    \n",
    "    df_general = df[df[\"Category\"]==\"general\"]\n",
    "    df_entrance = df[df[\"Category\"]==[\"entrance\"]]\n",
    "    df_bathroom = df[df[\"Category\"]==[\"bathroom\"]]\n",
    "    df_transport = df[df[\"Category\"]==[\"transport\"]]\n",
    "    df_stairs = df[df[\"Category\"]==[\"stairs\"]]\n",
    "    df_space = df[df[\"Category\"]==[\"space\"]]\n",
    "    df_staff = df[df[\"Category\"]==[\"staff\"]]\n",
    "    \n",
    "    return df_general, df_entrance, df_bathroom, df_transport, df_stairs, df_space, df_staff\n",
    "\n",
    "def ex_summarization(text):\n",
    "    '''Concatenate reviews into one document.\n",
    "    Summarize document based on length of document'''\n",
    "    \n",
    "    # Concatenate seperate sentences into one document\n",
    "    doc = str()\n",
    "    sentence_count = 0\n",
    "    \n",
    "    for sentence in text:\n",
    "        sentence_count += 1\n",
    "        sentence = str(sentence)\n",
    "        sentence = sentence.capitalize()\n",
    "        doc = doc + ' ' + sentence\n",
    "    \n",
    "    #print(\"Sentence count:\", sentence_count)\n",
    "    #print(\"Original Document:\", doc)\n",
    "    \n",
    "    if sentence_count > 2:\n",
    "        venue_summary = summarize(doc, ratio=0.5)\n",
    "    else:\n",
    "        venue_summary = doc\n",
    "    \n",
    "    #print(\"Summary:\", venue_summary)\n",
    "    \n",
    "    return venue_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "05e32374",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_general, df_entrance, df_bathroom, df_transport, df_stairs, df_space, df_staff = conc_sentences(df1_filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "id": "95c3412f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/rn/k7vf4gp97fb11dzjbcr7w8bc0000gn/T/ipykernel_9062/3360370633.py:11: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  df_summary = df_summary.append(temp_df, ignore_index = True)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Category</th>\n",
       "      <th>Summary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Stroom</td>\n",
       "      <td>entrance</td>\n",
       "      <td>Great food, good coffee, amazing outdoor seat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Stroom</td>\n",
       "      <td>bathroom</td>\n",
       "      <td>Even tho we were sitting next to the bathroom...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Stroom</td>\n",
       "      <td>stairs</td>\n",
       "      <td>If you sit upstairs, you will have a view ove...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Stroom</td>\n",
       "      <td>staff</td>\n",
       "      <td>Friendly and nice staff and good coffee.\\nLaid...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Name  Category                                            Summary\n",
       "0  Stroom  entrance   Great food, good coffee, amazing outdoor seat...\n",
       "1  Stroom  bathroom   Even tho we were sitting next to the bathroom...\n",
       "2  Stroom    stairs   If you sit upstairs, you will have a view ove...\n",
       "3  Stroom     staff  Friendly and nice staff and good coffee.\\nLaid..."
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_categories = [df_general, df_entrance, df_bathroom, df_transport, df_stairs, df_space, df_staff]\n",
    "df_summary = pd.DataFrame(columns=[\"Name\", \"Category\", \"Summary\"])\n",
    "temp_df = pd.DataFrame(columns=[\"Name\", \"Category\", \"Summary\"])\n",
    "\n",
    "for df in df_categories:\n",
    "    if df.empty == False:\n",
    "        temp_df.loc[0, \"Name\"] = df[\"Name\"].iloc[0]\n",
    "        temp_df.loc[0, \"Category\"] = df[\"Category\"].iloc[0]\n",
    "        sentence_list = list(df[\"Sentence\"])        \n",
    "        temp_df.loc[0, \"Summary\"] = ex_summarization(sentence_list)\n",
    "        df_summary = df_summary.append(temp_df, ignore_index = True)\n",
    "\n",
    "df_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "id": "f04bbe00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Friendly and nice staff and good coffee.\\nLaid-back  atmosphere and very nice staff.\\nthe staff is so nice and kind.\\nDelicious food, however the service is always very slow.\\nGood breakfast/brunch place, excellent service and very welcoming to children!\\nVery good food - extremely friendly staff - i can recommend this place!\\nSlow service due to crowds, excellent food.\\nGreat friendly staff!\\nRelaxed and peaceful with attentive and friendly staff.\\nTasty breakfasts and friendly staff.\\nPleasant atmosphere and friendly staff.\\ntasty sandwiches and good service.'"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_summary.iloc[3][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 606,
   "id": "08045ab7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' If you sit upstairs, you will have a view over the street and the people sitting outside.'"
      ]
     },
     "execution_count": 606,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_summary.iloc[2][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "111960ba",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "module 'pytextrank' has no attribute 'TextRank'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Input \u001b[0;32mIn [232]\u001b[0m, in \u001b[0;36m<cell line: 11>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m nlp \u001b[38;5;241m=\u001b[39m spacy\u001b[38;5;241m.\u001b[39mload(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124men_core_web_sm\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     10\u001b[0m \u001b[38;5;66;03m# add PyTextRank to the spaCy pipeline\u001b[39;00m\n\u001b[0;32m---> 11\u001b[0m tr \u001b[38;5;241m=\u001b[39m \u001b[43mpytextrank\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTextRank\u001b[49m()\n\u001b[1;32m     12\u001b[0m nlp\u001b[38;5;241m.\u001b[39madd_pipe(tr\u001b[38;5;241m.\u001b[39mPipelineComponent, name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtextrank\u001b[39m\u001b[38;5;124m\"\u001b[39m, last\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m     14\u001b[0m doc \u001b[38;5;241m=\u001b[39m nlp(text)\n",
      "\u001b[0;31mAttributeError\u001b[0m: module 'pytextrank' has no attribute 'TextRank'"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "import pytextrank\n",
    "\n",
    "# example text\n",
    "text = \"Compatibility of systems of linear constraints over the set of natural numbers. Criteria of compatibility of a system of linear Diophantine equations, strict inequations, and nonstrict inequations are considered. Upper bounds for components of a minimal set of solutions and algorithms of construction of minimal generating sets of solutions for all types of systems are given. These criteria and the corresponding algorithms for constructing a minimal supporting set of solutions can be used in solving all the considered types systems and systems of mixed types.\"\n",
    "\n",
    "# load a spaCy model, depending on language, scale, etc.\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "# add PyTextRank to the spaCy pipeline\n",
    "tr = pytextrank.TextRank()\n",
    "nlp.add_pipe(tr.PipelineComponent, name=\"textrank\", last=True)\n",
    "\n",
    "doc = nlp(text)\n",
    "\n",
    "# examine the top-ranked phrases in the document\n",
    "for p in doc._.phrases:\n",
    "    print(\"{:.4f} {:5d}  {}\".format(p.rank, p.count, p.text))\n",
    "    print(p.chunks)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}