Hackathon_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hackathon"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Imports:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\T460\\AppData\\Roaming\\Python\\Python39\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\T460\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\T460\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\T460\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import string\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# Libraries for data preparation and model building\n",
    "from sklearn.metrics import classification_report, f1_score, precision_score, recall_score, confusion_matrix  # Classification metrics\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV  # Train-test split and grid search\n",
    "from sklearn.linear_model import LogisticRegression  # Logistic Regression classifier for machine learning\n",
    "from sklearn.tree import DecisionTreeClassifier  # Decision Tree classifier for machine learning\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier  # Random Forest classifier for machine learning\n",
    "from sklearn.svm import LinearSVC, SVC  # Support Vector Machine classifiers\n",
    "from sklearn.naive_bayes import GaussianNB, MultinomialNB  # Naive Bayes classifiers\n",
    "from sklearn.ensemble import BaggingClassifier  # Bagging classifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier  # Extra Trees classifier\n",
    "from sklearn.ensemble import VotingClassifier  # Voting classifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from xgboost import XGBClassifier  # Efficient and flexible gradient boosting library\n",
    "from catboost import CatBoostClassifier  # High-performance gradient boosting on decision trees library\n",
    "from scipy.sparse import hstack  # Used for stacking sparse matrices horizontally\n",
    "import pickle  # Serialization library\n",
    "from sklearn.utils import resample  # Resampling tool\n",
    "from sklearn import feature_selection  # Feature selection module\n",
    "from sklearn.feature_selection import f_classif  # Feature selection using F-statistic\n",
    "from mlxtend.feature_selection import SequentialFeatureSelector  # Sequential feature selection\n",
    "from sklearn import preprocessing  # Data preprocessing\n",
    "import pickle  # Serialization library\n",
    "import tensorflow as tf  # TensorFlow for neural net\n",
    "from tensorflow.keras.layers import Dense  # Dense layer in Keras\n",
    "from tensorflow.keras.models import Sequential  # Sequential model in Keras\n",
    "from tensorflow.keras.utils import to_categorical  # Conversion to categorical data\n",
    "from scipy.sparse import issparse\n",
    "from sklearn import metrics\n",
    "\n",
    "# Feature selection Libraries:\n",
    "from sklearn.feature_selection import SelectKBest  # To reduce features\n",
    "from sklearn.feature_selection import chi2  # Used to estimate which features are most impactful\n",
    "\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "\n",
    "# Flags for notebook Execution\n",
    "VECTORIZER_TO_USE = \"count\"  # Chooses between TfIDF vectorizer or Count Vectorizer - accepted values are \"tfidf\" or \"count\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Load DataFrames:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read training data from CSV file into a DataFrame\n",
    "df_train = pd.read_csv('train_set.csv')\n",
    "\n",
    "# Read test data without labels from CSV file into a DataFrame\n",
    "df_test = pd.read_csv('test_set.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Cleaning Data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(33000, 2)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5682, 2)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def nlp_preprocessing(texts):\n",
    "    # Convert to lowercase\n",
    "    texts = [text.lower() for text in texts]\n",
    "\n",
    "    # Remove numbers\n",
    "    texts = [re.sub(r'\\d+', '', text) for text in texts]\n",
    "\n",
    "    # Remove punctuation\n",
    "    texts = [text.translate(str.maketrans(\"\", \"\", string.punctuation)) for text in texts]\n",
    "\n",
    "    # Tokenize the text\n",
    "    words = [word_tokenize(text) for text in texts]\n",
    "\n",
    "    # Remove stopwords\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    words = [[word for word in doc if word not in stop_words] for doc in words]\n",
    "\n",
    "    # Lemmatization\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    words = [[lemmatizer.lemmatize(word) for word in doc] for doc in words]\n",
    "\n",
    "    # Join the cleaned words back into a single string\n",
    "    cleaned_texts = [' '.join(doc) for doc in words]\n",
    "\n",
    "    return cleaned_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['text'] = nlp_preprocessing(df_train['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang_id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>xho</td>\n",
       "      <td>umgaqosiseko wenza amalungiselelo kumaziko axh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>xho</td>\n",
       "      <td>idha iya kuba nobulumko bokubeka umsebenzi nap...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eng</td>\n",
       "      <td>province kwazulunatal department transport inv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>nso</td>\n",
       "      <td>netefatša gore ba file dilo ka moka tše le dum...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ven</td>\n",
       "      <td>khomishini ya ndinganyiso ya mbeu yo ewa maana...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32995</th>\n",
       "      <td>tsn</td>\n",
       "      <td>popo ya dipolateforomo tse ke go tlisa boetele...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32996</th>\n",
       "      <td>sot</td>\n",
       "      <td>modise mosadi na ntse sa utlwe hore thabang ra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32997</th>\n",
       "      <td>eng</td>\n",
       "      <td>closing date submission completed tender augus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32998</th>\n",
       "      <td>xho</td>\n",
       "      <td>nawuphina umntu ofunyenwe enetyala phantsi kwa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32999</th>\n",
       "      <td>sot</td>\n",
       "      <td>mafapha mang le ona lokela ho etsa ditlaleho t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>33000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      lang_id                                               text\n",
       "0         xho  umgaqosiseko wenza amalungiselelo kumaziko axh...\n",
       "1         xho  idha iya kuba nobulumko bokubeka umsebenzi nap...\n",
       "2         eng  province kwazulunatal department transport inv...\n",
       "3         nso  netefatša gore ba file dilo ka moka tše le dum...\n",
       "4         ven  khomishini ya ndinganyiso ya mbeu yo ewa maana...\n",
       "...       ...                                                ...\n",
       "32995     tsn  popo ya dipolateforomo tse ke go tlisa boetele...\n",
       "32996     sot  modise mosadi na ntse sa utlwe hore thabang ra...\n",
       "32997     eng  closing date submission completed tender augus...\n",
       "32998     xho  nawuphina umntu ofunyenwe enetyala phantsi kwa...\n",
       "32999     sot  mafapha mang le ona lokela ho etsa ditlaleho t...\n",
       "\n",
       "[33000 rows x 2 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test['text'] = nlp_preprocessing(df_test['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>mmasepala fa maemo kgethegileng letlelela kgat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>uzakwaziswa ngokufaneleko nakungafuneka eminye...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>tshivhumbeo tshi fana na ngano dza vhathu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>kube inja nelikati betingevakala kutsi titsini...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>winste op buitelandse valuta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5677</th>\n",
       "      <td>5678</td>\n",
       "      <td>mark ballot private</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5678</th>\n",
       "      <td>5679</td>\n",
       "      <td>ge ka kgetha ka bowena go se šomiše mofani ka ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5679</th>\n",
       "      <td>5680</td>\n",
       "      <td>e ka kopo etsa kgetho ya hao ka hloko hobane h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5680</th>\n",
       "      <td>5681</td>\n",
       "      <td>tb ke bokudi ba pmb mme morero tla lefella tlh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5681</th>\n",
       "      <td>5682</td>\n",
       "      <td>vakatjhela iwebhusayidi yethu kuwww</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5682 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index                                               text\n",
       "0         1  mmasepala fa maemo kgethegileng letlelela kgat...\n",
       "1         2  uzakwaziswa ngokufaneleko nakungafuneka eminye...\n",
       "2         3          tshivhumbeo tshi fana na ngano dza vhathu\n",
       "3         4  kube inja nelikati betingevakala kutsi titsini...\n",
       "4         5                       winste op buitelandse valuta\n",
       "...     ...                                                ...\n",
       "5677   5678                                mark ballot private\n",
       "5678   5679  ge ka kgetha ka bowena go se šomiše mofani ka ...\n",
       "5679   5680  e ka kopo etsa kgetho ya hao ka hloko hobane h...\n",
       "5680   5681  tb ke bokudi ba pmb mme morero tla lefella tlh...\n",
       "5681   5682                vakatjhela iwebhusayidi yethu kuwww\n",
       "\n",
       "[5682 rows x 2 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label Mapping: {'afr': 0, 'eng': 1, 'nbl': 2, 'nso': 3, 'sot': 4, 'ssw': 5, 'tsn': 6, 'tso': 7, 'ven': 8, 'xho': 9, 'zul': 10}\n"
     ]
    }
   ],
   "source": [
    "# Assuming df_train is your DataFrame and 'lang_id' is the column you want to encode\n",
    "le = LabelEncoder()\n",
    "df_train['lang_id_encoded'] = le.fit_transform(df_train['lang_id'])\n",
    "\n",
    "# Display the mapping between original categories and encoded values\n",
    "label_mapping = dict(zip(le.classes_, le.transform(le.classes_)))\n",
    "print(\"Label Mapping:\", label_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang_id</th>\n",
       "      <th>text</th>\n",
       "      <th>lang_id_encoded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>xho</td>\n",
       "      <td>umgaqosiseko wenza amalungiselelo kumaziko axh...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>xho</td>\n",
       "      <td>idha iya kuba nobulumko bokubeka umsebenzi nap...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eng</td>\n",
       "      <td>province kwazulunatal department transport inv...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>nso</td>\n",
       "      <td>netefatša gore ba file dilo ka moka tše le dum...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ven</td>\n",
       "      <td>khomishini ya ndinganyiso ya mbeu yo ewa maana...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32995</th>\n",
       "      <td>tsn</td>\n",
       "      <td>popo ya dipolateforomo tse ke go tlisa boetele...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32996</th>\n",
       "      <td>sot</td>\n",
       "      <td>modise mosadi na ntse sa utlwe hore thabang ra...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32997</th>\n",
       "      <td>eng</td>\n",
       "      <td>closing date submission completed tender augus...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32998</th>\n",
       "      <td>xho</td>\n",
       "      <td>nawuphina umntu ofunyenwe enetyala phantsi kwa...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32999</th>\n",
       "      <td>sot</td>\n",
       "      <td>mafapha mang le ona lokela ho etsa ditlaleho t...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>33000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      lang_id                                               text  \\\n",
       "0         xho  umgaqosiseko wenza amalungiselelo kumaziko axh...   \n",
       "1         xho  idha iya kuba nobulumko bokubeka umsebenzi nap...   \n",
       "2         eng  province kwazulunatal department transport inv...   \n",
       "3         nso  netefatša gore ba file dilo ka moka tše le dum...   \n",
       "4         ven  khomishini ya ndinganyiso ya mbeu yo ewa maana...   \n",
       "...       ...                                                ...   \n",
       "32995     tsn  popo ya dipolateforomo tse ke go tlisa boetele...   \n",
       "32996     sot  modise mosadi na ntse sa utlwe hore thabang ra...   \n",
       "32997     eng  closing date submission completed tender augus...   \n",
       "32998     xho  nawuphina umntu ofunyenwe enetyala phantsi kwa...   \n",
       "32999     sot  mafapha mang le ona lokela ho etsa ditlaleho t...   \n",
       "\n",
       "       lang_id_encoded  \n",
       "0                    9  \n",
       "1                    9  \n",
       "2                    1  \n",
       "3                    3  \n",
       "4                    8  \n",
       "...                ...  \n",
       "32995                6  \n",
       "32996                4  \n",
       "32997                1  \n",
       "32998                9  \n",
       "32999                4  \n",
       "\n",
       "[33000 rows x 3 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>xho</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>xho</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eng</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>nso</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32995</th>\n",
       "      <td>tsn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32996</th>\n",
       "      <td>sot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32997</th>\n",
       "      <td>eng</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32998</th>\n",
       "      <td>xho</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32999</th>\n",
       "      <td>sot</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>33000 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      lang_id\n",
       "0         xho\n",
       "1         xho\n",
       "2         eng\n",
       "3         nso\n",
       "4         ven\n",
       "...       ...\n",
       "32995     tsn\n",
       "32996     sot\n",
       "32997     eng\n",
       "32998     xho\n",
       "32999     sot\n",
       "\n",
       "[33000 rows x 1 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract index that will be used when submitting predictions based on the evaluate dataset\n",
    "df_index_submission = pd.DataFrame(df_train['lang_id'])\n",
    "df_index_submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>lang_id_encoded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>umgaqosiseko wenza amalungiselelo kumaziko axh...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>idha iya kuba nobulumko bokubeka umsebenzi nap...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>province kwazulunatal department transport inv...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>netefatša gore ba file dilo ka moka tše le dum...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>khomishini ya ndinganyiso ya mbeu yo ewa maana...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  lang_id_encoded\n",
       "0  umgaqosiseko wenza amalungiselelo kumaziko axh...                9\n",
       "1  idha iya kuba nobulumko bokubeka umsebenzi nap...                9\n",
       "2  province kwazulunatal department transport inv...                1\n",
       "3  netefatša gore ba file dilo ka moka tše le dum...                3\n",
       "4  khomishini ya ndinganyiso ya mbeu yo ewa maana...                8"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dropping the message column since it has been vectorized\n",
    "df_train = df_train.drop(\"lang_id\", axis='columns')\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize and fit specified Vectorizer\n",
    "if VECTORIZER_TO_USE == \"tfidf\":\n",
    "\n",
    "    # Initialize Vectorizer\n",
    "    tfid_train = TfidfVectorizer(ngram_range=(4,5), analyzer='char', min_df=1, max_df =0.45, max_features=50000) # Change max features to include more data\n",
    "\n",
    "    # Fit vectoriser on text data:\n",
    "    vec_text_train = tfid_train.fit_transform(df_train[\"text\"])\n",
    "\n",
    "    # Transform both test set for 'tfid'\n",
    "    vec_text_test = tfid_train.transform(df_test[\"text\"])\n",
    "    \n",
    "elif VECTORIZER_TO_USE == \"count\":\n",
    "\n",
    "    # Initialize Vectorizer\n",
    "    count_vec_train = CountVectorizer(ngram_range=(4,5), analyzer='char', min_df=1, max_df =0.45, max_features=50000) # Change max features to include more data\n",
    "\n",
    "    # Fit vectoriser on text data:\n",
    "    vec_text_train = count_vec_train.fit_transform(df_train[\"text\"])\n",
    "\n",
    "    # Transform both test set for 'count'\n",
    "    vec_text_test = count_vec_train.transform(df_test[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aan</th>\n",
       "      <th>aan</th>\n",
       "      <th>aand</th>\n",
       "      <th>aang</th>\n",
       "      <th>aans</th>\n",
       "      <th>aant</th>\n",
       "      <th>aanv</th>\n",
       "      <th>aba</th>\n",
       "      <th>aba</th>\n",
       "      <th>abab</th>\n",
       "      <th>...</th>\n",
       "      <th>ṱo ḓ</th>\n",
       "      <th>ṱo ḓa</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱun</th>\n",
       "      <th>ṱun ḓ</th>\n",
       "      <th>ṱuwe</th>\n",
       "      <th>ṱuwed</th>\n",
       "      <th>ṱuṱu</th>\n",
       "      <th>ṱuṱuw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    aan   aan    aand   aang   aans   aant   aanv   aba   aba    abab  ...  \\\n",
       "0     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "1     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "2     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "3     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "4     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "\n",
       "   ṱo ḓ  ṱo ḓa  ṱuku  ṱuku   ṱun   ṱun ḓ  ṱuwe  ṱuwed  ṱuṱu  ṱuṱuw  \n",
       "0     0      0     0      0     0      0     0      0     0      0  \n",
       "1     0      0     0      0     0      0     0      0     0      0  \n",
       "2     0      0     0      0     0      0     0      0     0      0  \n",
       "3     0      0     0      0     0      0     0      0     0      0  \n",
       "4     0      0     0      0     0      0     0      0     0      0  \n",
       "\n",
       "[5 rows x 50000 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Converting vectorized text into sparse dataframe\n",
    "if VECTORIZER_TO_USE == \"tfidf\":\n",
    "    sparse_vec_msg_train = pd.DataFrame.sparse.from_spmatrix(vec_text_train, columns = tfid_train.get_feature_names_out())\n",
    "elif VECTORIZER_TO_USE == \"count\":\n",
    "    sparse_vec_msg_train = pd.DataFrame.sparse.from_spmatrix(vec_text_train, columns = count_vec_train.get_feature_names_out())\n",
    "sparse_vec_msg_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aan</th>\n",
       "      <th>aan</th>\n",
       "      <th>aand</th>\n",
       "      <th>aang</th>\n",
       "      <th>aans</th>\n",
       "      <th>aant</th>\n",
       "      <th>aanv</th>\n",
       "      <th>aba</th>\n",
       "      <th>aba</th>\n",
       "      <th>abab</th>\n",
       "      <th>...</th>\n",
       "      <th>ṱo ḓ</th>\n",
       "      <th>ṱo ḓa</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱun</th>\n",
       "      <th>ṱun ḓ</th>\n",
       "      <th>ṱuwe</th>\n",
       "      <th>ṱuwed</th>\n",
       "      <th>ṱuṱu</th>\n",
       "      <th>ṱuṱuw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    aan   aan    aand   aang   aans   aant   aanv   aba   aba    abab  ...  \\\n",
       "0     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "1     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "2     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "3     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "4     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "\n",
       "   ṱo ḓ  ṱo ḓa  ṱuku  ṱuku   ṱun   ṱun ḓ  ṱuwe  ṱuwed  ṱuṱu  ṱuṱuw  \n",
       "0     0      0     0      0     0      0     0      0     0      0  \n",
       "1     0      0     0      0     0      0     0      0     0      0  \n",
       "2     0      0     0      0     0      0     0      0     0      0  \n",
       "3     0      0     0      0     0      0     0      0     0      0  \n",
       "4     0      0     0      0     0      0     0      0     0      0  \n",
       "\n",
       "[5 rows x 50000 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Converting vectorized text into sparse dataframe\n",
    "if VECTORIZER_TO_USE == \"tfidf\":\n",
    "    sparse_vec_msg_test = pd.DataFrame.sparse.from_spmatrix(vec_text_test, columns = tfid_train.get_feature_names_out())\n",
    "elif VECTORIZER_TO_USE == \"count\":\n",
    "    sparse_vec_msg_test = pd.DataFrame.sparse.from_spmatrix(vec_text_test, columns = count_vec_train.get_feature_names_out())\n",
    "sparse_vec_msg_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>lang_id_encoded</th>\n",
       "      <th>aan</th>\n",
       "      <th>aan</th>\n",
       "      <th>aand</th>\n",
       "      <th>aang</th>\n",
       "      <th>aans</th>\n",
       "      <th>aant</th>\n",
       "      <th>aanv</th>\n",
       "      <th>aba</th>\n",
       "      <th>...</th>\n",
       "      <th>ṱo ḓ</th>\n",
       "      <th>ṱo ḓa</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱun</th>\n",
       "      <th>ṱun ḓ</th>\n",
       "      <th>ṱuwe</th>\n",
       "      <th>ṱuwed</th>\n",
       "      <th>ṱuṱu</th>\n",
       "      <th>ṱuṱuw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>umgaqosiseko wenza amalungiselelo kumaziko axh...</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>idha iya kuba nobulumko bokubeka umsebenzi nap...</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>province kwazulunatal department transport inv...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>netefatša gore ba file dilo ka moka tše le dum...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>khomishini ya ndinganyiso ya mbeu yo ewa maana...</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50002 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  lang_id_encoded   aan  \\\n",
       "0  umgaqosiseko wenza amalungiselelo kumaziko axh...                9     0   \n",
       "1  idha iya kuba nobulumko bokubeka umsebenzi nap...                9     0   \n",
       "2  province kwazulunatal department transport inv...                1     0   \n",
       "3  netefatša gore ba file dilo ka moka tše le dum...                3     0   \n",
       "4  khomishini ya ndinganyiso ya mbeu yo ewa maana...                8     0   \n",
       "\n",
       "    aan    aand   aang   aans   aant   aanv   aba  ...  ṱo ḓ  ṱo ḓa  ṱuku  \\\n",
       "0      0      0      0      0      0      0     0  ...     0      0     0   \n",
       "1      0      0      0      0      0      0     0  ...     0      0     0   \n",
       "2      0      0      0      0      0      0     0  ...     0      0     0   \n",
       "3      0      0      0      0      0      0     0  ...     0      0     0   \n",
       "4      0      0      0      0      0      0     0  ...     0      0     0   \n",
       "\n",
       "   ṱuku   ṱun   ṱun ḓ  ṱuwe  ṱuwed  ṱuṱu  ṱuṱuw  \n",
       "0      0     0      0     0      0     0      0  \n",
       "1      0     0      0     0      0     0      0  \n",
       "2      0     0      0     0      0     0      0  \n",
       "3      0     0      0     0      0     0      0  \n",
       "4      0     0      0     0      0     0      0  \n",
       "\n",
       "[5 rows x 50002 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Adding the dataframes together\n",
    "df_vectorized_clean = pd.concat([df_train.reset_index(drop=True), sparse_vec_msg_train.reset_index(drop=True)], axis=1)\n",
    "df_vectorized_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>text</th>\n",
       "      <th>aan</th>\n",
       "      <th>aan</th>\n",
       "      <th>aand</th>\n",
       "      <th>aang</th>\n",
       "      <th>aans</th>\n",
       "      <th>aant</th>\n",
       "      <th>aanv</th>\n",
       "      <th>aba</th>\n",
       "      <th>...</th>\n",
       "      <th>ṱo ḓ</th>\n",
       "      <th>ṱo ḓa</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱun</th>\n",
       "      <th>ṱun ḓ</th>\n",
       "      <th>ṱuwe</th>\n",
       "      <th>ṱuwed</th>\n",
       "      <th>ṱuṱu</th>\n",
       "      <th>ṱuṱuw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>mmasepala fa maemo kgethegileng letlelela kgat...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>uzakwaziswa ngokufaneleko nakungafuneka eminye...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>tshivhumbeo tshi fana na ngano dza vhathu</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>kube inja nelikati betingevakala kutsi titsini...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>winste op buitelandse valuta</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50002 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   index                                               text   aan   aan   \\\n",
       "0      1  mmasepala fa maemo kgethegileng letlelela kgat...     0      0   \n",
       "1      2  uzakwaziswa ngokufaneleko nakungafuneka eminye...     0      0   \n",
       "2      3          tshivhumbeo tshi fana na ngano dza vhathu     0      0   \n",
       "3      4  kube inja nelikati betingevakala kutsi titsini...     0      0   \n",
       "4      5                       winste op buitelandse valuta     0      0   \n",
       "\n",
       "    aand   aang   aans   aant   aanv   aba  ...  ṱo ḓ  ṱo ḓa  ṱuku  ṱuku   \\\n",
       "0      0      0      0      0      0     0  ...     0      0     0      0   \n",
       "1      0      0      0      0      0     0  ...     0      0     0      0   \n",
       "2      0      0      0      0      0     0  ...     0      0     0      0   \n",
       "3      0      0      0      0      0     0  ...     0      0     0      0   \n",
       "4      0      0      0      0      0     0  ...     0      0     0      0   \n",
       "\n",
       "   ṱun   ṱun ḓ  ṱuwe  ṱuwed  ṱuṱu  ṱuṱuw  \n",
       "0     0      0     0      0     0      0  \n",
       "1     0      0     0      0     0      0  \n",
       "2     0      0     0      0     0      0  \n",
       "3     0      0     0      0     0      0  \n",
       "4     0      0     0      0     0      0  \n",
       "\n",
       "[5 rows x 50002 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Adding the dataframes together\n",
    "df_vectorized_test_clean = pd.concat([df_test.reset_index(drop=True), sparse_vec_msg_test.reset_index(drop=True)], axis=1)\n",
    "df_vectorized_test_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang_id_encoded</th>\n",
       "      <th>aan</th>\n",
       "      <th>aan</th>\n",
       "      <th>aand</th>\n",
       "      <th>aang</th>\n",
       "      <th>aans</th>\n",
       "      <th>aant</th>\n",
       "      <th>aanv</th>\n",
       "      <th>aba</th>\n",
       "      <th>aba</th>\n",
       "      <th>...</th>\n",
       "      <th>ṱo ḓ</th>\n",
       "      <th>ṱo ḓa</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱun</th>\n",
       "      <th>ṱun ḓ</th>\n",
       "      <th>ṱuwe</th>\n",
       "      <th>ṱuwed</th>\n",
       "      <th>ṱuṱu</th>\n",
       "      <th>ṱuṱuw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   lang_id_encoded   aan   aan    aand   aang   aans   aant   aanv   aba  \\\n",
       "0                9     0      0      0      0      0      0      0     0   \n",
       "1                9     0      0      0      0      0      0      0     0   \n",
       "2                1     0      0      0      0      0      0      0     0   \n",
       "3                3     0      0      0      0      0      0      0     0   \n",
       "4                8     0      0      0      0      0      0      0     0   \n",
       "\n",
       "    aba   ...  ṱo ḓ  ṱo ḓa  ṱuku  ṱuku   ṱun   ṱun ḓ  ṱuwe  ṱuwed  ṱuṱu  ṱuṱuw  \n",
       "0      0  ...     0      0     0      0     0      0     0      0     0      0  \n",
       "1      0  ...     0      0     0      0     0      0     0      0     0      0  \n",
       "2      0  ...     0      0     0      0     0      0     0      0     0      0  \n",
       "3      0  ...     0      0     0      0     0      0     0      0     0      0  \n",
       "4      0  ...     0      0     0      0     0      0     0      0     0      0  \n",
       "\n",
       "[5 rows x 50001 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dropping the message column since it has been vectorized\n",
    "df_vectorized_clean = df_vectorized_clean.drop(\"text\", axis='columns')\n",
    "df_vectorized_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>aan</th>\n",
       "      <th>aan</th>\n",
       "      <th>aand</th>\n",
       "      <th>aang</th>\n",
       "      <th>aans</th>\n",
       "      <th>aant</th>\n",
       "      <th>aanv</th>\n",
       "      <th>aba</th>\n",
       "      <th>aba</th>\n",
       "      <th>...</th>\n",
       "      <th>ṱo ḓ</th>\n",
       "      <th>ṱo ḓa</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱun</th>\n",
       "      <th>ṱun ḓ</th>\n",
       "      <th>ṱuwe</th>\n",
       "      <th>ṱuwed</th>\n",
       "      <th>ṱuṱu</th>\n",
       "      <th>ṱuṱuw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   index   aan   aan    aand   aang   aans   aant   aanv   aba   aba   ...  \\\n",
       "0      1     0      0      0      0      0      0      0     0      0  ...   \n",
       "1      2     0      0      0      0      0      0      0     0      0  ...   \n",
       "2      3     0      0      0      0      0      0      0     0      0  ...   \n",
       "3      4     0      0      0      0      0      0      0     0      0  ...   \n",
       "4      5     0      0      0      0      0      0      0     0      0  ...   \n",
       "\n",
       "   ṱo ḓ  ṱo ḓa  ṱuku  ṱuku   ṱun   ṱun ḓ  ṱuwe  ṱuwed  ṱuṱu  ṱuṱuw  \n",
       "0     0      0     0      0     0      0     0      0     0      0  \n",
       "1     0      0     0      0     0      0     0      0     0      0  \n",
       "2     0      0     0      0     0      0     0      0     0      0  \n",
       "3     0      0     0      0     0      0     0      0     0      0  \n",
       "4     0      0     0      0     0      0     0      0     0      0  \n",
       "\n",
       "[5 rows x 50001 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dropping the message column since it has been vectorized\n",
    "df_vectorized_test_clean = df_vectorized_test_clean.drop(\"text\", axis='columns')\n",
    "df_vectorized_test_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5677</th>\n",
       "      <td>5678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5678</th>\n",
       "      <td>5679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5679</th>\n",
       "      <td>5680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5680</th>\n",
       "      <td>5681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5681</th>\n",
       "      <td>5682</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5682 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index\n",
       "0         1\n",
       "1         2\n",
       "2         3\n",
       "3         4\n",
       "4         5\n",
       "...     ...\n",
       "5677   5678\n",
       "5678   5679\n",
       "5679   5680\n",
       "5680   5681\n",
       "5681   5682\n",
       "\n",
       "[5682 rows x 1 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract index that will be used when submitting predictions based on the evaluate dataset\n",
    "df_index_submission = pd.DataFrame(df_vectorized_test_clean['index'])\n",
    "df_index_submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aan</th>\n",
       "      <th>aan</th>\n",
       "      <th>aand</th>\n",
       "      <th>aang</th>\n",
       "      <th>aans</th>\n",
       "      <th>aant</th>\n",
       "      <th>aanv</th>\n",
       "      <th>aba</th>\n",
       "      <th>aba</th>\n",
       "      <th>abab</th>\n",
       "      <th>...</th>\n",
       "      <th>ṱo ḓ</th>\n",
       "      <th>ṱo ḓa</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱuku</th>\n",
       "      <th>ṱun</th>\n",
       "      <th>ṱun ḓ</th>\n",
       "      <th>ṱuwe</th>\n",
       "      <th>ṱuwed</th>\n",
       "      <th>ṱuṱu</th>\n",
       "      <th>ṱuṱuw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 50000 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    aan   aan    aand   aang   aans   aant   aanv   aba   aba    abab  ...  \\\n",
       "0     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "1     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "2     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "3     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "4     0      0      0      0      0      0      0     0      0      0  ...   \n",
       "\n",
       "   ṱo ḓ  ṱo ḓa  ṱuku  ṱuku   ṱun   ṱun ḓ  ṱuwe  ṱuwed  ṱuṱu  ṱuṱuw  \n",
       "0     0      0     0      0     0      0     0      0     0      0  \n",
       "1     0      0     0      0     0      0     0      0     0      0  \n",
       "2     0      0     0      0     0      0     0      0     0      0  \n",
       "3     0      0     0      0     0      0     0      0     0      0  \n",
       "4     0      0     0      0     0      0     0      0     0      0  \n",
       "\n",
       "[5 rows x 50000 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove index form both training and test datasets\n",
    "df_vectorized_test_clean = df_vectorized_test_clean.drop(\"index\", axis=1)\n",
    "\n",
    "df_vectorized_test_clean.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Pre-processing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extracting the response variable from the training dataset\n",
    "y_train_og = df_vectorized_clean[:len(df_vectorized_clean)][['lang_id_encoded']]\n",
    "\n",
    "# Separating predictor variables from response variable\n",
    "x_train = df_vectorized_clean[:len(df_vectorized_clean)].drop(\"lang_id_encoded\", axis = \"columns\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitting into our training and validation data subsets\n",
    "X_train, X_validate, y_train, y_validate = train_test_split(x_train, y_train_og, test_size=0.2, random_state=42, stratify=y_train_og)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Models:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multinominal Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\T460\\Documents\\Anaconda\\lib\\site-packages\\sklearn\\utils\\validation.py:993: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=0.1)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a new instance of the Multinomial Naive Bayes model with the best hyperparameter\n",
    "best_multinom_nb = MultinomialNB(alpha=0.1)\n",
    "\n",
    "# Fitting the model to our training data subset\n",
    "best_multinom_nb.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00       600\n",
      "           1       1.00      1.00      1.00       600\n",
      "           2       1.00      1.00      1.00       600\n",
      "           3       1.00      1.00      1.00       600\n",
      "           4       1.00      1.00      1.00       600\n",
      "           5       1.00      1.00      1.00       600\n",
      "           6       1.00      1.00      1.00       600\n",
      "           7       1.00      1.00      1.00       600\n",
      "           8       1.00      1.00      1.00       600\n",
      "           9       1.00      1.00      1.00       600\n",
      "          10       1.00      1.00      1.00       600\n",
      "\n",
      "    accuracy                           1.00      6600\n",
      "   macro avg       1.00      1.00      1.00      6600\n",
      "weighted avg       1.00      1.00      1.00      6600\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Predict values of the testing subset\n",
    "pred_multinom_nb = best_multinom_nb.predict(X_validate)\n",
    "\n",
    "# Let's produce a classification report of the model as is:\n",
    "print(classification_report(y_validate, pred_multinom_nb)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "multinom_nb_cm = confusion_matrix(y_validate, pred_multinom_nb)\n",
    "multinom_nb_f1 = f1_score(y_validate, pred_multinom_nb, average=\"weighted\") \n",
    "multinom_nb_precision = precision_score(y_validate, pred_multinom_nb, average='weighted')\n",
    "multinom_nb_recall = recall_score(y_validate, pred_multinom_nb, average=\"weighted\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\T460\\Documents\\Anaconda\\lib\\site-packages\\sklearn\\utils\\validation.py:993: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    }
   ],
   "source": [
    "# Initiate final model\n",
    "final_multinom_nb = MultinomialNB(alpha=0.1)\n",
    "\n",
    "# Train on all available data\n",
    "final_multinom_nb.fit(x_train, y_train_og)\n",
    "\n",
    "# Generate predictions for the evaluation dataset\n",
    "multinom_nb_predictions = final_multinom_nb.predict(df_vectorized_test_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>lang_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>tsn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>nbl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>ven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>ssw</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>afr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5677</th>\n",
       "      <td>5678</td>\n",
       "      <td>eng</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5678</th>\n",
       "      <td>5679</td>\n",
       "      <td>nso</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5679</th>\n",
       "      <td>5680</td>\n",
       "      <td>sot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5680</th>\n",
       "      <td>5681</td>\n",
       "      <td>sot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5681</th>\n",
       "      <td>5682</td>\n",
       "      <td>nbl</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5682 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index lang_id\n",
       "0         1     tsn\n",
       "1         2     nbl\n",
       "2         3     ven\n",
       "3         4     ssw\n",
       "4         5     afr\n",
       "...     ...     ...\n",
       "5677   5678     eng\n",
       "5678   5679     nso\n",
       "5679   5680     sot\n",
       "5680   5681     sot\n",
       "5681   5682     nbl\n",
       "\n",
       "[5682 rows x 2 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make submissions into dataframe\n",
    "multi_nb_predictions = pd.DataFrame(multinom_nb_predictions, columns=[\"lang_id_encoded\"])\n",
    "\n",
    "# Convert lang_id_encoded to integer\n",
    "multi_nb_predictions['lang_id_encoded'] = multi_nb_predictions['lang_id_encoded'].astype('int')\n",
    "\n",
    "# Assuming multi_nb_predictions is your DataFrame and 'lang_id_encoded' is the encoded column\n",
    "lang_id = le.inverse_transform(multi_nb_predictions['lang_id_encoded'])\n",
    "\n",
    "# Create a new column in the DataFrame with the original values\n",
    "multi_nb_predictions['lang_id'] = lang_id\n",
    "\n",
    "# Dropping the message column since it has been vectorized\n",
    "multi_nb_predictions = multi_nb_predictions.drop(\"lang_id_encoded\", axis='columns')\n",
    "\n",
    "# Match submissions to index\n",
    "multi_nb_submission = pd.concat([df_index_submission.reset_index(drop=True), multi_nb_predictions.reset_index(drop=True)], axis=1,)\n",
    "\n",
    "\n",
    "#Save submission as csv\n",
    "multi_nb_submission.to_csv('Multinominal_Naive_Bayes_Predictions.csv', index=False)\n",
    "\n",
    "multi_nb_submission"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}