BERT_LV1_freezed-pad25.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aef26e7a",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2024-08-17T19:51:55.351934Z",
     "iopub.status.busy": "2024-08-17T19:51:55.351624Z",
     "iopub.status.idle": "2024-08-17T19:51:56.117207Z",
     "shell.execute_reply": "2024-08-17T19:51:56.116192Z"
    },
    "papermill": {
     "duration": 0.7816,
     "end_time": "2024-08-17T19:51:56.119209",
     "exception": false,
     "start_time": "2024-08-17T19:51:55.337609",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/dataset-tachygraphy/Tachygraphy_dataset_main.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ee28f8e3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:51:56.144060Z",
     "iopub.status.busy": "2024-08-17T19:51:56.143387Z",
     "iopub.status.idle": "2024-08-17T19:52:00.000016Z",
     "shell.execute_reply": "2024-08-17T19:51:59.999023Z"
    },
    "papermill": {
     "duration": 3.871293,
     "end_time": "2024-08-17T19:52:00.002308",
     "exception": false,
     "start_time": "2024-08-17T19:51:56.131015",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ebcf291d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:00.027053Z",
     "iopub.status.busy": "2024-08-17T19:52:00.026644Z",
     "iopub.status.idle": "2024-08-17T19:52:01.718636Z",
     "shell.execute_reply": "2024-08-17T19:52:01.717698Z"
    },
    "papermill": {
     "duration": 1.706862,
     "end_time": "2024-08-17T19:52:01.721084",
     "exception": false,
     "start_time": "2024-08-17T19:52:00.014222",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f01b20a5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:01.746874Z",
     "iopub.status.busy": "2024-08-17T19:52:01.746319Z",
     "iopub.status.idle": "2024-08-17T19:52:01.819437Z",
     "shell.execute_reply": "2024-08-17T19:52:01.818621Z"
    },
    "papermill": {
     "duration": 0.088456,
     "end_time": "2024-08-17T19:52:01.821372",
     "exception": false,
     "start_time": "2024-08-17T19:52:01.732916",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /usr/share/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "db606431",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:01.846303Z",
     "iopub.status.busy": "2024-08-17T19:52:01.845759Z",
     "iopub.status.idle": "2024-08-17T19:52:01.887092Z",
     "shell.execute_reply": "2024-08-17T19:52:01.886247Z"
    },
    "papermill": {
     "duration": 0.055877,
     "end_time": "2024-08-17T19:52:01.888988",
     "exception": false,
     "start_time": "2024-08-17T19:52:01.833111",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4958, 3)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"/kaggle/input/dataset-tachygraphy/Tachygraphy_dataset_main.csv\")\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5c18969e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:01.914073Z",
     "iopub.status.busy": "2024-08-17T19:52:01.913403Z",
     "iopub.status.idle": "2024-08-17T19:52:01.929278Z",
     "shell.execute_reply": "2024-08-17T19:52:01.928480Z"
    },
    "papermill": {
     "duration": 0.030376,
     "end_time": "2024-08-17T19:52:01.931186",
     "exception": false,
     "start_time": "2024-08-17T19:52:01.900810",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Meaning</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Last session of the day http://twitpic.com/67ezh</td>\n",
       "      <td>Last session of the day http://twitpic.com/67ezh</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Shanghai is also really exciting (precisely --...</td>\n",
       "      <td>Shanghai is also really exciting (precisely --...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>submit the report ASAP!</td>\n",
       "      <td>submit the report as soon as possilbe</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>happy bday!</td>\n",
       "      <td>Happy Birthday!</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The OGs - I like it!!</td>\n",
       "      <td>The original gangsters - i like it!</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text  \\\n",
       "0   Last session of the day http://twitpic.com/67ezh   \n",
       "1  Shanghai is also really exciting (precisely --...   \n",
       "2                            submit the report ASAP!   \n",
       "3                                        happy bday!   \n",
       "4                              The OGs - I like it!!   \n",
       "\n",
       "                                             Meaning Sentiment  \n",
       "0   Last session of the day http://twitpic.com/67ezh   neutral  \n",
       "1  Shanghai is also really exciting (precisely --...  positive  \n",
       "2              submit the report as soon as possilbe  negative  \n",
       "3                                    Happy Birthday!  positive  \n",
       "4                The original gangsters - i like it!  positive  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "716364d2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:01.957406Z",
     "iopub.status.busy": "2024-08-17T19:52:01.956871Z",
     "iopub.status.idle": "2024-08-17T19:52:01.964656Z",
     "shell.execute_reply": "2024-08-17T19:52:01.963793Z"
    },
    "papermill": {
     "duration": 0.022311,
     "end_time": "2024-08-17T19:52:01.966412",
     "exception": false,
     "start_time": "2024-08-17T19:52:01.944101",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df=df.drop('Meaning',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "74d1cb69",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:01.991754Z",
     "iopub.status.busy": "2024-08-17T19:52:01.991204Z",
     "iopub.status.idle": "2024-08-17T19:52:01.996380Z",
     "shell.execute_reply": "2024-08-17T19:52:01.995591Z"
    },
    "papermill": {
     "duration": 0.020109,
     "end_time": "2024-08-17T19:52:01.998434",
     "exception": false,
     "start_time": "2024-08-17T19:52:01.978325",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4958, 2)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6f379649",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.023856Z",
     "iopub.status.busy": "2024-08-17T19:52:02.023384Z",
     "iopub.status.idle": "2024-08-17T19:52:02.027826Z",
     "shell.execute_reply": "2024-08-17T19:52:02.026977Z"
    },
    "papermill": {
     "duration": 0.019127,
     "end_time": "2024-08-17T19:52:02.029675",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.010548",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df.rename(columns={'Text': 'tweet'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fe689a1a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.054907Z",
     "iopub.status.busy": "2024-08-17T19:52:02.054633Z",
     "iopub.status.idle": "2024-08-17T19:52:02.058879Z",
     "shell.execute_reply": "2024-08-17T19:52:02.058083Z"
    },
    "papermill": {
     "duration": 0.018968,
     "end_time": "2024-08-17T19:52:02.060691",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.041723",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df.rename(columns={'Sentiment': 'class'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fa8baf78",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.085959Z",
     "iopub.status.busy": "2024-08-17T19:52:02.085708Z",
     "iopub.status.idle": "2024-08-17T19:52:02.101309Z",
     "shell.execute_reply": "2024-08-17T19:52:02.100495Z"
    },
    "papermill": {
     "duration": 0.030387,
     "end_time": "2024-08-17T19:52:02.103302",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.072915",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "class\n",
       "neutral     1743\n",
       "positive    1643\n",
       "negative    1572\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['class'] = df['class'].str.lower()\n",
    "sentiment_counts = df['class'].value_counts()\n",
    "sentiment_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "668e272f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.129350Z",
     "iopub.status.busy": "2024-08-17T19:52:02.128833Z",
     "iopub.status.idle": "2024-08-17T19:52:02.134203Z",
     "shell.execute_reply": "2024-08-17T19:52:02.133405Z"
    },
    "papermill": {
     "duration": 0.0203,
     "end_time": "2024-08-17T19:52:02.136036",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.115736",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df[df['class'] != \"neutral\"]  ###droping neutral columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "afb29bf8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.161845Z",
     "iopub.status.busy": "2024-08-17T19:52:02.161424Z",
     "iopub.status.idle": "2024-08-17T19:52:02.167527Z",
     "shell.execute_reply": "2024-08-17T19:52:02.166753Z"
    },
    "papermill": {
     "duration": 0.020974,
     "end_time": "2024-08-17T19:52:02.169279",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.148305",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "label_encoder = LabelEncoder()\n",
    "label_encoder.fit(df['class'])\n",
    "encoded_labels = label_encoder.transform(df['class'])\n",
    "df['class'] = encoded_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4815a7dc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.194720Z",
     "iopub.status.busy": "2024-08-17T19:52:02.194285Z",
     "iopub.status.idle": "2024-08-17T19:52:02.202027Z",
     "shell.execute_reply": "2024-08-17T19:52:02.201193Z"
    },
    "papermill": {
     "duration": 0.022369,
     "end_time": "2024-08-17T19:52:02.203904",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.181535",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Shanghai is also really exciting (precisely --...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>submit the report ASAP!</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>happy bday!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The OGs - I like it!!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>that`s great!! weee!! visitors!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  class\n",
       "1  Shanghai is also really exciting (precisely --...      1\n",
       "2                            submit the report ASAP!      0\n",
       "3                                        happy bday!      1\n",
       "4                              The OGs - I like it!!      1\n",
       "5                    that`s great!! weee!! visitors!      1"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "64fd8a25",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.229656Z",
     "iopub.status.busy": "2024-08-17T19:52:02.229152Z",
     "iopub.status.idle": "2024-08-17T19:52:02.236553Z",
     "shell.execute_reply": "2024-08-17T19:52:02.235719Z"
    },
    "papermill": {
     "duration": 0.022346,
     "end_time": "2024-08-17T19:52:02.238466",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.216120",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tweet    1\n",
       "class    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "894eb81f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.264727Z",
     "iopub.status.busy": "2024-08-17T19:52:02.264485Z",
     "iopub.status.idle": "2024-08-17T19:52:02.268998Z",
     "shell.execute_reply": "2024-08-17T19:52:02.268192Z"
    },
    "papermill": {
     "duration": 0.019848,
     "end_time": "2024-08-17T19:52:02.270885",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.251037",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df=df.fillna('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c33a68a7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.297901Z",
     "iopub.status.busy": "2024-08-17T19:52:02.297633Z",
     "iopub.status.idle": "2024-08-17T19:52:02.305565Z",
     "shell.execute_reply": "2024-08-17T19:52:02.304730Z"
    },
    "papermill": {
     "duration": 0.022894,
     "end_time": "2024-08-17T19:52:02.307401",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.284507",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Shanghai is also really exciting (precisely --...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>submit the report ASAP!</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>happy bday!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The OGs - I like it!!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>that`s great!! weee!! visitors!</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  class\n",
       "1  Shanghai is also really exciting (precisely --...      1\n",
       "2                            submit the report ASAP!      0\n",
       "3                                        happy bday!      1\n",
       "4                              The OGs - I like it!!      1\n",
       "5                    that`s great!! weee!! visitors!      1"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "452f84c3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.334210Z",
     "iopub.status.busy": "2024-08-17T19:52:02.333585Z",
     "iopub.status.idle": "2024-08-17T19:52:02.509368Z",
     "shell.execute_reply": "2024-08-17T19:52:02.508622Z"
    },
    "papermill": {
     "duration": 0.191482,
     "end_time": "2024-08-17T19:52:02.511630",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.320148",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def text_cleaning(text):\n",
    "    text = str(text)\n",
    "    soup = BeautifulSoup(text, \"html.parser\")    ###removing html tages\n",
    "    text = re.sub(r'\\[[^]]*\\]', '', soup.get_text())      ##removing text within square brackets\n",
    "    pattern = r\"[^a-zA-Z0-9\\s,']\"                  # Removing unwanted characters\n",
    "    text = re.sub(pattern, '', text)\n",
    "    url_pattern = r'http\\S+|www\\S+'            ###removing urls\n",
    "    text = re.sub(url_pattern, '', text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c2bebd41",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.539793Z",
     "iopub.status.busy": "2024-08-17T19:52:02.539292Z",
     "iopub.status.idle": "2024-08-17T19:52:02.720646Z",
     "shell.execute_reply": "2024-08-17T19:52:02.719759Z"
    },
    "papermill": {
     "duration": 0.196912,
     "end_time": "2024-08-17T19:52:02.722613",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.525701",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_24/2827669698.py:6: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  soup = BeautifulSoup(text, \"html.parser\")    ###removing html tages\n"
     ]
    }
   ],
   "source": [
    "df['tweet'] = df['tweet'].apply(text_cleaning).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a40ff539",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.749826Z",
     "iopub.status.busy": "2024-08-17T19:52:02.749299Z",
     "iopub.status.idle": "2024-08-17T19:52:02.758245Z",
     "shell.execute_reply": "2024-08-17T19:52:02.757467Z"
    },
    "papermill": {
     "duration": 0.024725,
     "end_time": "2024-08-17T19:52:02.760340",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.735615",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Shanghai is also really exciting precisely  sk...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>submit the report ASAP</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>happy bday</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The OGs  I like it</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>thats great weee visitors</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>I THINK EVERYONE HATES ME ON HERE lol</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>soooooo wish i could, but im in school and mys...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>My bike was put on holdshould have known that ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Im in VA for the weekend, my youngest son turn...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Its coming out the socket I feel like my phone...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                tweet  class\n",
       "1   Shanghai is also really exciting precisely  sk...      1\n",
       "2                              submit the report ASAP      0\n",
       "3                                          happy bday      1\n",
       "4                                  The OGs  I like it      1\n",
       "5                           thats great weee visitors      1\n",
       "6               I THINK EVERYONE HATES ME ON HERE lol      0\n",
       "7   soooooo wish i could, but im in school and mys...      0\n",
       "10  My bike was put on holdshould have known that ...      0\n",
       "13  Im in VA for the weekend, my youngest son turn...      0\n",
       "14  Its coming out the socket I feel like my phone...      0"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f419507d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.788163Z",
     "iopub.status.busy": "2024-08-17T19:52:02.787531Z",
     "iopub.status.idle": "2024-08-17T19:52:02.794617Z",
     "shell.execute_reply": "2024-08-17T19:52:02.793791Z"
    },
    "papermill": {
     "duration": 0.022809,
     "end_time": "2024-08-17T19:52:02.796498",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.773689",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "lens=[len(i.split()) for i in df['tweet']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ef2d349f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:02.823775Z",
     "iopub.status.busy": "2024-08-17T19:52:02.823138Z",
     "iopub.status.idle": "2024-08-17T19:52:03.114798Z",
     "shell.execute_reply": "2024-08-17T19:52:03.113826Z"
    },
    "papermill": {
     "duration": 0.307384,
     "end_time": "2024-08-17T19:52:03.116757",
     "exception": false,
     "start_time": "2024-08-17T19:52:02.809373",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([135., 493., 641., 585., 429., 438., 237., 174.,  74.,   9.]),\n",
       " array([ 0. ,  3.2,  6.4,  9.6, 12.8, 16. , 19.2, 22.4, 25.6, 28.8, 32. ]),\n",
       " <BarContainer object of 10 artists>)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkEElEQVR4nO3df3DU9Z3H8VdCkuXnbgyQXXIkEKsVUgFr0LDV2lZyBJo6eMQ76aU27TEwpYlXSKWSDoKlHcPQnlo6CP114s1Jabk59IQBxVBCT5YoUUYETcGLF7ywiZXLLsTmB8nn/mj5XleCuJDk+0nyfMx8Z8j3+9nd937nO5PnbHaXBGOMEQAAgEUS3R4AAADgwwgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANZJcnuAK9Hd3a3GxkaNGTNGCQkJbo8DAAA+BmOMzp49q4yMDCUmfvRrJAMyUBobG5WZmen2GAAA4AqcOnVKEydO/Mg1AzJQxowZI+lPT9Dr9bo8DQAA+Dii0agyMzOd3+MfZUAGyoU/63i9XgIFAIAB5uO8PYM3yQIAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDpJbg+AoWvyyl1ujxC3d9YVuj0CAAwJvIICAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOvEHSj/8z//o6985SsaO3asRowYoWnTpunw4cPOcWOMVq9erQkTJmjEiBHKz8/XiRMnYu7jzJkzKi4ultfrVWpqqhYtWqRz585d/bMBAACDQlyB8r//+7+67bbblJycrN27d+v48eP6p3/6J11zzTXOmvXr12vDhg3avHmzampqNGrUKBUUFKitrc1ZU1xcrGPHjmnv3r3auXOnDhw4oCVLlvTeswIAAANagjHGfNzFK1eu1EsvvaTf/e53PR43xigjI0Pf/va39cADD0iSIpGI/H6/tmzZooULF+rNN99UTk6OXnnlFc2cOVOStGfPHn3xi1/Uu+++q4yMjMvOEY1G5fP5FIlE5PV6P+74sMzklbvcHiFu76wrdHsEABiw4vn9HdcrKP/xH/+hmTNn6m//9m+Vnp6uT3/60/r5z3/uHK+vr1c4HFZ+fr6zz+fzKS8vT6FQSJIUCoWUmprqxIkk5efnKzExUTU1NT0+bnt7u6LRaMwGAAAGr7gC5b/+67+0adMmXX/99Xr++ee1dOlS/eM//qOeeuopSVI4HJYk+f3+mNv5/X7nWDgcVnp6eszxpKQkpaWlOWs+rLKyUj6fz9kyMzPjGRsAAAwwcQVKd3e3br75Zj3yyCP69Kc/rSVLlmjx4sXavHlzX80nSaqoqFAkEnG2U6dO9enjAQAAd8UVKBMmTFBOTk7MvqlTp6qhoUGSFAgEJElNTU0xa5qampxjgUBAzc3NMcfPnz+vM2fOOGs+zOPxyOv1xmwAAGDwiitQbrvtNtXV1cXs+/3vf69JkyZJkrKzsxUIBFRVVeUcj0ajqqmpUTAYlCQFg0G1tLSotrbWWbNv3z51d3crLy/vip8IAAAYPJLiWbx8+XJ95jOf0SOPPKK/+7u/08svv6yf/exn+tnPfiZJSkhI0LJly/SDH/xA119/vbKzs/XQQw8pIyNDd999t6Q/veIyd+5c509DnZ2dKisr08KFCz/WJ3gAAMDgF1eg3HLLLdqxY4cqKiq0du1aZWdn6/HHH1dxcbGz5jvf+Y5aW1u1ZMkStbS06Pbbb9eePXs0fPhwZ83TTz+tsrIyzZ49W4mJiSoqKtKGDRt671kBAIABLa7vQbEF34MyOPA9KAAwtPTZ96AAAAD0h7j+xAMMdbzqAwD9g1dQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGCduALl4YcfVkJCQsw2ZcoU53hbW5tKS0s1duxYjR49WkVFRWpqaoq5j4aGBhUWFmrkyJFKT0/XihUrdP78+d55NgAAYFBIivcGn/rUp/Tiiy/+/x0k/f9dLF++XLt27dL27dvl8/lUVlamBQsW6KWXXpIkdXV1qbCwUIFAQAcPHtTp06f11a9+VcnJyXrkkUd64ekAAIDBIO5ASUpKUiAQuGh/JBLRL3/5S23dulV33nmnJOnJJ5/U1KlTdejQIc2aNUsvvPCCjh8/rhdffFF+v1833XSTvv/97+vBBx/Uww8/rJSUlKt/RgAAYMCL+z0oJ06cUEZGhq699loVFxeroaFBklRbW6vOzk7l5+c7a6dMmaKsrCyFQiFJUigU0rRp0+T3+501BQUFikajOnbs2CUfs729XdFoNGYDAACDV1yBkpeXpy1btmjPnj3atGmT6uvr9dnPflZnz55VOBxWSkqKUlNTY27j9/sVDoclSeFwOCZOLhy/cOxSKisr5fP5nC0zMzOesQEAwAAT15945s2b5/x7+vTpysvL06RJk/Sb3/xGI0aM6PXhLqioqFB5ebnzczQaJVIAABjErupjxqmpqfrkJz+pkydPKhAIqKOjQy0tLTFrmpqanPesBAKBiz7Vc+Hnnt7XcoHH45HX643ZAADA4HVVgXLu3Dm9/fbbmjBhgnJzc5WcnKyqqirneF1dnRoaGhQMBiVJwWBQR48eVXNzs7Nm79698nq9ysnJuZpRAADAIBLXn3geeOAB3XXXXZo0aZIaGxu1Zs0aDRs2TF/+8pfl8/m0aNEilZeXKy0tTV6vV/fff7+CwaBmzZolSZozZ45ycnJ03333af369QqHw1q1apVKS0vl8Xj65AkOFZNX7nJ7BAAAek1cgfLuu+/qy1/+st5//32NHz9et99+uw4dOqTx48dLkh577DElJiaqqKhI7e3tKigo0BNPPOHcftiwYdq5c6eWLl2qYDCoUaNGqaSkRGvXru3dZwUAAAa0BGOMcXuIeEWjUfl8PkUiEd6P8me8goJLeWddodsjAICk+H5/83/xAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwTpLbAwDAh01eucvtEeL2zrpCt0cABhVeQQEAANYhUAAAgHUIFAAAYB0CBQAAWIc3yQKD3EB8wykA8AoKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArHNVgbJu3TolJCRo2bJlzr62tjaVlpZq7NixGj16tIqKitTU1BRzu4aGBhUWFmrkyJFKT0/XihUrdP78+asZBQAADCJXHCivvPKKfvrTn2r69Okx+5cvX67nnntO27dvV3V1tRobG7VgwQLneFdXlwoLC9XR0aGDBw/qqaee0pYtW7R69eorfxYAAGBQuaJAOXfunIqLi/Xzn/9c11xzjbM/Eonol7/8pR599FHdeeedys3N1ZNPPqmDBw/q0KFDkqQXXnhBx48f17/+67/qpptu0rx58/T9739fGzduVEdHR+88KwAAMKBdUaCUlpaqsLBQ+fn5Mftra2vV2dkZs3/KlCnKyspSKBSSJIVCIU2bNk1+v99ZU1BQoGg0qmPHjl3JOAAAYJBJivcG27Zt06uvvqpXXnnlomPhcFgpKSlKTU2N2e/3+xUOh501fxknF45fONaT9vZ2tbe3Oz9Ho9F4xwYAAANIXK+gnDp1St/61rf09NNPa/jw4X0100UqKyvl8/mcLTMzs98eGwAA9L+4AqW2tlbNzc26+eablZSUpKSkJFVXV2vDhg1KSkqS3+9XR0eHWlpaYm7X1NSkQCAgSQoEAhd9qufCzxfWfFhFRYUikYiznTp1Kp6xAQDAABNXoMyePVtHjx7VkSNHnG3mzJkqLi52/p2cnKyqqirnNnV1dWpoaFAwGJQkBYNBHT16VM3Nzc6avXv3yuv1Kicnp8fH9Xg88nq9MRsAABi84noPypgxY3TjjTfG7Bs1apTGjh3r7F+0aJHKy8uVlpYmr9er+++/X8FgULNmzZIkzZkzRzk5Obrvvvu0fv16hcNhrVq1SqWlpfJ4PL30tAAAwEAW95tkL+exxx5TYmKiioqK1N7eroKCAj3xxBPO8WHDhmnnzp1aunSpgsGgRo0apZKSEq1du7a3RwEAAANUgjHGuD1EvKLRqHw+nyKRCH/u+bPJK3e5PQIwpL2zrtDtEQDrxfP7m/+LBwAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGCduAJl06ZNmj59urxer7xer4LBoHbv3u0cb2trU2lpqcaOHavRo0erqKhITU1NMffR0NCgwsJCjRw5Uunp6VqxYoXOnz/fO88GAAAMCnEFysSJE7Vu3TrV1tbq8OHDuvPOOzV//nwdO3ZMkrR8+XI999xz2r59u6qrq9XY2KgFCxY4t+/q6lJhYaE6Ojp08OBBPfXUU9qyZYtWr17du88KAAAMaAnGGHM1d5CWlqYf/vCHuueeezR+/Hht3bpV99xzjyTprbfe0tSpUxUKhTRr1izt3r1bX/rSl9TY2Ci/3y9J2rx5sx588EG99957SklJ+ViPGY1G5fP5FIlE5PV6r2b8QWPyyl1ujwAMae+sK3R7BMB68fz+vuL3oHR1dWnbtm1qbW1VMBhUbW2tOjs7lZ+f76yZMmWKsrKyFAqFJEmhUEjTpk1z4kSSCgoKFI1GnVdhAAAAkuK9wdGjRxUMBtXW1qbRo0drx44dysnJ0ZEjR5SSkqLU1NSY9X6/X+FwWJIUDodj4uTC8QvHLqW9vV3t7e3Oz9FoNN6xAQDAABL3Kyg33HCDjhw5opqaGi1dulQlJSU6fvx4X8zmqKyslM/nc7bMzMw+fTwAAOCuuAMlJSVF1113nXJzc1VZWakZM2boxz/+sQKBgDo6OtTS0hKzvqmpSYFAQJIUCAQu+lTPhZ8vrOlJRUWFIpGIs506dSresQEAwABy1d+D0t3drfb2duXm5io5OVlVVVXOsbq6OjU0NCgYDEqSgsGgjh49qubmZmfN3r175fV6lZOTc8nH8Hg8zkebL2wAAGDwius9KBUVFZo3b56ysrJ09uxZbd26Vfv379fzzz8vn8+nRYsWqby8XGlpafJ6vbr//vsVDAY1a9YsSdKcOXOUk5Oj++67T+vXr1c4HNaqVatUWloqj8fTJ08QAAAMPHEFSnNzs7761a/q9OnT8vl8mj59up5//nn99V//tSTpscceU2JiooqKitTe3q6CggI98cQTzu2HDRumnTt3aunSpQoGgxo1apRKSkq0du3a3n1WAABgQLvq70FxA9+DcjG+BwVwF9+DAlxev3wPCgAAQF8hUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWCfJ7QEAYDCYvHKX2yPE7Z11hW6PAFwSr6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALBOXIFSWVmpW265RWPGjFF6erruvvtu1dXVxaxpa2tTaWmpxo4dq9GjR6uoqEhNTU0xaxoaGlRYWKiRI0cqPT1dK1as0Pnz56/+2QAAgEEhrkCprq5WaWmpDh06pL1796qzs1Nz5sxRa2urs2b58uV67rnntH37dlVXV6uxsVELFixwjnd1damwsFAdHR06ePCgnnrqKW3ZskWrV6/uvWcFAAAGtARjjLnSG7/33ntKT09XdXW17rjjDkUiEY0fP15bt27VPffcI0l66623NHXqVIVCIc2aNUu7d+/Wl770JTU2Nsrv90uSNm/erAcffFDvvfeeUlJSLvu40WhUPp9PkUhEXq/3SscfVCav3OX2CAAGmHfWFbo9AoaYeH5/X9V7UCKRiCQpLS1NklRbW6vOzk7l5+c7a6ZMmaKsrCyFQiFJUigU0rRp05w4kaSCggJFo1EdO3asx8dpb29XNBqN2QAAwOB1xYHS3d2tZcuW6bbbbtONN94oSQqHw0pJSVFqamrMWr/fr3A47Kz5yzi5cPzCsZ5UVlbK5/M5W2Zm5pWODQAABoArDpTS0lK98cYb2rZtW2/O06OKigpFIhFnO3XqVJ8/JgAAcE/SldyorKxMO3fu1IEDBzRx4kRnfyAQUEdHh1paWmJeRWlqalIgEHDWvPzyyzH3d+FTPhfWfJjH45HH47mSUQEAwAAU1ysoxhiVlZVpx44d2rdvn7Kzs2OO5+bmKjk5WVVVVc6+uro6NTQ0KBgMSpKCwaCOHj2q5uZmZ83evXvl9XqVk5NzNc8FAAAMEnG9glJaWqqtW7fq2Wef1ZgxY5z3jPh8Po0YMUI+n0+LFi1SeXm50tLS5PV6df/99ysYDGrWrFmSpDlz5ignJ0f33Xef1q9fr3A4rFWrVqm0tJRXSQAAgKQ4A2XTpk2SpM9//vMx+5988kl97WtfkyQ99thjSkxMVFFRkdrb21VQUKAnnnjCWTts2DDt3LlTS5cuVTAY1KhRo1RSUqK1a9de3TMBAACDxlV9D4pb+B6Ui/E9KADixfegoL/12/egAAAA9AUCBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1rug/CwQADHwD8Qse+XK5oYNXUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWSXJ7ABtNXrnL7REAABjSeAUFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWiTtQDhw4oLvuuksZGRlKSEjQM888E3PcGKPVq1drwoQJGjFihPLz83XixImYNWfOnFFxcbG8Xq9SU1O1aNEinTt37qqeCAAAGDziDpTW1lbNmDFDGzdu7PH4+vXrtWHDBm3evFk1NTUaNWqUCgoK1NbW5qwpLi7WsWPHtHfvXu3cuVMHDhzQkiVLrvxZAACAQSUp3hvMmzdP8+bN6/GYMUaPP/64Vq1apfnz50uS/uVf/kV+v1/PPPOMFi5cqDfffFN79uzRK6+8opkzZ0qSfvKTn+iLX/yifvSjHykjI+Mqng4AABgMevU9KPX19QqHw8rPz3f2+Xw+5eXlKRQKSZJCoZBSU1OdOJGk/Px8JSYmqqampsf7bW9vVzQajdkAAMDg1auBEg6HJUl+vz9mv9/vd46Fw2Glp6fHHE9KSlJaWpqz5sMqKyvl8/mcLTMzszfHBgAAlhkQn+KpqKhQJBJxtlOnTrk9EgAA6EO9GiiBQECS1NTUFLO/qanJORYIBNTc3Bxz/Pz58zpz5oyz5sM8Ho+8Xm/MBgAABq9eDZTs7GwFAgFVVVU5+6LRqGpqahQMBiVJwWBQLS0tqq2tddbs27dP3d3dysvL681xAADAABX3p3jOnTunkydPOj/X19fryJEjSktLU1ZWlpYtW6Yf/OAHuv7665Wdna2HHnpIGRkZuvvuuyVJU6dO1dy5c7V48WJt3rxZnZ2dKisr08KFC/kEDwAAkHQFgXL48GF94QtfcH4uLy+XJJWUlGjLli36zne+o9bWVi1ZskQtLS26/fbbtWfPHg0fPty5zdNPP62ysjLNnj1biYmJKioq0oYNG3rh6QAAgMEgwRhj3B4iXtFoVD6fT5FIpE/ejzJ55a5ev08AwNV7Z12h2yPgKsTz+3tAfIoHAAAMLQQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOvE/U2yAAC4ZSB+kSZfLndleAUFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1klyewAAAAazySt3uT3CFXlnXaGrj+/qKygbN27U5MmTNXz4cOXl5enll192cxwAAGAJ1wLl17/+tcrLy7VmzRq9+uqrmjFjhgoKCtTc3OzWSAAAwBKuBcqjjz6qxYsX6+tf/7pycnK0efNmjRw5Uv/8z//s1kgAAMASrrwHpaOjQ7W1taqoqHD2JSYmKj8/X6FQ6KL17e3tam9vd36ORCKSpGg02ifzdbd/0Cf3CwDAQNEXv2Mv3Kcx5rJrXQmUP/zhD+rq6pLf74/Z7/f79dZbb120vrKyUt/73vcu2p+ZmdlnMwIAMJT5Hu+7+z579qx8Pt9HrhkQn+KpqKhQeXm583N3d7fOnDmjsWPHKiEhoVcfKxqNKjMzU6dOnZLX6+3V+x7IOC+XxrnpGeelZ5yXS+Pc9GwwnRdjjM6ePauMjIzLrnUlUMaNG6dhw4apqakpZn9TU5MCgcBF6z0ejzweT8y+1NTUvhxRXq93wF8IfYHzcmmcm55xXnrGebk0zk3PBst5udwrJxe48ibZlJQU5ebmqqqqytnX3d2tqqoqBYNBN0YCAAAWce1PPOXl5SopKdHMmTN166236vHHH1dra6u+/vWvuzUSAACwhGuBcu+99+q9997T6tWrFQ6HddNNN2nPnj0XvXG2v3k8Hq1Zs+aiPykNdZyXS+Pc9Izz0jPOy6Vxbno2VM9Lgvk4n/UBAADoR/xngQAAwDoECgAAsA6BAgAArEOgAAAA6xAof2Hjxo2aPHmyhg8frry8PL388stuj+S6hx9+WAkJCTHblClT3B6r3x04cEB33XWXMjIylJCQoGeeeSbmuDFGq1ev1oQJEzRixAjl5+frxIkT7gzbzy53br72ta9ddA3NnTvXnWH7UWVlpW655RaNGTNG6enpuvvuu1VXVxezpq2tTaWlpRo7dqxGjx6toqKii77AcrD5OOfl85///EXXzDe+8Q2XJu4/mzZt0vTp050vZAsGg9q9e7dzfKhdLwTKn/36179WeXm51qxZo1dffVUzZsxQQUGBmpub3R7NdZ/61Kd0+vRpZ/vP//xPt0fqd62trZoxY4Y2btzY4/H169drw4YN2rx5s2pqajRq1CgVFBSora2tnyftf5c7N5I0d+7cmGvoV7/6VT9O6I7q6mqVlpbq0KFD2rt3rzo7OzVnzhy1trY6a5YvX67nnntO27dvV3V1tRobG7VgwQIXp+57H+e8SNLixYtjrpn169e7NHH/mThxotatW6fa2lodPnxYd955p+bPn69jx45JGoLXi4Exxphbb73VlJaWOj93dXWZjIwMU1lZ6eJU7luzZo2ZMWOG22NYRZLZsWOH83N3d7cJBALmhz/8obOvpaXFeDwe86tf/cqFCd3z4XNjjDElJSVm/vz5rsxjk+bmZiPJVFdXG2P+dI0kJyeb7du3O2vefPNNI8mEQiG3xux3Hz4vxhjzuc99znzrW99ybyiLXHPNNeYXv/jFkLxeeAVFUkdHh2pra5Wfn+/sS0xMVH5+vkKhkIuT2eHEiRPKyMjQtddeq+LiYjU0NLg9klXq6+sVDodjrh+fz6e8vDyunz/bv3+/0tPTdcMNN2jp0qV6//333R6p30UiEUlSWlqaJKm2tladnZ0x182UKVOUlZU1pK6bD5+XC55++mmNGzdON954oyoqKvTBBx+4MZ5rurq6tG3bNrW2tioYDA7J62VA/G/Gfe0Pf/iDurq6LvoWW7/fr7feesulqeyQl5enLVu26IYbbtDp06f1ve99T5/97Gf1xhtvaMyYMW6PZ4VwOCxJPV4/F44NZXPnztWCBQuUnZ2tt99+W9/97nc1b948hUIhDRs2zO3x+kV3d7eWLVum2267TTfeeKOkP103KSkpF/3Hp0PpuunpvEjS3//932vSpEnKyMjQ66+/rgcffFB1dXX693//dxen7R9Hjx5VMBhUW1ubRo8erR07dignJ0dHjhwZctcLgYKPNG/ePOff06dPV15eniZNmqTf/OY3WrRokYuTYaBYuHCh8+9p06Zp+vTp+sQnPqH9+/dr9uzZLk7Wf0pLS/XGG28MyfdvfZRLnZclS5Y4/542bZomTJig2bNn6+2339YnPvGJ/h6zX91www06cuSIIpGI/u3f/k0lJSWqrq52eyxX8CceSePGjdOwYcMuejd0U1OTAoGAS1PZKTU1VZ/85Cd18uRJt0exxoVrhOvn47n22ms1bty4IXMNlZWVaefOnfrtb3+riRMnOvsDgYA6OjrU0tISs36oXDeXOi89ycvLk6Qhcc2kpKTouuuuU25uriorKzVjxgz9+Mc/HpLXC4GiP10Qubm5qqqqcvZ1d3erqqpKwWDQxcnsc+7cOb399tuaMGGC26NYIzs7W4FAIOb6iUajqqmp4frpwbvvvqv3339/0F9DxhiVlZVpx44d2rdvn7Kzs2OO5+bmKjk5Oea6qaurU0NDw6C+bi53Xnpy5MgRSRr010xPuru71d7ePjSvF7ffpWuLbdu2GY/HY7Zs2WKOHz9ulixZYlJTU004HHZ7NFd9+9vfNvv37zf19fXmpZdeMvn5+WbcuHGmubnZ7dH61dmzZ81rr71mXnvtNSPJPProo+a1114z//3f/22MMWbdunUmNTXVPPvss+b111838+fPN9nZ2eaPf/yjy5P3vY86N2fPnjUPPPCACYVCpr6+3rz44ovm5ptvNtdff71pa2tze/Q+tXTpUuPz+cz+/fvN6dOnne2DDz5w1nzjG98wWVlZZt++febw4cMmGAyaYDDo4tR973Ln5eTJk2bt2rXm8OHDpr6+3jz77LPm2muvNXfccYfLk/e9lStXmurqalNfX29ef/11s3LlSpOQkGBeeOEFY8zQu14IlL/wk5/8xGRlZZmUlBRz6623mkOHDrk9kuvuvfdeM2HCBJOSkmL+6q/+ytx7773m5MmTbo/V7377298aSRdtJSUlxpg/fdT4oYceMn6/33g8HjN79mxTV1fn7tD95KPOzQcffGDmzJljxo8fb5KTk82kSZPM4sWLh0T493ROJJknn3zSWfPHP/7RfPOb3zTXXHONGTlypPmbv/kbc/r0afeG7geXOy8NDQ3mjjvuMGlpacbj8ZjrrrvOrFixwkQiEXcH7wf/8A//YCZNmmRSUlLM+PHjzezZs504MWboXS8JxhjTf6/XAAAAXB7vQQEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFjn/wCE2vB4RlGAmgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.hist(lens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b1b12407",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:03.185735Z",
     "iopub.status.busy": "2024-08-17T19:52:03.185366Z",
     "iopub.status.idle": "2024-08-17T19:52:04.215422Z",
     "shell.execute_reply": "2024-08-17T19:52:04.214484Z"
    },
    "papermill": {
     "duration": 1.047633,
     "end_time": "2024-08-17T19:52:04.217794",
     "exception": false,
     "start_time": "2024-08-17T19:52:03.170161",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "import transformers\n",
    "from transformers import AutoModel, BertTokenizerFast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "68644a29",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:04.246441Z",
     "iopub.status.busy": "2024-08-17T19:52:04.246151Z",
     "iopub.status.idle": "2024-08-17T19:52:04.258124Z",
     "shell.execute_reply": "2024-08-17T19:52:04.257286Z"
    },
    "papermill": {
     "duration": 0.028393,
     "end_time": "2024-08-17T19:52:04.260141",
     "exception": false,
     "start_time": "2024-08-17T19:52:04.231748",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# split into train, validation and test sets in the ration 70 : 15 : 15\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_text, temp_text, train_labels, temp_labels = train_test_split(df['tweet'], df['class'], \n",
    "                                                                    random_state=2021, \n",
    "                                                                    test_size=0.2, \n",
    "                                                                    stratify=df['class'])\n",
    "\n",
    "\n",
    "val_text, test_text, val_labels, test_labels = train_test_split(temp_text, temp_labels, \n",
    "                                                                random_state=2021, \n",
    "                                                                test_size=0.5, \n",
    "                                                                stratify=temp_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f9f6b432",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:04.288809Z",
     "iopub.status.busy": "2024-08-17T19:52:04.288533Z",
     "iopub.status.idle": "2024-08-17T19:52:07.556925Z",
     "shell.execute_reply": "2024-08-17T19:52:07.555919Z"
    },
    "papermill": {
     "duration": 3.285306,
     "end_time": "2024-08-17T19:52:07.559345",
     "exception": false,
     "start_time": "2024-08-17T19:52:04.274039",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c74d5699d0ee4574904752328957f40a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "544a69bcc7314f3eafd8a8ff944a3a2c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/440M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "bert = AutoModel.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "51e29c2c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:07.589523Z",
     "iopub.status.busy": "2024-08-17T19:52:07.589068Z",
     "iopub.status.idle": "2024-08-17T19:52:08.502575Z",
     "shell.execute_reply": "2024-08-17T19:52:08.501585Z"
    },
    "papermill": {
     "duration": 0.931403,
     "end_time": "2024-08-17T19:52:08.505156",
     "exception": false,
     "start_time": "2024-08-17T19:52:07.573753",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0c18211ccd74b1baed0ae75a5414c65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/48.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8602c1ba4faa49f59b69aa308a4e5fda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0fcd8376d34b434a9be85cc9dfafd04b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "5d56a536",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:08.535886Z",
     "iopub.status.busy": "2024-08-17T19:52:08.535158Z",
     "iopub.status.idle": "2024-08-17T19:52:08.796369Z",
     "shell.execute_reply": "2024-08-17T19:52:08.795491Z"
    },
    "papermill": {
     "duration": 0.278531,
     "end_time": "2024-08-17T19:52:08.798278",
     "exception": false,
     "start_time": "2024-08-17T19:52:08.519747",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([192., 469., 502., 424., 315., 268., 194., 145.,  55.,   8.]),\n",
       " array([ 1. ,  4.1,  7.2, 10.3, 13.4, 16.5, 19.6, 22.7, 25.8, 28.9, 32. ]),\n",
       " <BarContainer object of 10 artists>)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAg4klEQVR4nO3de2xUdf7/8VdvM1xnaoHO0KXl4gWoXIxVy8TLutClstXgUhN0CVaXQGQHInRloRsExc2W4EYUw2Wz64KbFVE2iwYIKBYpWRluVSKCNEDqFlOmRU1noNoL7fn9sb/OfkeQOlB6Pm2fj+Qk9Jwz0/ccT9Knp2emcZZlWQIAADBIvN0DAAAAfB+BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4iXYPcDVaWlpUVVWlvn37Ki4uzu5xAADAj2BZls6fP6+0tDTFx1/5GkmnDJSqqiqlp6fbPQYAALgKZ86c0aBBg664T6cMlL59+0r67wt0uVw2TwMAAH6McDis9PT0yM/xK+mUgdL6ax2Xy0WgAADQyfyY2zO4SRYAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGCcmALlueeeU1xcXNQyYsSIyPb6+nr5/X7169dPffr0UX5+vqqrq6Oeo7KyUnl5eerVq5dSU1O1YMECXbx4sX1eDQAA6BJi/mvGt956qz744IP/PUHi/55i/vz52r59uzZv3iy32605c+ZoypQp+uijjyRJzc3NysvLk9fr1b59+3T27Fk9/vjjSkpK0h//+Md2eDkAAKAriDlQEhMT5fV6L1kfCoX02muvaePGjRo/frwkaf369Ro5cqT279+vcePG6f3339fx48f1wQcfyOPx6LbbbtMLL7yghQsX6rnnnpPD4bj2V4ROY8ii7XaPELMvlufZPQIAdAsx34Ny8uRJpaWladiwYZo2bZoqKyslSWVlZWpqalJOTk5k3xEjRigjI0OBQECSFAgENHr0aHk8nsg+ubm5CofDOnbs2A9+z4aGBoXD4agFAAB0XTEFSnZ2tjZs2KCdO3dq7dq1qqio0L333qvz588rGAzK4XAoOTk56jEej0fBYFCSFAwGo+KkdXvrth9SXFwst9sdWdLT02MZGwAAdDIx/Ypn0qRJkX+PGTNG2dnZGjx4sN5++2317Nmz3YdrVVRUpMLCwsjX4XCYSAEAoAu7prcZJycn65ZbbtGpU6fk9XrV2Nio2traqH2qq6sj96x4vd5L3tXT+vXl7mtp5XQ65XK5ohYAANB1XVOgXLhwQadPn9bAgQOVlZWlpKQklZSURLaXl5ersrJSPp9PkuTz+XT06FHV1NRE9tm1a5dcLpcyMzOvZRQAANCFxPQrnmeeeUYPPfSQBg8erKqqKi1dulQJCQl67LHH5Ha7NWPGDBUWFiolJUUul0tz586Vz+fTuHHjJEkTJ05UZmampk+frhUrVigYDGrx4sXy+/1yOp3X5QUCAIDOJ6ZA+fLLL/XYY4/p66+/1oABA3TPPfdo//79GjBggCRp5cqVio+PV35+vhoaGpSbm6s1a9ZEHp+QkKBt27Zp9uzZ8vl86t27twoKCrRs2bL2fVUAAKBTi7Msy7J7iFiFw2G53W6FQiHuR+nE+BwUAOheYvn5zd/iAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcRLtHgDtY8ii7XaPAABAu+EKCgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjJNo9ANCZDFm03e4RYvbF8jy7RwCAmHEFBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYJxrCpTly5crLi5O8+bNi6yrr6+X3+9Xv3791KdPH+Xn56u6ujrqcZWVlcrLy1OvXr2UmpqqBQsW6OLFi9cyCgAA6EKuOlAOHTqkP//5zxozZkzU+vnz52vr1q3avHmzSktLVVVVpSlTpkS2Nzc3Ky8vT42Njdq3b59ef/11bdiwQUuWLLn6VwEAALqUqwqUCxcuaNq0afrLX/6iG264IbI+FArptdde00svvaTx48crKytL69ev1759+7R//35J0vvvv6/jx4/rH//4h2677TZNmjRJL7zwglavXq3Gxsb2eVUAAKBTu6pA8fv9ysvLU05OTtT6srIyNTU1Ra0fMWKEMjIyFAgEJEmBQECjR4+Wx+OJ7JObm6twOKxjx45d9vs1NDQoHA5HLQAAoOtKjPUBmzZt0scff6xDhw5dsi0YDMrhcCg5OTlqvcfjUTAYjOzzf+OkdXvrtsspLi7W888/H+uoAACgk4rpCsqZM2f09NNP64033lCPHj2u10yXKCoqUigUiixnzpzpsO8NAAA6XkyBUlZWppqaGt1+++1KTExUYmKiSktLtWrVKiUmJsrj8aixsVG1tbVRj6uurpbX65Ukeb3eS97V0/p16z7f53Q65XK5ohYAANB1xRQoEyZM0NGjR3XkyJHIcscdd2jatGmRfyclJamkpCTymPLyclVWVsrn80mSfD6fjh49qpqamsg+u3btksvlUmZmZju9LAAA0JnFdA9K3759NWrUqKh1vXv3Vr9+/SLrZ8yYocLCQqWkpMjlcmnu3Lny+XwaN26cJGnixInKzMzU9OnTtWLFCgWDQS1evFh+v19Op7OdXhYAAOjMYr5Jti0rV65UfHy88vPz1dDQoNzcXK1ZsyayPSEhQdu2bdPs2bPl8/nUu3dvFRQUaNmyZe09CgAA6KTiLMuy7B4iVuFwWG63W6FQiPtR/r8hi7bbPQIM9cXyPLtHAABJsf385m/xAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMk2j3AACuryGLtts9Qsy+WJ5n9wgAbMYVFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYJ6ZAWbt2rcaMGSOXyyWXyyWfz6cdO3ZEttfX18vv96tfv37q06eP8vPzVV1dHfUclZWVysvLU69evZSamqoFCxbo4sWL7fNqAABAlxBToAwaNEjLly9XWVmZDh8+rPHjx2vy5Mk6duyYJGn+/PnaunWrNm/erNLSUlVVVWnKlCmRxzc3NysvL0+NjY3at2+fXn/9dW3YsEFLlixp31cFAAA6tTjLsqxreYKUlBS9+OKLeuSRRzRgwABt3LhRjzzyiCTpxIkTGjlypAKBgMaNG6cdO3bowQcfVFVVlTwejyRp3bp1Wrhwoc6dOyeHw/Gjvmc4HJbb7VYoFJLL5bqW8buMzvhhXMAP4YPagK4plp/fV30PSnNzszZt2qS6ujr5fD6VlZWpqalJOTk5kX1GjBihjIwMBQIBSVIgENDo0aMjcSJJubm5CofDkaswl9PQ0KBwOBy1AACArivmQDl69Kj69Okjp9Opp556Slu2bFFmZqaCwaAcDoeSk5Oj9vd4PAoGg5KkYDAYFSet21u3/ZDi4mK53e7Ikp6eHuvYAACgE4k5UIYPH64jR47owIEDmj17tgoKCnT8+PHrMVtEUVGRQqFQZDlz5sx1/X4AAMBeMf+xQIfDoZtuukmSlJWVpUOHDumVV17R1KlT1djYqNra2qirKNXV1fJ6vZIkr9ergwcPRj1f67t8Wve5HKfTKafTGeuoAACgk7rmz0FpaWlRQ0ODsrKylJSUpJKSksi28vJyVVZWyufzSZJ8Pp+OHj2qmpqayD67du2Sy+VSZmbmtY4CAAC6iJiuoBQVFWnSpEnKyMjQ+fPntXHjRu3Zs0fvvfee3G63ZsyYocLCQqWkpMjlcmnu3Lny+XwaN26cJGnixInKzMzU9OnTtWLFCgWDQS1evFh+v58rJAAAICKmQKmpqdHjjz+us2fPyu12a8yYMXrvvff085//XJK0cuVKxcfHKz8/Xw0NDcrNzdWaNWsij09ISNC2bds0e/Zs+Xw+9e7dWwUFBVq2bFn7vioAANCpXfPnoNiBz0G5FJ+Dgq6Ez0EBuqYO+RwUAACA64VAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcRLtHgAAvm/Iou12jxCzL5bn2T0C0KVwBQUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcWIKlOLiYt15553q27evUlNT9fDDD6u8vDxqn/r6evn9fvXr1099+vRRfn6+qquro/aprKxUXl6eevXqpdTUVC1YsEAXL1689lcDAAC6hJgCpbS0VH6/X/v379euXbvU1NSkiRMnqq6uLrLP/PnztXXrVm3evFmlpaWqqqrSlClTItubm5uVl5enxsZG7du3T6+//ro2bNigJUuWtN+rAgAAnVqcZVnW1T743LlzSk1NVWlpqe677z6FQiENGDBAGzdu1COPPCJJOnHihEaOHKlAIKBx48Zpx44devDBB1VVVSWPxyNJWrdunRYuXKhz587J4XC0+X3D4bDcbrdCoZBcLtfVjt+lDFm03e4RgG7ti+V5do8AGC+Wn9/XdA9KKBSSJKWkpEiSysrK1NTUpJycnMg+I0aMUEZGhgKBgCQpEAho9OjRkTiRpNzcXIXDYR07duyy36ehoUHhcDhqAQAAXddVB0pLS4vmzZunu+++W6NGjZIkBYNBORwOJScnR+3r8XgUDAYj+/zfOGnd3rrtcoqLi+V2uyNLenr61Y4NAAA6gasOFL/fr88++0ybNm1qz3kuq6ioSKFQKLKcOXPmun9PAABgn8SredCcOXO0bds27d27V4MGDYqs93q9amxsVG1tbdRVlOrqanm93sg+Bw8ejHq+1nf5tO7zfU6nU06n82pGBQAAnVBMV1Asy9KcOXO0ZcsW7d69W0OHDo3anpWVpaSkJJWUlETWlZeXq7KyUj6fT5Lk8/l09OhR1dTURPbZtWuXXC6XMjMzr+W1AACALiKmKyh+v18bN27Uu+++q759+0buGXG73erZs6fcbrdmzJihwsJCpaSkyOVyae7cufL5fBo3bpwkaeLEicrMzNT06dO1YsUKBYNBLV68WH6/n6skADqtzvhOOt55BJPFFChr166VJN1///1R69evX68nnnhCkrRy5UrFx8crPz9fDQ0Nys3N1Zo1ayL7JiQkaNu2bZo9e7Z8Pp969+6tgoICLVu27NpeCQAA6DKu6XNQ7HK9PwelM/6fEADEiiso6Ggd9jkoAAAA1wOBAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4iXYPAACwx5BF2+0eIWZfLM+zewR0EK6gAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAOAQKAAAwDoECAACMQ6AAAADjECgAAMA4BAoAADAOgQIAAIxDoAAAAOMQKAAAwDgECgAAMA6BAgAAjEOgAAAA4xAoAADAODEHyt69e/XQQw8pLS1NcXFxeuedd6K2W5alJUuWaODAgerZs6dycnJ08uTJqH2++eYbTZs2TS6XS8nJyZoxY4YuXLhwTS8EAAB0HTEHSl1dncaOHavVq1dfdvuKFSu0atUqrVu3TgcOHFDv3r2Vm5ur+vr6yD7Tpk3TsWPHtGvXLm3btk179+7VrFmzrv5VAACALiUx1gdMmjRJkyZNuuw2y7L08ssva/HixZo8ebIk6e9//7s8Ho/eeecdPfroo/r888+1c+dOHTp0SHfccYck6dVXX9UvfvEL/elPf1JaWto1vBwAANAVtOs9KBUVFQoGg8rJyYmsc7vdys7OViAQkCQFAgElJydH4kSScnJyFB8frwMHDlz2eRsaGhQOh6MWAADQdbVroASDQUmSx+OJWu/xeCLbgsGgUlNTo7YnJiYqJSUlss/3FRcXy+12R5b09PT2HBsAABimU7yLp6ioSKFQKLKcOXPG7pEAAMB11K6B4vV6JUnV1dVR66urqyPbvF6vampqorZfvHhR33zzTWSf73M6nXK5XFELAADouto1UIYOHSqv16uSkpLIunA4rAMHDsjn80mSfD6famtrVVZWFtln9+7damlpUXZ2dnuOAwAAOqmY38Vz4cIFnTp1KvJ1RUWFjhw5opSUFGVkZGjevHn6wx/+oJtvvllDhw7Vs88+q7S0ND388MOSpJEjR+qBBx7QzJkztW7dOjU1NWnOnDl69NFHeQcPAACQdBWBcvjwYf3sZz+LfF1YWChJKigo0IYNG/S73/1OdXV1mjVrlmpra3XPPfdo586d6tGjR+Qxb7zxhubMmaMJEyYoPj5e+fn5WrVqVTu8HAAA0BXEWZZl2T1ErMLhsNxut0Kh0HW5H2XIou3t/pwAgGv3xfI8u0fANYjl53eneBcPAADoXggUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcRLtHgAAgB9ryKLtdo8Qsy+W59k9QqfEFRQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHEIFAAAYBwCBQAAGIdAAQAAxiFQAACAcRLtHgAAgK5syKLtdo9wVb5Ynmfr97f1Csrq1as1ZMgQ9ejRQ9nZ2Tp48KCd4wAAAEPYFihvvfWWCgsLtXTpUn388ccaO3ascnNzVVNTY9dIAADAELYFyksvvaSZM2fqySefVGZmptatW6devXrpb3/7m10jAQAAQ9hyD0pjY6PKyspUVFQUWRcfH6+cnBwFAoFL9m9oaFBDQ0Pk61AoJEkKh8PXZb6Whm+vy/MCANBZXI+fsa3PaVlWm/vaEihfffWVmpub5fF4otZ7PB6dOHHikv2Li4v1/PPPX7I+PT39us0IAEB35n75+j33+fPn5Xa7r7hPp3gXT1FRkQoLCyNft7S06JtvvlG/fv0UFxd3yf7hcFjp6ek6c+aMXC5XR47aaXCM2sYxujKOT9s4Rm3jGF1ZVzs+lmXp/PnzSktLa3NfWwKlf//+SkhIUHV1ddT66upqeb3eS/Z3Op1yOp1R65KTk9v8Pi6Xq0v8B72eOEZt4xhdGcenbRyjtnGMrqwrHZ+2rpy0suUmWYfDoaysLJWUlETWtbS0qKSkRD6fz46RAACAQWz7FU9hYaEKCgp0xx136K677tLLL7+suro6Pfnkk3aNBAAADGFboEydOlXnzp3TkiVLFAwGddttt2nnzp2X3Dh7NZxOp5YuXXrJr4XwPxyjtnGMrozj0zaOUds4RlfWnY9PnPVj3usDAADQgfhjgQAAwDgECgAAMA6BAgAAjEOgAAAA43TJQFm9erWGDBmiHj16KDs7WwcPHrR7JGM899xziouLi1pGjBhh91i22bt3rx566CGlpaUpLi5O77zzTtR2y7K0ZMkSDRw4UD179lROTo5Onjxpz7A2aesYPfHEE5ecUw888IA9w9qguLhYd955p/r27avU1FQ9/PDDKi8vj9qnvr5efr9f/fr1U58+fZSfn3/JB1V2ZT/mGN1///2XnEdPPfWUTRN3vLVr12rMmDGRD2Tz+XzasWNHZHt3PIe6XKC89dZbKiws1NKlS/Xxxx9r7Nixys3NVU1Njd2jGePWW2/V2bNnI8u///1vu0eyTV1dncaOHavVq1dfdvuKFSu0atUqrVu3TgcOHFDv3r2Vm5ur+vr6Dp7UPm0dI0l64IEHos6pN998swMntFdpaan8fr/279+vXbt2qampSRMnTlRdXV1kn/nz52vr1q3avHmzSktLVVVVpSlTptg4dcf6McdIkmbOnBl1Hq1YscKmiTveoEGDtHz5cpWVlenw4cMaP368Jk+erGPHjknqpueQ1cXcddddlt/vj3zd3NxspaWlWcXFxTZOZY6lS5daY8eOtXsMI0mytmzZEvm6paXF8nq91osvvhhZV1tbazmdTuvNN9+0YUL7ff8YWZZlFRQUWJMnT7ZlHhPV1NRYkqzS0lLLsv57ziQlJVmbN2+O7PP5559bkqxAIGDXmLb6/jGyLMv66U9/aj399NP2DWWgG264wfrrX//abc+hLnUFpbGxUWVlZcrJyYmsi4+PV05OjgKBgI2TmeXkyZNKS0vTsGHDNG3aNFVWVto9kpEqKioUDAajzie3263s7GzOp+/Zs2ePUlNTNXz4cM2ePVtff/213SPZJhQKSZJSUlIkSWVlZWpqaoo6j0aMGKGMjIxuex59/xi1euONN9S/f3+NGjVKRUVF+vbbb+0Yz3bNzc3atGmT6urq5PP5uu051Cn+mvGP9dVXX6m5ufmST6P1eDw6ceKETVOZJTs7Wxs2bNDw4cN19uxZPf/887r33nv12WefqW/fvnaPZ5RgMChJlz2fWrfhv7/emTJlioYOHarTp0/r97//vSZNmqRAIKCEhAS7x+tQLS0tmjdvnu6++26NGjVK0n/PI4fDcckfOO2u59HljpEk/epXv9LgwYOVlpamTz/9VAsXLlR5ebn+9a9/2Thtxzp69Kh8Pp/q6+vVp08fbdmyRZmZmTpy5Ei3PIe6VKCgbZMmTYr8e8yYMcrOztbgwYP19ttva8aMGTZOhs7q0Ucfjfx79OjRGjNmjG688Ubt2bNHEyZMsHGyjuf3+/XZZ5916/u62vJDx2jWrFmRf48ePVoDBw7UhAkTdPr0ad14440dPaYthg8friNHjigUCumf//ynCgoKVFpaavdYtulSv+Lp37+/EhISLrmzubq6Wl6v16apzJacnKxbbrlFp06dsnsU47SeM5xPsRk2bJj69+/f7c6pOXPmaNu2bfrwww81aNCgyHqv16vGxkbV1tZG7d8dz6MfOkaXk52dLUnd6jxyOBy66aablJWVpeLiYo0dO1avvPJKtz2HulSgOBwOZWVlqaSkJLKupaVFJSUl8vl8Nk5mrgsXLuj06dMaOHCg3aMYZ+jQofJ6vVHnUzgc1oEDBzifruDLL7/U119/3W3OKcuyNGfOHG3ZskW7d+/W0KFDo7ZnZWUpKSkp6jwqLy9XZWVltzmP2jpGl3PkyBFJ6jbn0eW0tLSooaGh+55Ddt+l2942bdpkOZ1Oa8OGDdbx48etWbNmWcnJyVYwGLR7NCP89re/tfbs2WNVVFRYH330kZWTk2P179/fqqmpsXs0W5w/f9765JNPrE8++cSSZL300kvWJ598Yv3nP/+xLMuyli9fbiUnJ1vvvvuu9emnn1qTJ0+2hg4dan333Xc2T95xrnSMzp8/bz3zzDNWIBCwKioqrA8++MC6/fbbrZtvvtmqr6+3e/QOMXv2bMvtdlt79uyxzp49G1m+/fbbyD5PPfWUlZGRYe3evds6fPiw5fP5LJ/PZ+PUHautY3Tq1Clr2bJl1uHDh62Kigrr3XfftYYNG2bdd999Nk/ecRYtWmSVlpZaFRUV1qeffmotWrTIiouLs95//33LsrrnOdTlAsWyLOvVV1+1MjIyLIfDYd11113W/v377R7JGFOnTrUGDhxoORwO6yc/+Yk1depU69SpU3aPZZsPP/zQknTJUlBQYFnWf99q/Oyzz1oej8dyOp3WhAkTrPLycnuH7mBXOkbffvutNXHiRGvAgAFWUlKSNXjwYGvmzJnd6n8ILndsJFnr16+P7PPdd99Zv/nNb6wbbrjB6tWrl/XLX/7SOnv2rH1Dd7C2jlFlZaV13333WSkpKZbT6bRuuukma8GCBVYoFLJ38A7061//2ho8eLDlcDisAQMGWBMmTIjEiWV1z3MozrIsq+Ou1wAAALStS92DAgAAugYCBQAAGIdAAQAAxiFQAACAcQgUAABgHAIFAAAYh0ABAADGIVAAAIBxCBQAAGAcAgUAABiHQAEAAMYhUAAAgHH+HyiwxoucFWB2AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_lens=[len(i.split()) for i in train_text]\n",
    "plt.hist(train_lens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0e5a91ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:08.829229Z",
     "iopub.status.busy": "2024-08-17T19:52:08.828912Z",
     "iopub.status.idle": "2024-08-17T19:52:08.844190Z",
     "shell.execute_reply": "2024-08-17T19:52:08.843280Z"
    },
    "papermill": {
     "duration": 0.032871,
     "end_time": "2024-08-17T19:52:08.846122",
     "exception": false,
     "start_time": "2024-08-17T19:52:08.813251",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count    3215.000000\n",
      "mean       12.223328\n",
      "std         6.223879\n",
      "min         0.000000\n",
      "25%         7.000000\n",
      "50%        11.000000\n",
      "75%        17.000000\n",
      "max        32.000000\n",
      "Name: tweet, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "sequence_lengths = df['tweet'].apply(lambda x: len(x.split()))\n",
    "print(sequence_lengths.describe())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "fa37822e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:08.876899Z",
     "iopub.status.busy": "2024-08-17T19:52:08.876652Z",
     "iopub.status.idle": "2024-08-17T19:52:08.879953Z",
     "shell.execute_reply": "2024-08-17T19:52:08.879166Z"
    },
    "papermill": {
     "duration": 0.020842,
     "end_time": "2024-08-17T19:52:08.881725",
     "exception": false,
     "start_time": "2024-08-17T19:52:08.860883",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pad_len = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "54599d98",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:08.912180Z",
     "iopub.status.busy": "2024-08-17T19:52:08.911934Z",
     "iopub.status.idle": "2024-08-17T19:52:09.026992Z",
     "shell.execute_reply": "2024-08-17T19:52:09.026098Z"
    },
    "papermill": {
     "duration": 0.132485,
     "end_time": "2024-08-17T19:52:09.029108",
     "exception": false,
     "start_time": "2024-08-17T19:52:08.896623",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2760: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# tokenizing and encoding sequences \n",
    "tokens_train = tokenizer.batch_encode_plus(\n",
    "    train_text.tolist(),\n",
    "    max_length = pad_len,\n",
    "    pad_to_max_length=True,\n",
    "    truncation=True\n",
    ")\n",
    "\n",
    "tokens_val = tokenizer.batch_encode_plus(\n",
    "    val_text.tolist(),\n",
    "    max_length = pad_len,\n",
    "    pad_to_max_length=True,\n",
    "    truncation=True\n",
    ")\n",
    "\n",
    "tokens_test = tokenizer.batch_encode_plus(\n",
    "    test_text.tolist(),\n",
    "    max_length = pad_len,\n",
    "    pad_to_max_length=True,\n",
    "    truncation=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "8bf89f79",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:09.060912Z",
     "iopub.status.busy": "2024-08-17T19:52:09.060174Z",
     "iopub.status.idle": "2024-08-17T19:52:09.119172Z",
     "shell.execute_reply": "2024-08-17T19:52:09.118495Z"
    },
    "papermill": {
     "duration": 0.076673,
     "end_time": "2024-08-17T19:52:09.121098",
     "exception": false,
     "start_time": "2024-08-17T19:52:09.044425",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_seq = torch.tensor(tokens_train['input_ids'])\n",
    "train_mask = torch.tensor(tokens_train['attention_mask'])\n",
    "train_y = torch.tensor(train_labels.tolist())\n",
    "\n",
    "val_seq = torch.tensor(tokens_val['input_ids'])\n",
    "val_mask = torch.tensor(tokens_val['attention_mask'])\n",
    "val_y = torch.tensor(val_labels.tolist())\n",
    "\n",
    "test_seq = torch.tensor(tokens_test['input_ids'])\n",
    "test_mask = torch.tensor(tokens_test['attention_mask'])\n",
    "test_y = torch.tensor(test_labels.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "eecdef64",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:09.151758Z",
     "iopub.status.busy": "2024-08-17T19:52:09.151489Z",
     "iopub.status.idle": "2024-08-17T19:52:09.157163Z",
     "shell.execute_reply": "2024-08-17T19:52:09.156393Z"
    },
    "papermill": {
     "duration": 0.023133,
     "end_time": "2024-08-17T19:52:09.158961",
     "exception": false,
     "start_time": "2024-08-17T19:52:09.135828",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "batch_size = 64  ##batch size can be adjusted or modified\n",
    "train_data = TensorDataset(train_seq, train_mask, train_y)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
    "val_data = TensorDataset(val_seq, val_mask, val_y)\n",
    "val_sampler = SequentialSampler(val_data)\n",
    "val_dataloader = DataLoader(val_data, sampler = val_sampler, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "031cf8b7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:09.189981Z",
     "iopub.status.busy": "2024-08-17T19:52:09.189707Z",
     "iopub.status.idle": "2024-08-17T19:52:09.194661Z",
     "shell.execute_reply": "2024-08-17T19:52:09.193795Z"
    },
    "papermill": {
     "duration": 0.02286,
     "end_time": "2024-08-17T19:52:09.196562",
     "exception": false,
     "start_time": "2024-08-17T19:52:09.173702",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for param in bert.parameters():\n",
    "    param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b0d86848",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:09.227070Z",
     "iopub.status.busy": "2024-08-17T19:52:09.226809Z",
     "iopub.status.idle": "2024-08-17T19:52:09.236560Z",
     "shell.execute_reply": "2024-08-17T19:52:09.235771Z"
    },
    "papermill": {
     "duration": 0.027045,
     "end_time": "2024-08-17T19:52:09.238405",
     "exception": false,
     "start_time": "2024-08-17T19:52:09.211360",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "##model building\n",
    "class SentimentClassifier(nn.Module):\n",
    "     \n",
    "    def __init__(self, n_classes):\n",
    "        super(SentimentClassifier, self).__init__()\n",
    "        self.bert = BertModel.from_pretrained(MODEL_NAME)\n",
    "        self.drop = nn.Dropout(p=0.3)\n",
    "        self.out = nn.Linear(self.bert.config.hidden_size, n_classes)\n",
    "    \n",
    "    # Forward propagaion\n",
    "    def forward(self, input_ids, attention_mask):\n",
    "        _, pooled_output = self.bert(\n",
    "          input_ids=input_ids,\n",
    "          attention_mask=attention_mask\n",
    "        ) \n",
    "        output = self.drop(pooled_output)  ##drop out layer\n",
    "        return self.out(output)\n",
    "class BERT_architecture(nn.Module):\n",
    "\n",
    "    def __init__(self, bert):\n",
    "        super(BERT_architecture, self).__init__()\n",
    "\n",
    "        self.bert = bert \n",
    "        \n",
    "        self.dropout = nn.Dropout(0.2) ##needs adjusment\n",
    "        self.relu = nn.ReLU() ##activation function\n",
    "        \n",
    "        self.fc1 = nn.Linear(768, 512) #dense layer\n",
    "        \n",
    "        # Dense layer 2 (Output layer)\n",
    "        self.fc2 = nn.Linear(512, 2)  #2 classes\n",
    "\n",
    "        # Softmax activation function\n",
    "        self.softmax = nn.LogSoftmax(dim=1)  \n",
    "\n",
    "    # Defining forward pass\n",
    "    def forward(self, sent_id, mask): \n",
    "        _, cls_hs = self.bert(sent_id, attention_mask=mask, return_dict=False)\n",
    "        x = self.fc1(cls_hs)\n",
    "        x = self.relu(x)\n",
    "        x = self.dropout(x)\n",
    "        # Output layer\n",
    "        x = self.fc2(x)\n",
    "        x = self.softmax(x)\n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "bf48a22f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:09.268830Z",
     "iopub.status.busy": "2024-08-17T19:52:09.268569Z",
     "iopub.status.idle": "2024-08-17T19:52:09.278490Z",
     "shell.execute_reply": "2024-08-17T19:52:09.277849Z"
    },
    "papermill": {
     "duration": 0.027273,
     "end_time": "2024-08-17T19:52:09.280314",
     "exception": false,
     "start_time": "2024-08-17T19:52:09.253041",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = BERT_architecture(bert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "098b5796",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:09.311464Z",
     "iopub.status.busy": "2024-08-17T19:52:09.311193Z",
     "iopub.status.idle": "2024-08-17T19:52:09.623333Z",
     "shell.execute_reply": "2024-08-17T19:52:09.622374Z"
    },
    "papermill": {
     "duration": 0.332474,
     "end_time": "2024-08-17T19:52:09.627726",
     "exception": false,
     "start_time": "2024-08-17T19:52:09.295252",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BERT_architecture(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSdpaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.2, inplace=False)\n",
       "  (relu): ReLU()\n",
       "  (fc1): Linear(in_features=768, out_features=512, bias=True)\n",
       "  (fc2): Linear(in_features=512, out_features=2, bias=True)\n",
       "  (softmax): LogSoftmax(dim=1)\n",
       ")"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_seq = test_seq.to(device)\n",
    "test_mask = test_mask.to(device)\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "05b2476a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:09.664156Z",
     "iopub.status.busy": "2024-08-17T19:52:09.663431Z",
     "iopub.status.idle": "2024-08-17T19:52:10.295601Z",
     "shell.execute_reply": "2024-08-17T19:52:10.294839Z"
    },
    "papermill": {
     "duration": 0.650596,
     "end_time": "2024-08-17T19:52:10.297945",
     "exception": false,
     "start_time": "2024-08-17T19:52:09.647349",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/transformers/optimization.py:591: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import AdamW\n",
    "optimizer = AdamW(model.parameters(),lr = 1e-3)  ## learning rate needs to be adjusted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "87103287",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:10.331571Z",
     "iopub.status.busy": "2024-08-17T19:52:10.331090Z",
     "iopub.status.idle": "2024-08-17T19:52:10.340399Z",
     "shell.execute_reply": "2024-08-17T19:52:10.339392Z"
    },
    "papermill": {
     "duration": 0.028435,
     "end_time": "2024-08-17T19:52:10.342542",
     "exception": false,
     "start_time": "2024-08-17T19:52:10.314107",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class weights are [1.02225755 0.97869102] for [0 1]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "# Compute the class weights for three classes\n",
    "class_weights = compute_class_weight(class_weight=\"balanced\",\n",
    "                                     classes=np.unique(train_labels),\n",
    "                                     y=train_labels)\n",
    "print(\"Class weights are {} for {}\".format(class_weights, np.unique(train_labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "403085c5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:10.375960Z",
     "iopub.status.busy": "2024-08-17T19:52:10.375690Z",
     "iopub.status.idle": "2024-08-17T19:52:10.384415Z",
     "shell.execute_reply": "2024-08-17T19:52:10.383547Z"
    },
    "papermill": {
     "duration": 0.027468,
     "end_time": "2024-08-17T19:52:10.386779",
     "exception": false,
     "start_time": "2024-08-17T19:52:10.359311",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_24/883685092.py:1: FutureWarning: pandas.value_counts is deprecated and will be removed in a future version. Use pd.Series(obj).value_counts() instead.\n",
      "  pd.value_counts(train_labels)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "class\n",
       "1    1314\n",
       "0    1258\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "6ad53baf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:10.420414Z",
     "iopub.status.busy": "2024-08-17T19:52:10.420138Z",
     "iopub.status.idle": "2024-08-17T19:52:10.426867Z",
     "shell.execute_reply": "2024-08-17T19:52:10.426173Z"
    },
    "papermill": {
     "duration": 0.025321,
     "end_time": "2024-08-17T19:52:10.428725",
     "exception": false,
     "start_time": "2024-08-17T19:52:10.403404",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "weights = torch.tensor(class_weights, dtype=torch.float)\n",
    "\n",
    "weights = weights.to(device)\n",
    "cross_entropy = nn.CrossEntropyLoss()\n",
    "\n",
    "epochs = 50 ##increase the no of epochs for better accuracy \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "af49dcf3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:10.461153Z",
     "iopub.status.busy": "2024-08-17T19:52:10.460887Z",
     "iopub.status.idle": "2024-08-17T19:52:10.476430Z",
     "shell.execute_reply": "2024-08-17T19:52:10.475732Z"
    },
    "papermill": {
     "duration": 0.033598,
     "end_time": "2024-08-17T19:52:10.478304",
     "exception": false,
     "start_time": "2024-08-17T19:52:10.444706",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train():\n",
    "    model.train()\n",
    "    total_loss, total_accuracy = 0, 0\n",
    "    total_preds, total_labels = [], []\n",
    "\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "        if step % 50 == 0 and not step == 0:\n",
    "            print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(train_dataloader)))\n",
    "\n",
    "        batch = [r.to(device) for r in batch]\n",
    "        sent_id, mask, labels = batch\n",
    "        model.zero_grad()\n",
    "        preds = model(sent_id, mask)\n",
    "        loss = cross_entropy(preds, labels)\n",
    "        total_loss = total_loss + loss.item()\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "        optimizer.step()\n",
    "\n",
    "        preds = preds.detach().cpu().numpy()\n",
    "        total_preds.append(preds)\n",
    "        total_labels.append(labels.detach().cpu().numpy())\n",
    "\n",
    "    avg_loss = total_loss / len(train_dataloader)\n",
    "    total_preds = np.concatenate(total_preds, axis=0)\n",
    "    total_labels = np.concatenate(total_labels, axis=0)\n",
    "    return avg_loss, total_preds, total_labels\n",
    "\n",
    "def evaluate():\n",
    "    print(\"\\nEvaluating after the test: \")\n",
    "    model.eval()\n",
    "    total_loss, total_accuracy = 0, 0\n",
    "    total_preds, total_labels = [], []\n",
    "\n",
    "    for step, batch in enumerate(val_dataloader):\n",
    "        if step % 50 == 0 and not step == 0:\n",
    "            print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(val_dataloader)))\n",
    "\n",
    "        batch = [t.to(device) for t in batch]\n",
    "        sent_id, mask, labels = batch\n",
    "\n",
    "        with torch.no_grad():\n",
    "            preds = model(sent_id, mask)\n",
    "            loss = cross_entropy(preds, labels)\n",
    "            total_loss = total_loss + loss.item()\n",
    "            preds = preds.detach().cpu().numpy()\n",
    "            total_preds.append(preds)\n",
    "            total_labels.append(labels.detach().cpu().numpy())\n",
    "\n",
    "    avg_loss = total_loss / len(val_dataloader)\n",
    "    total_preds = np.concatenate(total_preds, axis=0)\n",
    "    total_labels = np.concatenate(total_labels, axis=0)\n",
    "    return avg_loss, total_preds, total_labels\n",
    "def calculate_accuracy(preds, labels):\n",
    "\n",
    "    preds_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    accuracy = np.sum(preds_flat == labels_flat) / len(labels_flat)\n",
    "    return accuracy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "b345fb9d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:52:10.510173Z",
     "iopub.status.busy": "2024-08-17T19:52:10.509928Z",
     "iopub.status.idle": "2024-08-17T19:55:28.543156Z",
     "shell.execute_reply": "2024-08-17T19:55:28.541978Z"
    },
    "papermill": {
     "duration": 198.051835,
     "end_time": "2024-08-17T19:55:28.545440",
     "exception": false,
     "start_time": "2024-08-17T19:52:10.493605",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Epoch 1 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.6900762930148985\n",
      "Validation Loss: 0.6261748671531677\n",
      "Training Accuracy: 55.72%\n",
      "Validation Accuracy: 70.40%\n",
      "\n",
      " Epoch 2 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.6510759650207147\n",
      "Validation Loss: 0.5993129511674246\n",
      "Training Accuracy: 61.78%\n",
      "Validation Accuracy: 68.85%\n",
      "\n",
      " Epoch 3 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.6681744950573619\n",
      "Validation Loss: 0.5631492485602697\n",
      "Training Accuracy: 61.70%\n",
      "Validation Accuracy: 74.77%\n",
      "\n",
      " Epoch 4 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.6004181133537758\n",
      "Validation Loss: 0.5341268181800842\n",
      "Training Accuracy: 67.22%\n",
      "Validation Accuracy: 73.21%\n",
      "\n",
      " Epoch 5 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5673466215773326\n",
      "Validation Loss: 0.49006179720163345\n",
      "Training Accuracy: 71.66%\n",
      "Validation Accuracy: 75.70%\n",
      "\n",
      " Epoch 6 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5608664678364266\n",
      "Validation Loss: 0.49525000154972076\n",
      "Training Accuracy: 70.92%\n",
      "Validation Accuracy: 73.52%\n",
      "\n",
      " Epoch 7 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.558301915482777\n",
      "Validation Loss: 0.5091917167107264\n",
      "Training Accuracy: 71.31%\n",
      "Validation Accuracy: 72.59%\n",
      "\n",
      " Epoch 8 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5635761247902382\n",
      "Validation Loss: 0.5272846519947052\n",
      "Training Accuracy: 71.85%\n",
      "Validation Accuracy: 68.54%\n",
      "\n",
      " Epoch 9 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5461868607416386\n",
      "Validation Loss: 0.6624768674373627\n",
      "Training Accuracy: 72.36%\n",
      "Validation Accuracy: 63.86%\n",
      "\n",
      " Epoch 10 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5353263834627663\n",
      "Validation Loss: 0.6157537003358206\n",
      "Training Accuracy: 72.94%\n",
      "Validation Accuracy: 62.62%\n",
      "\n",
      " Epoch 11 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5379354764775532\n",
      "Validation Loss: 0.5744081338246664\n",
      "Training Accuracy: 73.41%\n",
      "Validation Accuracy: 67.60%\n",
      "\n",
      " Epoch 12 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5416994552786757\n",
      "Validation Loss: 0.47616448005040485\n",
      "Training Accuracy: 72.74%\n",
      "Validation Accuracy: 77.88%\n",
      "\n",
      " Epoch 13 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.517698191288041\n",
      "Validation Loss: 0.4682033533851306\n",
      "Training Accuracy: 73.87%\n",
      "Validation Accuracy: 75.70%\n",
      "\n",
      " Epoch 14 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5038963112889266\n",
      "Validation Loss: 0.4402507742245992\n",
      "Training Accuracy: 76.56%\n",
      "Validation Accuracy: 78.19%\n",
      "\n",
      " Epoch 15 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5222645787204184\n",
      "Validation Loss: 0.5182084441184998\n",
      "Training Accuracy: 75.58%\n",
      "Validation Accuracy: 71.03%\n",
      "\n",
      " Epoch 16 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5241442133740681\n",
      "Validation Loss: 0.44865844771265984\n",
      "Training Accuracy: 74.03%\n",
      "Validation Accuracy: 77.88%\n",
      "\n",
      " Epoch 17 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5234433012764629\n",
      "Validation Loss: 0.4388359611233075\n",
      "Training Accuracy: 74.11%\n",
      "Validation Accuracy: 76.32%\n",
      "\n",
      " Epoch 18 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5119843962715893\n",
      "Validation Loss: 0.4469211126367251\n",
      "Training Accuracy: 75.51%\n",
      "Validation Accuracy: 76.32%\n",
      "\n",
      " Epoch 19 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5023794748434206\n",
      "Validation Loss: 0.5430166274309158\n",
      "Training Accuracy: 75.89%\n",
      "Validation Accuracy: 71.34%\n",
      "\n",
      " Epoch 20 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5196188106769468\n",
      "Validation Loss: 0.4274896023174127\n",
      "Training Accuracy: 74.11%\n",
      "Validation Accuracy: 79.13%\n",
      "\n",
      " Epoch 21 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5023521584708516\n",
      "Validation Loss: 0.4341018423438072\n",
      "Training Accuracy: 74.96%\n",
      "Validation Accuracy: 77.57%\n",
      "\n",
      " Epoch 22 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5011459218292702\n",
      "Validation Loss: 0.44647400577863056\n",
      "Training Accuracy: 76.52%\n",
      "Validation Accuracy: 77.26%\n",
      "\n",
      " Epoch 23 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.49775207260759863\n",
      "Validation Loss: 0.43901722381512326\n",
      "Training Accuracy: 76.87%\n",
      "Validation Accuracy: 76.95%\n",
      "\n",
      " Epoch 24 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5049241526824672\n",
      "Validation Loss: 0.4324878354867299\n",
      "Training Accuracy: 74.65%\n",
      "Validation Accuracy: 78.19%\n",
      "\n",
      " Epoch 25 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5293281041994328\n",
      "Validation Loss: 0.4981830269098282\n",
      "Training Accuracy: 73.29%\n",
      "Validation Accuracy: 73.83%\n",
      "\n",
      " Epoch 26 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.49973371188815047\n",
      "Validation Loss: 0.4365982860326767\n",
      "Training Accuracy: 76.44%\n",
      "Validation Accuracy: 77.88%\n",
      "\n",
      " Epoch 27 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5160873285154017\n",
      "Validation Loss: 0.43580905348062515\n",
      "Training Accuracy: 75.08%\n",
      "Validation Accuracy: 77.26%\n",
      "\n",
      " Epoch 28 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4803820426871137\n",
      "Validation Loss: 0.4306392843524615\n",
      "Training Accuracy: 77.80%\n",
      "Validation Accuracy: 76.32%\n",
      "\n",
      " Epoch 29 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4788141599515589\n",
      "Validation Loss: 0.47754525144894916\n",
      "Training Accuracy: 77.29%\n",
      "Validation Accuracy: 74.14%\n",
      "\n",
      " Epoch 30 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4837568925648201\n",
      "Validation Loss: 0.4207671595116456\n",
      "Training Accuracy: 77.26%\n",
      "Validation Accuracy: 79.75%\n",
      "\n",
      " Epoch 31 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.49838850701727516\n",
      "Validation Loss: 0.4587252040704091\n",
      "Training Accuracy: 75.54%\n",
      "Validation Accuracy: 75.39%\n",
      "\n",
      " Epoch 32 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4708641651200085\n",
      "Validation Loss: 0.4236501107613246\n",
      "Training Accuracy: 78.19%\n",
      "Validation Accuracy: 77.57%\n",
      "\n",
      " Epoch 33 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.49603056762276626\n",
      "Validation Loss: 0.4420521731177966\n",
      "Training Accuracy: 76.48%\n",
      "Validation Accuracy: 76.95%\n",
      "\n",
      " Epoch 34 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4877778030023342\n",
      "Validation Loss: 0.4461430236697197\n",
      "Training Accuracy: 76.71%\n",
      "Validation Accuracy: 76.64%\n",
      "\n",
      " Epoch 35 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.49152462220773463\n",
      "Validation Loss: 0.4530780166387558\n",
      "Training Accuracy: 76.17%\n",
      "Validation Accuracy: 75.39%\n",
      "\n",
      " Epoch 36 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.48399662898807994\n",
      "Validation Loss: 0.4209860985477765\n",
      "Training Accuracy: 77.80%\n",
      "Validation Accuracy: 77.57%\n",
      "\n",
      " Epoch 37 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5315913514393132\n",
      "Validation Loss: 0.43901066730419797\n",
      "Training Accuracy: 73.13%\n",
      "Validation Accuracy: 76.95%\n",
      "\n",
      " Epoch 38 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4957613552488932\n",
      "Validation Loss: 0.4962748655428489\n",
      "Training Accuracy: 76.44%\n",
      "Validation Accuracy: 71.03%\n",
      "\n",
      " Epoch 39 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.49862739007647444\n",
      "Validation Loss: 0.41316789637009305\n",
      "Training Accuracy: 76.87%\n",
      "Validation Accuracy: 78.82%\n",
      "\n",
      " Epoch 40 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.5005994838912312\n",
      "Validation Loss: 0.43031319230794907\n",
      "Training Accuracy: 74.22%\n",
      "Validation Accuracy: 77.57%\n",
      "\n",
      " Epoch 41 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4855219205705131\n",
      "Validation Loss: 0.42283207426468533\n",
      "Training Accuracy: 77.64%\n",
      "Validation Accuracy: 77.88%\n",
      "\n",
      " Epoch 42 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4878284902107425\n",
      "Validation Loss: 0.4287916695078214\n",
      "Training Accuracy: 76.40%\n",
      "Validation Accuracy: 77.88%\n",
      "\n",
      " Epoch 43 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4724769875770662\n",
      "Validation Loss: 0.4173559099435806\n",
      "Training Accuracy: 77.60%\n",
      "Validation Accuracy: 79.44%\n",
      "\n",
      " Epoch 44 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4804889646972098\n",
      "Validation Loss: 0.4316040128469467\n",
      "Training Accuracy: 76.98%\n",
      "Validation Accuracy: 79.44%\n",
      "\n",
      " Epoch 45 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.48855039331971145\n",
      "Validation Loss: 0.4182642636199792\n",
      "Training Accuracy: 76.63%\n",
      "Validation Accuracy: 77.88%\n",
      "\n",
      " Epoch 46 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4765173531160122\n",
      "Validation Loss: 0.45524823913971585\n",
      "Training Accuracy: 76.21%\n",
      "Validation Accuracy: 76.01%\n",
      "\n",
      " Epoch 47 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4827725400285023\n",
      "Validation Loss: 0.46655935049057007\n",
      "Training Accuracy: 76.94%\n",
      "Validation Accuracy: 74.45%\n",
      "\n",
      " Epoch 48 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.4753300438566906\n",
      "Validation Loss: 0.4428781320651372\n",
      "Training Accuracy: 76.56%\n",
      "Validation Accuracy: 77.26%\n",
      "\n",
      " Epoch 49 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.500854236323659\n",
      "Validation Loss: 0.4516540008286635\n",
      "Training Accuracy: 75.58%\n",
      "Validation Accuracy: 76.32%\n",
      "\n",
      " Epoch 50 / 50\n",
      "\n",
      "Evaluating after the test: \n",
      "\n",
      "Training Loss: 0.47759734421241573\n",
      "Validation Loss: 0.4152484213312467\n",
      "Training Accuracy: 76.91%\n",
      "Validation Accuracy: 80.06%\n"
     ]
    }
   ],
   "source": [
    "best_valid_loss = float('inf')\n",
    "train_losses = []\n",
    "valid_losses = []\n",
    "train_accuracies = []\n",
    "valid_accuracies = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    print('\\n Epoch {:} / {:}'.format(epoch + 1, epochs))\n",
    "    \n",
    "    train_loss, train_preds, train_labels = train()\n",
    "    valid_loss, valid_preds, valid_labels = evaluate()\n",
    "    train_accuracy = calculate_accuracy(train_preds, train_labels)\n",
    "    valid_accuracy = calculate_accuracy(valid_preds, valid_labels)\n",
    "\n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), 'saved_weights.pt')\n",
    "    \n",
    "    train_losses.append(train_loss)\n",
    "    valid_losses.append(valid_loss)\n",
    "    train_accuracies.append(train_accuracy)\n",
    "    valid_accuracies.append(valid_accuracy)\n",
    "    \n",
    "    print('\\nTraining Loss: {}'.format(train_loss))\n",
    "    print('Validation Loss: {}'.format(valid_loss))\n",
    "    print('Training Accuracy: {:.2f}%'.format(train_accuracy * 100))\n",
    "    print('Validation Accuracy: {:.2f}%'.format(valid_accuracy * 100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "74f15c81",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:55:28.593813Z",
     "iopub.status.busy": "2024-08-17T19:55:28.593507Z",
     "iopub.status.idle": "2024-08-17T19:55:29.000335Z",
     "shell.execute_reply": "2024-08-17T19:55:28.999227Z"
    },
    "papermill": {
     "duration": 0.43335,
     "end_time": "2024-08-17T19:55:29.002497",
     "exception": false,
     "start_time": "2024-08-17T19:55:28.569147",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.79      0.76       157\n",
      "           1       0.78      0.72      0.75       165\n",
      "\n",
      "    accuracy                           0.75       322\n",
      "   macro avg       0.76      0.76      0.75       322\n",
      "weighted avg       0.76      0.75      0.75       322\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "with torch.no_grad():\n",
    "  preds = model(test_seq.to(device), test_mask.to(device))\n",
    "  preds = preds.detach().cpu().numpy()\n",
    "pred = np.argmax(preds, axis = 1)\n",
    "print(classification_report(test_y, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "4a41e1c6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-17T19:55:29.052169Z",
     "iopub.status.busy": "2024-08-17T19:55:29.051893Z",
     "iopub.status.idle": "2024-08-17T19:55:29.083767Z",
     "shell.execute_reply": "2024-08-17T19:55:29.082870Z"
    },
    "papermill": {
     "duration": 0.058169,
     "end_time": "2024-08-17T19:55:29.085705",
     "exception": false,
     "start_time": "2024-08-17T19:55:29.027536",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted sentiment: Negative\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2760: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "def predict_user_input(input_text, model, tokenizer, device):\n",
    "    inputs = tokenizer.encode_plus(\n",
    "        input_text,\n",
    "        add_special_tokens=True,\n",
    "        max_length=max_seq_len,\n",
    "        pad_to_max_length=True,\n",
    "        return_attention_mask=True,\n",
    "        return_tensors='pt',\n",
    "    )\n",
    "    input_ids = inputs['input_ids'].to(device)\n",
    "    attention_mask = inputs['attention_mask'].to(device)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        outputs = model(input_ids, attention_mask)\n",
    "        preds = torch.argmax(outputs, dim=1)\n",
    "    \n",
    "    return preds.item()\n",
    "max_seq_len=25\n",
    "input_text = \"I been told you that yo folks dont care about you . Im starving and my dad deadass didnt make me a plate\"\n",
    "predicted_class = predict_user_input(input_text, model, tokenizer, device)\n",
    "if(predicted_class==0):\n",
    "    print('Predicted sentiment: Negative')\n",
    "else:\n",
    "    print('Predicted sentiment: Positive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07eb0841",
   "metadata": {
    "papermill": {
     "duration": 0.023326,
     "end_time": "2024-08-17T19:55:29.132947",
     "exception": false,
     "start_time": "2024-08-17T19:55:29.109621",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "nvidiaTeslaT4",
   "dataSources": [
    {
     "datasetId": 5557640,
     "sourceId": 9193227,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30747,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 219.31771,
   "end_time": "2024-08-17T19:55:31.836177",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-08-17T19:51:52.518467",
   "version": "2.5.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "079b2770a9064eef9509b2ae0b97134f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "0b382c1cf1fa4fd7a44bf0b5be997cd8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "0ed9a5a0326042059fd42f0129d2f8b8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d502fee6f5ae468d9addcbef5c550702",
       "placeholder": "​",
       "style": "IPY_MODEL_dfe62a6e7a814a7d8576fd4060d90c06",
       "value": " 570/570 [00:00&lt;00:00, 48.9kB/s]"
      }
     },
     "0fcd8376d34b434a9be85cc9dfafd04b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_291ff460abf24317b3ac5f8c4755b07a",
        "IPY_MODEL_530f26b1f26f4228851c085a956905d1",
        "IPY_MODEL_c129a925c1fe4c1e9e92905ff56319ea"
       ],
       "layout": "IPY_MODEL_850862a3246244199e1d197b02e7b8b6"
      }
     },
     "119ca2e2452b4b528401bb5625fca0d0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_2d3bd3446fb645a8b0e512e5fc6f0297",
       "max": 48.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_521b5a603f1546e4b1dc7a7a237cfff0",
       "value": 48.0
      }
     },
     "19bb2c72c369476c92bd2ecf92df0c8c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_40ab98afe5ca444e9218656baf849250",
       "max": 231508.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_9255854878384b3daa7b4023906e0ae6",
       "value": 231508.0
      }
     },
     "1bae0c839953469399291d0f6903fbdc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_60a70e5f2a4c4b5ca796b39d036ce7f4",
       "max": 440449768.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_d9f9ed0ac1b64e1894d3156127a3c955",
       "value": 440449768.0
      }
     },
     "2611cc1ac3e34e01af10f83f2a775f8b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "2698093c578042bba98dedacb5f6c38a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "291ff460abf24317b3ac5f8c4755b07a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_cb4bbffe425842f09e5b6bd5d91d561f",
       "placeholder": "​",
       "style": "IPY_MODEL_2f473e5a9a284357ac436f05cb300e46",
       "value": "tokenizer.json: 100%"
      }
     },
     "2d3bd3446fb645a8b0e512e5fc6f0297": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "2e0d319a9e9d4b91871b49b9458da023": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "2ebe4c97bc76415cbd873dbd3e595287": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "2f473e5a9a284357ac436f05cb300e46": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "3363ebb899574a35b0a071a649c32c57": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "389d6339ffa247b9bd163cdac4ea9bb5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "40ab98afe5ca444e9218656baf849250": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "41c9a2f4ca6b4e42b447ed8d1a512f06": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4210d21dc02a4a939f7e8ea0845c4759": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "443cfe0b5674450cb17406c7a7597788": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_2698093c578042bba98dedacb5f6c38a",
       "placeholder": "​",
       "style": "IPY_MODEL_2ebe4c97bc76415cbd873dbd3e595287",
       "value": " 48.0/48.0 [00:00&lt;00:00, 4.64kB/s]"
      }
     },
     "4ab238906d40402287e103d29888f7de": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_daeb6f7677314410bd171c1044ebb06a",
       "max": 570.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_0b382c1cf1fa4fd7a44bf0b5be997cd8",
       "value": 570.0
      }
     },
     "4ba7567f121842549eec4e19c66b441f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "521b5a603f1546e4b1dc7a7a237cfff0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "530f26b1f26f4228851c085a956905d1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_389d6339ffa247b9bd163cdac4ea9bb5",
       "max": 466062.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_2611cc1ac3e34e01af10f83f2a775f8b",
       "value": 466062.0
      }
     },
     "544a69bcc7314f3eafd8a8ff944a3a2c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_6cc5838e81504ee1998ab310e2c867e0",
        "IPY_MODEL_1bae0c839953469399291d0f6903fbdc",
        "IPY_MODEL_d5d1673fd66d47d6b93cfc62a65e9009"
       ],
       "layout": "IPY_MODEL_cffd9275adf74001831915d284cd6d50"
      }
     },
     "60a70e5f2a4c4b5ca796b39d036ce7f4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6cc5838e81504ee1998ab310e2c867e0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_b889ed5b5b474494b77f030f21c8a10c",
       "placeholder": "​",
       "style": "IPY_MODEL_f3c5b9b120bf4c89823da0a104ee12ee",
       "value": "model.safetensors: 100%"
      }
     },
     "70930dee0b644e36a5e8d185367b0608": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "749c17ba137b45d1bd8d96f386cbad5a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "850862a3246244199e1d197b02e7b8b6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8602c1ba4faa49f59b69aa308a4e5fda": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_a7d27a351a9345638ba69b5345cd3f32",
        "IPY_MODEL_19bb2c72c369476c92bd2ecf92df0c8c",
        "IPY_MODEL_c242c34bb57d4bb4844a51d4af06e983"
       ],
       "layout": "IPY_MODEL_f82cde0e015543b4b32cd744d759ba7e"
      }
     },
     "9255854878384b3daa7b4023906e0ae6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "93bba7c7c0514c89b0471c1f8a17f4ea": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "96c69f8be9214cf28417a91e34357361": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e8768e39fb6e4ae891dd07f1a3e537c3",
       "placeholder": "​",
       "style": "IPY_MODEL_c75d0e00fcfe42eb824b31dd8dc15ced",
       "value": "config.json: 100%"
      }
     },
     "98e9239825e9438b9a4882a63ea95a3b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "a7d27a351a9345638ba69b5345cd3f32": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_3363ebb899574a35b0a071a649c32c57",
       "placeholder": "​",
       "style": "IPY_MODEL_98e9239825e9438b9a4882a63ea95a3b",
       "value": "vocab.txt: 100%"
      }
     },
     "b0f4c75368f14106b2d37a188afbab92": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b889ed5b5b474494b77f030f21c8a10c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c129a925c1fe4c1e9e92905ff56319ea": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_2e0d319a9e9d4b91871b49b9458da023",
       "placeholder": "​",
       "style": "IPY_MODEL_93bba7c7c0514c89b0471c1f8a17f4ea",
       "value": " 466k/466k [00:00&lt;00:00, 7.44MB/s]"
      }
     },
     "c242c34bb57d4bb4844a51d4af06e983": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_41c9a2f4ca6b4e42b447ed8d1a512f06",
       "placeholder": "​",
       "style": "IPY_MODEL_749c17ba137b45d1bd8d96f386cbad5a",
       "value": " 232k/232k [00:00&lt;00:00, 3.60MB/s]"
      }
     },
     "c74d5699d0ee4574904752328957f40a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_96c69f8be9214cf28417a91e34357361",
        "IPY_MODEL_4ab238906d40402287e103d29888f7de",
        "IPY_MODEL_0ed9a5a0326042059fd42f0129d2f8b8"
       ],
       "layout": "IPY_MODEL_dc8a03e27d3a4501b4a9c21a479baffd"
      }
     },
     "c75d0e00fcfe42eb824b31dd8dc15ced": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "cb4bbffe425842f09e5b6bd5d91d561f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cffd9275adf74001831915d284cd6d50": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d502fee6f5ae468d9addcbef5c550702": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d5d1673fd66d47d6b93cfc62a65e9009": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_079b2770a9064eef9509b2ae0b97134f",
       "placeholder": "​",
       "style": "IPY_MODEL_70930dee0b644e36a5e8d185367b0608",
       "value": " 440M/440M [00:01&lt;00:00, 281MB/s]"
      }
     },
     "d6e9e0d99beb4f9ebb800e259891d307": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_4210d21dc02a4a939f7e8ea0845c4759",
       "placeholder": "​",
       "style": "IPY_MODEL_4ba7567f121842549eec4e19c66b441f",
       "value": "tokenizer_config.json: 100%"
      }
     },
     "d9f9ed0ac1b64e1894d3156127a3c955": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "daeb6f7677314410bd171c1044ebb06a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "dc8a03e27d3a4501b4a9c21a479baffd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "dfe62a6e7a814a7d8576fd4060d90c06": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "e8768e39fb6e4ae891dd07f1a3e537c3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f0c18211ccd74b1baed0ae75a5414c65": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_d6e9e0d99beb4f9ebb800e259891d307",
        "IPY_MODEL_119ca2e2452b4b528401bb5625fca0d0",
        "IPY_MODEL_443cfe0b5674450cb17406c7a7597788"
       ],
       "layout": "IPY_MODEL_b0f4c75368f14106b2d37a188afbab92"
      }
     },
     "f3c5b9b120bf4c89823da0a104ee12ee": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f82cde0e015543b4b32cd744d759ba7e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}