Basic EDA, BERT sentiment classification and NER.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Анализ сантимента (эмоциональной окраски) твитов."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(Kaggle kernel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Импорт и установка необходимых библиотек."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install '/kaggle/input/simple-transformers-pypi/seqeval-0.0.12-py3-none-any.whl' -q\n",
    "!pip install '/kaggle/input/simple-transformers-pypi/simpletransformers-0.22.1-py3-none-any.whl' -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import gc\n",
    "import time\n",
    "import string\n",
    "import random\n",
    "import datetime\n",
    "import itertools\n",
    "import collections\n",
    "import h5py\n",
    "import typing\n",
    "import nltk\n",
    "import json\n",
    "import sklearn\n",
    "import transformers\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from transformers import BertTokenizer, BertForSequenceClassification, AdamW, BertConfig, get_linear_schedule_with_warmup,  RobertaModel, RobertaTokenizer\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "from torch.multiprocessing import Pipe, Process\n",
    "from torch.utils import data\n",
    "import allennlp\n",
    "from allennlp.modules.elmo import Elmo, batch_to_ids\n",
    "from allennlp.commands.elmo import ElmoEmbedder\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from nltk.corpus import stopwords\n",
    "import spacy\n",
    "import warnings\n",
    "from sklearn.utils import shuffle\n",
    "from keras.utils import to_categorical\n",
    "from keras.preprocessing.sequence import pad_sequences as pad\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def debug_memory():\n",
    "    import collections, gc, resource, torch\n",
    "    print('maxrss = {}'.format(\n",
    "        resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))\n",
    "    tensors = collections.Counter((str(o.device), o.dtype, tuple(o.shape))\n",
    "                                  for o in gc.get_objects()\n",
    "                                  if torch.is_tensor(o))\n",
    "    for line in sorted(tensors.items()):\n",
    "        print('{}\\t{}'.format(*line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Объявление функций, необходимых для использования spacy\n",
    "# Grac https://www.kaggle.com/rohitsingh9990/ner-training-using-spacy-ensemble\n",
    "def save_model(output_dir, nlp, new_model_name):\n",
    "    \"\"\"\n",
    "    Saves model given the directory where to save, model and the name that should be used for this model\n",
    "    \"\"\"\n",
    "    output_dir = f'../working/{output_dir}'\n",
    "    if output_dir is not None:       # In case some sad shit happens  \n",
    "        if not os.path.exists(output_dir):\n",
    "            os.makedirs(output_dir)\n",
    "        nlp.meta[\"name\"] = new_model_name\n",
    "        nlp.to_disk(output_dir)\n",
    "        print(\"Saved model to\", output_dir)\n",
    "\n",
    "        \n",
    "        \n",
    "def train(train_data, output_dir, n_iter=20, model=None):\n",
    "    \"\"\"\n",
    "    Load the model, set up the pipeline and train the entity recognizer.\n",
    "    \"\"\"\n",
    "    if model is not None:\n",
    "        nlp = spacy.load(output_dir)  # load existing spaCy model\n",
    "        print(\"Loaded model '%s'\" % model)\n",
    "    else:\n",
    "        nlp = spacy.blank(\"en\")  # create blank Language class\n",
    "        print(\"Created blank 'en' model\")\n",
    "    \n",
    "    # create the built-in pipeline components and add them to the pipeline\n",
    "    # nlp.create_pipe works for built-ins that are registered with spaCy\n",
    "    if \"ner\" not in nlp.pipe_names:\n",
    "        ner = nlp.create_pipe(\"ner\")\n",
    "        nlp.add_pipe(ner, last=True)\n",
    "    # otherwise, get it so we can add labels\n",
    "    else:\n",
    "        ner = nlp.get_pipe(\"ner\")\n",
    "    \n",
    "    # add labels\n",
    "    for _, annotations in train_data:\n",
    "        for ent in annotations.get(\"entities\"):\n",
    "            ner.add_label(ent[2])\n",
    "\n",
    "    # get names of other pipes to disable them during training\n",
    "    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != \"ner\"]\n",
    "    with nlp.disable_pipes(*other_pipes):  # only train NER\n",
    "        # sizes = compounding(1.0, 4.0, 1.001)\n",
    "        # batch up the examples using spaCy's minibatch\n",
    "        if model is None:\n",
    "            nlp.begin_training()\n",
    "        else:\n",
    "            nlp.resume_training()\n",
    "\n",
    "        for itn in tqdm(range(n_iter)):\n",
    "            random.shuffle(train_data)\n",
    "            batches = minibatch(train_data, size=compounding(4.0, 500.0, 1.001))    \n",
    "            losses = {}\n",
    "            for batch in batches:\n",
    "                texts, annotations = zip(*batch)\n",
    "                nlp.update(\n",
    "                    texts,  # batch of texts\n",
    "                    annotations,  # batch of annotations\n",
    "                    drop=0.5,   # dropout - make it harder to memorise data\n",
    "                    losses=losses, \n",
    "                )\n",
    "            \n",
    "            print(\"Losses\", losses)\n",
    "    save_model(output_dir, nlp, 'st_ner')\n",
    "    \n",
    "    \n",
    "def get_model_out_path(sentiment:str) -> str:\n",
    "    \"\"\"\n",
    "    Simply sets model path based on the sentiment\n",
    "    \"\"\"\n",
    "    model_out_path = None\n",
    "    if sentiment == 'positive':\n",
    "        model_out_path = 'models/model_pos'\n",
    "    elif sentiment == 'negative':\n",
    "        model_out_path = 'models/model_neg'\n",
    "    else:\n",
    "        model_out_path = 'models/model_neu'\n",
    "    return model_out_path\n",
    "\n",
    "def get_training_data(train_df, sentiment:str) -> list:\n",
    "    \"\"\"\n",
    "    Prepairs data in acceptible by spacy models format\n",
    "    \"\"\"\n",
    "    train_data = []\n",
    "    for index, row in train_df.iterrows():\n",
    "        if row.sentiment == sentiment:\n",
    "            selected_text = str(row.selected_text)\n",
    "            text = str(row.text)\n",
    "            start = text.find(selected_text)\n",
    "            end = start + len(selected_text)\n",
    "            train_data.append((text, {\"entities\": [[start, end, 'selected_text']]}))\n",
    "    return train_data\n",
    "\n",
    "def print_metric(data, batch, epoch, start, end, metric, typ):\n",
    "    t = typ, metric, \"%s\", data, \"%s\"\n",
    "    if typ == \"Train\": pre = \"BATCH \" + str(batch-1) + \" \"\n",
    "    if typ == \"Val\": pre = \"\\nEPOCH \" + str(epoch+1) + \" \"\n",
    "    time = np.round(end - start, 1); time = \"Time: %s{}%s s\".format(time)\n",
    "    \n",
    "    print(pre  + \"{} {}: {}{}{}\".format(*t)  + \"  \" + time )\n",
    "    \n",
    "    \n",
    "def find_all(input_str, search_str):\n",
    "    l1 = []\n",
    "    length = len(input_str)\n",
    "    index = 0\n",
    "    while index < length:\n",
    "        i = input_str.find(search_str, index)\n",
    "        if i == -1:\n",
    "            return l1\n",
    "        l1.append(i)\n",
    "        index = i + 1\n",
    "    return l1\n",
    "\n",
    "def do_qa_train(train):\n",
    "\n",
    "    output = []\n",
    "    for line in train:\n",
    "        context = line[1]\n",
    "\n",
    "        qas = []\n",
    "        question = line[-1]\n",
    "        qid = line[0]\n",
    "        answers = []\n",
    "        answer = line[2]\n",
    "        if type(answer) != str or type(context) != str or type(question) != str:\n",
    "            print(context, type(context))\n",
    "            print(answer, type(answer))\n",
    "            print(question, type(question))\n",
    "            continue\n",
    "        answer_starts = find_all(context, answer)\n",
    "        for answer_start in answer_starts:\n",
    "            answers.append({'answer_start': answer_start, 'text': answer.lower()})\n",
    "            break\n",
    "        qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})\n",
    "\n",
    "        output.append({'context': context.lower(), 'qas': qas})\n",
    "        \n",
    "    return output\n",
    "\n",
    "def do_qa_test(test):\n",
    "    output = []\n",
    "    for line in test:\n",
    "        context = line[1]\n",
    "        qas = []\n",
    "        question = line[-1]\n",
    "        qid = line[0]\n",
    "        if type(context) != str or type(question) != str:\n",
    "            print(context, type(context))\n",
    "            print(answer, type(answer))\n",
    "            print(question, type(question))\n",
    "            continue\n",
    "        answers = []\n",
    "        answers.append({'answer_start': 1000000, 'text': '__None__'})\n",
    "        qas.append({'question': question, 'id': qid, 'is_impossible': False, 'answers': answers})\n",
    "        output.append({'context': context.lower(), 'qas': qas})\n",
    "    return output\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def basic_cleaning(text):\n",
    "    text=re.sub(r'https?://www\\.\\S+\\.com','',text)\n",
    "    text=re.sub(r'[^A-Za-z|\\s]','',text)\n",
    "    return text\n",
    "\n",
    "def clean(df):\n",
    "    for col in ['text','selected_text']:\n",
    "        df[col]=df[col].astype(str).apply(lambda x:basic_cleaning(x))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_time(elapsed):\n",
    "    '''\n",
    "    Takes a time in seconds and returns a string hh:mm:ss\n",
    "    '''\n",
    "    # Round to the nearest second.\n",
    "    elapsed_rounded = int(round((elapsed)))\n",
    "    \n",
    "    # Format as hh:mm:ss\n",
    "    return str(datetime.timedelta(seconds=elapsed_rounded))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set_style(\"darkgrid\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "# Импортируем данные для обучения модели.\n",
    "train_df = pd.read_csv(\"/kaggle/input/tweet-sentiment-extraction/train.csv\")\n",
    "test_df = pd.read_csv(\"/kaggle/input/tweet-sentiment-extraction/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Уникальные эмоциональные окраски текста:\n",
    "df['sentiment'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(16,10))\n",
    "sns.categorical.countplot(df['sentiment'])\n",
    "plt.title(\"Distribution of number of tweets prior to sentiment\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Наблюдается некоторый дисбаланс классов."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups = df.groupby('sentiment')['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(16,10))\n",
    "for sentiment in df['sentiment'].unique():\n",
    "    sns.distplot(groups.get_group(sentiment).apply(lambda item: len(item) if type(item) == str else len(str(item))),\n",
    "                hist=False,\n",
    "                 kde=True,\n",
    "                 bins=20,\n",
    "                hist_kws={'edgecolor':'black'},\n",
    "                kde_kws={'linewidth': 4},\n",
    "                label=sentiment);\n",
    "#plt.axvline(np.quantile(df['text'].apply(lambda item: len(item) if type(item) == str else len(str(item))), 0.05), 0, 17, linewidth=4, color=\"red\")\n",
    "#plt.axvline(np.quantile(df['text'].apply(lambda item: len(item) if type(item) == str else len(str(item))), 0.95), 0, 17, linewidth=4, color=\"red\")\n",
    "plt.title('Distribution KDE of number of characters in tweets with respect to sentiment')\n",
    "plt.xlabel('Lengths of tweets')\n",
    "plt.ylabel('Density');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiments = df['sentiment'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig,ax = plt.subplots(1,3,figsize=(12,6))\n",
    "for i, sentiment in enumerate(sentiments):\n",
    "    sns.distplot(groups.get_group(sentiment).str.split().str.len(), ax=ax[i], hist=True, kde=True)\n",
    "    ax[i].set_ylim(0, 0.06)\n",
    "    ax[i].set_title(sentiment)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Как видим, пока что не было замечено никаких аномалий в данных."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "puncts = set(string.punctuation)\n",
    "\n",
    "df['punct'] = df['text'].apply(lambda row: list(filter(lambda x: x in puncts, str(row))))\n",
    "groups_1 = df.groupby('sentiment')['punct']\n",
    "fig,ax = plt.subplots(1, 3,figsize=(40,10))\n",
    "for axis, sentiment in zip(ax, df['sentiment'].unique()):\n",
    "    sns.distplot(groups_1.get_group(sentiment).apply(len),\n",
    "                hist=False,\n",
    "                 kde=True,\n",
    "                 bins=20,\n",
    "                hist_kws={'edgecolor':'black'},\n",
    "                kde_kws={'linewidth': 4},\n",
    "                ax=axis)\n",
    "    axis.set_xlabel(\"Number of punctuations\", fontsize=18)\n",
    "    axis.set_ylabel(\"Density\", fontsize=18)\n",
    "    axis.set_title(sentiment.capitalize(), fontsize=22)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Легко заметить, что в текстах с позитивным сантиментом встречается больше знаков пунктуации."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Попробуем найти закономерности в том, какие элементы чаще всего встречаются в твитах: проведем частотный анализ по знакам пунктуации и токенам.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from allennlp.data.tokenizers.word_splitter import SpacyWordSplitter\n",
    "from allennlp.data.token_indexers import SingleIdTokenIndexer\n",
    "\n",
    "# the token indexer is responsible for mapping tokens to integers\n",
    "token_indexer = SingleIdTokenIndexer()\n",
    "\n",
    "def tokenizer(x: str):\n",
    "    return [w.text for w in\n",
    "            SpacyWordSplitter(language='en_core_web_sm', \n",
    "                              pos_tags=False).split_words(x)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['tokens'] = df['text'].apply(lambda row:  str(row).split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups2 = df.groupby(\"sentiment\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Распределение знаков пунктуации:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 3, figsize=(22, 10))\n",
    "for axis, sentiment in zip(ax, sentiments):\n",
    "    x, y = zip(*collections.Counter(list(itertools.chain.from_iterable(groups2.get_group(sentiment)['punct'].tolist()))).most_common())\n",
    "    axis.bar(x, y)\n",
    "    axis.set_xlabel(\"Punctuation signs\")\n",
    "    axis.set_ylabel(\"Number of occurences\")\n",
    "    axis.set_title(sentiment.capitalize())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Распределение всех символов:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 3, figsize=(22, 10))\n",
    "for axis, sentiment in zip(ax, sentiments):\n",
    "    x, y = zip(*collections.Counter(list(itertools.chain.from_iterable(groups2.get_group(sentiment)['tokens'].tolist()))).most_common()[:20])\n",
    "    axis.bar(x, y)\n",
    "    axis.set_xlabel(\"Number of occurences\")\n",
    "    axis.set_ylabel(\"Words\")\n",
    "    axis.set_title(sentiment.capitalize())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Распределение слов:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['tokens_no_punct'] = df['text'].apply(lambda row: tokenizer(basic_cleaning(str(row))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups3 = df.groupby(\"sentiment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_tweets(dataframe: pd.core.frame.DataFrame, n:int = 3, column: str = 'text') -> list:\n",
    "    return dataframe[column].apply(lambda row: (lambda arr: list(filter(lambda x: len(x) >= n and len(puncts.intersection(x)) == 0, arr)))(tokenizer(str(row)))).tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## При наименьшей длинне слова, равной 3м символам:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 3, figsize=(22, 10))\n",
    "for axis, sentiment in zip(ax, sentiments):\n",
    "    #x, y = zip(*collections.Counter(list(itertools.chain.from_iterable(groups3.get_group(sentiment)['tokens_no_punct'].tolist()))).most_common()[:20])\n",
    "    x, y = zip(*collections.Counter(itertools.chain.from_iterable(preprocess_tweets(groups3.get_group(sentiment), n=3))).most_common()[:20])\n",
    "    sns.barplot( x = y, y = list(x), ax=axis)\n",
    "    axis.set_xlabel(\"Number of occurences\")\n",
    "    axis.set_ylabel(\"Words\")\n",
    "    axis.set_title(sentiment.capitalize())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## При наименьшей длинне слова, равной 4м символам:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 3, figsize=(22, 10))\n",
    "for axis, sentiment in zip(ax, sentiments):\n",
    "    #x, y = zip(*collections.Counter(list(itertools.chain.from_iterable(groups3.get_group(sentiment)['tokens_no_punct'].tolist()))).most_common()[:20])\n",
    "    x, y = zip(*collections.Counter(itertools.chain.from_iterable(preprocess_tweets(groups3.get_group(sentiment), n=4))).most_common()[:20])\n",
    "    sns.barplot( x = y, y = list(x), ax=axis)\n",
    "    axis.set_xlabel(\"Number of occurences\")\n",
    "    axis.set_ylabel(\"Words\")\n",
    "    axis.set_title(sentiment.capitalize())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## При наименьшей длинне слова, равной 5ти символам:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 3, figsize=(22, 10))\n",
    "for axis, sentiment in zip(ax, sentiments):\n",
    "    #x, y = zip(*collections.Counter(list(itertools.chain.from_iterable(groups3.get_group(sentiment)['tokens_no_punct'].tolist()))).most_common()[:20])\n",
    "    x, y = zip(*collections.Counter(itertools.chain.from_iterable(preprocess_tweets(groups3.get_group(sentiment), n=5))).most_common()[:20])\n",
    "    sns.barplot( x = y, y = list(x), ax=axis)\n",
    "    axis.set_xlabel(\"Number of occurences\")\n",
    "    axis.set_ylabel(\"Words\")\n",
    "    axis.set_title(sentiment.capitalize())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Можно видеть, что при увеличении длинны слов становится видна разница между оттенками наиболее часто встречающийся слов."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Теперь, когда мы взглянули на данные, предобработаем данные и построим модель для решения задачи."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, x, y ):\n",
    "        self.sentences = x\n",
    "        self.target = torch.tensor(y).long()\n",
    "    def __len__(self) -> int:\n",
    "        return len(self.target)\n",
    "        \n",
    "    def __getitem__(self, index:int) -> tuple:\n",
    "        x = self.sentences[index]\n",
    "        y = self.target[index]\n",
    "        \n",
    "        return (x, y[0])\n",
    "    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ELMOSentenceClassifier(nn.Module):\n",
    "    \"\"\"\n",
    "    A simple classifier based on ELMo transformer\n",
    "    \"\"\"\n",
    "    def __init__(self, dropout=0.5):\n",
    "        \n",
    "        super(ELMOSentenceClassifier, self).__init__()\n",
    "        self.dropout = dropout\n",
    "        self.device = torch.device(\"cpu\") #torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\") Пизда полнейшая, ебаный торч не хочет работать с гпу от кегла\n",
    "        options_file = \"https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json\"\n",
    "        weight_file = \"https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5\"\n",
    "        self.elmo = Elmo(options_file, weight_file, 3, dropout=0)\n",
    "\n",
    "        self.fc1 = nn.Linear(1024, 512)\n",
    "        self.fc2 = nn.Linear(512, 3)\n",
    "        \n",
    "        \n",
    "        \n",
    "    def forward(self, sentences):\n",
    "        \"\"\"\n",
    "        Input:\n",
    "        ::param sentences - list(of lists())// Tensor / numpy array, containing a batch of tokenized sentences.\n",
    "        \"\"\"\n",
    "        x = batch_to_ids(sentences) \n",
    "        x = x.to(self.device)\n",
    "        x = self.elmo(x)\n",
    "        x = (x['elmo_representations'][0] + x['elmo_representations'][1] + x['elmo_representations'][2])/3 # хуй знает почему, но тут не получается усреднять с помощью torch.mean, ебись в рот создатели AllenNLP\n",
    "        x = self.fc1(x)\n",
    "        x = self.fc2(x)\n",
    "        x = F.softmax(x, dim=0) # TODO: понять, на кой ляд здесь dim; без него не работает\n",
    "        #print(len(x), x[0].size())\n",
    "        return x[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, train_tuple, test_tuple, n_epochs,  learning_rate=1e-4, param_dict = {'batch_size': 64,'shuffle': True,'num_workers': 1}):\n",
    "    device = torch.device(\"cpu\")#torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(f\"starting training on device: {device}\")\n",
    "    train_dataset = CustomDataset(*train_tuple)\n",
    "    test_dataset = CustomDataset(*test_tuple)\n",
    "    train_dataloader = data.DataLoader(train_dataset, **param_dict)\n",
    "    test_dataloader = data.DataLoader(test_dataset, **param_dict)\n",
    "    model.to(device)\n",
    "    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "    criterion = nn. CrossEntropyLoss()\n",
    "    start = time.time()\n",
    "    train_losses = []\n",
    "    test_losses = []\n",
    "    for epoch in range(n_epochs):\n",
    "    \n",
    "        print(f\"[{time.time() - start}] On epoch {epoch}\")\n",
    "        running_loss = 0.0\n",
    "        i = 0\n",
    "        for local_features, local_target in train_dataloader:\n",
    "            optimizer.zero_grad()\n",
    "            local_target.to(device)\n",
    "            outputs = model(local_features)\n",
    "            outputs.to(device)\n",
    "            #print(local_target.size())\n",
    "            loss = criterion(outputs , local_target )\n",
    "            running_loss += loss\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            i += 1\n",
    "            if i % 10 == 0:\n",
    "                \n",
    "                print(f\"[{time.time() - start}] Epoch {epoch}, step {i}/{len(train_dataloader)}, loss: {running_loss/i}\")\n",
    "        train_losses.append(running_loss/len(train_dataloader))\n",
    "        #print(f\"[{time.time() - start}] Train loss: {running_loss/len(train_dataloader)}\")  \n",
    "        with torch.set_grad_enabled(False):\n",
    "            test_loss = 0.0\n",
    "            \n",
    "            for local_features, local_target in test_dataloader:\n",
    "                outputs = model(local_features)\n",
    "                outputs.to(device)\n",
    "                local_target.to(device)\n",
    "                \n",
    "                loss = criterion(outputs , local_target)\n",
    "                test_loss += loss\n",
    "            test_losses.append(test_loss/len(test_dataloader))\n",
    "            print(f\"[{time.time() - start}] Test loss: {test_loss/len(test_dataloader)}\")\n",
    "        \n",
    "    return model, train_losses, test_lossss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"model, train_losses, test_losses = train_model(model=ELMOSentenceClassifier(dropout=0.0),\n",
    "                    train_tuple=(x_train, y_train),\n",
    "                    test_tuple=(x_test, y_test),\n",
    "                   n_epochs=10,\n",
    "                   )\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Модель BERT требует специфичной предобработки данных."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "df['tokens'].apply(lambda row: len(row)).max()\n",
    "\n",
    "bertTokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
    "\n",
    "tokenizer_param_dict={\n",
    "    \"add_special_tokens\": True, # Add '[CLS]' and '[SEP]'\n",
    "    \"max_length\": 106,  # Pad & truncate all sentences.\n",
    "    \"pad_to_max_length\": True,\n",
    "    \"return_attention_mask\": True,   # Construct attn. masks.\n",
    "    \"return_tensors\": 'pt'     # Return pytorch tensors.\n",
    "}\n",
    "\n",
    "sent_ids = []\n",
    "sent_masks = []\n",
    "\n",
    "for sentence in df['text']:\n",
    "    encoded_dict = bertTokenizer.encode_plus(str(sentence), add_special_tokens=True, max_length=106, return_attention_mask=True, pad_to_max_length=True,return_tensors='pt' )\n",
    "    sent_ids.append(encoded_dict['input_ids'])\n",
    "    sent_masks.append(encoded_dict['attention_mask'])\n",
    "    \n",
    "sent_ids = torch.cat(sent_ids, dim=0)\n",
    "sent_masks = torch.cat(sent_masks, dim=0)\n",
    "\n",
    "\n",
    "labels = sklearn.preprocessing.OrdinalEncoder().fit_transform(df['sentiment'].to_numpy().reshape(-1,1))\n",
    "\n",
    "labels = torch.tensor(labels).long()\n",
    "\n",
    "dataset = torch.utils.data.TensorDataset(sent_ids, sent_masks, labels)\n",
    "train_size = int(0.9 * len(dataset))\n",
    "val_size = len(dataset) - train_size\n",
    "train_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size])\n",
    "print('{:>5,} training samples'.format(train_size))\n",
    "print('{:>5,} validation samples'.format(val_size))\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "\n",
    "train_dataloader = torch.utils.data.DataLoader(\n",
    "            train_dataset,  # The training samples.\n",
    "            sampler = torch.utils.data.RandomSampler(train_dataset), # Select batches randomly\n",
    "            batch_size = batch_size # Trains with this batch size.\n",
    "        )\n",
    "validation_dataloader = torch.utils.data.DataLoader(\n",
    "            val_dataset, # The validation samples.\n",
    "            sampler = torch.utils.data.SequentialSampler(val_dataset), # Pull out batches sequentially.\n",
    "            batch_size = batch_size # Evaluate with this batch size.\n",
    "        )\n",
    "\n",
    "model = BertForSequenceClassification.from_pretrained(\n",
    "    \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
    "    num_labels = 3, # The number of output labels--2 for binary classification. \n",
    "    output_attentions = False,\n",
    "    output_hidden_states = False,\n",
    ")\n",
    "model.cuda()\n",
    "\n",
    "optimizer = AdamW(model.parameters(),\n",
    "                  lr = 1e-4, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
    "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
    "                )\n",
    "\n",
    "\n",
    "# В оригинальной статье \n",
    "epochs = 4\n",
    "\n",
    "# Total number of training steps is [number of batches] x [number of epochs]. \n",
    "# (Note that this is not the same as the number of training samples).\n",
    "total_steps = len(train_dataloader) * epochs\n",
    "\n",
    "# Create the learning rate scheduler.\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
    "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
    "                                            num_training_steps = total_steps)\n",
    "\n",
    "def one_hot(y):\n",
    "    tmp = np.zeros(3)\n",
    "    tmp[y[0]] = 1\n",
    "    return tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Следуя рекоммендациям авторов оригинальной статьи о BERT, выберем размер минибатча, равный 32."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "# This training code is based on the `run_glue.py` script here:\n",
    "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
    "\n",
    "\n",
    "# We'll store a number of quantities such as training and validation loss, \n",
    "# validation accuracy, and timings.\n",
    "training_stats = []\n",
    "\n",
    "# Measure the total training time for the whole run.\n",
    "total_t0 = time.time()\n",
    "\n",
    "# For each epoch...\n",
    "for epoch_i in range(0, epochs):\n",
    "    # ========================================\n",
    "    #               Training\n",
    "    # ========================================\n",
    "    \n",
    "    # Perform one full pass over the training set.\n",
    "\n",
    "    print(\"\")\n",
    "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
    "    print('Training...')\n",
    "\n",
    "    # Measure how long the training epoch takes.\n",
    "    t0 = time.time()\n",
    "\n",
    "    # Reset the total loss for this epoch.\n",
    "    total_train_loss = 0\n",
    "\n",
    "    # Put the model into training mode. Don't be mislead--the call to \n",
    "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
    "    # `dropout` and `batchnorm` layers behave differently during training\n",
    "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
    "    model.train()\n",
    "\n",
    "    # For each batch of training data...\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "\n",
    "        # Progress update every 40 batches.\n",
    "        if step % 40 == 0 and not step == 0:\n",
    "            # Calculate elapsed time in minutes.\n",
    "            elapsed = format_time(time.time() - t0)\n",
    "            \n",
    "            # Report progress.\n",
    "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
    "\n",
    "        # Unpack this training batch from our dataloader. \n",
    "        #\n",
    "        # As we unpack the batch, we'll also copy each tensor to the GPU using the \n",
    "        # `to` method.\n",
    "        #\n",
    "        # `batch` contains three pytorch tensors:\n",
    "        #   [0]: input ids \n",
    "        #   [1]: attention masks\n",
    "        #   [2]: labels \n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "        #print(b_labels.size())\n",
    "        # Always clear any previously calculated gradients before performing a\n",
    "        # backward pass. PyTorch doesn't do this automatically because \n",
    "        # accumulating the gradients is \"convenient while training RNNs\". \n",
    "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
    "        model.zero_grad()        \n",
    "\n",
    "        # Perform a forward pass (evaluate the model on this training batch).\n",
    "        # The documentation for this `model` function is here: \n",
    "        # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
    "        # It returns different numbers of parameters depending on what arguments\n",
    "        # are given and what flags are set. For our useage here, it returns\n",
    "        # the loss (because we provided labels) and the \"logits\"--the model\n",
    "        # outputs prior to activation.\n",
    "        loss, logits = model(b_input_ids, \n",
    "                             token_type_ids=None, \n",
    "                             attention_mask=b_input_mask, \n",
    "                             labels=b_labels)\n",
    "\n",
    "        # Accumulate the training loss over all of the batches so that we can\n",
    "        # calculate the average loss at the end. `loss` is a Tensor containing a\n",
    "        # single value; the `.item()` function just returns the Python value \n",
    "        # from the tensor.\n",
    "        total_train_loss += loss.item()\n",
    "\n",
    "        # Perform a backward pass to calculate the gradients.\n",
    "        loss.backward()\n",
    "\n",
    "        # Clip the norm of the gradients to 1.0.\n",
    "        # This is to help prevent the \"exploding gradients\" problem.\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "\n",
    "        # Update parameters and take a step using the computed gradient.\n",
    "        # The optimizer dictates the \"update rule\"--how the parameters are\n",
    "        # modified based on their gradients, the learning rate, etc.\n",
    "        optimizer.step()\n",
    "\n",
    "        # Update the learning rate.\n",
    "        scheduler.step()\n",
    "\n",
    "    # Calculate the average loss over all of the batches.\n",
    "    avg_train_loss = total_train_loss / len(train_dataloader)            \n",
    "    \n",
    "    # Measure how long this epoch took.\n",
    "    training_time = format_time(time.time() - t0)\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
    "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
    "        \n",
    "    # ========================================\n",
    "    #               Validation\n",
    "    # ========================================\n",
    "    # After the completion of each training epoch, measure our performance on\n",
    "    # our validation set.\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Running Validation...\")\n",
    "\n",
    "    t0 = time.time()\n",
    "\n",
    "    # Put the model in evaluation mode--the dropout layers behave differently\n",
    "    # during evaluation.\n",
    "    model.eval()\n",
    "\n",
    "    # Tracking variables \n",
    "    total_eval_accuracy = 0\n",
    "    total_eval_loss = 0\n",
    "    nb_eval_steps = 0\n",
    "\n",
    "    # Evaluate data for one epoch\n",
    "    for batch in validation_dataloader:\n",
    "        \n",
    "        # Unpack this training batch from our dataloader. \n",
    "        #\n",
    "        # As we unpack the batch, we'll also copy each tensor to the GPU using \n",
    "        # the `to` method.\n",
    "        #\n",
    "        # `batch` contains three pytorch tensors:\n",
    "        #   [0]: input ids \n",
    "        #   [1]: attention masks\n",
    "        #   [2]: labels \n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "        #print(b_labels.size())\n",
    "        # Tell pytorch not to bother with constructing the compute graph during\n",
    "        # the forward pass, since this is only needed for backprop (training).\n",
    "        with torch.no_grad():        \n",
    "\n",
    "            # Forward pass, calculate logit predictions.\n",
    "            # token_type_ids is the same as the \"segment ids\", which \n",
    "            # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
    "            # The documentation for this `model` function is here: \n",
    "            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
    "            # Get the \"logits\" output by the model. The \"logits\" are the output\n",
    "            # values prior to applying an activation function like the softmax.\n",
    "            (loss, logits) = model(b_input_ids, \n",
    "                                   token_type_ids=None, \n",
    "                                   attention_mask=b_input_mask,\n",
    "                                   labels=b_labels)\n",
    "            \n",
    "        # Accumulate the validation loss.\n",
    "        total_eval_loss += loss.item()\n",
    "        \n",
    "        # Move logits and labels to CPU\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "\n",
    "        # Calculate the accuracy for this batch of test sentences, and\n",
    "        # accumulate it over all batches.\n",
    "        #print(list(map(lambda item: one_hot(item), label_ids)))\n",
    "        total_eval_accuracy += sklearn.metrics.f1_score(label_ids, np.argmax(logits, axis=1), average='weighted')\n",
    "        \n",
    "\n",
    "    # Report the final accuracy for this validation run.\n",
    "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
    "    print(\"  F1-score: {0:.2f}\".format(avg_val_accuracy))\n",
    "\n",
    "    # Calculate the average loss over all of the batches.\n",
    "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
    "    \n",
    "    # Measure how long the validation run took.\n",
    "    validation_time = format_time(time.time() - t0)\n",
    "    \n",
    "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
    "    print(\"  Validation took: {:}\".format(validation_time))\n",
    "\n",
    "    # Record all statistics from this epoch.\n",
    "    training_stats.append(\n",
    "        {\n",
    "            'epoch': epoch_i + 1,\n",
    "            'Training Loss': avg_train_loss,\n",
    "            'Valid. Loss': avg_val_loss,\n",
    "            'Valid. Accur.': avg_val_accuracy,\n",
    "            'Training Time': training_time,\n",
    "            'Validation Time': validation_time\n",
    "        }\n",
    "    )\n",
    "\n",
    "print(\"\")\n",
    "print(\"Training complete!\")\n",
    "\n",
    "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Теперь, когда мы позанимались какой-то хуйней, давайте сделаем что-то полезное - а именно, прикрутим два NERа к этой задаче; для neutral будем давать само предложение."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups = df.groupby('sentiment')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pos = groups.get_group('positive')\n",
    "df_neg = groups.get_group('negative')\n",
    "df_neu = groups.get_group('neutral')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.array([1, 2, 3 ,4 , 5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set([1, 3, 4]).intersection(set([1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_jaccard_score(sents, masks, targets):\n",
    "    \"\"\"\n",
    "    ::param sents - array of initial sentences, tokenized.\n",
    "\n",
    "    \"\"\"\n",
    "    assert len(sents) == len(masks) and len(targets) == len(masks), f\"Input arrays lengths do not match, got {len(sents)}, {len(masks)}, {len(targets)}\"\n",
    "    \n",
    "    dists = []\n",
    "    for sentence, source_mask, target_mask in zip(sents, np.asarray(masks, dtype=bool), np.asarray(targets, dtype=bool)):\n",
    "        source_set = set(np.asarray(sentence)[source_mask[:len(sentence)]])\n",
    "        target_set = set(np.asarray(sentence)[target_mask[:len(sentence)]])\n",
    "        denominator = len(source_set.union(target_set))\n",
    "        if denominator == 0:\n",
    "            dists.append(1)\n",
    "        else:\n",
    "            dists.append(len(source_set.intersection(target_set))/denominator)\n",
    "    return np.mean(dists)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_int_mask(source_array,target_array, max_len=106):\n",
    "    mask = [0]*max_len\n",
    "    for i in range(max_len):\n",
    "        if i < len(source_array) and source_array[i] in target_array:\n",
    "            mask[i] = 1\n",
    "            \n",
    "    return mask\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepair_dataloaders(dataframe, test_size=0.1, batch_size=32):\n",
    "    sent_ids = []\n",
    "    sent_masks = []\n",
    "    targets = []\n",
    "    robertaTokenizer = transformers.RobertaTokenizer.from_pretrained('roberta-base', do_lower_case=True)\n",
    "    for sentence, target in zip(dataframe['text'], dataframe['selected_text']):\n",
    "        encoded_dict = robertaTokenizer.encode_plus(str(sentence), add_special_tokens=True, max_length=54, return_attention_mask=True, pad_to_max_length=True,return_tensors='pt' )\n",
    "        sent_ids.append(encoded_dict['input_ids'])\n",
    "        sent_masks.append(encoded_dict['attention_mask'])\n",
    "        targets.append(make_int_mask(sentence.split(), target.split(), max_len=54))\n",
    "    sent_ids = torch.cat(sent_ids, dim=0)\n",
    "    sent_masks = torch.cat(sent_masks, dim=0)\n",
    "    targets = torch.tensor(targets).long()\n",
    "    dataset = torch.utils.data.TensorDataset(sent_ids, sent_masks, targets)\n",
    "    \n",
    "    \n",
    "    \n",
    "    train_size = int((1-test_size) * len(dataset))\n",
    "    val_size = len(dataset) - train_size\n",
    "    \n",
    "    \n",
    "    train_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size])\n",
    "    \n",
    "    print('{:>5,} training samples'.format(train_size))\n",
    "    print('{:>5,} validation samples'.format(val_size))\n",
    "    \n",
    "    train_dataloader = torch.utils.data.DataLoader(\n",
    "            train_dataset,  # The training samples.\n",
    "            sampler = torch.utils.data.RandomSampler(train_dataset), # Select batches randomly\n",
    "            batch_size = batch_size # Trains with this batch size.\n",
    "        )\n",
    "    validation_dataloader = torch.utils.data.DataLoader(\n",
    "            val_dataset, # The validation samples.\n",
    "            sampler = torch.utils.data.SequentialSampler(val_dataset), # Pull out batches sequentially.\n",
    "            batch_size = batch_size # Evaluate with this batch size.\n",
    "        )\n",
    "    \n",
    "    return train_dataloader, validation_dataloader, robertaTokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Сначала тренируем модель для позитивных примеров."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataloader, validation_dataloader, tokenizer = prepair_dataloaders(df_pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = transformers.RobertaForTokenClassification.from_pretrained('roberta-base', output_attentions=False, output_hidden_states=False )\n",
    "model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = AdamW(model.parameters(),\n",
    "                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
    "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
    "                )\n",
    "# В оригинальной статье сказано 2-4\n",
    "epochs = 4\n",
    "\n",
    "# Total number of training steps is [number of batches] x [number of epochs]. \n",
    "# (Note that this is not the same as the number of training samples).\n",
    "total_steps = len(train_dataloader) * epochs\n",
    "\n",
    "# Create the learning rate scheduler.\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
    "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
    "                                            num_training_steps = total_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "flag = True\n",
    "# This training code is based on the `run_glue.py` script here:\n",
    "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
    "\n",
    "\n",
    "# We'll store a number of quantities such as training and validation loss, \n",
    "# validation accuracy, and timings.\n",
    "training_stats = []\n",
    "\n",
    "# Measure the total training time for the whole run.\n",
    "total_t0 = time.time()\n",
    "\n",
    "# For each epoch...\n",
    "for epoch_i in range(0, epochs):\n",
    "    # ========================================\n",
    "    #               Training\n",
    "    # ========================================\n",
    "    \n",
    "    # Perform one full pass over the training set.\n",
    "\n",
    "    print(\"\")\n",
    "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
    "    print('Training...')\n",
    "\n",
    "    # Measure how long the training epoch takes.\n",
    "    t0 = time.time()\n",
    "\n",
    "    # Reset the total loss for this epoch.\n",
    "    total_train_loss = 0\n",
    "\n",
    "    # Put the model into training mode. Don't be mislead--the call to \n",
    "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
    "    # `dropout` and `batchnorm` layers behave differently during training\n",
    "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
    "    model.train()\n",
    "\n",
    "    # For each batch of training data...\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "\n",
    "        # Progress update every 40 batches.\n",
    "        if step % 40 == 0 and not step == 0:\n",
    "            # Calculate elapsed time in minutes.\n",
    "            elapsed = format_time(time.time() - t0)\n",
    "            \n",
    "            # Report progress.\n",
    "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
    "\n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "        #print(b_labels.size())\n",
    "        # Always clear any previously calculated gradients before performing a\n",
    "        # backward pass. PyTorch doesn't do this automatically because \n",
    "        # accumulating the gradients is \"convenient while training RNNs\". \n",
    "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
    "        model.zero_grad()        \n",
    "\n",
    "        # Perform a forward pass (evaluate the model on this training batch).\n",
    "        # The documentation for this `model` function is here: \n",
    "        # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
    "        # It returns different numbers of parameters depending on what arguments\n",
    "        # are given and what flags are set. For our useage here, it returns\n",
    "        # the loss (because we provided labels) and the \"logits\"--the model\n",
    "        # outputs prior to activation.\n",
    "        loss, logits = model(b_input_ids, \n",
    "                             token_type_ids=None, \n",
    "                             attention_mask=b_input_mask, \n",
    "                             labels=b_labels)\n",
    "\n",
    "        # Accumulate the training loss over all of the batches so that we can\n",
    "        # calculate the average loss at the end. `loss` is a Tensor containing a\n",
    "        # single value; the `.item()` function just returns the Python value \n",
    "        # from the tensor.\n",
    "        total_train_loss += loss.item()\n",
    "\n",
    "        # Perform a backward pass to calculate the gradients.\n",
    "        loss.backward()\n",
    "\n",
    "        # Clip the norm of the gradients to 1.0.\n",
    "        # This is to help prevent the \"exploding gradients\" problem.\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "\n",
    "        # Update parameters and take a step using the computed gradient.\n",
    "        # The optimizer dictates the \"update rule\"--how the parameters are\n",
    "        # modified based on their gradients, the learning rate, etc.\n",
    "        optimizer.step()\n",
    "\n",
    "        # Update the learning rate.\n",
    "        scheduler.step()\n",
    "\n",
    "    # Calculate the average loss over all of the batches.\n",
    "    avg_train_loss = total_train_loss / len(train_dataloader)            \n",
    "    \n",
    "    # Measure how long this epoch took.\n",
    "    training_time = format_time(time.time() - t0)\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
    "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
    "        \n",
    "    # ========================================\n",
    "    #               Validation\n",
    "    # ========================================\n",
    "    # After the completion of each training epoch, measure our performance on\n",
    "    # our validation set.\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Running Validation...\")\n",
    "\n",
    "    t0 = time.time()\n",
    "\n",
    "    # Put the model in evaluation mode--the dropout layers behave differently\n",
    "    # during evaluation.\n",
    "    model.eval()\n",
    "\n",
    "    # Tracking variables \n",
    "    total_eval_accuracy = 0\n",
    "    total_eval_loss = 0\n",
    "    nb_eval_steps = 0\n",
    "\n",
    "    # Evaluate data for one epoch\n",
    "    for batch in validation_dataloader:\n",
    "        \n",
    "        # Unpack this training batch from our dataloader. \n",
    "        #\n",
    "        # As we unpack the batch, we'll also copy each tensor to the GPU using \n",
    "        # the `to` method.\n",
    "        #\n",
    "        # `batch` contains three pytorch tensors:\n",
    "        #   [0]: input ids \n",
    "        #   [1]: attention masks\n",
    "        #   [2]: labels \n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "        #print(b_labels.size())\n",
    "        # Tell pytorch not to bother with constructing the compute graph during\n",
    "        # the forward pass, since this is only needed for backprop (training).\n",
    "        with torch.no_grad():        \n",
    "\n",
    "\n",
    "            (loss, outputs) = model(b_input_ids, \n",
    "                                   token_type_ids=None, \n",
    "                                   attention_mask=b_input_mask,\n",
    "                                   labels=b_labels)\n",
    "            \n",
    "        # Accumulate the validation loss.\n",
    "        total_eval_loss += loss.item()\n",
    "        \n",
    "        # Move logits and labels to CPU\n",
    "        outputs = outputs.detach().cpu()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "        \n",
    "        # Calculate the accuracy for this batch of test sentences, and\n",
    "        # accumulate it over all batches.\n",
    "        #print(list(map(lambda item: one_hot(item), label_ids)))\n",
    "        predictions = torch.argmax(outputs, dim=2)\n",
    "        if flag:\n",
    "            flag = False\n",
    "            #print(predictions)\n",
    "        predictions = predictions.cpu().numpy()\n",
    "        source_sents = list(map(lambda sent: list(map(lambda item: item.lower(), tokenizer.decode(sent, skip_special_tokens=True).split())), b_input_ids))\n",
    "        #print(source_sents)\n",
    "        total_eval_accuracy += calculate_jaccard_score(source_sents, predictions, label_ids) #sklearn.metrics.f1_score(label_ids, np.argmax(logits, axis=1), average='weighted')\n",
    "        \n",
    "\n",
    "    # Report the final accuracy for this validation run.\n",
    "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
    "    print(\"  Jaccard score: {0:.2f}\".format(avg_val_accuracy))\n",
    "\n",
    "    # Calculate the average loss over all of the batches.\n",
    "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
    "    \n",
    "    # Measure how long the validation run took.\n",
    "    validation_time = format_time(time.time() - t0)\n",
    "    \n",
    "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
    "    print(\"  Validation took: {:}\".format(validation_time))\n",
    "\n",
    "    # Record all statistics from this epoch.\n",
    "    training_stats.append(\n",
    "        {\n",
    "            'epoch': epoch_i + 1,\n",
    "            'Training Loss': avg_train_loss,\n",
    "            'Valid. Loss': avg_val_loss,\n",
    "            'Valid. Accur.': avg_val_accuracy,\n",
    "            'Training Time': training_time,\n",
    "            'Validation Time': validation_time\n",
    "        }\n",
    "    )\n",
    "\n",
    "print(\"\")\n",
    "print(\"Training complete!\")\n",
    "\n",
    "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "И проделаем то же самое с негативным классом."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_pos = model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataloader, validation_dataloader, tokenizer = prepair_dataloaders(df_neg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = transformers.RobertaForTokenClassification.from_pretrained('roberta-base', output_attentions=False, output_hidden_states=False )\n",
    "model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = AdamW(model.parameters(),\n",
    "                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
    "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
    "                )\n",
    "# В оригинальной статье сказано 2-4\n",
    "epochs = 4\n",
    "\n",
    "# Total number of training steps is [number of batches] x [number of epochs]. \n",
    "# (Note that this is not the same as the number of training samples).\n",
    "total_steps = len(train_dataloader) * epochs\n",
    "\n",
    "# Create the learning rate scheduler.\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
    "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
    "                                            num_training_steps = total_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "flag = True\n",
    "# This training code is based on the `run_glue.py` script here:\n",
    "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
    "\n",
    "\n",
    "# We'll store a number of quantities such as training and validation loss, \n",
    "# validation accuracy, and timings.\n",
    "training_stats = []\n",
    "\n",
    "# Measure the total training time for the whole run.\n",
    "total_t0 = time.time()\n",
    "\n",
    "# For each epoch...\n",
    "for epoch_i in range(0, epochs):\n",
    "    # ========================================\n",
    "    #               Training\n",
    "    # ========================================\n",
    "    \n",
    "    # Perform one full pass over the training set.\n",
    "\n",
    "    print(\"\")\n",
    "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
    "    print('Training...')\n",
    "\n",
    "    # Measure how long the training epoch takes.\n",
    "    t0 = time.time()\n",
    "\n",
    "    # Reset the total loss for this epoch.\n",
    "    total_train_loss = 0\n",
    "\n",
    "    # Put the model into training mode. Don't be mislead--the call to \n",
    "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
    "    # `dropout` and `batchnorm` layers behave differently during training\n",
    "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
    "    model.train()\n",
    "\n",
    "    # For each batch of training data...\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "\n",
    "        # Progress update every 40 batches.\n",
    "        if step % 40 == 0 and not step == 0:\n",
    "            # Calculate elapsed time in minutes.\n",
    "            elapsed = format_time(time.time() - t0)\n",
    "            \n",
    "            # Report progress.\n",
    "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
    "\n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "        #print(b_labels.size())\n",
    "        # Always clear any previously calculated gradients before performing a\n",
    "        # backward pass. PyTorch doesn't do this automatically because \n",
    "        # accumulating the gradients is \"convenient while training RNNs\". \n",
    "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
    "        model.zero_grad()        \n",
    "\n",
    "        # Perform a forward pass (evaluate the model on this training batch).\n",
    "        # The documentation for this `model` function is here: \n",
    "        # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
    "        # It returns different numbers of parameters depending on what arguments\n",
    "        # are given and what flags are set. For our useage here, it returns\n",
    "        # the loss (because we provided labels) and the \"logits\"--the model\n",
    "        # outputs prior to activation.\n",
    "        loss, logits = model(b_input_ids, \n",
    "                             token_type_ids=None, \n",
    "                             attention_mask=b_input_mask, \n",
    "                             labels=b_labels)\n",
    "\n",
    "        # Accumulate the training loss over all of the batches so that we can\n",
    "        # calculate the average loss at the end. `loss` is a Tensor containing a\n",
    "        # single value; the `.item()` function just returns the Python value \n",
    "        # from the tensor.\n",
    "        total_train_loss += loss.item()\n",
    "\n",
    "        # Perform a backward pass to calculate the gradients.\n",
    "        loss.backward()\n",
    "\n",
    "        # Clip the norm of the gradients to 1.0.\n",
    "        # This is to help prevent the \"exploding gradients\" problem.\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "\n",
    "        # Update parameters and take a step using the computed gradient.\n",
    "        # The optimizer dictates the \"update rule\"--how the parameters are\n",
    "        # modified based on their gradients, the learning rate, etc.\n",
    "        optimizer.step()\n",
    "\n",
    "        # Update the learning rate.\n",
    "        scheduler.step()\n",
    "\n",
    "    # Calculate the average loss over all of the batches.\n",
    "    avg_train_loss = total_train_loss / len(train_dataloader)            \n",
    "    \n",
    "    # Measure how long this epoch took.\n",
    "    training_time = format_time(time.time() - t0)\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
    "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
    "        \n",
    "    # ========================================\n",
    "    #               Validation\n",
    "    # ========================================\n",
    "    # After the completion of each training epoch, measure our performance on\n",
    "    # our validation set.\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Running Validation...\")\n",
    "\n",
    "    t0 = time.time()\n",
    "\n",
    "    # Put the model in evaluation mode--the dropout layers behave differently\n",
    "    # during evaluation.\n",
    "    model.eval()\n",
    "\n",
    "    # Tracking variables \n",
    "    total_eval_accuracy = 0\n",
    "    total_eval_loss = 0\n",
    "    nb_eval_steps = 0\n",
    "\n",
    "    # Evaluate data for one epoch\n",
    "    for batch in validation_dataloader:\n",
    "        \n",
    "        # Unpack this training batch from our dataloader. \n",
    "        #\n",
    "        # As we unpack the batch, we'll also copy each tensor to the GPU using \n",
    "        # the `to` method.\n",
    "        #\n",
    "        # `batch` contains three pytorch tensors:\n",
    "        #   [0]: input ids \n",
    "        #   [1]: attention masks\n",
    "        #   [2]: labels \n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "        #print(b_labels.size())\n",
    "        # Tell pytorch not to bother with constructing the compute graph during\n",
    "        # the forward pass, since this is only needed for backprop (training).\n",
    "        with torch.no_grad():        \n",
    "\n",
    "\n",
    "            (loss, outputs) = model(b_input_ids, \n",
    "                                   token_type_ids=None, \n",
    "                                   attention_mask=b_input_mask,\n",
    "                                   labels=b_labels)\n",
    "            \n",
    "        # Accumulate the validation loss.\n",
    "        total_eval_loss += loss.item()\n",
    "        \n",
    "        # Move logits and labels to CPU\n",
    "        outputs = outputs.detach().cpu()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "        \n",
    "        # Calculate the accuracy for this batch of test sentences, and\n",
    "        # accumulate it over all batches.\n",
    "        #print(list(map(lambda item: one_hot(item), label_ids)))\n",
    "        predictions = torch.argmax(outputs, dim=2)\n",
    "        if flag:\n",
    "            flag = False\n",
    "            #print(predictions)\n",
    "        predictions = predictions.cpu().numpy()\n",
    "        source_sents = list(map(lambda sent: list(map(lambda item: item.lower(), tokenizer.decode(sent, skip_special_tokens=True).split())), b_input_ids))\n",
    "        #print(source_sents)\n",
    "        total_eval_accuracy += calculate_jaccard_score(source_sents, predictions, label_ids) #sklearn.metrics.f1_score(label_ids, np.argmax(logits, axis=1), average='weighted')\n",
    "        \n",
    "\n",
    "    # Report the final accuracy for this validation run.\n",
    "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
    "    print(\"  Jaccard score: {0:.2f}\".format(avg_val_accuracy))\n",
    "\n",
    "    # Calculate the average loss over all of the batches.\n",
    "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
    "    \n",
    "    # Measure how long the validation run took.\n",
    "    validation_time = format_time(time.time() - t0)\n",
    "    \n",
    "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
    "    print(\"  Validation took: {:}\".format(validation_time))\n",
    "\n",
    "    # Record all statistics from this epoch.\n",
    "    training_stats.append(\n",
    "        {\n",
    "            'epoch': epoch_i + 1,\n",
    "            'Training Loss': avg_train_loss,\n",
    "            'Valid. Loss': avg_val_loss,\n",
    "            'Valid. Accur.': avg_val_accuracy,\n",
    "            'Training Time': training_time,\n",
    "            'Validation Time': validation_time\n",
    "        }\n",
    "    )\n",
    "\n",
    "print(\"\")\n",
    "print(\"Training complete!\")\n",
    "\n",
    "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_neg = model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clf(text ,sentiment):\n",
    "    with torch.no_grad():\n",
    "        if sentiment == \"positive\":\n",
    "            encoded_dict = tokenizer.encode_plus(text, add_special_tokens=True, max_length=54, return_attention_mask=True, pad_to_max_length=True,return_tensors='pt' )\n",
    "            #print(inds)\n",
    "            input_ids = torch.tensor(tokenizer.encode(text, add_special_tokens=True)).unsqueeze(0)\n",
    "            labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) \n",
    "            outputs = model_pos(input_ids.cuda(), labels=labels.cuda())\n",
    "            \n",
    "            predictions = torch.argmax(outputs[1], dim=2).cpu()[0]\n",
    "            #print(predictions)\n",
    "            output_labels = \" \".join(np.asarray(text.split())[np.asarray(predictions.cpu(), dtype=bool)[:len(text.split())]])\n",
    "            return output_labels.strip()\n",
    "        elif sentiment == 'negative':\n",
    "            encoded_dict = tokenizer.encode_plus(text, add_special_tokens=True, max_length=54, return_attention_mask=True, pad_to_max_length=True,return_tensors='pt' )\n",
    "            #print(inds)\n",
    "            input_ids = torch.tensor(tokenizer.encode(text, add_special_tokens=True)).unsqueeze(0)\n",
    "            labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) \n",
    "            outputs = model_neg(input_ids.cuda(), labels=labels.cuda())\n",
    "            \n",
    "            predictions = torch.argmax(outputs[1], dim=2).cpu()[0]\n",
    "            #print(predictions)\n",
    "            output_labels = \" \".join(np.asarray(text.split())[np.asarray(predictions.cpu(), dtype=bool)[:len(text.split())]]).strip()\n",
    "            return output_labels\n",
    "        else:\n",
    "            return text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(lambda i: clf(df['text'][i],df['sentiment'][i] ))(12), df['selected_text'][12], df['sentiment'][12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds =list(map(lambda i: clf(test_df.iloc[i]['text'], test_df.iloc[i]['sentiment']), range(len(test_df)) ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df.iloc[1]['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample.loc[:, 'selected_text'] = preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Попробуем другую модель из семейства BERT - roBERT от Facebook AI.\n",
    "Также, немного улучшим класс Датасета."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TweetDataset(torch.utils.data.Dataset):\n",
    "    \"\"\"\n",
    "    Улучшенный датасет.\n",
    "    Параметры:\n",
    "    ::param data - датафрейм с данными\n",
    "    ::param tokenizer - токкенайзер, который будет применяться к тексту\n",
    "    \n",
    "    \"\"\"\n",
    "    def __init__(self, data, tokenizer):\n",
    "        self.data = data\n",
    "        self.text = data.text\n",
    "        self.tokenizer = tokenizer\n",
    "        self.sentiment = data.sentiment\n",
    "        self.sentiment_dict = {\"positive\": 0, \"neutral\": 1, \"negative\": 2}\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "\n",
    "    def __getitem__(self, i:int) -> tuple:\n",
    "        start, finish = 0, 2\n",
    "        pg, tg = 'post', 'post'\n",
    "        tweet = str(self.text[i]).strip()\n",
    "        tweet_ids = self.tokenizer.encode(tweet)\n",
    "\n",
    "        attention_mask_idx = len(tweet_ids) - 1\n",
    "        if start not in tweet_ids: tweet_ids = start + tweet_ids\n",
    "        tweet_ids = pad([tweet_ids], maxlen=MAXLEN, value=1, padding=pg, truncating=tg)\n",
    "\n",
    "        attention_mask = np.zeros(MAXLEN)\n",
    "        attention_mask[1:attention_mask_idx] = 1\n",
    "        attention_mask = attention_mask.reshape((1, -1))\n",
    "        if finish not in tweet_ids: tweet_ids[-1], attention_mask[-1] = finish, start\n",
    "            \n",
    "        sentiment = [self.sentiment_dict[self.sentiment[i]]]\n",
    "        sentiment = torch.FloatTensor(to_categorical(sentiment, num_classes=3))\n",
    "        return sentiment, torch.LongTensor(tweet_ids), torch.LongTensor(attention_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Roberta(nn.Module):\n",
    "    \"\"\"\n",
    "    An implementation of roBERTa model.\n",
    "    It \n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        super(Roberta, self).__init__()\n",
    "        self.softmax = nn.Softmax(dim=1)\n",
    "        self.drop = nn.Dropout(DROP_RATE)\n",
    "        self.roberta = RobertaModel.from_pretrained(model)\n",
    "        self.dense = nn.Linear(ROBERTA_UNITS, OUTPUT_UNITS)\n",
    "        \n",
    "    def forward(self, inp, att):\n",
    "        inp = inp.view(-1, MAXLEN)\n",
    "        _, self.feat = self.roberta(inp, att)\n",
    "        return self.softmax(self.dense(self.drop(self.feat)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCHS = 20\n",
    "SPLIT = 0.8\n",
    "MAXLEN = 48\n",
    "DROP_RATE = 0.3\n",
    "np.random.seed(42)\n",
    "\n",
    "OUTPUT_UNITS = 3\n",
    "BATCH_SIZE = 256\n",
    "LR = (4e-5, 1e-2)\n",
    "ROBERTA_UNITS = 768\n",
    "VAL_BATCH_SIZE = 384\n",
    "MODEL_SAVE_PATH = 'sentiment_model.pt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = 'roberta-base'\n",
    "tokenizer = RobertaTokenizer.from_pretrained(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cel(inp, target):\n",
    "    _, labels = target.max(dim=1)\n",
    "    return nn.CrossEntropyLoss()(inp, labels)*len(inp)\n",
    "\n",
    "def accuracy(inp, target):\n",
    "    inp_ind = inp.max(axis=1).indices\n",
    "    target_ind = target.max(axis=1).indices\n",
    "    return (inp_ind == target_ind).float().sum(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_roberta():\n",
    "    size = 1\n",
    "    torch.manual_seed(42)\n",
    "    train_df = pd.read_csv('../input/tweet-sentiment-extraction/train.csv')\n",
    "\n",
    "    train_df = shuffle(train_df)\n",
    "    split = np.int32(SPLIT*len(train_df))\n",
    "    val_df, train_df = train_df[split:], train_df[:split]\n",
    "\n",
    "    val_df = val_df.reset_index(drop=True)\n",
    "    val_set = TweetDataset(val_df, tokenizer)\n",
    "    val_sampler = torch.utils.data.RandomSampler(val_set)\n",
    "\n",
    "    train_df = train_df.reset_index(drop=True)\n",
    "    train_set = TweetDataset(train_df, tokenizer)\n",
    "    train_sampler = torch.utils.data.RandomSampler(train_set)\n",
    "    \n",
    "    val_loader = torch.utils.data.DataLoader(val_set,\n",
    "                            batch_size = VAL_BATCH_SIZE,\n",
    "                            sampler=val_sampler)\n",
    "\n",
    "    train_loader = torch.utils.data.DataLoader(train_set,\n",
    "                                               batch_size = BATCH_SIZE,\n",
    "                              sampler=train_sampler)\n",
    "\n",
    "    device = torch.device(\"cuda\")\n",
    "    network = Roberta().to(device)\n",
    "    optimizer = optim.Adam([{'params': network.dense.parameters(), 'lr': LR[1]*size},\n",
    "                      {'params': network.roberta.parameters(), 'lr': LR[0]*size}])\n",
    "\n",
    "    val_losses, val_accuracies = [], []\n",
    "    train_losses, train_accuracies = [], []\n",
    "    \n",
    "    start = time.time()\n",
    "    for epoch in range(EPOCHS):\n",
    "\n",
    "        batch = 1\n",
    "        network.train()\n",
    "        #fonts = (fg(48), attr('reset'))\n",
    "        #xm.master_print((\"EPOCH %s\" + str(epoch+1) + \"%s\") % fonts)\n",
    "\n",
    "        #val_parallel = pl.ParallelLoader(val_loader, [device]).per_device_loader(device)\n",
    "        #train_parallel = pl.ParallelLoader(train_loader, [device]).per_device_loader(device)\n",
    "        \n",
    "        for train_batch in train_loader:\n",
    "            train_targ, train_in, train_att = train_batch\n",
    "            \n",
    "            network = network.to(device)\n",
    "            train_in = train_in.to(device)\n",
    "            train_att = train_att.to(device)\n",
    "            train_targ = train_targ.to(device)\n",
    "\n",
    "            train_preds = network.forward(train_in, train_att)\n",
    "            train_loss = cel(train_preds, train_targ.squeeze(dim=1))/len(train_in)\n",
    "            train_accuracy = accuracy(train_preds, train_targ.squeeze(dim=1))/len(train_in)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            train_loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            end = time.time()\n",
    "            batch = batch + 1\n",
    "            acc = np.round(train_accuracy.item(), 3)\n",
    "            print(f\"Training accuracy on batch {batch}: {np.round(acc, 3)}, time: {np.round(end - start, 1)}s\")\n",
    "            #print_metric(acc, batch, None, start, end, metric=\"acc\", typ=\"Train\")\n",
    "\n",
    "        val_loss, val_accuracy, val_points = 0, 0, 0\n",
    "\n",
    "        network.eval()\n",
    "        with torch.no_grad():\n",
    "            for val_batch in val_loader:\n",
    "                targ, val_in, val_att = val_batch\n",
    "\n",
    "                targ = targ.to(device)\n",
    "                val_in = val_in.to(device)\n",
    "                val_att = val_att.to(device)\n",
    "                network = network.to(device)\n",
    "            \n",
    "                val_points += len(targ)\n",
    "                pred = network.forward(val_in, val_att)\n",
    "                val_loss += cel(pred, targ.squeeze(dim=1)).item()\n",
    "                val_accuracy += accuracy(pred, targ.squeeze(dim=1)).item()\n",
    "        \n",
    "        end = time.time()\n",
    "        val_loss /= val_points\n",
    "        val_accuracy /= val_points\n",
    "        #acc = xm.mesh_reduce('acc', val_accuracy, lambda x: sum(x)/len(x))\n",
    "        print(f\"Validation accuracy: {np.round(np.mean(val_accuracy), 3)}\")\n",
    "        #print_metric(np.round(acc, 3), None, epoch, start, end, metric=\"acc\", typ=\"Val\")\n",
    "    \n",
    "        print(\"\")\n",
    "        val_losses.append(val_loss)\n",
    "        train_losses.append(train_loss.item())\n",
    "        val_accuracies.append(val_accuracy) \n",
    "        train_accuracies.append(train_accuracy.item())\n",
    "\n",
    "    print(\"ENDING TRAINING ...\")\n",
    "    torch.save(network.state_dict(), MODEL_SAVE_PATH); del network; gc.collect()\n",
    "\n",
    "    metric_names = ['val_loss_', 'train_loss_', 'val_acc_', 'train_acc_']\n",
    "    metric_lists = [val_losses, train_losses, val_accuracies, train_accuracies]\n",
    "    \n",
    "    for i, metric_list in enumerate(metric_lists):\n",
    "        for j, metric_value in enumerate(metric_list):\n",
    "            torch.save(metric_value, metric_names[i] + str(j) + '.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_roberta()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Для выделения слов, образующих и определяющих эмоциональную окраску предложений, мы будем использовать модель DistilBERT, предобученную на датасете SQuAD. Для лаконичности, будем использовать обертку над библиотекой transformers от HuggingFace, которая называется simpletransformers."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Для начала, переведем наши данные в подходящий для тренировки вопросно-ответной системы формат вида: \n",
    "```\n",
    "train_data = [\n",
    "    {\n",
    "        'context': \"This tweet sentiment extraction challenge is great\",\n",
    "        'qas': [\n",
    "            {\n",
    "                'id': \"00001\",\n",
    "                'question': \"positive\",\n",
    "                'answers': [\n",
    "                    {\n",
    "                        'text': \"is great\",\n",
    "                        'answer_start': 43\n",
    "                    }\n",
    "                ]\n",
    "            }\n",
    "        ]\n",
    "    }\n",
    "    ]\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/train.csv')\n",
    "test_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')\n",
    "sub_df = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv')\n",
    "\n",
    "train = np.array(train_df)\n",
    "test = np.array(test_df)\n",
    "\n",
    "qa_train = do_qa_train(train)\n",
    "\n",
    "with open('data/train.json', 'w') as outfile:\n",
    "    json.dump(qa_train, outfile)\n",
    "    \n",
    "qa_test = do_qa_test(test)\n",
    "\n",
    "with open('data/test.json', 'w') as outfile:\n",
    "    json.dump(qa_test, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from simpletransformers.question_answering import QuestionAnsweringModel\n",
    "\n",
    "MODEL_PATH = '/kaggle/input/transformers-pretrained-distilbert/distilbert-base-uncased-distilled-squad/'\n",
    "\n",
    "# Create the QuestionAnsweringModel\n",
    "model = QuestionAnsweringModel('distilbert', \n",
    "                               MODEL_PATH, \n",
    "                               args={'reprocess_input_data': True,\n",
    "                                     'overwrite_output_dir': True,\n",
    "                                     'learning_rate': 5e-5,\n",
    "                                     'num_train_epochs': 3,\n",
    "                                     'max_seq_length': 192,\n",
    "                                     'doc_stride': 64,\n",
    "                                     'fp16': False,\n",
    "                                    },\n",
    "                              use_cuda=True)\n",
    "\n",
    "model.train_model('data/train.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = model.predict(qa_test)\n",
    "predictions_df = pd.DataFrame.from_dict(predictions)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}