POS-Intervention.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Current setup using this notebook\n",
    "<p>label - POS</p>\n",
    "<p>Fine tune on: Raw-train</p>\n",
    "<p>Evaluate on: Yap-dev (and Yap-test, but not reported)</p>\n",
    "<p>Classification by: whole word (as opposed to prefix/host)</p>\n",
    "<p>Morphologically informed labels? Yes </p>\n",
    "<p>Shuffle/Sort? shuffle </p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm, trange\n",
    "import bclm\n",
    "\n",
    "import torch\n",
    "from torch.optim import Adam\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertConfig\n",
    "from pytorch_pretrained_bert import BertForTokenClassification, BertAdam"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Manually setting seeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(3)\n",
    "np.random.seed(3)\n",
    "torch.cuda.manual_seed_all(3)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data\n",
    "`bclm.read_dataframe('spmrl', subset='train')` - gives the gold-segmented tokens <br>\n",
    "`bclm.get_token_df(train, ['upostag'])` - gives the raw tokens<br>\n",
    "`bclm.read_dataframe('yap_dev')` - gives the YAP tokenization (only available on `yap_dev` and `yap_test`. No `yap_train`!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = bclm.read_dataframe('spmrl', subset='train')\n",
    "train_df = bclm.get_token_df(train, ['upostag'])\n",
    "train_df['token_str'] = train_df['token_str'].str.replace('”','\"')\n",
    "\n",
    "dev_df = bclm.read_dataframe('yap_dev')\n",
    "# dev_df = bclm.get_token_df(dev, ['upostag'])\n",
    "dev_df['form'] = dev_df['form'].str.replace('”','\"')\n",
    "\n",
    "test_df = bclm.read_dataframe('yap_test')\n",
    "# test_df = bclm.get_token_df(test, ['upostag'])\n",
    "test_df['form'] = test_df['form'].str.replace('”','\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Uniform column names\n",
    "Note that the column names in yap dfs can be slightly different from spmrl dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For Evaluating on Raw-dev/Raw-test\n",
    "train_df.rename(columns = {\"token_str\": \"form\"}, inplace = True)\n",
    "# dev_df.rename(columns = {\"token_str\": \"form\"}, inplace = True)\n",
    "# test_df.rename(columns = {\"token_str\": \"form\"}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## For Evaluating on Yap-dev/Yap-test\n",
    "dev_df.rename(columns = {\"misc_token_id\": \"token_id\"}, inplace = True)\n",
    "test_df.rename(columns = {\"misc_token_id\": \"token_id\"}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>form</th>\n",
       "      <th>lemma</th>\n",
       "      <th>upostag</th>\n",
       "      <th>xpostag</th>\n",
       "      <th>feats</th>\n",
       "      <th>token_id</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>global_sent_id</th>\n",
       "      <th>...</th>\n",
       "      <th>deps</th>\n",
       "      <th>misc</th>\n",
       "      <th>ner_escaped</th>\n",
       "      <th>set</th>\n",
       "      <th>duplicate_sent_id</th>\n",
       "      <th>very_similar_sent_id</th>\n",
       "      <th>biose_layer0</th>\n",
       "      <th>biose_layer1</th>\n",
       "      <th>biose_layer2</th>\n",
       "      <th>biose_layer3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11301</th>\n",
       "      <td>1</td>\n",
       "      <td>\"</td>\n",
       "      <td>_</td>\n",
       "      <td>yyQUOT</td>\n",
       "      <td>yyQUOT</td>\n",
       "      <td>_</td>\n",
       "      <td>0</td>\n",
       "      <td>501</td>\n",
       "      <td>\"</td>\n",
       "      <td>501</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11302</th>\n",
       "      <td>2</td>\n",
       "      <td>תהיה</td>\n",
       "      <td>היה</td>\n",
       "      <td>COP</td>\n",
       "      <td>COP</td>\n",
       "      <td>gen=F|num=S|per=3</td>\n",
       "      <td>1</td>\n",
       "      <td>501</td>\n",
       "      <td>תהיה</td>\n",
       "      <td>501</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11303</th>\n",
       "      <td>3</td>\n",
       "      <td>נקמה</td>\n",
       "      <td>נקמה</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>gen=F|num=S</td>\n",
       "      <td>2</td>\n",
       "      <td>501</td>\n",
       "      <td>נקמה</td>\n",
       "      <td>501</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11304</th>\n",
       "      <td>4</td>\n",
       "      <td>ו</td>\n",
       "      <td>ו</td>\n",
       "      <td>CONJ</td>\n",
       "      <td>CONJ</td>\n",
       "      <td>_</td>\n",
       "      <td>3</td>\n",
       "      <td>501</td>\n",
       "      <td>ובגדול</td>\n",
       "      <td>501</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11305</th>\n",
       "      <td>5</td>\n",
       "      <td>בגדול</td>\n",
       "      <td>בגדול</td>\n",
       "      <td>RB</td>\n",
       "      <td>RB</td>\n",
       "      <td>_</td>\n",
       "      <td>3</td>\n",
       "      <td>501</td>\n",
       "      <td>ובגדול</td>\n",
       "      <td>501</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11306</th>\n",
       "      <td>6</td>\n",
       "      <td>.</td>\n",
       "      <td>_</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>_</td>\n",
       "      <td>4</td>\n",
       "      <td>501</td>\n",
       "      <td>.</td>\n",
       "      <td>501</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11307</th>\n",
       "      <td>1</td>\n",
       "      <td>גם</td>\n",
       "      <td>גם</td>\n",
       "      <td>RB</td>\n",
       "      <td>RB</td>\n",
       "      <td>_</td>\n",
       "      <td>0</td>\n",
       "      <td>502</td>\n",
       "      <td>גם</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11308</th>\n",
       "      <td>2</td>\n",
       "      <td>יהודים</td>\n",
       "      <td>יהודי</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>gen=M|num=P</td>\n",
       "      <td>1</td>\n",
       "      <td>502</td>\n",
       "      <td>יהודים</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11309</th>\n",
       "      <td>3</td>\n",
       "      <td>נמצאים</td>\n",
       "      <td>נמצא</td>\n",
       "      <td>BN</td>\n",
       "      <td>BN</td>\n",
       "      <td>gen=M|num=P|per=A|HebBinyan=NIFAL</td>\n",
       "      <td>2</td>\n",
       "      <td>502</td>\n",
       "      <td>נמצאים</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11310</th>\n",
       "      <td>4</td>\n",
       "      <td>עתה</td>\n",
       "      <td>עתה</td>\n",
       "      <td>RB</td>\n",
       "      <td>RB</td>\n",
       "      <td>_</td>\n",
       "      <td>3</td>\n",
       "      <td>502</td>\n",
       "      <td>עתה</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11311</th>\n",
       "      <td>5</td>\n",
       "      <td>על</td>\n",
       "      <td>על</td>\n",
       "      <td>IN</td>\n",
       "      <td>IN</td>\n",
       "      <td>_</td>\n",
       "      <td>4</td>\n",
       "      <td>502</td>\n",
       "      <td>על</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11312</th>\n",
       "      <td>6</td>\n",
       "      <td>ה</td>\n",
       "      <td>ה</td>\n",
       "      <td>DEF</td>\n",
       "      <td>DEF</td>\n",
       "      <td>_</td>\n",
       "      <td>5</td>\n",
       "      <td>502</td>\n",
       "      <td>הכוונת</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11313</th>\n",
       "      <td>7</td>\n",
       "      <td>כוונת</td>\n",
       "      <td>כוונת</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>gen=F|num=S</td>\n",
       "      <td>5</td>\n",
       "      <td>502</td>\n",
       "      <td>הכוונת</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11314</th>\n",
       "      <td>8</td>\n",
       "      <td>\"</td>\n",
       "      <td>_</td>\n",
       "      <td>yyQUOT</td>\n",
       "      <td>yyQUOT</td>\n",
       "      <td>_</td>\n",
       "      <td>6</td>\n",
       "      <td>502</td>\n",
       "      <td>\"</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11315</th>\n",
       "      <td>9</td>\n",
       "      <td>אמרו</td>\n",
       "      <td>אמר</td>\n",
       "      <td>VB</td>\n",
       "      <td>VB</td>\n",
       "      <td>gen=F|gen=M|num=P|per=3|tense=PAST|HebBinyan=PAAL</td>\n",
       "      <td>7</td>\n",
       "      <td>502</td>\n",
       "      <td>אמרו</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11316</th>\n",
       "      <td>10</td>\n",
       "      <td>אנשי</td>\n",
       "      <td>איש</td>\n",
       "      <td>NNT</td>\n",
       "      <td>NNT</td>\n",
       "      <td>gen=M|num=P</td>\n",
       "      <td>8</td>\n",
       "      <td>502</td>\n",
       "      <td>אנשי</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11317</th>\n",
       "      <td>11</td>\n",
       "      <td>כך</td>\n",
       "      <td>כך</td>\n",
       "      <td>NNP</td>\n",
       "      <td>NNP</td>\n",
       "      <td>_</td>\n",
       "      <td>9</td>\n",
       "      <td>502</td>\n",
       "      <td>כך</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>ORG</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S-ORG</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11318</th>\n",
       "      <td>12</td>\n",
       "      <td>ב</td>\n",
       "      <td>ב</td>\n",
       "      <td>PREPOSITION</td>\n",
       "      <td>PREPOSITION</td>\n",
       "      <td>_</td>\n",
       "      <td>10</td>\n",
       "      <td>502</td>\n",
       "      <td>בעת</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11319</th>\n",
       "      <td>13</td>\n",
       "      <td>עת</td>\n",
       "      <td>עת</td>\n",
       "      <td>NNT</td>\n",
       "      <td>NNT</td>\n",
       "      <td>gen=F|num=S</td>\n",
       "      <td>10</td>\n",
       "      <td>502</td>\n",
       "      <td>בעת</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11320</th>\n",
       "      <td>14</td>\n",
       "      <td>מסע</td>\n",
       "      <td>מסע</td>\n",
       "      <td>NNT</td>\n",
       "      <td>NNT</td>\n",
       "      <td>gen=M|num=S</td>\n",
       "      <td>11</td>\n",
       "      <td>502</td>\n",
       "      <td>מסע</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11321</th>\n",
       "      <td>15</td>\n",
       "      <td>ה</td>\n",
       "      <td>ה</td>\n",
       "      <td>DEF</td>\n",
       "      <td>DEF</td>\n",
       "      <td>_</td>\n",
       "      <td>12</td>\n",
       "      <td>502</td>\n",
       "      <td>ההלווייה</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11322</th>\n",
       "      <td>16</td>\n",
       "      <td>הלווייה</td>\n",
       "      <td>הלוויה</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>gen=F|num=S</td>\n",
       "      <td>12</td>\n",
       "      <td>502</td>\n",
       "      <td>ההלווייה</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11323</th>\n",
       "      <td>17</td>\n",
       "      <td>.</td>\n",
       "      <td>_</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>_</td>\n",
       "      <td>13</td>\n",
       "      <td>502</td>\n",
       "      <td>.</td>\n",
       "      <td>502</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11324</th>\n",
       "      <td>1</td>\n",
       "      <td>איש</td>\n",
       "      <td>איש</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>gen=M|num=S</td>\n",
       "      <td>0</td>\n",
       "      <td>503</td>\n",
       "      <td>איש</td>\n",
       "      <td>503</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11325</th>\n",
       "      <td>2</td>\n",
       "      <td>לא</td>\n",
       "      <td>לא</td>\n",
       "      <td>RB</td>\n",
       "      <td>RB</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>503</td>\n",
       "      <td>לא</td>\n",
       "      <td>503</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11326</th>\n",
       "      <td>3</td>\n",
       "      <td>ניסה</td>\n",
       "      <td>ניסה</td>\n",
       "      <td>VB</td>\n",
       "      <td>VB</td>\n",
       "      <td>gen=M|num=S|per=3|tense=PAST|HebBinyan=PIEL</td>\n",
       "      <td>2</td>\n",
       "      <td>503</td>\n",
       "      <td>ניסה</td>\n",
       "      <td>503</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11327</th>\n",
       "      <td>4</td>\n",
       "      <td>להסתיר</td>\n",
       "      <td>הסתיר</td>\n",
       "      <td>VB</td>\n",
       "      <td>VB</td>\n",
       "      <td>tense=TOINFINITIVE|HebBinyan=HIFIL</td>\n",
       "      <td>3</td>\n",
       "      <td>503</td>\n",
       "      <td>להסתיר</td>\n",
       "      <td>503</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11328</th>\n",
       "      <td>5</td>\n",
       "      <td>זאת</td>\n",
       "      <td>זאת</td>\n",
       "      <td>PRP</td>\n",
       "      <td>PRP</td>\n",
       "      <td>gen=F|num=S|per=3</td>\n",
       "      <td>4</td>\n",
       "      <td>503</td>\n",
       "      <td>זאת</td>\n",
       "      <td>503</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11329</th>\n",
       "      <td>6</td>\n",
       "      <td>.</td>\n",
       "      <td>_</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>_</td>\n",
       "      <td>5</td>\n",
       "      <td>503</td>\n",
       "      <td>.</td>\n",
       "      <td>503</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11330</th>\n",
       "      <td>1</td>\n",
       "      <td>לא</td>\n",
       "      <td>לא</td>\n",
       "      <td>RB</td>\n",
       "      <td>RB</td>\n",
       "      <td>_</td>\n",
       "      <td>0</td>\n",
       "      <td>504</td>\n",
       "      <td>לא</td>\n",
       "      <td>504</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>train</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       id     form   lemma      upostag      xpostag  \\\n",
       "11301   1        \"       _       yyQUOT       yyQUOT   \n",
       "11302   2     תהיה     היה          COP          COP   \n",
       "11303   3     נקמה    נקמה           NN           NN   \n",
       "11304   4        ו       ו         CONJ         CONJ   \n",
       "11305   5    בגדול   בגדול           RB           RB   \n",
       "11306   6        .       _        yyDOT        yyDOT   \n",
       "11307   1       גם      גם           RB           RB   \n",
       "11308   2   יהודים   יהודי           NN           NN   \n",
       "11309   3   נמצאים    נמצא           BN           BN   \n",
       "11310   4      עתה     עתה           RB           RB   \n",
       "11311   5       על      על           IN           IN   \n",
       "11312   6        ה       ה          DEF          DEF   \n",
       "11313   7    כוונת   כוונת           NN           NN   \n",
       "11314   8        \"       _       yyQUOT       yyQUOT   \n",
       "11315   9     אמרו     אמר           VB           VB   \n",
       "11316  10     אנשי     איש          NNT          NNT   \n",
       "11317  11       כך      כך          NNP          NNP   \n",
       "11318  12        ב       ב  PREPOSITION  PREPOSITION   \n",
       "11319  13       עת      עת          NNT          NNT   \n",
       "11320  14      מסע     מסע          NNT          NNT   \n",
       "11321  15        ה       ה          DEF          DEF   \n",
       "11322  16  הלווייה  הלוויה           NN           NN   \n",
       "11323  17        .       _        yyDOT        yyDOT   \n",
       "11324   1      איש     איש           NN           NN   \n",
       "11325   2       לא      לא           RB           RB   \n",
       "11326   3     ניסה    ניסה           VB           VB   \n",
       "11327   4   להסתיר   הסתיר           VB           VB   \n",
       "11328   5      זאת     זאת          PRP          PRP   \n",
       "11329   6        .       _        yyDOT        yyDOT   \n",
       "11330   1       לא      לא           RB           RB   \n",
       "\n",
       "                                                   feats  token_id  sent_id  \\\n",
       "11301                                                  _         0      501   \n",
       "11302                                  gen=F|num=S|per=3         1      501   \n",
       "11303                                        gen=F|num=S         2      501   \n",
       "11304                                                  _         3      501   \n",
       "11305                                                  _         3      501   \n",
       "11306                                                  _         4      501   \n",
       "11307                                                  _         0      502   \n",
       "11308                                        gen=M|num=P         1      502   \n",
       "11309                  gen=M|num=P|per=A|HebBinyan=NIFAL         2      502   \n",
       "11310                                                  _         3      502   \n",
       "11311                                                  _         4      502   \n",
       "11312                                                  _         5      502   \n",
       "11313                                        gen=F|num=S         5      502   \n",
       "11314                                                  _         6      502   \n",
       "11315  gen=F|gen=M|num=P|per=3|tense=PAST|HebBinyan=PAAL         7      502   \n",
       "11316                                        gen=M|num=P         8      502   \n",
       "11317                                                  _         9      502   \n",
       "11318                                                  _        10      502   \n",
       "11319                                        gen=F|num=S        10      502   \n",
       "11320                                        gen=M|num=S        11      502   \n",
       "11321                                                  _        12      502   \n",
       "11322                                        gen=F|num=S        12      502   \n",
       "11323                                                  _        13      502   \n",
       "11324                                        gen=M|num=S         0      503   \n",
       "11325                                                  _         1      503   \n",
       "11326        gen=M|num=S|per=3|tense=PAST|HebBinyan=PIEL         2      503   \n",
       "11327                 tense=TOINFINITIVE|HebBinyan=HIFIL         3      503   \n",
       "11328                                  gen=F|num=S|per=3         4      503   \n",
       "11329                                                  _         5      503   \n",
       "11330                                                  _         0      504   \n",
       "\n",
       "      token_str  global_sent_id  ...  deps misc ner_escaped    set  \\\n",
       "11301         \"             501  ...     _    _           _  train   \n",
       "11302      תהיה             501  ...     _    _           _  train   \n",
       "11303      נקמה             501  ...     _    _           _  train   \n",
       "11304    ובגדול             501  ...     _    _           _  train   \n",
       "11305    ובגדול             501  ...     _    _           _  train   \n",
       "11306         .             501  ...     _    _           _  train   \n",
       "11307        גם             502  ...     _    _           _  train   \n",
       "11308    יהודים             502  ...     _    _           _  train   \n",
       "11309    נמצאים             502  ...     _    _           _  train   \n",
       "11310       עתה             502  ...     _    _           _  train   \n",
       "11311        על             502  ...     _    _           _  train   \n",
       "11312    הכוונת             502  ...     _    _           _  train   \n",
       "11313    הכוונת             502  ...     _    _           _  train   \n",
       "11314         \"             502  ...     _    _           _  train   \n",
       "11315      אמרו             502  ...     _    _           _  train   \n",
       "11316      אנשי             502  ...     _    _           _  train   \n",
       "11317        כך             502  ...     _    _         ORG  train   \n",
       "11318       בעת             502  ...     _    _           _  train   \n",
       "11319       בעת             502  ...     _    _           _  train   \n",
       "11320       מסע             502  ...     _    _           _  train   \n",
       "11321  ההלווייה             502  ...     _    _           _  train   \n",
       "11322  ההלווייה             502  ...     _    _           _  train   \n",
       "11323         .             502  ...     _    _           _  train   \n",
       "11324       איש             503  ...     _    _           _  train   \n",
       "11325        לא             503  ...     _    _           _  train   \n",
       "11326      ניסה             503  ...     _    _           _  train   \n",
       "11327    להסתיר             503  ...     _    _           _  train   \n",
       "11328       זאת             503  ...     _    _           _  train   \n",
       "11329         .             503  ...     _    _           _  train   \n",
       "11330        לא             504  ...     _    _           _  train   \n",
       "\n",
       "      duplicate_sent_id  very_similar_sent_id biose_layer0 biose_layer1  \\\n",
       "11301               NaN                   NaN            O            O   \n",
       "11302               NaN                   NaN            O            O   \n",
       "11303               NaN                   NaN            O            O   \n",
       "11304               NaN                   NaN            O            O   \n",
       "11305               NaN                   NaN            O            O   \n",
       "11306               NaN                   NaN            O            O   \n",
       "11307               NaN                   NaN            O            O   \n",
       "11308               NaN                   NaN            O            O   \n",
       "11309               NaN                   NaN            O            O   \n",
       "11310               NaN                   NaN            O            O   \n",
       "11311               NaN                   NaN            O            O   \n",
       "11312               NaN                   NaN            O            O   \n",
       "11313               NaN                   NaN            O            O   \n",
       "11314               NaN                   NaN            O            O   \n",
       "11315               NaN                   NaN            O            O   \n",
       "11316               NaN                   NaN            O            O   \n",
       "11317               NaN                   NaN        S-ORG            O   \n",
       "11318               NaN                   NaN            O            O   \n",
       "11319               NaN                   NaN            O            O   \n",
       "11320               NaN                   NaN            O            O   \n",
       "11321               NaN                   NaN            O            O   \n",
       "11322               NaN                   NaN            O            O   \n",
       "11323               NaN                   NaN            O            O   \n",
       "11324               NaN                   NaN            O            O   \n",
       "11325               NaN                   NaN            O            O   \n",
       "11326               NaN                   NaN            O            O   \n",
       "11327               NaN                   NaN            O            O   \n",
       "11328               NaN                   NaN            O            O   \n",
       "11329               NaN                   NaN            O            O   \n",
       "11330               NaN                   NaN            O            O   \n",
       "\n",
       "      biose_layer2 biose_layer3  \n",
       "11301            O            O  \n",
       "11302            O            O  \n",
       "11303            O            O  \n",
       "11304            O            O  \n",
       "11305            O            O  \n",
       "11306            O            O  \n",
       "11307            O            O  \n",
       "11308            O            O  \n",
       "11309            O            O  \n",
       "11310            O            O  \n",
       "11311            O            O  \n",
       "11312            O            O  \n",
       "11313            O            O  \n",
       "11314            O            O  \n",
       "11315            O            O  \n",
       "11316            O            O  \n",
       "11317            O            O  \n",
       "11318            O            O  \n",
       "11319            O            O  \n",
       "11320            O            O  \n",
       "11321            O            O  \n",
       "11322            O            O  \n",
       "11323            O            O  \n",
       "11324            O            O  \n",
       "11325            O            O  \n",
       "11326            O            O  \n",
       "11327            O            O  \n",
       "11328            O            O  \n",
       "11329            O            O  \n",
       "11330            O            O  \n",
       "\n",
       "[30 rows x 27 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def renumber_tokenid(df):\n",
    "    for index in df.index:\n",
    "        df.at[index, 'token_id'] = df.at[index, 'token_id'] - 1\n",
    "        \n",
    "renumber_tokenid(train)\n",
    "renumber_tokenid(dev_df)\n",
    "renumber_tokenid(test_df)\n",
    "train.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>form</th>\n",
       "      <th>lemma</th>\n",
       "      <th>upostag</th>\n",
       "      <th>xpostag</th>\n",
       "      <th>head</th>\n",
       "      <th>deprel</th>\n",
       "      <th>deps</th>\n",
       "      <th>misc</th>\n",
       "      <th>sent</th>\n",
       "      <th>token_id</th>\n",
       "      <th>misc_token_str</th>\n",
       "      <th>feats_gen</th>\n",
       "      <th>feats_num</th>\n",
       "      <th>feats_per</th>\n",
       "      <th>feats_tense</th>\n",
       "      <th>feats_suf_gen</th>\n",
       "      <th>feats_suf_num</th>\n",
       "      <th>feats_suf_per</th>\n",
       "      <th>sent_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>עשרות</td>\n",
       "      <td>עשר</td>\n",
       "      <td>CDT</td>\n",
       "      <td>CDT</td>\n",
       "      <td>2</td>\n",
       "      <td>num</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>עשרות</td>\n",
       "      <td>F</td>\n",
       "      <td>P</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>אנשים</td>\n",
       "      <td>איש</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>3</td>\n",
       "      <td>subj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>אנשים</td>\n",
       "      <td>M</td>\n",
       "      <td>P</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>מגיעים</td>\n",
       "      <td>הגיע</td>\n",
       "      <td>BN</td>\n",
       "      <td>BN</td>\n",
       "      <td>14</td>\n",
       "      <td>conj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>מגיעים</td>\n",
       "      <td>M</td>\n",
       "      <td>P</td>\n",
       "      <td>A</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>מ</td>\n",
       "      <td>מ</td>\n",
       "      <td>PREPOSITION</td>\n",
       "      <td>PREPOSITION</td>\n",
       "      <td>3</td>\n",
       "      <td>comp</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>מתאילנד</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>תאילנד</td>\n",
       "      <td>תאילנד</td>\n",
       "      <td>NNP</td>\n",
       "      <td>NNP</td>\n",
       "      <td>4</td>\n",
       "      <td>pobj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>מתאילנד</td>\n",
       "      <td>F</td>\n",
       "      <td>S</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id    form   lemma      upostag      xpostag  head deprel deps misc  sent  \\\n",
       "0   1   עשרות     עשר          CDT          CDT     2    num    _    _     1   \n",
       "1   2   אנשים     איש           NN           NN     3   subj    _    _     1   \n",
       "2   3  מגיעים    הגיע           BN           BN    14   conj    _    _     1   \n",
       "3   4       מ       מ  PREPOSITION  PREPOSITION     3   comp    _    _     1   \n",
       "4   5  תאילנד  תאילנד          NNP          NNP     4   pobj    _    _     1   \n",
       "\n",
       "   token_id misc_token_str feats_gen feats_num feats_per feats_tense  \\\n",
       "0         0          עשרות         F         P       NaN         NaN   \n",
       "1         1          אנשים         M         P       NaN         NaN   \n",
       "2         2         מגיעים         M         P         A         NaN   \n",
       "3         3        מתאילנד       NaN       NaN       NaN         NaN   \n",
       "4         3        מתאילנד         F         S       NaN         NaN   \n",
       "\n",
       "  feats_suf_gen feats_suf_num  feats_suf_per  sent_id  \n",
       "0           NaN           NaN            NaN        1  \n",
       "1           NaN           NaN            NaN        1  \n",
       "2           NaN           NaN            NaN        1  \n",
       "3           NaN           NaN            NaN        1  \n",
       "4           NaN           NaN            NaN        1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add sorting on the dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df['sent_len'] = train_df.groupby('sent_id').id.transform('size')\n",
    "s = train_df.sort_values(by=['sent_len', 'sent_id', 'id']).index\n",
    "train_df_sorted = train_df.reindex(s)\n",
    "train_df_sorted.groupby('sent_id', sort=False).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['הם', 'התבקשו', 'לדווח', 'למשטרה', 'על', 'תנועותיהם', '.']\n",
      "['PRP', 'VB', 'VB', 'PREPOSITION^DEF^NN', 'IN', 'NN', 'yyDOT']\n",
      "490\n",
      "702\n"
     ]
    }
   ],
   "source": [
    "class sentenceGetter(object):\n",
    "    def __init__(self, data, max_sent=None):\n",
    "        self.index = 0\n",
    "        self.max_sent = max_sent\n",
    "        self.tokens = data['form']\n",
    "        self.labels = data['upostag']\n",
    "        #for evaluating by word-accuracy\n",
    "        self.correspondingToken = data['token_id']\n",
    "        self.orig_sent_id = data['sent_id']\n",
    "    \n",
    "    def sentences(self):\n",
    "        sent = []\n",
    "        counter = 0\n",
    "        \n",
    "        for token,label, corres_tok, sent_id in zip(self.tokens, self.labels, self.correspondingToken, \n",
    "                                                    self.orig_sent_id):\n",
    "            sent.append((token, label, corres_tok, sent_id))\n",
    "            if token.strip() == \".\":\n",
    "                yield sent\n",
    "                sent = []\n",
    "                counter += 1\n",
    "            if self.max_sent is not None and counter >= self.max_sent:\n",
    "                return\n",
    "\n",
    "train_getter = sentenceGetter(train_df)\n",
    "dev_getter = sentenceGetter(dev_df)\n",
    "test_getter = sentenceGetter(test_df)\n",
    "\n",
    "train_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in train_getter.sentences()]\n",
    "train_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in train_getter.sentences()]\n",
    "\n",
    "dev_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "dev_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "dev_corresTokens = [[corres_tok for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "dev_sent_ids = [[sent_id for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "\n",
    "test_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "test_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "test_corresTokens = [[corres_tok for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "test_sent_ids = [[sent_id for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "\n",
    "print(train_sentences[10])\n",
    "print(train_labels[10])\n",
    "\n",
    "print(len(dev_sentences))\n",
    "print(len(test_sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Get the longest sentences in the dev and test sets\n",
    "longest_sent_len = 0\n",
    "for sent in dev_sentences:\n",
    "    if len(sent) >= longest_sent_len:\n",
    "        print(len(sent))\n",
    "        longest_sent_len = len(sent)\n",
    "        print(\"index of longest sentence:{} \".format(dev_sentences.index(sent)))\n",
    "        \n",
    "longest_sent_len = 0\n",
    "for sent in test_sentences:\n",
    "    if len(sent) >= longest_sent_len:\n",
    "        print(len(sent))\n",
    "        longest_sent_len = len(sent)\n",
    "        print(\"index of longest sentence:{} \".format(test_sentences.index(sent)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Remove too long sentences\n",
    "\n",
    "# del dev_sentences[296]\n",
    "# del dev_labels[296]\n",
    "# del dev_corresTokens[296]\n",
    "# del dev_sent_ids[296]\n",
    "\n",
    "# del dev_sentences[226]\n",
    "# del dev_labels[226]\n",
    "# del dev_corresTokens[226]\n",
    "# del dev_sent_ids[226]\n",
    "\n",
    "# del dev_sentences[57]\n",
    "# del dev_labels[57]\n",
    "# del dev_corresTokens[57]\n",
    "# del dev_sent_ids[57]\n",
    "\n",
    "# del dev_sentences[49]\n",
    "# del dev_labels[49]\n",
    "# del dev_corresTokens[49]\n",
    "# del dev_sent_ids[49]\n",
    "\n",
    "\n",
    "# del test_sentences[396]\n",
    "# del test_labels[396]\n",
    "# del test_corresTokens[396]\n",
    "# del test_sent_ids[396]\n",
    "\n",
    "# del test_sentences[164]\n",
    "# del test_labels[164]\n",
    "# del test_corresTokens[164]\n",
    "# del test_sent_ids[164]\n",
    "\n",
    "# del test_sentences[157]\n",
    "# del test_labels[157]\n",
    "# del test_corresTokens[157]\n",
    "# del test_sent_ids[157]\n",
    "\n",
    "# del test_sentences[151]\n",
    "# del test_labels[151]\n",
    "# del test_corresTokens[151]\n",
    "# del test_sent_ids[151]\n",
    "\n",
    "## YAP deletions\n",
    "del dev_sentences[296]\n",
    "del dev_labels[296]\n",
    "del dev_corresTokens[296]\n",
    "del dev_sent_ids[296]\n",
    "\n",
    "del dev_sentences[226]\n",
    "del dev_labels[226]\n",
    "del dev_corresTokens[226]\n",
    "del dev_sent_ids[226]\n",
    "\n",
    "del dev_sentences[57]\n",
    "del dev_labels[57]\n",
    "del dev_corresTokens[57]\n",
    "del dev_sent_ids[57]\n",
    "\n",
    "del dev_sentences[49]\n",
    "del dev_labels[49]\n",
    "del dev_corresTokens[49]\n",
    "del dev_sent_ids[49]\n",
    "\n",
    "del dev_sentences[24]\n",
    "del dev_labels[24]\n",
    "del dev_corresTokens[24]\n",
    "del dev_sent_ids[24]\n",
    "\n",
    "del dev_sentences[22]\n",
    "del dev_labels[22]\n",
    "del dev_corresTokens[22]\n",
    "del dev_sent_ids[22]\n",
    "\n",
    "del dev_sentences[12]\n",
    "del dev_labels[12]\n",
    "del dev_corresTokens[12]\n",
    "del dev_sent_ids[12]\n",
    "\n",
    "del dev_sentences[9]\n",
    "del dev_labels[9]\n",
    "del dev_corresTokens[9]\n",
    "del dev_sent_ids[9]\n",
    "\n",
    "del dev_sentences[5]\n",
    "del dev_labels[5]\n",
    "del dev_corresTokens[5]\n",
    "del dev_sent_ids[5]\n",
    "\n",
    "del test_sentences[386]\n",
    "del test_labels[386]\n",
    "del test_corresTokens[386]\n",
    "del test_sent_ids[386]\n",
    "\n",
    "del test_sentences[384]\n",
    "del test_labels[384]\n",
    "del test_corresTokens[384]\n",
    "del test_sent_ids[384]\n",
    "\n",
    "del test_sentences[377]\n",
    "del test_labels[377]\n",
    "del test_corresTokens[377]\n",
    "del test_sent_ids[377]\n",
    "\n",
    "del test_sentences[213]\n",
    "del test_labels[213]\n",
    "del test_corresTokens[213]\n",
    "del test_sent_ids[213]\n",
    "\n",
    "del test_sentences[141]\n",
    "del test_labels[141]\n",
    "del test_corresTokens[141]\n",
    "del test_sent_ids[141]\n",
    "\n",
    "del test_sentences[124]\n",
    "del test_labels[124]\n",
    "del test_corresTokens[124]\n",
    "del test_sent_ids[124]\n",
    "\n",
    "del test_sentences[45]\n",
    "del test_labels[45]\n",
    "del test_corresTokens[45]\n",
    "del test_sent_ids[45]\n",
    "\n",
    "del test_sentences[35]\n",
    "del test_labels[35]\n",
    "del test_corresTokens[35]\n",
    "del test_sent_ids[35]\n",
    "\n",
    "del test_sentences[23]\n",
    "del test_labels[23]\n",
    "del test_corresTokens[23]\n",
    "del test_sent_ids[23]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_getter = sentenceGetter(train)\n",
    "dev_getter = sentenceGetter(dev)\n",
    "test_getter = sentenceGetter(test)\n",
    "\n",
    "gold_train_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in train_getter.sentences()]\n",
    "gold_train_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in train_getter.sentences()]\n",
    "gold_train_corresTokens = [[corres_tok for token, label, corres_tok, sent_id in sent] for sent in train_getter.sentences()]\n",
    "\n",
    "gold_dev_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "gold_dev_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "gold_dev_corresTokens = [[corres_tok for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "gold_dev_sent_ids = [[sent_id for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "\n",
    "gold_test_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "gold_test_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "gold_test_corresTokens = [[corres_tok for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "gold_test_sent_ids = [[sent_id for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "\n",
    "print(gold_train_sentences[10])\n",
    "print(gold_train_labels[10])\n",
    "print(gold_train_corresTokens[10])\n",
    "\n",
    "print(len(gold_dev_sentences))\n",
    "print(len(gold_test_sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Delete the same gold sentences as before"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()\n",
    "torch.cuda.set_device(1)\n",
    "\n",
    "print(\"Device: \" + str(device))\n",
    "print(\"Number of gpus: \" + str(n_gpu))\n",
    "print(\"Name of gpu: \" + torch.cuda.get_device_name(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN = 150\n",
    "bs = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')\n",
    "def tokenize(sentences, orig_labels, gold_sentences, gold_labels, gold_ranges):\n",
    "    tokenized_texts = []\n",
    "    labels = []\n",
    "    for sent, sent_labels, gold_sent, gold_sent_labels, gold_range in zip(sentences, \n",
    "                                                                           orig_labels, \n",
    "                                                                           gold_sentences, \n",
    "                                                                           gold_labels, \n",
    "                                                                           gold_ranges):\n",
    "        bert_tokens = []\n",
    "        bert_labels = []\n",
    "        \n",
    "        for ix, (orig_token, orig_label) in enumerate(zip(sent, sent_labels)):\n",
    "            b_tokens = tokenizer.tokenize(orig_token)\n",
    "            bert_tokens.extend(b_tokens)\n",
    "            orig_label_split = orig_label.split('^')\n",
    "            \n",
    "#             print(ix, b_tokens, orig_token, orig_label, gold_sent, gold_sent_labels, gold_range)\n",
    "#             print('\\n')\n",
    "            \n",
    "            if len(orig_label_split) > 1 and len(b_tokens) > 1:\n",
    "                tokenized_labels = tokenize_by_gold(b_tokens, orig_token, gold_sent, gold_sent_labels, gold_range, ix)\n",
    "#                 print(len(tokenized_labels), len(b_tokens))\n",
    "#                 print(tokenized_labels, b_tokens)\n",
    "#                 print(gold_sent)\n",
    "                assert len(tokenized_labels)== len(b_tokens)\n",
    "                \n",
    "                bert_labels.extend(tokenized_labels)\n",
    "                \n",
    "            else:\n",
    "                for b_token in b_tokens:\n",
    "                    bert_labels.append(orig_label)\n",
    "\n",
    "            \n",
    "        tokenized_texts.append(bert_tokens)\n",
    "        labels.append(bert_labels)\n",
    "\n",
    "#         print(len(bert_tokens), len(bert_labels))\n",
    "        assert len(bert_tokens) == len(bert_labels)\n",
    "        \n",
    "    \n",
    "    return tokenized_texts, labels\n",
    "\n",
    "\n",
    "def tokenize_by_gold(bert_tokenized_wordps, orig_token, gold_sent, gold_sent_label, gold_range, ix):\n",
    "    gold_tokenized_labels = []\n",
    "    len_bert_tokenized_wordps = len(bert_tokenized_wordps)\n",
    "    \n",
    "    relevant_gold_tokens = []\n",
    "    relevant_gold_labels = []\n",
    "    gold_token_ix = 0\n",
    "    \n",
    "    for token, label, corres_token in zip(gold_sent, gold_sent_label, gold_range):\n",
    "#         print(token, label, corres_token)\n",
    "        if ix == corres_token:\n",
    "            relevant_gold_tokens.append(token)\n",
    "#             print(relevant_gold_tokens)\n",
    "            relevant_gold_labels.append(label)\n",
    "#             print(relevant_gold_labels)\n",
    "#     print(\"End of sentence tokens and labels\")\n",
    "#     print(relevant_gold_tokens, relevant_gold_labels)\n",
    "            \n",
    "    for wordp_ix, wordp in enumerate(bert_tokenized_wordps):\n",
    "#         print(relevant_gold_tokens, relevant_gold_labels, bert_tokenized_wordps)\n",
    "\n",
    "        if wordp.startswith(\"##\"):\n",
    "            wordp = wordp[2:]\n",
    "\n",
    "            \n",
    "        if len(relevant_gold_labels) == 1:\n",
    "            gold_tokenized_labels.extend(relevant_gold_labels)\n",
    "#             print(\"from last relevant label\")\n",
    "#             print(gold_tokenized_labels)\n",
    "            continue\n",
    "    \n",
    "        if gold_token_ix +1 < len(relevant_gold_tokens):\n",
    "            if (relevant_gold_labels[gold_token_ix] == 'PREPOSITION' and \n",
    "                relevant_gold_labels[gold_token_ix+1] == 'DEF'):\n",
    "                joint_label = relevant_gold_labels[gold_token_ix]\n",
    "                joint_label += '^'\n",
    "                joint_label += relevant_gold_labels[gold_token_ix+1]                \n",
    "                gold_tokenized_labels.append(joint_label)\n",
    "    #             print(\"from next-label is DEF\")\n",
    "    #             print(gold_tokenized_labels)\n",
    "\n",
    "                relevant_gold_tokens.pop(gold_token_ix+1)\n",
    "                relevant_gold_labels.pop(gold_token_ix+1)\n",
    "                relevant_gold_tokens.pop(gold_token_ix)\n",
    "                relevant_gold_labels.pop(gold_token_ix)\n",
    "    #             print(\"from next-label is DEF - remaining relevant gold labels\")\n",
    "    #             print(relevant_gold_labels)\n",
    "\n",
    "                gold_token_ix += 1\n",
    "                continue\n",
    "        \n",
    "        ## word piece is identical and aligned to the gold morpheme\n",
    "        if gold_token_ix +1 < len(relevant_gold_tokens):\n",
    "            if wordp == relevant_gold_tokens[gold_token_ix]:\n",
    "\n",
    "                gold_tokenized_labels.append(relevant_gold_labels[gold_token_ix])\n",
    "#                 print(\"from wordp identical to morpheme\")\n",
    "#                 print(gold_tokenized_labels)\n",
    "\n",
    "                relevant_gold_tokens.pop(gold_token_ix)\n",
    "                relevant_gold_labels.pop(gold_token_ix)\n",
    "\n",
    "    #             print(\"from wordp identical to morpheme - remaining relevant gold labels\")\n",
    "    #             print(relevant_gold_labels)\n",
    "\n",
    "                gold_token_ix += 1\n",
    "                continue\n",
    "\n",
    "\n",
    "        ## word piece is contained in one morpheme\n",
    "        if gold_token_ix +1 < len(relevant_gold_tokens):\n",
    "            if wordp in relevant_gold_tokens[gold_token_ix]:\n",
    "                gold_tokenized_labels.append(relevant_gold_labels[gold_token_ix])\n",
    "#                 print(\"from wordp contained in morpheme\")\n",
    "#                 print(gold_tokenized_labels)\n",
    "    #             print(\"from wordp contained in morphemes - remaining relevant gold labels\")\n",
    "    #             print(relevant_gold_labels)\n",
    "                continue\n",
    "\n",
    "\n",
    "        if gold_token_ix +1 < len(relevant_gold_tokens):\n",
    "            this_gold_and_next_gold = relevant_gold_tokens[gold_token_ix] + relevant_gold_tokens[gold_token_ix+1]\n",
    "            \n",
    "            ## word piece is the start of two consequtive morphemes - only the first morpheme is removed\n",
    "            if this_gold_and_next_gold.startswith(wordp):\n",
    "                joint_label = relevant_gold_labels[gold_token_ix]\n",
    "                joint_label += '^'\n",
    "                joint_label += relevant_gold_labels[gold_token_ix+1]\n",
    "                gold_tokenized_labels.append(joint_label)\n",
    "#                 print(\"from wordp is the start of two consequtive morphemes\")\n",
    "#                 print(gold_tokenized_labels)\n",
    "                if relevant_gold_labels[gold_token_ix+1] == 'PREPOSITION':\n",
    "                    relevant_gold_tokens.pop(gold_token_ix+1)\n",
    "                    relevant_gold_labels.pop(gold_token_ix+1)\n",
    "                relevant_gold_tokens.pop(gold_token_ix)\n",
    "                relevant_gold_labels.pop(gold_token_ix)\n",
    "    #             print(\"from wordp is the start of two consequtive morphemes - remaining relevant gold labels\")\n",
    "    #             print(relevant_gold_labels)\n",
    "\n",
    "    #             gold_token_ix += 1\n",
    "                continue\n",
    "\n",
    "            ## word piece is contained consequtive morphemes - only the first morpheme is removed\n",
    "            if wordp in this_gold_and_next_gold:\n",
    "                joint_label = relevant_gold_labels[gold_token_ix]\n",
    "                joint_label += '^'\n",
    "                joint_label += relevant_gold_labels[gold_token_ix+1]\n",
    "                gold_tokenized_labels.append(joint_label)\n",
    "#                 print(\"from wordp is contained in two consequtive morphemes\")\n",
    "#                 print(gold_tokenized_labels)\n",
    "                if relevant_gold_labels[gold_token_ix+1] == 'PREPOSITION':\n",
    "                    relevant_gold_tokens.pop(gold_token_ix+1)\n",
    "                relevant_gold_tokens.pop(gold_token_ix)\n",
    "                relevant_gold_labels.pop(gold_token_ix)\n",
    "    #             print(\"from wordp is contained in two consequtive morphemes - remaining relevant gold labels\")\n",
    "    #             print(relevant_gold_labels)\n",
    "                gold_token_ix += 1\n",
    "                continue\n",
    "\n",
    "#             if wordp_ix == len_bert_tokenized_wordps - 1:\n",
    "#                 last_tag = '^'.join(relevant_gold_labels)\n",
    "#                 gold_tokenized_labels.append(last_tag)\n",
    "#                 print(\"from last wordp\")\n",
    "#                 print(gold_tokenized_labels)\n",
    "#     #             print(\"from last wordp - remaining relevant gold labels\")\n",
    "#     #             print(relevant_gold_labels)\n",
    "#                 break\n",
    "    \n",
    "    \n",
    "        last_tag = '^'.join(relevant_gold_labels)\n",
    "        gold_tokenized_labels.append(last_tag)\n",
    "#         print(\"from else\")\n",
    "#         print(gold_tokenized_labels)\n",
    "#         print(\"from else - remaining relevant gold labels\")\n",
    "#         print(relevant_gold_labels)\n",
    "        \n",
    "        \n",
    "#         return gold_tokenized_labels\n",
    "    \n",
    "    \n",
    "#     print(\"##### Tokenization By Gold #####\")\n",
    "#     print(bert_tokenized_wordps)\n",
    "#     print(orig_token)\n",
    "#     print(gold_tokenized_labels)\n",
    "#     print(\"##### END #####\")\n",
    "#     print('\\n')\n",
    "        \n",
    "    \n",
    "    return gold_tokenized_labels\n",
    "\n",
    "train_tokenized_texts, train_tokenized_labels = tokenize(train_sentences[0:],\n",
    "                                                         train_labels[0:],\n",
    "                                                         gold_train_sentences[0:], \n",
    "                                                         gold_train_labels[0:], \n",
    "                                                         gold_train_corresTokens[0:])\n",
    "print(train_tokenized_texts[10])\n",
    "print(train_tokenized_labels[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train_tokenized_texts[55])\n",
    "print(train_tokenized_labels[55])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data = train_df\n",
    "tag_vals = list(set(data[\"upostag\"].values))\n",
    "tags = ['PAD'] + tag_vals\n",
    "tag2idx = {tag:idx for idx, tag in enumerate(tags)}\n",
    "idx2tag = {idx:tag for idx, tag in enumerate(tags)}\n",
    "\n",
    "print(tag2idx)\n",
    "# print(idx2tag)\n",
    "print(len(tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentences_and_labels(tokenized_texts, labels):\n",
    "    input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
    "                              maxlen = MAX_LEN, dtype = \"float32\", truncating = \"post\", padding = \"post\", value = tag2idx['PAD'])\n",
    "    tags = pad_sequences([[tag2idx.get(l) for l in lab] for lab in labels], \n",
    "                         maxlen = MAX_LEN, value = tag2idx['PAD'], padding = \"post\",\n",
    "                        dtype = \"float32\", truncating = \"post\")\n",
    "    attention_masks = [[float(i>0) for i in ii] for ii in input_ids]\n",
    "    return input_ids, tags, attention_masks\n",
    "\n",
    "input_ids, tags, attention_masks = pad_sentences_and_labels(train_tokenized_texts, train_tokenized_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_inputs = torch.tensor(input_ids, dtype=torch.long)\n",
    "tr_tags = torch.tensor(tags, dtype=torch.long)\n",
    "tr_masks = torch.tensor(attention_masks, dtype=torch.long)\n",
    "\n",
    "train_data = TensorDataset(tr_inputs, tr_masks, tr_tags)\n",
    "train_dataloader = DataLoader(train_data, batch_size=bs, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model = BertForTokenClassification.from_pretrained('bert-base-multilingual-cased', num_labels=len(tag2idx))\n",
    "model.cuda()\n",
    "FULL_FINETUNING = True\n",
    "if FULL_FINETUNING:\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.01},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.0}\n",
    "    ]\n",
    "else:\n",
    "    param_optimizer = list(model.classifier.named_parameters())\n",
    "    optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n",
    "\n",
    "optimizer = Adam(optimizer_grouped_parameters, lr=3e-5)\n",
    "\n",
    "from seqeval.metrics import f1_score\n",
    "\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=2).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "#     print (pred_flat, labels_flat)\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
    "\n",
    "epochs = 15\n",
    "max_grad_norm = 1.0\n",
    "\n",
    "for _ in trange(epochs, desc=\"Epoch\"):\n",
    "    # TRAIN loop\n",
    "    model.train()\n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "        # add batch to gpu\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "        # forward pass\n",
    "        loss = model(b_input_ids, token_type_ids=None,\n",
    "                     attention_mask=b_input_mask, labels=b_labels)\n",
    "        # backward pass\n",
    "        loss.backward()\n",
    "        # track train loss\n",
    "        tr_loss += loss.item()\n",
    "        nb_tr_examples += b_input_ids.size(0)\n",
    "        nb_tr_steps += 1\n",
    "        # gradient clipping\n",
    "        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n",
    "        # update parameters\n",
    "        optimizer.step()\n",
    "        model.zero_grad()\n",
    "    # print train loss per epoch\n",
    "    print(\"Train loss: {}\".format(tr_loss / nb_tr_steps))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function receives a sentence with its labels, and the tokenized sentence and labels\n",
    "def aggr_toks_labels_tags(orig_words, orig_labels, tok_wordps, tok_labels, predicted_tags):\n",
    "    \n",
    "    joint_tokens = []\n",
    "    joint_labels = []\n",
    "    joint_predicted = []\n",
    "#     joint_test = []\n",
    "    \n",
    "    for word in orig_words:\n",
    "        aggregated_tokenized = \"\"\n",
    "        aggregated_label = \"\"\n",
    "        aggregated_predicted = \"\"\n",
    "        aggregated_test = \"\"\n",
    "        \n",
    "        while aggregated_tokenized != word:\n",
    "#             print(len(tok_sent))\n",
    "            tmpTok = tok_wordps.pop(0)\n",
    "#             print(tmpTok)\n",
    "#             print(joint_tokens)\n",
    "            if tmpTok.startswith(\"##\"):\n",
    "                tmpTok = tmpTok[2:]\n",
    "                \n",
    "            tmpLab = tok_labels.pop(0)\n",
    "#             if aggregated_label == \"\":\n",
    "            aggregated_label += '^'\n",
    "            aggregated_label += tmpLab\n",
    "\n",
    "                \n",
    "            tmpPred = predicted_tags.pop(0)\n",
    "#             print(tmpPred)\n",
    "\n",
    "            aggregated_predicted += '^'\n",
    "            aggregated_predicted += tmpPred\n",
    "#             if aggregated_predicted == \"\":\n",
    "#                 aggregated_predicted = tmpPred\n",
    "                \n",
    "#             tmpTest = test_tags.pop(0)\n",
    "#             if aggregated_test == \"\":\n",
    "#                 aggregated_test = tmpTest\n",
    "                \n",
    "            aggregated_tokenized += tmpTok\n",
    "#             print(aggregated_tokenized)\n",
    "            \n",
    "        joint_tokens.append(aggregated_tokenized)\n",
    "        joint_labels.append(aggregated_label)\n",
    "        joint_predicted.append(aggregated_predicted)\n",
    "#         joint_test.append(aggregated_test)\n",
    "        \n",
    "    assert len(joint_tokens) == len(orig_words)\n",
    "    assert len(joint_tokens) == len(joint_predicted)\n",
    "    return joint_tokens, joint_labels, joint_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=2).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
    "\n",
    "def delete_pads_from_preds(predicted_tags, test_tags):\n",
    "    clean_predicted = []\n",
    "    clean_test = []\n",
    "    \n",
    "    for ix in range(0, len(test_tags)):\n",
    "        if test_tags[ix] != 'PAD':\n",
    "            clean_predicted.append(predicted_tags[ix])\n",
    "            clean_test.append(test_tags[ix])\n",
    "            \n",
    "    return clean_predicted, clean_test\n",
    "\n",
    "def delete_pads(predicted_tags):\n",
    "    clean_predicted = []    \n",
    "    for ix in range(0, len(test_tags)):\n",
    "        if test_tags[ix] != 'PAD':\n",
    "            clean_predicted.append(predicted_tags[ix])            \n",
    "    return clean_predicted\n",
    "\n",
    "    \n",
    "def calculate_accuracy(df):\n",
    "    numOfCorrectPredictions = 0\n",
    "    for index in df.index:\n",
    "        orig_pos = df.at[index, 'test_tag']\n",
    "        pred_pos = df.at[index, 'predicted_tag']\n",
    "        if orig_pos == pred_pos:\n",
    "            numOfCorrectPredictions += 1\n",
    "    return numOfCorrectPredictions/len(df)\n",
    "                \n",
    "def test_model(sentence, labels, tok_sent, tok_labels, corres_tokens, sent_id):\n",
    "    input_ids, tags, attention_masks = pad_sentences_and_labels([tok_sent], [tok_labels])\n",
    "\n",
    "    val_inputs = torch.tensor(input_ids, dtype=torch.long)\n",
    "    val_tags = torch.tensor(tags, dtype=torch.long)\n",
    "    val_masks = torch.tensor(attention_masks, dtype=torch.long)\n",
    "\n",
    "    test_data = TensorDataset(val_inputs, val_masks, val_tags)\n",
    "    test_sampler = SequentialSampler(test_data)\n",
    "    test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=bs)\n",
    "\n",
    "    model.eval()\n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    nb_eval_steps, nb_eval_examples = 0, 0\n",
    "    predictions, true_labels = [], []\n",
    "    counter = 0\n",
    "    for batch in test_dataloader:\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "\n",
    "        with torch.no_grad():\n",
    "            tmp_eval_loss = model(b_input_ids, token_type_ids=None,\n",
    "                                attention_mask=b_input_mask, labels=b_labels)\n",
    "            logits = model(b_input_ids, token_type_ids=None,\n",
    "                         attention_mask=b_input_mask)\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "        predictions.append([list(p) for p in np.argmax(logits, axis=2)])\n",
    "        \n",
    "        true_labels.append(label_ids)\n",
    "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
    "\n",
    "        eval_loss += tmp_eval_loss.mean().item()\n",
    "        eval_accuracy += tmp_eval_accuracy\n",
    "\n",
    "        nb_eval_examples += b_input_ids.size(0)\n",
    "        nb_eval_steps += 1\n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    \n",
    "    pred_tags = [idx2tag[p_ii] for p in predictions for p_i in p for p_ii in p_i]\n",
    "#     test_tags = [idx2tag[l_ii] for l in true_labels for l_i in l for l_ii in l_i]\n",
    "#     print(list(zip(pred_tags, test_tags)))\n",
    "    # -----------------------------------------------------------------------\n",
    "#     clean_predicted, clean_test = delete_pads_from_preds(pred_tags, test_tags)\n",
    "#     clean_predicted = delete_pads(pred_tags)\n",
    "\n",
    "#     joint_tokenized, joint_labels, preds, tests = aggr_toks_labels_tags(sentence, labels, tok_sent, tok_labels, \n",
    "#                                                                         clean_predicted, clean_test)\n",
    "    joint_tokenized, joint_labels, preds = aggr_toks_labels_tags(sentence, labels, tok_sent, tok_labels, \n",
    "                                                                        pred_tags)\n",
    "    \n",
    "    tmp = {'word': sentence, 'orig_label': labels, 'predicted_tag': preds, \n",
    "           'corresToken': corres_tokens, 'sent_id': sent_id}\n",
    "    tmp_df = pd.DataFrame(data=tmp)\n",
    "    # -----------------------------------------------------------------------\n",
    "    \n",
    "#     y_true = pd.Series(test_tags)\n",
    "#     y_pred = pd.Series(pred_tags)\n",
    "#     cross_tab = pd.crosstab(y_true, y_pred, rownames=['Real Label'], colnames=['Prediction'], margins=True)\n",
    "#     report = classification_report(y_true, y_pred)\n",
    "#     print(report)\n",
    "#     print(tmp_df)\n",
    "    return tmp_df\n",
    "\n",
    "\n",
    "\n",
    "# full_df\n",
    "# f1_accuracy = calculate_accuracy(full_df)\n",
    "# print(\"Accuracy (F1): = {}\".format(f1_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_dev_df = pd.DataFrame()\n",
    "dev_tokenized_texts, dev_tokenized_labels = tokenize(dev_sentences, dev_labels, \n",
    "                                                     gold_dev_sentences, gold_dev_labels, gold_dev_corresTokens)\n",
    "\n",
    "# print(len(dev_tokenized_texts), len(dev_tokenized_labels), len(dev_sentences))\n",
    "for sent, label, tok_sent, tok_label, corresTokens, sent_id in zip(dev_sentences, \n",
    "                                                                   dev_labels, \n",
    "                                                                   dev_tokenized_texts, \n",
    "                                                                   dev_tokenized_labels, \n",
    "                                                                   dev_corresTokens, \n",
    "                                                                   dev_sent_ids):\n",
    "    eval_df = test_model(sent, label, tok_sent, tok_label, corresTokens, sent_id)\n",
    "    full_dev_df = full_dev_df.append(eval_df, ignore_index=True, sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_test_df = pd.DataFrame()\n",
    "test_tokenized_texts, test_tokenized_labels = tokenize(test_sentences, test_labels, \n",
    "                                                     gold_test_sentences, gold_test_labels, gold_test_corresTokens)\n",
    "\n",
    "# print(len(dev_tokenized_texts), len(dev_tokenized_labels), len(dev_sentences))\n",
    "for sent, label, tok_sent, tok_label, corresTokens, sent_id in zip(test_sentences, \n",
    "                                                                   test_labels, \n",
    "                                                                   test_tokenized_texts, \n",
    "                                                                   test_tokenized_labels, \n",
    "                                                                   test_corresTokens, \n",
    "                                                                   test_sent_ids):\n",
    "    eval_df = test_model(sent, label, tok_sent, tok_label, corresTokens, sent_id)\n",
    "    full_test_df = full_test_df.append(eval_df, ignore_index=True, sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "full_dev_df.iloc[-50:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from more_itertools import unique_everseen\n",
    "\n",
    "def unique_vals_to_list(df):\n",
    "    for index in df.index:\n",
    "        joint_pred = df.at[index, 'predicted_tag']\n",
    "        joint_orig = df.at[index, 'orig_label']\n",
    "        \n",
    "        predicted_tag_list = joint_pred.split('^')\n",
    "        predicted_tag_list_no_empty = list(filter(None, predicted_tag_list))\n",
    "        original_tag_list = joint_orig.split('^')\n",
    "        original_tag_list_no_empty = list(filter(None, original_tag_list))\n",
    "\n",
    "        \n",
    "        df.at[index, 'predicted_tag'] = list(unique_everseen(predicted_tag_list_no_empty))\n",
    "        df.at[index, 'orig_label'] = list(unique_everseen(original_tag_list_no_empty))\n",
    "        \n",
    "        \n",
    "unique_vals_to_list(full_dev_df)\n",
    "unique_vals_to_list(full_test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "full_test_df.tail(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exact_match_accuracy(df):\n",
    "    exact_matches = 0\n",
    "    for index in df.index:\n",
    "        if df.at[index, 'orig_label'] == df.at[index, 'predicted_tag']:\n",
    "            exact_matches += 1\n",
    "            \n",
    "    return exact_matches\n",
    "\n",
    "print(\"DEV - Exact Match Accuracy = {0:.2f}%\".format(exact_match_accuracy(full_dev_df)/len(full_dev_df) * 100))\n",
    "print(\"TEST - Exact Match Accuracy = {0:.2f}%\".format(exact_match_accuracy(full_test_df)/len(full_test_df) * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def existence_accuracy(df):\n",
    "    # correct tag = appeared in predicted and in gold\n",
    "    total_orig_num_of_labels = 0\n",
    "    total_predicted_num_of_labels = 0\n",
    "    total_num_of_correct_tags = 0\n",
    "    \n",
    "    for index in df.index:\n",
    "        orig_list = df.at[index, 'orig_label']\n",
    "        predicted_list = df.at[index, 'predicted_tag']\n",
    "        total_orig_num_of_labels += len(orig_list)\n",
    "        total_predicted_num_of_labels += len(predicted_list)\n",
    "        total_num_of_correct_tags += len(set(orig_list).intersection(set(predicted_list)))\n",
    "        \n",
    "    precision = total_num_of_correct_tags / total_predicted_num_of_labels * 100\n",
    "    recall = total_num_of_correct_tags / total_orig_num_of_labels * 100\n",
    "    f1 = 2*precision*recall/(precision+recall)\n",
    "    \n",
    "    print(\"Precision: {0:.2f}%\".format(precision))\n",
    "    print(\"Recall: {0:.2f}%\".format(recall))\n",
    "    print(\"F1: {0:.2f}%\".format(f1))\n",
    "    \n",
    "print(\"DEV:\")\n",
    "existence_accuracy(full_dev_df)\n",
    "print(\"TEST:\")\n",
    "existence_accuracy(full_test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "morphbert",
   "language": "python",
   "name": "morphbert"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}