train_and_eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "variable-cross",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import BertTokenizer, BertModel\n",
    "import torch\n",
    "import pandas as pd\n",
    "import csv\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "from torch.utils import data\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from sklearn.metrics import classification_report\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "available-wages",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN SAMPLES: 540\n",
      "TEST SAMPLES: 135\n"
     ]
    }
   ],
   "source": [
    "#Uncomment for Aspect Category Experiments\n",
    "# train = 'data/aspect_final_train.csv'\n",
    "# test = 'data/aspect_final_test.csv'\n",
    "\n",
    "#Uncomment for Polarity Experiment\n",
    "train = 'data/polarity_final_train.csv'\n",
    "test = 'data/polarity_final_test.csv'\n",
    "\n",
    "with open(train) as fp:\n",
    "    reader = csv.reader(fp, delimiter=\",\", quotechar='\"')\n",
    "    train_data = [row for row in reader]\n",
    "with open(test) as fp:\n",
    "    reader = csv.reader(fp, delimiter=\",\", quotechar='\"')\n",
    "    test_data = [row for row in reader]\n",
    "    \n",
    "print(\"TRAIN SAMPLES: {}\".format(len(train_data)))\n",
    "print(\"TEST SAMPLES: {}\".format(len(test_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "altered-asian",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_aspect(index_string, full_text):\n",
    "    indices = index_string.replace('[','').replace(']','').split(',')\n",
    "    aspect = ''\n",
    "    for index in indices:\n",
    "        aspect+= full_text.split()[int(index)] + ' '\n",
    "    return aspect.strip()\n",
    "\n",
    "def get_pos(sent_list,aspect_list):\n",
    "    first_pos = sent_list.index(aspect_list[0])\n",
    "    final_pos = []\n",
    "    for i in range(0,len(aspect_list)):\n",
    "        final_pos.append(first_pos+i)\n",
    "    return final_pos    \n",
    "\n",
    "class TalkLitDataset(data.Dataset):\n",
    "    def __init__(self, tagged_sents):\n",
    "        sents, aspects, tags = [], [], [] # list of lists\n",
    "        for sent in tagged_sents:\n",
    "            sent_tokens = tokenizer.encode(sent[0])\n",
    "            try:\n",
    "                aspect_tokens = tokenizer.encode(get_aspect(sent[1],sent[0]))[1:-1]\n",
    "            except:\n",
    "                print(sent[1])\n",
    "                print(sent[0])\n",
    "                continue\n",
    "            pos_aspects = get_pos(sent_tokens, aspect_tokens)\n",
    "            tag = sent[2]\n",
    "            sents.append(sent_tokens)\n",
    "            aspects.append(pos_aspects)\n",
    "            tags.append(tag)\n",
    "            \n",
    "        self.sents, self.aspects, self.tags = sents, aspects, tags\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.sents)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        words, aspects, tags = self.sents[idx], self.aspects[idx], tag2idx[self.tags[idx]] # words, tags: string list\n",
    "        return words, aspects, tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "municipal-delaware",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Positive': 0, 'Negative': 1}\n"
     ]
    }
   ],
   "source": [
    "tags = []\n",
    "for dat in train_data:\n",
    "    tags.append(dat[2])\n",
    "    \n",
    "tags = list(set(tags))\n",
    "\",\".join(tags)\n",
    "tag2idx = {tag:idx for idx, tag in enumerate(tags)}\n",
    "idx2tag = {idx:tag for idx, tag in enumerate(tags)}\n",
    "print(tag2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "absolute-ceremony",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "renewable-valley",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-german-cased\")\n",
    "model = BertModel.from_pretrained(\"bert-base-german-cased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "comparable-knock",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, vocab_size=None):\n",
    "        super().__init__()\n",
    "        self.model = BertModel.from_pretrained(\"bert-base-german-cased\", output_hidden_states=True)\n",
    "        self.fc = nn.Linear(768, vocab_size)\n",
    "        self.device = device\n",
    "\n",
    "    def forward(self, sent, aspects, y):\n",
    "        sent = torch.LongTensor(sent).to(device)\n",
    "        y = torch.LongTensor(y).to(device)\n",
    "        input_ids = sent.unsqueeze(0)  # Batch size 1\n",
    "        with torch.no_grad():\n",
    "            outputs = self.model(input_ids)\n",
    "            last_hidden_states = outputs.last_hidden_state[0]  # The last hidden-state is the first element of the output tuple\n",
    "            start = 0\n",
    "            end = len(last_hidden_states)-1\n",
    "            context_window = 5\n",
    "            \n",
    "            if aspects[0]-context_window>0:\n",
    "                start = aspects[0]-context_window\n",
    "            if aspects[-1]+context_window<len(last_hidden_states)-1:\n",
    "                end = aspects[-1]+context_window\n",
    "                \n",
    "            all_aspects = []\n",
    "            for i in range(start,end):\n",
    "                all_aspects.append(i)\n",
    "            bert_embeds = torch.zeros(len(all_aspects),768)\n",
    "            for i, aspect in enumerate(all_aspects):\n",
    "                bert_embeds[i] = last_hidden_states[aspect]\n",
    "                \n",
    "            bert_embeds = torch.mean(bert_embeds, axis=0)\n",
    "        logits = self.fc(bert_embeds)\n",
    "        y_hat = logits.argmax(-1)\n",
    "        return logits, y, y_hat, bert_embeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cosmetic-french",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = TalkLitDataset(train_data)\n",
    "train_iter = data.DataLoader(dataset=train_dataset,\n",
    "                             batch_size=1,\n",
    "                             shuffle=True,\n",
    "                             num_workers=0)\n",
    "\n",
    "test_dataset = TalkLitDataset(test_data)\n",
    "test_iter = data.DataLoader(dataset=test_dataset,\n",
    "                             batch_size=1,\n",
    "                             shuffle=False,\n",
    "                             num_workers=0)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "veterinary-saying",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "540it [01:11,  7.60it/s]\n",
      "135it [00:16,  8.17it/s]\n"
     ]
    }
   ],
   "source": [
    "model = Net(vocab_size=len(tag2idx))\n",
    "model.to(device)\n",
    "model = nn.DataParallel(model)\n",
    "\n",
    "X_train = []\n",
    "y_train = []\n",
    "X_test = []\n",
    "y_test = []\n",
    "\n",
    "for i, batch in tqdm.tqdm(enumerate(train_iter)):\n",
    "        words, aspects, y = batch\n",
    "        _y = y\n",
    "        logits, y, _, bert_embeds = model(words, aspects, y)\n",
    "        logits = logits.view(-1, logits.shape[-1])\n",
    "        y = y.view(-1)\n",
    "        bert_embeds = bert_embeds.cpu().numpy()\n",
    "        y = int(y.cpu().numpy()[0])\n",
    "        X_train.append(bert_embeds)\n",
    "        y_train.append(y)\n",
    "        \n",
    "for i, batch in tqdm.tqdm(enumerate(test_iter)):\n",
    "        words, aspects, y = batch\n",
    "        _y = y\n",
    "        logits, y, _, bert_embeds = model(words, aspects, y)\n",
    "        logits = logits.view(-1, logits.shape[-1])\n",
    "        y = y.view(-1)\n",
    "        bert_embeds = bert_embeds.cpu().numpy()\n",
    "        y = int(y.cpu().numpy()[0])\n",
    "        X_test.append(bert_embeds)\n",
    "        y_test.append(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "seven-change",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CLASSIFIER: Linear SVM\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.68      0.85      0.75        79\n",
      "           1       0.67      0.43      0.52        56\n",
      "\n",
      "    accuracy                           0.67       135\n",
      "   macro avg       0.67      0.64      0.64       135\n",
      "weighted avg       0.67      0.67      0.66       135\n",
      "\n",
      "CLASSIFIER: Neural Net\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      0.77      0.73        79\n",
      "           1       0.61      0.50      0.55        56\n",
      "\n",
      "    accuracy                           0.66       135\n",
      "   macro avg       0.65      0.64      0.64       135\n",
      "weighted avg       0.65      0.66      0.65       135\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.gaussian_process import GaussianProcessClassifier\n",
    "from sklearn.gaussian_process.kernels import RBF\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "import random\n",
    "\n",
    "names = [\"Linear SVM\", \"Neural Net\"]\n",
    "\n",
    "\n",
    "classifiers = [\n",
    "    SVC(kernel=\"linear\", C=0.025),\n",
    "    MLPClassifier(alpha=1, max_iter=1000),\n",
    "    AdaBoostClassifier()\n",
    "]\n",
    "\n",
    "for name, clf in zip(names, classifiers):\n",
    "    clf.fit(X_train, y_train)\n",
    "    predicted = clf.predict(X_test)\n",
    "    print(\"CLASSIFIER: {}\".format(name))\n",
    "    print(classification_report(y_test, predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "lined-wiring",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_______________________________________________________\n",
      "TEXT: Armer #Nolte liest nach einem Zwischenfall - Kollaps eines Gastes - weiter. Das hat der Autor nicht verdient- egal wie gut der Text ist, der Fluss ist dahin.  #tddl\n",
      "Armer #Nolte liest nach einem Zwischenfall - Kollaps eines Gastes - weiter. Das hat der Autor nicht verdient- egal wie gut der Text ist, der Fluss ist dahin.  #tddl\n",
      "ASPECT: #Nolte\n",
      "PREDICTION: Negative\n",
      "ACTUAL: Negative\n",
      "******************\n",
      "Armer #Nolte liest nach einem Zwischenfall - Kollaps eines Gastes - weiter. Das hat der Autor nicht verdient- egal wie gut der Text ist, der Fluss ist dahin.  #tddl\n",
      "ASPECT: Fluss\n",
      "PREDICTION: Negative\n",
      "ACTUAL: Negative\n",
      "******************\n",
      "_______________________________________________________\n",
      "TEXT: Kann die @DB_Bahn bitte daf�r sorgen, dass morgen im ICE das WLAN so gut ist, dass ich #tddl schauen kann? \n",
      "Ja? \n",
      "Cool, danke! :)\n",
      "Kann die @DB_Bahn bitte daf�r sorgen, dass morgen im ICE das WLAN so gut ist, dass ich #tddl schauen kann? \n",
      "Ja? \n",
      "Cool, danke! :)\n",
      "ASPECT: #tddl\n",
      "PREDICTION: Positive\n",
      "ACTUAL: Positive\n",
      "******************\n",
      "_______________________________________________________\n",
      "TEXT: Kollektives Aufatmen: Endlich mal eine klassisch erz�hlte Geschichte. #tddl\n",
      "Kollektives Aufatmen: Endlich mal eine klassisch erz�hlte Geschichte. #tddl\n",
      "ASPECT: klassisch erz�hlte Geschichte.\n",
      "PREDICTION: Negative\n",
      "ACTUAL: Positive\n",
      "******************\n",
      "_______________________________________________________\n",
      "TEXT: @Wolfseule jedes jahr denke ich mir \"ach, es ist wieder #tddl und niemand hat mich eingeladen!\" und eigentlich sollte mich beides nicht wundern\n",
      "@Wolfseule jedes jahr denke ich mir \"ach, es ist wieder #tddl und niemand hat mich eingeladen!\" und eigentlich sollte mich beides nicht wundern\n",
      "ASPECT: #tddl\n",
      "PREDICTION: Positive\n",
      "ACTUAL: Negative\n",
      "******************\n",
      "_______________________________________________________\n",
      "TEXT: JETZT live im Deutschlandfunk: Literaturredakteur @gerritbartels3 (@Tagesspiegel) und Fotograf Michael J. Stephan streiten �ber die Frage \"Braucht Literatur Wettbewerbe?\"\n",
      "\n",
      "Mit dabei: ein Wutausbruch von Bewerbs-Teilnehmerin Katharina Schultens. #tddl #tddl2019\n",
      "JETZT live im Deutschlandfunk: Literaturredakteur @gerritbartels3 (@Tagesspiegel) und Fotograf Michael J. Stephan streiten �ber die Frage \"Braucht Literatur Wettbewerbe?\"\n",
      "\n",
      "Mit dabei: ein Wutausbruch von Bewerbs-Teilnehmerin Katharina Schultens. #tddl #tddl2019\n",
      "ASPECT: Wutausbruch von Bewerbs-Teilnehmerin Katharina Schultens.\n",
      "PREDICTION: Positive\n",
      "ACTUAL: Negative\n",
      "******************\n"
     ]
    }
   ],
   "source": [
    "test_set_outputs = []\n",
    "for i, row in enumerate(test_data):\n",
    "    row.extend([predicted[i]])\n",
    "    test_set_outputs.append(row)\n",
    "    \n",
    "for i in range(0,5):\n",
    "    example = random.choice(test_set_outputs)\n",
    "    print(\"_______________________________________________________\")\n",
    "    print(\"TEXT: {}\".format(example[0]))\n",
    "    for j in test_set_outputs:\n",
    "        if j[0] == example[0]:\n",
    "            print(j[0])\n",
    "            print(\"ASPECT: {}\".format(get_aspect(j[1],example[0])))\n",
    "            print(\"PREDICTION: {}\".format(idx2tag[j[-1]]))\n",
    "            print(\"ACTUAL: {}\".format(j[2]))\n",
    "            print(\"******************\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aboriginal-cholesterol",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}