BERT_Embeddings_with_CNN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT_Embeddings_with_CNN.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0rJZiQ0HQkC2"
      },
      "source": [
        "## Imports and Installs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ftv4ES5VRQtQ"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ltx6VL6LR4ig"
      },
      "source": [
        "!pip install tensorflow\n",
        "!pip install transformers\n",
        "!pip install keras\n",
        "!pip install torch"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o31hsdTb3cmB"
      },
      "source": [
        "import os\n",
        "import re\n",
        "import csv\n",
        "import codecs\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem import SnowballStemmer\n",
        "from string import punctuation\n",
        "\n",
        "from gensim.models import KeyedVectors\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation\n",
        "from keras.layers.merge import concatenate\n",
        "from keras.models import Model\n",
        "from keras.layers.normalization import BatchNormalization\n",
        "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
        "from keras.layers import Embedding\n",
        "from keras.layers import Dense, Input, Flatten\n",
        "from keras.layers import Conv1D, MaxPooling1D, Embedding, Dropout, LSTM, GRU, Bidirectional\n",
        "from keras.models import Model\n",
        "from keras import backend as K\n",
        "from keras.engine.topology import Layer, InputSpec\n",
        "from keras import initializers\n",
        "import tensorflow as tf\n",
        "\n",
        "from transformers import BertTokenizer\n",
        "from transformers.tokenization_bert import BertTokenizer\n",
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "import torch.nn as nn\n",
        "from torch.nn import CrossEntropyLoss\n",
        "from sklearn.metrics import classification_report"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WSZlgvciQn0h"
      },
      "source": [
        "## Load data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "axxoIfftlqnj"
      },
      "source": [
        "train_data = pd.read_csv('/content/gdrive/MyDrive/Combined_FAANG_percentage_2.2.csv', sep=',')\n",
        "print(\"Number of training examples {}\".format(len(train_data)))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qk8e-ylyGwRo"
      },
      "source": [
        "Drop rows before a particular date, according to the duration considered\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ik0PYqBY3cln"
      },
      "source": [
        "train_data.drop(train_data[train_data['Date'] <= '2018-07-20'].index, inplace = True)\n",
        "train_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tgOOKFFBGxw8"
      },
      "source": [
        "Drop neutrals and replace negative label -1 to 0\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tru24p1m3clp"
      },
      "source": [
        "train_data.drop(train_data[train_data['label'] == 0].index, inplace = True)\n",
        "train_data[\"label\"].replace({-1: 0}, inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J8WiTjXZGzbi"
      },
      "source": [
        "Consider whole data and shuffle it\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_9tZfBMK3clr"
      },
      "source": [
        "train_data = train_data.sample(frac=1)\n",
        "train_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_L2mSrvbG1DM"
      },
      "source": [
        "Drop NaN messages\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PMdpvq743clt"
      },
      "source": [
        "train_data = train_data.dropna(subset=['message'])\n",
        "train_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "esxPuIss0RpY"
      },
      "source": [
        "## Get BERT Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rVbGsHVLB5eV"
      },
      "source": [
        "model1 = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels = 2, output_attentions = False, output_hidden_states = False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZgNEv-HuG2sa"
      },
      "source": [
        "Get word embeddings from the model\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RuHo6RunB9we"
      },
      "source": [
        "bert_embeddings1 = list(model1.children())[0]\n",
        "word_embeddings1 = list(bert_embeddings1.children())[0]\n",
        "word_embeddings = word_embeddings1.word_embeddings.weight.data.numpy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "89iFPAqf4FiY"
      },
      "source": [
        "print(word_embeddings.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4wAaFbVMRPMC"
      },
      "source": [
        "## Split data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XQpR-zuj3cl8"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_test, y_train, y_test = train_test_split(train_data['message'], train_data['label'], test_size=0.1, random_state=42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VZnjAhssG4X-"
      },
      "source": [
        "Create dataframes for training messages, training labels and testing messages\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Upp1V4Ics_8i"
      },
      "source": [
        "df_train = pd.DataFrame(X_train,columns=['message'])\n",
        "df_test = pd.DataFrame(X_test,columns=['message'])\n",
        "y_train = pd.DataFrame(y_train,columns=['label'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2t1T9bwPs9jg"
      },
      "source": [
        "print(\"Number of training examples {}\".format(len(X_train)))\n",
        "print(\"Number of testing examples {}\".format(len(X_test)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cf6qL9e23cl-"
      },
      "source": [
        "df_train = pd.DataFrame(y_train, columns=['label'])\n",
        "df_train['label'].value_counts()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "USanarOs3cmA"
      },
      "source": [
        "df_test = pd.DataFrame(y_test,columns=['label'])\n",
        "df_test['label'].value_counts()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "61z_5eEkSIzw"
      },
      "source": [
        "## Prepare data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lYGuNGkM3cmD"
      },
      "source": [
        "MAX_SEQUENCE_LENGTH = 160 # size of vector\n",
        "MAX_VOCAB_SIZE = word_embeddings.shape[0] # number of unique words to use\n",
        "EMBEDDING_DIM = word_embeddings.shape[1] # size of embedding\n",
        "VALIDATION_SPLIT = 0.1 # 10% of training data is used for validation"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DtEgptJbG6Sl"
      },
      "source": [
        "Create BERT Tokenizer\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RdqSUifIUXb9"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XxS9fYOZG7mM"
      },
      "source": [
        "Pad training data \r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ccRrilphpYbj"
      },
      "source": [
        "train_cnn_data = pad_sequences([tokenizer.convert_tokens_to_ids(tokenizer.tokenize(txt)) for txt in X_train], maxlen = MAX_SEQUENCE_LENGTH, dtype = \"long\", truncating = \"post\", padding = \"post\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RnOd4OvOvA10"
      },
      "source": [
        "train_cnn_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nCfOOm4RG9Vh"
      },
      "source": [
        "Pad testing data\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WkPTwY4wq4y1"
      },
      "source": [
        "test_cnn_data = pad_sequences([tokenizer.convert_tokens_to_ids(tokenizer.tokenize(txt)) for txt in X_test],maxlen=MAX_SEQUENCE_LENGTH, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "test_cnn_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cpF10oBjAS_c"
      },
      "source": [
        "df_train = train_cnn_data\n",
        "y_tr = y_train.label.values\n",
        "y_tr"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n_vzi6yBtRgz"
      },
      "source": [
        "word_index = tokenizer.get_vocab()\n",
        "print(word_index)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cU50ezwGG-nv"
      },
      "source": [
        "Assign embedding matrix\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NaFTEwtBsu2h"
      },
      "source": [
        "embedding_matrix = word_embeddings\n",
        "embedding_matrix"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ITXY9HzNSv2i"
      },
      "source": [
        "## Convolutional Neural Network (CNN)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oHLk9oFM5FVE"
      },
      "source": [
        "from keras.layers import concatenate\n",
        "\n",
        "def ConvNet(embeddings, max_sequence_length, num_words, embedding_dim, labels_index, trainable=False, extra_conv=True):\n",
        "    \n",
        "    embedding_layer = Embedding(num_words,\n",
        "                            embedding_dim,\n",
        "                            weights=[embedding_matrix],\n",
        "                            input_length=max_sequence_length,\n",
        "                            trainable=trainable)\n",
        "\n",
        "    sequence_input = Input(shape=(max_sequence_length,), dtype='int32')\n",
        "    embedded_sequences = embedding_layer(sequence_input)\n",
        "\n",
        "    # Yoon Kim model (https://arxiv.org/abs/1408.5882)\n",
        "    convs = []\n",
        "    filter_sizes = [3,4,5]\n",
        "\n",
        "    for filter_size in filter_sizes:\n",
        "        l_conv = Conv1D(filters=128, kernel_size=filter_size, activation='relu')(embedded_sequences)\n",
        "        l_pool = MaxPooling1D(pool_size=3)(l_conv)\n",
        "        convs.append(l_pool)\n",
        "\n",
        "    l_merge = concatenate(convs, axis = 1)\n",
        "\n",
        "    # Add one dimensional convolutional network with global maxpooling, instead of \n",
        "    conv = Conv1D(filters=128, kernel_size=3, activation='relu')(embedded_sequences)\n",
        "    pool = MaxPooling1D(pool_size=3)(conv)\n",
        "\n",
        "    if extra_conv==True:\n",
        "        x = Dropout(0.5)(l_merge)  \n",
        "    else:\n",
        "        # Yoon Kim model\n",
        "        x = Dropout(0.5)(pool)\n",
        "    x = Flatten()(x)\n",
        "    x = Dense(128, activation='relu')(x)\n",
        "    x = Dropout(0.5)(x)\n",
        "\n",
        "    # Feed output to sigmoid layer to compress output between 0 and 1 for binary classification\n",
        "    preds = Dense(labels_index, activation='sigmoid')(x)\n",
        "\n",
        "    model = Model(sequence_input, preds)\n",
        "    model.compile(loss='binary_crossentropy',\n",
        "                  optimizer='adam',\n",
        "                  metrics=['acc'])\n",
        "    model.summary()\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ohtOdhV15VWl"
      },
      "source": [
        "label_names = ['label']\n",
        "model = ConvNet(embedding_matrix, MAX_SEQUENCE_LENGTH, MAX_VOCAB_SIZE, EMBEDDING_DIM, \n",
        "                len(list(label_names)), False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wqr_Nr-XHDdK"
      },
      "source": [
        "Define callbacks\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0u_35HmU6f4j"
      },
      "source": [
        "early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.01, patience=4, verbose=1)\n",
        "callbacks_list = [early_stopping]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F3UVChnMHE8u"
      },
      "source": [
        "Train the model\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "pGVFty-K6xc2"
      },
      "source": [
        "hist = model.fit(df_train, y_tr, epochs=2, callbacks=callbacks_list, validation_split=0.1, shuffle=True, batch_size=32)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZB5--G3bHGp2"
      },
      "source": [
        "Predict on test set and print classification report\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9a5y3_-B73tu"
      },
      "source": [
        "preds = model.predict(test_cnn_data, batch_size=8, verbose=1)\n",
        "preds"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g7lQ3hqR07BA"
      },
      "source": [
        "y_pred = (preds > 0.53) * 1.0 # classify prediction above 0.53 as class 1, else class 0\n",
        "y_pred.sum()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eqLMlmxr1BiF"
      },
      "source": [
        "print(classification_report(y_test, y_pred,zero_division=1))"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}