Word2Vec_CNN_Model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Word2Vec_CNN_Model.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mjag7682/CS9-1-NLP-for-Twitter-Data-for-predicting-stocks/blob/master/Word2Vec_CNN_Model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lDxpNBpz4Eqi"
      },
      "source": [
        "##Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wnwKpL4FQYSX"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import torch\n",
        "import tensorflow as tf"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6SZr2opcMfS6"
      },
      "source": [
        "# Mounting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qjEx8w8rMf7V",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6a587041-ce0c-4131-ae9d-43a189637c48"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive',force_remount=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PskirMDs4NI1"
      },
      "source": [
        "#Load FAANG Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xPB2nwZTQdrp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "be2ad969-8af9-493b-9cd1-ec144a876b27"
      },
      "source": [
        "import numpy as np\n",
        "# data = pd.read_csv('/content/drive/My Drive/Data/Combined_FAANG_percentage_2.2.csv')\n",
        "data = pd.read_csv('/content/drive/My Drive/Data/Combined_FAANG_binary_previous.csv')\n",
        "\n",
        "print(\"Number of examples {}\".format(len(data)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of examples 2566858\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8jz-yD_xhrDZ"
      },
      "source": [
        "Limiting dataset to 1 year"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tq9iNdXvQjhh",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "5d3122b2-ef71-4260-c269-d4db3ea3b9bb"
      },
      "source": [
        "# data = data[data[\"symbol\"].isin(['AAPL', 'FB', 'AMZN', 'NFLX', 'GOOGL'])]\n",
        "\n",
        "data.drop(data[data['Date'] < '2019-07-20'].index, inplace = True)\n",
        "data"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>symbol</th>\n",
              "      <th>message</th>\n",
              "      <th>datetime</th>\n",
              "      <th>user</th>\n",
              "      <th>message_id</th>\n",
              "      <th>Date</th>\n",
              "      <th>Time</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>AAPL</td>\n",
              "      <td>aapl</td>\n",
              "      <td>2020-01-27 07:07:03</td>\n",
              "      <td>1229493.0</td>\n",
              "      <td>191978042.0</td>\n",
              "      <td>2020-01-27</td>\n",
              "      <td>07:07:03</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>AAPL</td>\n",
              "      <td>qq became euphoric calls exp week aiming ath f...</td>\n",
              "      <td>2020-05-13 02:13:00</td>\n",
              "      <td>2250451.0</td>\n",
              "      <td>212222428.0</td>\n",
              "      <td>2020-05-13</td>\n",
              "      <td>02:13:00</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>AAPL</td>\n",
              "      <td>spy novices like davey day trader lose money s...</td>\n",
              "      <td>2020-06-24 11:12:09</td>\n",
              "      <td>543250.0</td>\n",
              "      <td>222404886.0</td>\n",
              "      <td>2020-06-24</td>\n",
              "      <td>11:12:09</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>AAPL</td>\n",
              "      <td>today aapl shows buy signal ta short term tech...</td>\n",
              "      <td>2019-09-11 09:33:43</td>\n",
              "      <td>700679.0</td>\n",
              "      <td>176835918.0</td>\n",
              "      <td>2019-09-11</td>\n",
              "      <td>09:33:43</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>AAPL</td>\n",
              "      <td>aapl let melt begin</td>\n",
              "      <td>2020-01-03 16:46:16</td>\n",
              "      <td>741099.0</td>\n",
              "      <td>188910094.0</td>\n",
              "      <td>2020-01-03</td>\n",
              "      <td>16:46:16</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2566840</th>\n",
              "      <td>NFLX</td>\n",
              "      <td>hope aside nyt putting also cnbc puts nflx take</td>\n",
              "      <td>2019-09-21 21:49:00</td>\n",
              "      <td>2807876.0</td>\n",
              "      <td>178253955.0</td>\n",
              "      <td>2019-09-21</td>\n",
              "      <td>21:49:00</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2566841</th>\n",
              "      <td>NFLX</td>\n",
              "      <td>amd msft nflx everything beautiful 39 way</td>\n",
              "      <td>2020-03-06 06:06:03</td>\n",
              "      <td>911299.0</td>\n",
              "      <td>198589415.0</td>\n",
              "      <td>2020-03-06</td>\n",
              "      <td>06:06:03</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2566848</th>\n",
              "      <td>NFLX</td>\n",
              "      <td>nflx full move key support</td>\n",
              "      <td>2020-03-12 16:52:43</td>\n",
              "      <td>677915.0</td>\n",
              "      <td>199933357.0</td>\n",
              "      <td>2020-03-12</td>\n",
              "      <td>16:52:43</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2566849</th>\n",
              "      <td>NFLX</td>\n",
              "      <td>spy spx nflx nvda virtually volume today absen...</td>\n",
              "      <td>2019-10-14 18:16:28</td>\n",
              "      <td>55818.0</td>\n",
              "      <td>180328889.0</td>\n",
              "      <td>2019-10-14</td>\n",
              "      <td>18:16:28</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2566857</th>\n",
              "      <td>NFLX</td>\n",
              "      <td>nflx stock unstable ups downs like ex girlfriend</td>\n",
              "      <td>2020-02-27 14:52:21</td>\n",
              "      <td>2153160.0</td>\n",
              "      <td>196927202.0</td>\n",
              "      <td>2020-02-27</td>\n",
              "      <td>14:52:21</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>514645 rows × 8 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        symbol  ... label\n",
              "3         AAPL  ...     0\n",
              "8         AAPL  ...     0\n",
              "9         AAPL  ...     0\n",
              "10        AAPL  ...     1\n",
              "11        AAPL  ...     0\n",
              "...        ...  ...   ...\n",
              "2566840   NFLX  ...     0\n",
              "2566841   NFLX  ...     0\n",
              "2566848   NFLX  ...     0\n",
              "2566849   NFLX  ...     1\n",
              "2566857   NFLX  ...     0\n",
              "\n",
              "[514645 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y2_PApn-hwSd"
      },
      "source": [
        "Dropping Neutral (wherever required)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JSaUZIGPQmUK"
      },
      "source": [
        "# data.drop(data[data['label'] == 0].index, inplace = True) \n",
        "# data[\"label\"].replace({-1: 0}, inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A6F6lxBsQntM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "9cd138b6-a3ff-42ae-f98a-84907e77a890"
      },
      "source": [
        "data = data.sample(frac=1)\n",
        "data"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>symbol</th>\n",
              "      <th>message</th>\n",
              "      <th>datetime</th>\n",
              "      <th>user</th>\n",
              "      <th>message_id</th>\n",
              "      <th>Date</th>\n",
              "      <th>Time</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2078476</th>\n",
              "      <td>NFLX</td>\n",
              "      <td>nflx back 335</td>\n",
              "      <td>2019-07-26 21:10:36</td>\n",
              "      <td>2125598.0</td>\n",
              "      <td>171955278.0</td>\n",
              "      <td>2019-07-26</td>\n",
              "      <td>21:10:36</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2168255</th>\n",
              "      <td>NFLX</td>\n",
              "      <td>8 latest short seller targets bx v nflx fdx bmy</td>\n",
              "      <td>2019-09-30 16:56:52</td>\n",
              "      <td>7108.0</td>\n",
              "      <td>179047530.0</td>\n",
              "      <td>2019-09-30</td>\n",
              "      <td>16:56:52</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1643232</th>\n",
              "      <td>AMZN</td>\n",
              "      <td>amzn disney shares fall earnings miss</td>\n",
              "      <td>2019-08-06 20:19:47</td>\n",
              "      <td>316034.0</td>\n",
              "      <td>173169674.0</td>\n",
              "      <td>2019-08-06</td>\n",
              "      <td>20:19:47</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>933245</th>\n",
              "      <td>AAPL</td>\n",
              "      <td>tsla aapl spy amd amzn lol course</td>\n",
              "      <td>2020-03-09 20:52:49</td>\n",
              "      <td>277257.0</td>\n",
              "      <td>199186640.0</td>\n",
              "      <td>2020-03-09</td>\n",
              "      <td>20:52:49</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2339611</th>\n",
              "      <td>NFLX</td>\n",
              "      <td>nflx long long time since rsi low</td>\n",
              "      <td>2019-09-19 13:55:28</td>\n",
              "      <td>391844.0</td>\n",
              "      <td>178015795.0</td>\n",
              "      <td>2019-09-19</td>\n",
              "      <td>13:55:28</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>521672</th>\n",
              "      <td>AAPL</td>\n",
              "      <td>spy djia ba aapl 39 weird people living payche...</td>\n",
              "      <td>2020-03-23 03:17:34</td>\n",
              "      <td>2266176.0</td>\n",
              "      <td>202033813.0</td>\n",
              "      <td>2020-03-23</td>\n",
              "      <td>03:17:34</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2141752</th>\n",
              "      <td>NFLX</td>\n",
              "      <td>nflx bounced support closed resistance expecti...</td>\n",
              "      <td>2019-07-29 03:08:46</td>\n",
              "      <td>2185664.0</td>\n",
              "      <td>172058434.0</td>\n",
              "      <td>2019-07-29</td>\n",
              "      <td>03:08:46</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1243666</th>\n",
              "      <td>AMZN</td>\n",
              "      <td>amzn new article amazon put tesla notice</td>\n",
              "      <td>2020-07-21 13:07:59</td>\n",
              "      <td>1555408.0</td>\n",
              "      <td>229495210.0</td>\n",
              "      <td>2020-07-21</td>\n",
              "      <td>13:07:59</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>907693</th>\n",
              "      <td>AAPL</td>\n",
              "      <td>aapl breaks 22 5 huge squeeze imminent</td>\n",
              "      <td>2019-09-30 14:32:49</td>\n",
              "      <td>2110951.0</td>\n",
              "      <td>179027010.0</td>\n",
              "      <td>2019-09-30</td>\n",
              "      <td>14:32:49</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>371891</th>\n",
              "      <td>AAPL</td>\n",
              "      <td>aapl power hour let us get strong close</td>\n",
              "      <td>2020-06-08 19:06:07</td>\n",
              "      <td>660144.0</td>\n",
              "      <td>218103000.0</td>\n",
              "      <td>2020-06-08</td>\n",
              "      <td>19:06:07</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>514645 rows × 8 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        symbol  ... label\n",
              "2078476   NFLX  ...     1\n",
              "2168255   NFLX  ...     1\n",
              "1643232   AMZN  ...     1\n",
              "933245    AAPL  ...     0\n",
              "2339611   NFLX  ...     0\n",
              "...        ...  ...   ...\n",
              "521672    AAPL  ...     0\n",
              "2141752   NFLX  ...     0\n",
              "1243666   AMZN  ...     0\n",
              "907693    AAPL  ...     1\n",
              "371891    AAPL  ...     1\n",
              "\n",
              "[514645 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PtHSNmPcQrJt",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "c3799e5a-e620-4813-b0ae-20eada38ffc7"
      },
      "source": [
        "data = data.dropna(subset=['message'])  \n",
        "data"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>symbol</th>\n",
              "      <th>message</th>\n",
              "      <th>datetime</th>\n",
              "      <th>user</th>\n",
              "      <th>message_id</th>\n",
              "      <th>Date</th>\n",
              "      <th>Time</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2078476</th>\n",
              "      <td>NFLX</td>\n",
              "      <td>nflx back 335</td>\n",
              "      <td>2019-07-26 21:10:36</td>\n",
              "      <td>2125598.0</td>\n",
              "      <td>171955278.0</td>\n",
              "      <td>2019-07-26</td>\n",
              "      <td>21:10:36</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2168255</th>\n",
              "      <td>NFLX</td>\n",
              "      <td>8 latest short seller targets bx v nflx fdx bmy</td>\n",
              "      <td>2019-09-30 16:56:52</td>\n",
              "      <td>7108.0</td>\n",
              "      <td>179047530.0</td>\n",
              "      <td>2019-09-30</td>\n",
              "      <td>16:56:52</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1643232</th>\n",
              "      <td>AMZN</td>\n",
              "      <td>amzn disney shares fall earnings miss</td>\n",
              "      <td>2019-08-06 20:19:47</td>\n",
              "      <td>316034.0</td>\n",
              "      <td>173169674.0</td>\n",
              "      <td>2019-08-06</td>\n",
              "      <td>20:19:47</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>933245</th>\n",
              "      <td>AAPL</td>\n",
              "      <td>tsla aapl spy amd amzn lol course</td>\n",
              "      <td>2020-03-09 20:52:49</td>\n",
              "      <td>277257.0</td>\n",
              "      <td>199186640.0</td>\n",
              "      <td>2020-03-09</td>\n",
              "      <td>20:52:49</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2339611</th>\n",
              "      <td>NFLX</td>\n",
              "      <td>nflx long long time since rsi low</td>\n",
              "      <td>2019-09-19 13:55:28</td>\n",
              "      <td>391844.0</td>\n",
              "      <td>178015795.0</td>\n",
              "      <td>2019-09-19</td>\n",
              "      <td>13:55:28</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>521672</th>\n",
              "      <td>AAPL</td>\n",
              "      <td>spy djia ba aapl 39 weird people living payche...</td>\n",
              "      <td>2020-03-23 03:17:34</td>\n",
              "      <td>2266176.0</td>\n",
              "      <td>202033813.0</td>\n",
              "      <td>2020-03-23</td>\n",
              "      <td>03:17:34</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2141752</th>\n",
              "      <td>NFLX</td>\n",
              "      <td>nflx bounced support closed resistance expecti...</td>\n",
              "      <td>2019-07-29 03:08:46</td>\n",
              "      <td>2185664.0</td>\n",
              "      <td>172058434.0</td>\n",
              "      <td>2019-07-29</td>\n",
              "      <td>03:08:46</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1243666</th>\n",
              "      <td>AMZN</td>\n",
              "      <td>amzn new article amazon put tesla notice</td>\n",
              "      <td>2020-07-21 13:07:59</td>\n",
              "      <td>1555408.0</td>\n",
              "      <td>229495210.0</td>\n",
              "      <td>2020-07-21</td>\n",
              "      <td>13:07:59</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>907693</th>\n",
              "      <td>AAPL</td>\n",
              "      <td>aapl breaks 22 5 huge squeeze imminent</td>\n",
              "      <td>2019-09-30 14:32:49</td>\n",
              "      <td>2110951.0</td>\n",
              "      <td>179027010.0</td>\n",
              "      <td>2019-09-30</td>\n",
              "      <td>14:32:49</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>371891</th>\n",
              "      <td>AAPL</td>\n",
              "      <td>aapl power hour let us get strong close</td>\n",
              "      <td>2020-06-08 19:06:07</td>\n",
              "      <td>660144.0</td>\n",
              "      <td>218103000.0</td>\n",
              "      <td>2020-06-08</td>\n",
              "      <td>19:06:07</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>514645 rows × 8 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        symbol  ... label\n",
              "2078476   NFLX  ...     1\n",
              "2168255   NFLX  ...     1\n",
              "1643232   AMZN  ...     1\n",
              "933245    AAPL  ...     0\n",
              "2339611   NFLX  ...     0\n",
              "...        ...  ...   ...\n",
              "521672    AAPL  ...     0\n",
              "2141752   NFLX  ...     0\n",
              "1243666   AMZN  ...     0\n",
              "907693    AAPL  ...     1\n",
              "371891    AAPL  ...     1\n",
              "\n",
              "[514645 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MQj-lp76paKw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2d7f000b-649d-4375-afef-3a5e2a3c462f"
      },
      "source": [
        "data.label.value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1    283541\n",
              "0    231104\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ABx-nUaQtOA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "703cab31-43d6-4ed8-b8be-ede4d2c61032"
      },
      "source": [
        "print('Positive samples: %d of %d (%.2f%%)' % (data.label.sum(), len(data.label), (data.label.sum() / len(data.label) * 100.0)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Positive samples: 283541 of 514645 (55.09%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RdlQJAA7h6rW"
      },
      "source": [
        "Splitting dataset into 90-10 "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KoN6oT8WDy9A"
      },
      "source": [
        " from sklearn.model_selection import train_test_split\n",
        " X_train, X_test, y_train, y_test = train_test_split(data['message'], data['label'], test_size=0.1, random_state=42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EIhBgVPtEtpr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "08991494-25c5-460d-c24f-a0bd9e7c1cba"
      },
      "source": [
        "print(\"Number of training examples {}\".format(len(X_train)))\n",
        "print(\"Number of testing examples {}\".format(len(X_test)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of training examples 463180\n",
            "Number of testing examples 51465\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YEVWuUmEV4vm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1e4b692a-50ac-414d-8d98-179642a1617b"
      },
      "source": [
        "y_test"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "612463     0\n",
              "2057093    1\n",
              "1344873    1\n",
              "1649199    0\n",
              "1192622    1\n",
              "          ..\n",
              "2340538    0\n",
              "593263     0\n",
              "1731787    1\n",
              "1913447    1\n",
              "27080      1\n",
              "Name: label, Length: 51465, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U8AvHLKyiA1z"
      },
      "source": [
        "Creating training and testing dataframes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pCy1hfeQTuUH",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "617ff06d-3469-48e1-a8f6-85a6040a04aa"
      },
      "source": [
        "df_train = pd.DataFrame(X_train,columns=['message'])\n",
        "df_train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>message</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2047154</th>\n",
              "      <td>would say amzn 39 essentials roku nflx</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1989285</th>\n",
              "      <td>nflx give</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1839479</th>\n",
              "      <td>amazon looks ready 40 rally says market watche...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1457195</th>\n",
              "      <td>amzn independent bookstores get creative survi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2009364</th>\n",
              "      <td>nflx first weekly close june 39 18 buying clim...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1728294</th>\n",
              "      <td>spy dis aapl tsla another fantastic testimonia...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1243445</th>\n",
              "      <td>amzn nothing better hours move amazon great in...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>832551</th>\n",
              "      <td>aapl apple tv premium shows comparison 39 watc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2088843</th>\n",
              "      <td>impressive aapl fb amzn nflx amp goog red appl...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2110691</th>\n",
              "      <td>nflx spxl rhinoceros nflx 275p mar 20 day trad...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>463180 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   message\n",
              "2047154             would say amzn 39 essentials roku nflx\n",
              "1989285                                          nflx give\n",
              "1839479  amazon looks ready 40 rally says market watche...\n",
              "1457195  amzn independent bookstores get creative survi...\n",
              "2009364  nflx first weekly close june 39 18 buying clim...\n",
              "...                                                    ...\n",
              "1728294  spy dis aapl tsla another fantastic testimonia...\n",
              "1243445  amzn nothing better hours move amazon great in...\n",
              "832551   aapl apple tv premium shows comparison 39 watc...\n",
              "2088843  impressive aapl fb amzn nflx amp goog red appl...\n",
              "2110691  nflx spxl rhinoceros nflx 275p mar 20 day trad...\n",
              "\n",
              "[463180 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jo3F53oeUczr",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "969207c2-9c21-4400-cacd-fbae5d977c12"
      },
      "source": [
        "df_test = pd.DataFrame(X_test,columns=['message'])\n",
        "df_test"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>message</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>612463</th>\n",
              "      <td>whole rally made 5 dead cats day amd I good go...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2057093</th>\n",
              "      <td>nflx buyer valuation growth good metric high c...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1344873</th>\n",
              "      <td>v amzn aapl pypl 22 sec ago europe must wean g...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1649199</th>\n",
              "      <td>uavs backed amzn 10 bagger easy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1192622</th>\n",
              "      <td>aapl really hope smart people bought beautiful...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2340538</th>\n",
              "      <td>nflx thinking 229 eod 336 eow</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>593263</th>\n",
              "      <td>aapl let us</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1731787</th>\n",
              "      <td>dis bounce coming</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1913447</th>\n",
              "      <td>nflx looking like buy imo</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27080</th>\n",
              "      <td>aapl well going</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>51465 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   message\n",
              "612463   whole rally made 5 dead cats day amd I good go...\n",
              "2057093  nflx buyer valuation growth good metric high c...\n",
              "1344873  v amzn aapl pypl 22 sec ago europe must wean g...\n",
              "1649199                    uavs backed amzn 10 bagger easy\n",
              "1192622  aapl really hope smart people bought beautiful...\n",
              "...                                                    ...\n",
              "2340538                      nflx thinking 229 eod 336 eow\n",
              "593263                                         aapl let us\n",
              "1731787                                  dis bounce coming\n",
              "1913447                          nflx looking like buy imo\n",
              "27080                                      aapl well going\n",
              "\n",
              "[51465 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8tr-iCDgYizl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "39f35175-b4ad-46a7-aeb6-054cfd00479d"
      },
      "source": [
        "y_train = pd.DataFrame(y_train,columns=['label'])\n",
        "y_train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2047154</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1989285</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1839479</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1457195</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2009364</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1728294</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1243445</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>832551</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2088843</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2110691</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>463180 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         label\n",
              "2047154      1\n",
              "1989285      0\n",
              "1839479      1\n",
              "1457195      1\n",
              "2009364      0\n",
              "...        ...\n",
              "1728294      1\n",
              "1243445      1\n",
              "832551       1\n",
              "2088843      0\n",
              "2110691      0\n",
              "\n",
              "[463180 rows x 1 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wQQaAV7Z4RVT"
      },
      "source": [
        "#Import Libraries for CNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kmwl6HZpE04x"
      },
      "source": [
        "import numpy as np # linear algebra\n",
        "import keras\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "from keras.layers import Dense, Input, Flatten, Dropout, Concatenate\n",
        "from keras.layers import Conv1D, MaxPooling1D, Embedding\n",
        "from keras.layers import LSTM, Bidirectional\n",
        "from keras.models import Model\n",
        "from keras.callbacks import EarlyStopping\n",
        "import gensim\n",
        "import nltk\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "import re\n",
        "import codecs\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2lwuesyyFg11"
      },
      "source": [
        "EMBEDDING_DIM = 300 \n",
        "MAX_VOCAB_SIZE = 175303 \n",
        "MAX_SEQUENCE_LENGTH = 160 \n",
        "\n",
        "#training params\n",
        "batch_size = 64 \n",
        "num_epochs = 2 "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8npk2I23Fjjy"
      },
      "source": [
        "tokenizer = RegexpTokenizer(r'\\w+')\n",
        "df_train[\"tokens\"] = df_train[\"message\"].apply(tokenizer.tokenize)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "toNOnQFbF70r",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "f3de3c50-6166-48cf-b755-09c7fb71677e"
      },
      "source": [
        "df_train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>message</th>\n",
              "      <th>tokens</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2047154</th>\n",
              "      <td>would say amzn 39 essentials roku nflx</td>\n",
              "      <td>[would, say, amzn, 39, essentials, roku, nflx]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1989285</th>\n",
              "      <td>nflx give</td>\n",
              "      <td>[nflx, give]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1839479</th>\n",
              "      <td>amazon looks ready 40 rally says market watche...</td>\n",
              "      <td>[amazon, looks, ready, 40, rally, says, market...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1457195</th>\n",
              "      <td>amzn independent bookstores get creative survi...</td>\n",
              "      <td>[amzn, independent, bookstores, get, creative,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2009364</th>\n",
              "      <td>nflx first weekly close june 39 18 buying clim...</td>\n",
              "      <td>[nflx, first, weekly, close, june, 39, 18, buy...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1728294</th>\n",
              "      <td>spy dis aapl tsla another fantastic testimonia...</td>\n",
              "      <td>[spy, dis, aapl, tsla, another, fantastic, tes...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1243445</th>\n",
              "      <td>amzn nothing better hours move amazon great in...</td>\n",
              "      <td>[amzn, nothing, better, hours, move, amazon, g...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>832551</th>\n",
              "      <td>aapl apple tv premium shows comparison 39 watc...</td>\n",
              "      <td>[aapl, apple, tv, premium, shows, comparison, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2088843</th>\n",
              "      <td>impressive aapl fb amzn nflx amp goog red appl...</td>\n",
              "      <td>[impressive, aapl, fb, amzn, nflx, amp, goog, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2110691</th>\n",
              "      <td>nflx spxl rhinoceros nflx 275p mar 20 day trad...</td>\n",
              "      <td>[nflx, spxl, rhinoceros, nflx, 275p, mar, 20, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>463180 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   message                                             tokens\n",
              "2047154             would say amzn 39 essentials roku nflx     [would, say, amzn, 39, essentials, roku, nflx]\n",
              "1989285                                          nflx give                                       [nflx, give]\n",
              "1839479  amazon looks ready 40 rally says market watche...  [amazon, looks, ready, 40, rally, says, market...\n",
              "1457195  amzn independent bookstores get creative survi...  [amzn, independent, bookstores, get, creative,...\n",
              "2009364  nflx first weekly close june 39 18 buying clim...  [nflx, first, weekly, close, june, 39, 18, buy...\n",
              "...                                                    ...                                                ...\n",
              "1728294  spy dis aapl tsla another fantastic testimonia...  [spy, dis, aapl, tsla, another, fantastic, tes...\n",
              "1243445  amzn nothing better hours move amazon great in...  [amzn, nothing, better, hours, move, amazon, g...\n",
              "832551   aapl apple tv premium shows comparison 39 watc...  [aapl, apple, tv, premium, shows, comparison, ...\n",
              "2088843  impressive aapl fb amzn nflx amp goog red appl...  [impressive, aapl, fb, amzn, nflx, amp, goog, ...\n",
              "2110691  nflx spxl rhinoceros nflx 275p mar 20 day trad...  [nflx, spxl, rhinoceros, nflx, 275p, mar, 20, ...\n",
              "\n",
              "[463180 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JcvAfTAxGXzc"
      },
      "source": [
        "df_test[\"tokens\"] = df_test[\"message\"].apply(tokenizer.tokenize)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hZJqcf-ZGkRc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "09ee6cac-f824-4010-80e6-5dcde20c5980"
      },
      "source": [
        "training_word_list = [word for tokens in df_train[\"tokens\"] for word in tokens]\n",
        "training_sentence_lengths = [len(tokens) for tokens in df_train[\"tokens\"]]\n",
        "vocab_train = sorted(list(set(training_word_list)))\n",
        "print(\"%s words total, with a vocabulary size of %s\" % (len(training_word_list), len(vocab_train)))\n",
        "print(\"Max sentence length is %s\" % max(training_sentence_lengths))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "6034876 words total, with a vocabulary size of 74460\n",
            "Max sentence length is 789\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dqNQGEzRGrvV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5bddbce3-a95f-49fd-dde4-5a858aee7ab9"
      },
      "source": [
        "test_word_list = [word for tokens in df_test[\"tokens\"] for word in tokens]\n",
        "test_sentence_lengths = [len(tokens) for tokens in df_test[\"tokens\"]]\n",
        "vocab_test = sorted(list(set(test_word_list)))\n",
        "print(\"%s words total, with a vocabulary size of %s\" % (len(test_word_list), len(vocab_test)))\n",
        "print(\"Max sentence length is %s\" % max(test_sentence_lengths))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "665914 words total, with a vocabulary size of 28309\n",
            "Max sentence length is 183\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ssk43QAPiITX"
      },
      "source": [
        "# Loading Word2Vec Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cEg1u-kiGy5Y"
      },
      "source": [
        "word2vec_path = \"/content/drive/My Drive/Word2Vec/GoogleNews-vectors-negative300.bin\"\n",
        "word2vec = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True)\n",
        "\n",
        "def get_average_word2vec(tokens_list, vector, generate_missing=False, k=300):\n",
        "    if len(tokens_list)<1:\n",
        "        return np.zeros(k)\n",
        "    if generate_missing:\n",
        "        vectorized = [vector[word] if word in vector else np.random.rand(k) for word in tokens_list]\n",
        "    else:\n",
        "        vectorized = [vector[word] if word in vector else np.zeros(k) for word in tokens_list]\n",
        "    length = len(vectorized)\n",
        "    summed = np.sum(vectorized, axis=0)\n",
        "    averaged = np.divide(summed, length)\n",
        "    return averaged\n",
        "\n",
        "def get_word2vec_embeddings(vectors, clean_message, generate_missing=False):\n",
        "    embeddings = clean_message['tokens'].apply(lambda x: get_average_word2vec(x, vectors, \n",
        "                                                                                generate_missing=generate_missing))\n",
        "    return list(embeddings)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bCcLqFrniPB5"
      },
      "source": [
        "Creating embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zO2iZqQkMl9x"
      },
      "source": [
        "training_embeddings = get_word2vec_embeddings(word2vec, df_train, generate_missing=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b1XwMeCuMmk9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4babb71f-25b7-4516-8416-05c328865ad7"
      },
      "source": [
        "tokenizer = Tokenizer(num_words=MAX_VOCAB_SIZE, lower=True, char_level=False)\n",
        "tokenizer.fit_on_texts(df_train['message'].tolist())\n",
        "training_sequences = tokenizer.texts_to_sequences(df_train[\"message\"].tolist())\n",
        "\n",
        "train_word_index = tokenizer.word_index\n",
        "print('Found %s unique tokens.' % len(train_word_index))\n",
        "\n",
        "train_cnn_data = pad_sequences(training_sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
        "\n",
        "train_embedding_weights = np.zeros((len(train_word_index)+1, EMBEDDING_DIM))\n",
        "for word,index in train_word_index.items():\n",
        "    train_embedding_weights[index,:] = word2vec[word] if word in word2vec else np.random.rand(EMBEDDING_DIM)\n",
        "print(train_embedding_weights.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 74389 unique tokens.\n",
            "(74390, 300)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ilYMuMmdr9bC",
        "outputId": "fb3b2719-6f28-470c-8ed9-28600fffac1b"
      },
      "source": [
        "train_embedding_weights"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
              "         0.        ,  0.        ],\n",
              "       [-0.04663086, -0.10058594, -0.06933594, ..., -0.02490234,\n",
              "        -0.15429688,  0.14941406],\n",
              "       [ 0.90292264,  0.53585593,  0.53837297, ...,  0.17690553,\n",
              "         0.76411943,  0.4151133 ],\n",
              "       ...,\n",
              "       [-0.1328125 ,  0.15039062,  0.03857422, ..., -0.24316406,\n",
              "        -0.11865234,  0.12792969],\n",
              "       [-0.08105469,  0.03515625,  0.03515625, ..., -0.05493164,\n",
              "        -0.06176758,  0.12353516],\n",
              "       [ 0.28320312, -0.05883789,  0.08056641, ..., -0.22265625,\n",
              "        -0.00367737,  0.20898438]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6SdTnNMXMrQe"
      },
      "source": [
        "test_sequences = tokenizer.texts_to_sequences(df_test[\"message\"].tolist())\n",
        "test_cnn_data = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U-d-MnRriRy9"
      },
      "source": [
        "# Building CNN model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GILVnHErMuY2"
      },
      "source": [
        "from keras.layers import concatenate\n",
        "def ConvNet(embeddings, max_sequence_length, num_words, embedding_dim, labels_index, trainable=False, extra_conv=True):\n",
        "    \n",
        "    embedding_layer = Embedding(num_words,\n",
        "                            embedding_dim,\n",
        "                            weights=[embeddings],\n",
        "                            input_length=max_sequence_length,\n",
        "                            trainable=trainable)\n",
        "\n",
        "    sequence_input = Input(shape=(max_sequence_length,), dtype='int32')\n",
        "    embedded_sequences = embedding_layer(sequence_input)\n",
        "\n",
        "    convs = []\n",
        "    filter_sizes = [3,4,5]\n",
        "\n",
        "    for filter_size in filter_sizes:\n",
        "        l_conv = Conv1D(filters=128, kernel_size=filter_size, activation='relu')(embedded_sequences)\n",
        "        l_pool = MaxPooling1D(pool_size=3)(l_conv)\n",
        "        convs.append(l_pool)\n",
        "\n",
        "    l_merge = concatenate(convs, axis = 1)\n",
        "    # add(mode='concat', concat_axis=1)(convs)\n",
        "\n",
        "    # add a 1D convnet with global maxpooling, instead of Yoon Kim model\n",
        "    conv = Conv1D(filters=128, kernel_size=3, activation='relu')(embedded_sequences)\n",
        "    pool = MaxPooling1D(pool_size=3)(conv)\n",
        "\n",
        "    if extra_conv==True:\n",
        "        x = Dropout(0.5)(l_merge)  \n",
        "    else:\n",
        "        # Original Yoon Kim model\n",
        "        x = Dropout(0.5)(pool)\n",
        "    x = Flatten()(x)\n",
        "    x = Dense(128, activation='relu')(x)\n",
        "    x = Dropout(0.5)(x)\n",
        "    # Finally, we feed the output into a Sigmoid layer.\n",
        "    # The reason why sigmoid is used is because we are trying to achieve a binary classification(1,0) \n",
        "    # for each of the 6 labels, and the sigmoid function will squash the output between the bounds of 0 and 1.\n",
        "    preds = Dense(labels_index, activation='sigmoid')(x)\n",
        "\n",
        "    model = Model(sequence_input, preds)\n",
        "    model.compile(loss='binary_crossentropy',\n",
        "                  optimizer='adam',\n",
        "                  metrics=['acc'])\n",
        "    model.summary()\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NImAc_21tLkw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "042728a1-9009-44d3-ae13-6f785dc8aa99"
      },
      "source": [
        "train_cnn_data"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[   0,    0,    0, ..., 5469,   94,    3],\n",
              "       [   0,    0,    0, ...,    0,    3,  268],\n",
              "       [   0,    0,    0, ...,    2,   18,   59],\n",
              "       ...,\n",
              "       [   0,    0,    0, ...,    7,   22, 1124],\n",
              "       [   0,    0,    0, ...,  420,  708,  415],\n",
              "       [   0,    0,    0, ..., 3951,  262, 2338]], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oOeNfFDHM1Ax"
      },
      "source": [
        "df_train = train_cnn_data\n",
        "y_tr = y_train.label.values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uso1wXwYY0mq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "17a84ac2-20f5-4287-a949-e607a608bd5a"
      },
      "source": [
        "y_tr"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 0, 1, ..., 1, 0, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rzAycJ1sVkC2"
      },
      "source": [
        "label_names=['label']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ai3iCVTcM4d2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "04725d78-2e92-4d84-afa6-cfee1d6f5613"
      },
      "source": [
        "model = ConvNet(train_embedding_weights, MAX_SEQUENCE_LENGTH, len(train_word_index)+1, EMBEDDING_DIM, \n",
        "                len(list(label_names)), False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 160)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding (Embedding)           (None, 160, 300)     29863800    input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv1d (Conv1D)                 (None, 158, 128)     115328      embedding[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv1d_1 (Conv1D)               (None, 157, 128)     153728      embedding[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv1d_2 (Conv1D)               (None, 156, 128)     192128      embedding[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling1d (MaxPooling1D)    (None, 52, 128)      0           conv1d[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling1d_1 (MaxPooling1D)  (None, 52, 128)      0           conv1d_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling1d_2 (MaxPooling1D)  (None, 52, 128)      0           conv1d_2[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 156, 128)     0           max_pooling1d[0][0]              \n",
            "                                                                 max_pooling1d_1[0][0]            \n",
            "                                                                 max_pooling1d_2[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "dropout (Dropout)               (None, 156, 128)     0           concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "flatten (Flatten)               (None, 19968)        0           dropout[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 128)          2556032     flatten[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dropout_1 (Dropout)             (None, 128)          0           dense[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 1)            129         dropout_1[0][0]                  \n",
            "==================================================================================================\n",
            "Total params: 32,881,145\n",
            "Trainable params: 3,017,345\n",
            "Non-trainable params: 29,863,800\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wdvxFbqkM488"
      },
      "source": [
        "\n",
        "early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.01, patience=4, verbose=1)\n",
        "callbacks_list = [early_stopping]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HfrnGFDriXU-"
      },
      "source": [
        "Model Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FNESfB2xM7iA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1668c00a-e555-4196-9cf0-603ccf29bdce"
      },
      "source": [
        "hist = model.fit(df_train, y_tr, epochs=num_epochs, callbacks=callbacks_list, validation_split=0.1, shuffle=True, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            "12951/12951 [==============================] - 2529s 195ms/step - loss: 0.6910 - acc: 0.5353 - val_loss: 0.6902 - val_acc: 0.5371\n",
            "Epoch 2/2\n",
            "12951/12951 [==============================] - 2503s 193ms/step - loss: 0.6901 - acc: 0.5373 - val_loss: 0.6895 - val_acc: 0.5371\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yX7sRXiHX8Ss",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1b13ba4a-6ca0-416a-97e9-0f810a2e6286"
      },
      "source": [
        "df_train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[    0,     0,     0, ...,   253, 12931, 12314],\n",
              "       [    0,     0,     0, ...,    92,    61,    27],\n",
              "       [    0,     0,     0, ...,    26,   445,   296],\n",
              "       ...,\n",
              "       [    0,     0,     0, ...,     0,     0,     3],\n",
              "       [    0,     0,     0, ...,  2884,   138,     1],\n",
              "       [    0,     0,     0, ...,  1961,  1523,  2049]], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k_BGwBK_M-46",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "08d15978-aeae-4526-c757-2ceb4a0a2d5c"
      },
      "source": [
        "y_predict_test = model.predict(test_cnn_data, batch_size=256, verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "400/400 [==============================] - 94s 235ms/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pY43WWtSmLi-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "64c5b744-bb93-4d19-9f70-521377582936"
      },
      "source": [
        "model.evaluate(test_cnn_data, y_test, verbose=0)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.6895556449890137, 0.5368821024894714]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b-c2WcmSoVi6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7482a6bb-dc6b-461c-cfe0-3208a547d8cf"
      },
      "source": [
        "y_predict_test"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.53484327],\n",
              "       [0.53484327],\n",
              "       [0.5911659 ],\n",
              "       ...,\n",
              "       [0.53601116],\n",
              "       [0.53484327],\n",
              "       [0.53484327]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_oq6Wdkeot_g",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "728d815d-4f73-45d5-9d55-9062c3488777"
      },
      "source": [
        "#y_pred = np.argmax(Y_pred, axis=1) - used for multiclass\n",
        "y_pred = (y_predict_test > 0.53) * 1.0\n",
        "# y_pred = y_pred.reshape(y_val.shape)\n",
        "y_pred.sum()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "96082.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hvxhrbJCiagD"
      },
      "source": [
        "# Classification Report"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lm6qUghhwijn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e45b8f20-9917-447b-e2b3-733d967841d7"
      },
      "source": [
        "from sklearn.metrics import classification_report\n",
        "print(classification_report(y_test, y_pred,zero_division=1))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.54      0.07      0.13     47417\n",
            "           1       0.54      0.95      0.69     54909\n",
            "\n",
            "    accuracy                           0.54    102326\n",
            "   macro avg       0.54      0.51      0.41    102326\n",
            "weighted avg       0.54      0.54      0.43    102326\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LgxpYg8xyFSO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c6807533-0906-4cc8-b9e6-a9345bfd5179"
      },
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "print('Confusion Matrix')\n",
        "print(confusion_matrix(y_test, y_pred))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Confusion Matrix\n",
            "[[12273  4675]\n",
            " [15347  8663]]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}