Labelling_Approach1_Binary_Classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Labelling_Approach1_Binary_Classification.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dYMyLfkPGyTj"
      },
      "source": [
        "## Imports and Installs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ob4jaL_oduCY"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y_p2JuHIrhEp"
      },
      "source": [
        "# Install yfinance API\n",
        "!pip install yfinance --upgrade --no-cache-dir"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I02KuiHhHdaO"
      },
      "source": [
        "import pandas as pd\n",
        "import datetime\n",
        "import yfinance as yf\n",
        "import numpy as np\n",
        "from pandas_datareader import data as pdr"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ajwodq2OG1HY"
      },
      "source": [
        "## Load a company's data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mo6yym-RzokJ"
      },
      "source": [
        "# Sample company - WMT\n",
        "company = 'WMT'\n",
        "df_tweets = pd.read_csv('/content/drive/My Drive/Data from 2010/stocktwits_' + company + '.csv')\n",
        "df_tweets.tail()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mbs3XdAZDgJd"
      },
      "source": [
        "Split datetime into date and time\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wMtN7d3LzvCU"
      },
      "source": [
        "df_tweets['datetime'] = df_tweets['datetime'].astype('datetime64[ns]')\n",
        "df_tweets['Date'] = [d.date() for d in df_tweets['datetime']]\n",
        "df_tweets['Time'] = [d.time() for d in df_tweets['datetime']]\n",
        "df_tweets"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sKnpDpMYiDOw"
      },
      "source": [
        "df_tweets['Day_of_week'] = df_tweets.datetime.dt.day_name()\n",
        "\n",
        "# Get End date\n",
        "if df_tweets.iloc[0][7] == 'Sunday':\n",
        "  endDate = df_tweets.iloc[0][5] + datetime.timedelta(days = 2)\n",
        "elif df_tweets.iloc[0][7] == 'Saturday':\n",
        "  endDate = df_tweets.iloc[0][5] + datetime.timedelta(days = 3)\n",
        "else:\n",
        "  endDate = df_tweets.iloc[0][5] + datetime.timedelta(days = 1)\n",
        "\n",
        "# Get start date\n",
        "if df_tweets.iloc[-1][7] == 'Sunday':\n",
        "  startDate = df_tweets.iloc[-1][5] - datetime.timedelta(days = 2)\n",
        "elif df_tweets.iloc[-1][7] == 'Saturday':\n",
        "  startDate = df_tweets.iloc[-1][5] - datetime.timedelta(days = 1)\n",
        "else:\n",
        "  startDate = df_tweets.iloc[-1][5]\n",
        "\n",
        "print(endDate) \n",
        "print(startDate) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oSNSlrMuHMWF"
      },
      "source": [
        "## Download a company's yahoo finance data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eCp6gDqzrYZF"
      },
      "source": [
        "yahoo_data = yf.download(company, start = startDate, end = endDate)\n",
        "yahoo_data.reset_index(level=0, inplace=True)\n",
        "yahoo_data.columns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g_wHK5MYy2D4"
      },
      "source": [
        "yahoo_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F6CXy7qJJFAA"
      },
      "source": [
        "## Fill missing data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uC9E5diyDjUI"
      },
      "source": [
        "Fill missing dates (weekends and public holidays)\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yOEIn8nTG0JC"
      },
      "source": [
        "start_date = startDate\n",
        "end_date = endDate\n",
        "\n",
        "df_prices = pd.DataFrame(yahoo_data)\n",
        "all_days = pd.date_range(start=start_date, end=end_date, freq='D')\n",
        "df_prices.info\n",
        "\n",
        "df_prices['Date'] =  pd.to_datetime(df_prices['Date'], format='%Y/%m/%d')\n",
        "df_prices = df_prices.sort_values(by=['Date'], ascending=[True])\n",
        "df_prices.set_index('Date', inplace=True)\n",
        "df_prices = df_prices.reindex(all_days).reset_index().rename(columns={\"index\":\"Date\"})\n",
        "df_prices"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZDFkz-_zDlW1"
      },
      "source": [
        "Remove last row with NaN\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UJzpr5QqI4gu"
      },
      "source": [
        "if np.isnan(df_prices.iloc[-1][1]):\n",
        "  df_prices.drop(df_prices.tail(1).index,inplace=True)\n",
        "df_prices"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ysajfuaDnFF"
      },
      "source": [
        "Fill missing values for missing dates by calculating mean of top and bottom row\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aq1keIXRnXOF"
      },
      "source": [
        "for i in range(len(df_prices)):\n",
        "  if np.isnan(df_prices.iloc[i][1]): # Continue if open price is NaN\n",
        "    counter = 1 \n",
        "    while counter != 0:\n",
        "      if ~np.isnan(df_prices.iloc[i+counter][1]): # Find the next non-NaN valued row using counter\n",
        "        for j in range(1, 7):\n",
        "          new = (df_prices.iloc[i - 1][j] + df_prices.iloc[i + counter][j]) / 2 # Calculate mean\n",
        "          df_prices.at[i, df_prices.columns[j]] = new\n",
        "        break\n",
        "      counter += 1\n",
        "df_prices.tail(15)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_eBM9jCMJOXq"
      },
      "source": [
        "## Calculate Price Change"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IhvLrUr3DpLh"
      },
      "source": [
        "Calculate Price Change using Close price and Open Price of Same day\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wXFGaXeEGpD5"
      },
      "source": [
        "# price_change = []\n",
        "# for i in range(len(df_prices)):\n",
        "#   if df_prices.iloc[i][4] > df_prices.iloc[i][1]:\n",
        "#     price_change.append(1)\n",
        "#   else:\n",
        "#     price_change.append(0)\n",
        "# df_prices['PriceChange'] = price_change\n",
        "# df_prices.tail(15)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7RnXnSC4DqzP"
      },
      "source": [
        "Calculate Price Change using Close price of Previous day with Close price of today\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h9Fl6zlITV7_"
      },
      "source": [
        "price_change = []\n",
        "for i in range(len(df_prices)):\n",
        "  if df_prices.iloc[i-1][4] < df_prices.iloc[i][4]:\n",
        "    price_change.append(1)\n",
        "  else:\n",
        "    price_change.append(0)\n",
        "df_prices['PriceChange'] = price_change\n",
        "df_prices.head(15)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XqkbXjIQJUIu"
      },
      "source": [
        "## Assign polarity / label to tweets"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QYX2hkTVDtBk"
      },
      "source": [
        "Function to return Price Change value for a particular date\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mvX962zKdQso"
      },
      "source": [
        "def get_polarity (date):\n",
        "  rslt_df = df_prices[df_prices['Date'] == date] \n",
        "  return rslt_df['PriceChange'].values[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uDTcoIH1DuXy"
      },
      "source": [
        "Assign polarity/label for each row using date\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gja-6z2ww0_v"
      },
      "source": [
        "for i in range(len(df_tweets)):\n",
        "  # print(i)\n",
        "  polarity = get_polarity(df_tweets.iloc[i][5].strftime(\"%Y-%m-%d\"))\n",
        "  df_tweets.at[i, 'polarity'] = polarity\n",
        "df_tweets[:100]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bzQPw-lwDwfB"
      },
      "source": [
        "Check null or NaN values\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YJlYhVqUcx2u"
      },
      "source": [
        "print(df_prices.isna().values.any()) #False\n",
        "print(df_prices.notnull().values.all()) #True\n",
        "print(df_prices.isnull().values.any()) #False"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YJFoFdApDx2g"
      },
      "source": [
        "Convert float values of polarity/label to integer \r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9SxAQbZP3pUl"
      },
      "source": [
        "df_tweets[\"polarity\"] = df_tweets[\"polarity\"].astype(float).astype(int)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8IPiS1h6I-Cf"
      },
      "source": [
        "## Save labelled company file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7pCx7j-lM3pq"
      },
      "source": [
        "df_tweets.to_csv('labelled_WMT.csv')\n",
        "!cp labelled_WMT.csv \"/content/drive/My Drive/Labelled (Binary - previous day open close)\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bpkiMfUzaoKB"
      },
      "source": [
        "df_prices.info()\n",
        "df_tweets.info()"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}