Finance_RNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch import optim\n",
    "import matplotlib\n",
    "import torch.nn.functional as F\n",
    "import datetime as dt\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyutilities as util\n",
    "%matplotlib inline\n",
    "import logging\n",
    "\n",
    "logger = logging.getLogger('F_RNN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class encoder(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, T, logger):\n",
    "        super().__init__()\n",
    "        # input size: number of underlying factors (81)\n",
    "        # T: number of time steps (10)\n",
    "        # hidden_size: dimension of the hidden state\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.T = T\n",
    "\n",
    "        self.logger = logger\n",
    "        self.lstm_layer = nn.LSTM(input_size= input_size,hidden_size = hidden_size, num_layers = 1 )\n",
    "        self.attn_linear = nn.Linear(in_features =2 * hidden_size * T -1, out_features=1)\n",
    "    def forward(self, input_data):\n",
    "        input_weighted = Variable(input_data.data.new(input_data.size(0), self.T - 1, self.input_size).zero_())\n",
    "        input_encoded = Variable(input_data.data.new(input_data.size(0), self.T - 1, self.hidden_size).zero_())\n",
    "        hidden = self.init_hidden(input_data)\n",
    "        cell = self.init_hidden(input_data)\n",
    "        \n",
    "        for t in range(self.T - 1):\n",
    "            x = torch.cat((hidden.repeat(self.input_size,1,1).permute(1,0,2),\n",
    "                          cell.repeat(self.input_size,1,1).permute(1,0,2),\n",
    "                          input_data.permute(0,2,1)), dim=2)\n",
    "            \n",
    "            x = self.attn_linear(x.view(-1,self.hidden_size *2 + self.T -1))\n",
    "            weights = F.softmax(X.view(-1, self.input_size))\n",
    "            weighted_input = torch.mul(attn_weights, input_data[:, t, :])\n",
    "            self.lstm_layer.flatten_parameters()\n",
    "            _, lstm_states = self.lstm_layer(weighted_input.unsqueeze(0), (hidden, cell))\n",
    "            hidden = lstm_states[0]\n",
    "            cell = lstm_states[1]\n",
    "            input_weighted[:,t,:] = weighted_input\n",
    "            input_encoded[:,t,:] = hidden\n",
    "        return input_weighted, input_encoded\n",
    "    def init_hidden(self,x):\n",
    "        return Variable(x.data.new(1, x.size(0), self.hidden_size).zero_())\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class decoder(nn.Module):\n",
    "    def __init__(self, encoder_hidden_size, decoder_hidden_size,T, logger):\n",
    "        super(decoder, self).__init__()\n",
    "        self.T = T\n",
    "        self.encoder_hidden_size = encoder_hidden_size\n",
    "        self.decoder_hidden_size = decoder_hidden_size\n",
    "\n",
    "        self.logger = logger\n",
    "\n",
    "        self.attn_layer = nn.Sequential(nn.Linear(2 * decoder_hidden_size + encoder_hidden_size, encoder_hidden_size),\n",
    "                                         nn.Tanh(), nn.Linear(encoder_hidden_size, 1))\n",
    "        self.lstm_layer = nn.LSTM(input_size = 1, hidden_size = decoder_hidden_size)\n",
    "        self.fc = nn.Linear(encoder_hidden_size + 1, 1)\n",
    "        self.fc_final = nn.Linear(decoder_hidden_size + encoder_hidden_size, 1)\n",
    "\n",
    "        self.fc.weight.data.normal_()\n",
    "    def forward(self, input_encoded, y_history):\n",
    "        # input_encoded: batch_size * T - 1 * encoder_hidden_size\n",
    "        # y_history: batch_size * (T-1)\n",
    "        # Initialize hidden and cell, 1 * batch_size * decoder_hidden_size\n",
    "        hidden = self.init_hidden(input_encoded)\n",
    "        cell = self.init_hidden(input_encoded)\n",
    "        # hidden.requires_grad = False\n",
    "        # cell.requires_grad = False\n",
    "        for t in range(self.T - 1):\n",
    "            # Eqn. 12-13: compute attention weights\n",
    "            ## batch_size * T * (2*decoder_hidden_size + encoder_hidden_size)\n",
    "            x = torch.cat((hidden.repeat(self.T - 1, 1, 1).permute(1, 0, 2),\n",
    "                           cell.repeat(self.T - 1, 1, 1).permute(1, 0, 2), input_encoded), dim = 2)\n",
    "            x = F.softmax(self.attn_layer(x.view(-1, 2 * self.decoder_hidden_size + self.encoder_hidden_size\n",
    "                                                )).view(-1, self.T - 1)) # batch_size * T - 1, row sum up to 1\n",
    "            # Eqn. 14: compute context vector\n",
    "            context = torch.bmm(x.unsqueeze(1), input_encoded)[:, 0, :] # batch_size * encoder_hidden_size\n",
    "            if t < self.T - 1:\n",
    "                # Eqn. 15\n",
    "                y_tilde = self.fc(torch.cat((context, y_history[:, t].unsqueeze(1)), dim = 1)) # batch_size * 1\n",
    "                # Eqn. 16: LSTM\n",
    "                self.lstm_layer.flatten_parameters()\n",
    "                _, lstm_output = self.lstm_layer(y_tilde.unsqueeze(0), (hidden, cell))\n",
    "                hidden = lstm_output[0] # 1 * batch_size * decoder_hidden_size\n",
    "                cell = lstm_output[1] # 1 * batch_size * decoder_hidden_size\n",
    "        # Eqn. 22: final output\n",
    "        y_pred = self.fc_final(torch.cat((hidden[0], context), dim = 1))\n",
    "        # self.logger.info(\"hidden %s context %s y_pred: %s\", hidden[0][0][:10], context[0][:10], y_pred[:10])\n",
    "        return y_pred\n",
    "\n",
    "    def init_hidden(self, x):\n",
    "        return Variable(x.data.new(1, x.size(0), self.decoder_hidden_size).zero_())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class da_rnn:\n",
    "    def __init__(self, file_data, logger, encoder_hidden_size = 64, decoder_hidden_size = 64, T = 10,\n",
    "                 learning_rate = 0.01, batch_size = 128, parallel = True, debug = False):\n",
    "        self.T = T\n",
    "        dat = pd.read_csv(file_data, nrows = 100 if debug else None)\n",
    "        self.logger = logger\n",
    "        self.logger.info(\"Shape of data: %s.\\nMissing in data: %s.\", dat.shape, dat.isnull().sum().sum())\n",
    "\n",
    "        self.X = dat.loc[:, [x for x in dat.columns.tolist() if x != 'NDX']].as_matrix()\n",
    "        self.y = np.array(dat.NDX)\n",
    "        self.batch_size = batch_size\n",
    "\n",
    "        self.encoder = encoder(input_size = self.X.shape[1], hidden_size = encoder_hidden_size, T = T,\n",
    "                              logger = logger).cuda()\n",
    "        self.decoder = decoder(encoder_hidden_size = encoder_hidden_size,\n",
    "                               decoder_hidden_size = decoder_hidden_size,\n",
    "                               T = T, logger = logger).cuda()\n",
    "\n",
    "        if parallel:\n",
    "            self.encoder = nn.DataParallel(self.encoder)\n",
    "            self.decoder = nn.DataParallel(self.decoder)\n",
    "\n",
    "        self.encoder_optimizer = optim.Adam(params = itertools.ifilter(lambda p: p.requires_grad, self.encoder.parameters()),\n",
    "                                           lr = learning_rate)\n",
    "        self.decoder_optimizer = optim.Adam(params = itertools.ifilter(lambda p: p.requires_grad, self.decoder.parameters()),\n",
    "                                           lr = learning_rate)\n",
    "        # self.learning_rate = learning_rate\n",
    "\n",
    "        self.train_size = int(self.X.shape[0] * 0.7)\n",
    "        self.y = self.y - np.mean(self.y[:self.train_size]) # Question: why Adam requires data to be normalized?\n",
    "        self.logger.info(\"Training size: %d.\", self.train_size)\n",
    "\n",
    "    def train(self, n_epochs = 10):\n",
    "        iter_per_epoch = int(np.ceil(self.train_size * 1. / self.batch_size))\n",
    "        logger.info(\"Iterations per epoch: %3.3f ~ %d.\", self.train_size * 1. / self.batch_size, iter_per_epoch)\n",
    "        self.iter_losses = np.zeros(n_epochs * iter_per_epoch)\n",
    "        self.epoch_losses = np.zeros(n_epochs)\n",
    "\n",
    "        self.loss_func = nn.MSELoss()\n",
    "\n",
    "        n_iter = 0\n",
    "\n",
    "        learning_rate = 1.\n",
    "\n",
    "        for i in range(n_epochs):\n",
    "            perm_idx = np.random.permutation(self.train_size - self.T)\n",
    "            j = 0\n",
    "            while j < self.train_size:\n",
    "                batch_idx = perm_idx[j:(j + self.batch_size)]\n",
    "                X = np.zeros((len(batch_idx), self.T - 1, self.X.shape[1]))\n",
    "                y_history = np.zeros((len(batch_idx), self.T - 1))\n",
    "                y_target = self.y[batch_idx + self.T]\n",
    "\n",
    "                for k in range(len(batch_idx)):\n",
    "                    X[k, :, :] = self.X[batch_idx[k] : (batch_idx[k] + self.T - 1), :]\n",
    "                    y_history[k, :] = self.y[batch_idx[k] : (batch_idx[k] + self.T - 1)]\n",
    "\n",
    "                loss = self.train_iteration(X, y_history, y_target)\n",
    "                self.iter_losses[i * iter_per_epoch + j / self.batch_size] = loss\n",
    "                #if (j / self.batch_size) % 50 == 0:\n",
    "                #    self.logger.info(\"Epoch %d, Batch %d: loss = %3.3f.\", i, j / self.batch_size, loss)\n",
    "                j += self.batch_size\n",
    "                n_iter += 1\n",
    "\n",
    "                if n_iter % 10000 == 0 and n_iter > 0:\n",
    "                    for param_group in self.encoder_optimizer.param_groups:\n",
    "                        param_group['lr'] = param_group['lr'] * 0.9\n",
    "                    for param_group in self.decoder_optimizer.param_groups:\n",
    "                        param_group['lr'] = param_group['lr'] * 0.9\n",
    "\n",
    "            self.epoch_losses[i] = np.mean(self.iter_losses[range(i * iter_per_epoch, (i + 1) * iter_per_epoch)])\n",
    "            if i % 10 == 0:\n",
    "                self.logger.info(\"Epoch %d, loss: %3.3f.\", i, self.epoch_losses[i])\n",
    "\n",
    "            if i % 10 == 0:\n",
    "                y_train_pred = self.predict(on_train = True)\n",
    "                y_test_pred = self.predict(on_train = False)\n",
    "                y_pred = np.concatenate((y_train_pred, y_test_pred))\n",
    "                plt.figure()\n",
    "                plt.plot(range(1, 1 + len(self.y)), self.y, label = \"True\")\n",
    "                plt.plot(range(self.T , len(y_train_pred) + self.T), y_train_pred, label = 'Predicted - Train')\n",
    "                plt.plot(range(self.T + len(y_train_pred) , len(self.y) + 1), y_test_pred, label = 'Predicted - Test')\n",
    "                plt.legend(loc = 'upper left')\n",
    "                plt.show()\n",
    "\n",
    "    def train_iteration(self, X, y_history, y_target):\n",
    "        self.encoder_optimizer.zero_grad()\n",
    "        self.decoder_optimizer.zero_grad()\n",
    "\n",
    "        input_weighted, input_encoded = self.encoder(Variable(torch.from_numpy(X).type(torch.FloatTensor).cuda()))\n",
    "        y_pred = self.decoder(input_encoded, Variable(torch.from_numpy(y_history).type(torch.FloatTensor).cuda()))\n",
    "\n",
    "        y_true = Variable(torch.from_numpy(y_target).type(torch.FloatTensor).cuda())\n",
    "        loss = self.loss_func(y_pred, y_true)\n",
    "        loss.backward()\n",
    "\n",
    "        self.encoder_optimizer.step()\n",
    "        self.decoder_optimizer.step()\n",
    "\n",
    "        return loss.data[0]\n",
    "\n",
    "    def predict(self, on_train = False):\n",
    "        if on_train:\n",
    "            y_pred = np.zeros(self.train_size - self.T + 1)\n",
    "        else:\n",
    "            y_pred = np.zeros(self.X.shape[0] - self.train_size)\n",
    "\n",
    "        i = 0\n",
    "        while i < len(y_pred):\n",
    "            batch_idx = np.array(range(len(y_pred)))[i : (i + self.batch_size)]\n",
    "            X = np.zeros((len(batch_idx), self.T - 1, self.X.shape[1]))\n",
    "            y_history = np.zeros((len(batch_idx), self.T - 1))\n",
    "            for j in range(len(batch_idx)):\n",
    "                if on_train:\n",
    "                    X[j, :, :] = self.X[range(batch_idx[j], batch_idx[j] + self.T - 1), :]\n",
    "                    y_history[j, :] = self.y[range(batch_idx[j],  batch_idx[j]+ self.T - 1)]\n",
    "                else:\n",
    "                    X[j, :, :] = self.X[range(batch_idx[j] + self.train_size - self.T, batch_idx[j] + self.train_size - 1), :]\n",
    "                    y_history[j, :] = self.y[range(batch_idx[j] + self.train_size - self.T,  batch_idx[j]+ self.train_size - 1)]\n",
    "\n",
    "            y_history = Variable(torch.from_numpy(y_history).type(torch.FloatTensor).cuda())\n",
    "            _, input_encoded = self.encoder(Variable(torch.from_numpy(X).type(torch.FloatTensor).cuda()))\n",
    "            y_pred[i:(i + self.batch_size)] = self.decoder(input_encoded, y_history).cpu().data.numpy()[:, 0]\n",
    "            i += self.batch_size\n",
    "        return y_pred\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'DataFrame' object has no attribute 'as_matrix'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-9af1a18b2d03>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mio_dir\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'Machine_Learning/nasdaq100'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m model = da_rnn(file_data = '{}/nasdaq100/small/nasdaq100_padding.csv'.format(io_dir), logger = logger, parallel = False,\n\u001b[0m\u001b[1;32m      4\u001b[0m               learning_rate = .001)\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-f66552db4bd2>\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, file_data, logger, encoder_hidden_size, decoder_hidden_size, T, learning_rate, batch_size, parallel, debug)\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Shape of data: %s.\\nMissing in data: %s.\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misnull\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'NDX'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNDX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.8/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   5458\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_info_axis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_can_hold_identifiers_and_holds_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5459\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5460\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5461\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5462\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'as_matrix'"
     ]
    }
   ],
   "source": [
    "io_dir = 'Machine_Learning/nasdaq100'\n",
    "\n",
    "model = da_rnn(file_data = '{}/nasdaq100/small/nasdaq100_padding.csv'.format(io_dir), logger = logger, parallel = False,\n",
    "              learning_rate = .001)\n",
    "\n",
    "model.train(n_epochs = 500)\n",
    "\n",
    "y_pred = model.predict()\n",
    "\n",
    "plt.figure()\n",
    "plt.semilogy(range(len(model.iter_losses)), model.iter_losses)\n",
    "plt.show()\n",
    "\n",
    "plt.figure()\n",
    "plt.semilogy(range(len(model.epoch_losses)), model.epoch_losses)\n",
    "plt.show()\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(y_pred, label = 'Predicted')\n",
    "plt.plot(model.y[model.train_size:], label = \"True\")\n",
    "plt.legend(loc = 'upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}