PytorchAE.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9fe7c43f",
   "metadata": {},
   "source": [
    "1. Preprocess\n",
    "2. Data Loader\n",
    "3. Model define"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27d8d602",
   "metadata": {},
   "source": [
    "# LSTM"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d9455ff",
   "metadata": {},
   "source": [
    "* https://github.com/SheezaShabbir/Time-series-Analysis-using-LSTM-RNN-and-GRU/blob/main/Pytorch_LSTMs%2CRNN%2CGRU_for_time_series_data.ipynb\n",
    "* https://github.com/vincrichard/LSTM-AutoEncoder-Unsupervised-Anomaly-Detection/blob/master/DataChallengeReport_VincentRichard.ipynb\n",
    "* https://github.com/yakhyo/pytorch-tutorials/blob/main/tutorials/03-intermediate/04-lstm-network/main.py\n",
    "* https://github.com/yakhyo/pytorch-tutorials/blob/main/tutorials/03-intermediate/05-var-auto-encode/main.py\n",
    "* https://github.com/vincrichard/LSTM-AutoEncoder-Unsupervised-Anomaly-Detection/blob/master/DataChallengeReport_VincentRichard.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "08d7a8a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader, TensorDataset\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch.nn as nn\n",
    "from torchsummary import summary\n",
    "import torch.optim as optim\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30431037",
   "metadata": {},
   "source": [
    "## 1. Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "b0a380f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./training_spiral.csv')\n",
    "feature_list = ['Fx','Fy','Fz','Mx','My']\n",
    "TIMESTEP = 50\n",
    "BATCH_SIZE = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "22aec7b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(357823, 20)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "93f445ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.iloc[:50000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "6fdf4c99",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 20)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "5043b68a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    50000\n",
       "Name: Case, dtype: int64"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.Case.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "61acb87f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def to_sequence(data, timesteps=1):\n",
    "#     n_features=data.shape[2]\n",
    "#     seq = []\n",
    "#     for i in range(len(data)-timesteps):\n",
    "#         # takes a window of data of specified timesteps\n",
    "#         temp = data[i:(i+timesteps)]\n",
    "#         temp = temp.reshape(timesteps, n_features)\n",
    "#         seq.append(temp)\n",
    "        \n",
    "#     return np.array(seq)\n",
    "\n",
    "def to_sequence(data, timesteps=1):\n",
    "    n_features=data.shape[2]\n",
    "    x = []\n",
    "    y = []\n",
    "    for i in range(len(data)-timesteps):\n",
    "        # takes a window of data of specified timesteps\n",
    "        \n",
    "        _x = data[i:(i+timesteps)]\n",
    "        _x = _x.reshape(timesteps, n_features)\n",
    "#         print(_x.shape)\n",
    "        _y = data[i+timesteps]\n",
    "        _y = _y.reshape(n_features)\n",
    "#         print(_y.shape)\n",
    "        x.append(_x)\n",
    "        y.append(_y)\n",
    "\n",
    "        \n",
    "    return np.array(x), np.array(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "4efa0cc3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(49961, 5)\n",
      "(49911, 50, 5)\n",
      "(49911, 5)\n"
     ]
    }
   ],
   "source": [
    "df_train = df[feature_list]\n",
    "window = 40\n",
    "df_train = df_train.rolling(window).mean()\n",
    "# due to the moving average we the first (window-1) rows become NaN so we remove them\n",
    "df_train = df_train.loc[window-1:]\n",
    "print(df_train.shape)\n",
    "train = np.expand_dims(df_train, axis=1)\n",
    "x_train, y_train = to_sequence(train, timesteps=TIMESTEP)\n",
    "print(x_train.shape)\n",
    "print(y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "2248c491",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([49911, 50, 5])\n",
      "torch.Size([49911, 5])\n"
     ]
    }
   ],
   "source": [
    "x_train_torch = torch.Tensor(x_train)\n",
    "y_train_torch = torch.Tensor(y_train)\n",
    "print(x_train_torch.shape)\n",
    "print(y_train_torch.shape)\n",
    "train = TensorDataset(x_train_torch, y_train_torch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "5b100fc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train, batch_size=BATCH_SIZE, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "22e5cbb1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature batch shape: torch.Size([32, 50, 5])\n",
      "Labels batch shape: torch.Size([32, 5])\n"
     ]
    }
   ],
   "source": [
    "train_features, train_labels = next(iter(train_loader))\n",
    "print(f\"Feature batch shape: {train_features.size()}\")\n",
    "print(f\"Labels batch shape: {train_labels.size()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "cf08ea80",
   "metadata": {},
   "outputs": [],
   "source": [
    "# x_train_torch = torch.Tensor(x_train)\n",
    "# print(x_train_torch.shape)\n",
    "# torch.save(x_train_torch, 'train_seq.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39aaaf8b",
   "metadata": {},
   "source": [
    "## 2. Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "ce32f9b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# class SensorData(Dataset):\n",
    "#     def __init__(self, file_path, file_type):\n",
    "        \n",
    "#         if file_type == 'csv':\n",
    "#             self.data = pd.read_csv(file_path, delimiter=',', nrows=None, header=None)\n",
    "#         elif file_type == 'pt':\n",
    "#             self.data = torch.load(file_path)\n",
    "#         else:\n",
    "#             raise ValueError('type value is wrong: ', file_type)\n",
    "#         self.file_type = file_type\n",
    "        \n",
    "#     def __len__(self):\n",
    "#         if self.file_type == 'csv':\n",
    "#             return len(self.data)\n",
    "        \n",
    "#         if self.file_type == 'pt':\n",
    "#             return self.data.shape[0]\n",
    "    \n",
    "#     def __getitem__(self, idx):\n",
    "#         if self.file_type == 'pt':\n",
    "#             return self.data[idx,:,:]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "017f58ec",
   "metadata": {},
   "source": [
    "### 2.1. Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "59274118",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_dataset = SensorData(file_path = './train_seq.pt', file_type='pt')\n",
    "# train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=False)\n",
    "# samples, timestep, features = train_dataset.data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "903665cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of Train Loader 1560 batches of 32\n"
     ]
    }
   ],
   "source": [
    "print(f\"Length of Train Loader {len(train_loader)} batches of {BATCH_SIZE}\")\n",
    "# print(f\"Data is of shape:\", train_dataset.data.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9221ed98",
   "metadata": {},
   "source": [
    "## 3. Model Building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "b1ac5198",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using: cpu\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "print(\"Using:\", device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d792f5d",
   "metadata": {},
   "source": [
    "## 3.1. LSTM \n",
    "LSTM: https://pytorch.org/docs/stable/generated/torch.nn.LSTM.html\n",
    "\n",
    "1. `input, (h_0, c_0)`\n",
    "    * input: `(N, L, Hin)`\n",
    "    * h_0: tensor of shape `(num_layers, N,hidden_size)`, defaults to zeros \n",
    "    * c_0: tensor of shape `(num_layers, N, Hcell)`\n",
    "2. `output, (h_n, c_n)`\n",
    "    * output: `(N,L,hidden_size)`\n",
    "    * h_n: containing final hidden state for each element in the sequence: `(num_layers, N, hidden_size)`\n",
    "    * c_0:  `(num_layers, N, Hcell)`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "45b536dc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class LSTMModel(nn.Module):\n",
    "    \"\"\"\n",
    "    Attributes:\n",
    "        input_size: number of expected features in X\n",
    "        hidden_size: how many LSTM cells are there in each hidden layer\n",
    "        num_layers: how many stacked LSTMs we want to use\n",
    "        output_dim: LSTM output shape\n",
    "    \"\"\"\n",
    "    def __init__(self, input_size, hidden_size, num_layers, output_dim, print_info):\n",
    "        super(LSTMModel, self).__init__()\n",
    "        self.print_info = print_info\n",
    "        \n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "                \n",
    "        # LSTM\n",
    "        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n",
    "        # Fully Connected\n",
    "        # Output_dim == number of features (1 pred per feature)\n",
    "        self.fc = nn.Linear(hidden_size, output_dim)\n",
    "        \n",
    "    def _init_hidden(self, X):\n",
    "        batch_size = X.size(0)\n",
    "        device = X.device\n",
    "        h0 = torch.zeros(self.num_layers, batch_size, self.hidden_size).to(device)\n",
    "        c0 = torch.zeros(self.num_layers, batch_size, self.hidden_size).to(device)\n",
    "        \n",
    "        return h0, c0\n",
    "        \n",
    "    def forward(self, X):\n",
    "        \"\"\"\n",
    "        input (X): should be of shape (batch_size, seq_length, hidden_size)\n",
    "        \"\"\"\n",
    "        # initialize hidden and cell states\n",
    "        h0, c0 = self._init_hidden(X)\n",
    "        # call lstm\n",
    "        out, (hn, cn) = self.lstm(X, (h0.detach(), c0.detach()))\n",
    "        # for plotting\n",
    "        init_out = out\n",
    "        # out: batch_size, seq_len, hidden_size\n",
    "        # out(N, 28, 128)\n",
    "        out = out[:,-1,:]\n",
    "        out_reshaped = out\n",
    "        # we want last timestep: out (N, 128)\n",
    "        out = self.fc(out)\n",
    "        # batch_size, output_dim\n",
    "        \n",
    "        if self.print_info:\n",
    "            print('X shape:', X.shape)\n",
    "            print(f\"h0 shape: {h0.shape}, c0 shape: {c0.shape}\")\n",
    "            print('init_out shape:', init_out.shape)\n",
    "            print('output_reshaped', out_reshaped.shape)\n",
    "            print('out shape:', out.shape)\n",
    "            \n",
    "        return out\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "9c08a06c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LSTMModel(\n",
      "  (lstm): LSTM(5, 32, batch_first=True)\n",
      "  (fc): Linear(in_features=32, out_features=5, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "#  input_size: number of expected features in X\n",
    "#  hidden_size: how many LSTM cells are there in each hidden layer\n",
    "#  num_layers: how many stacked LSTMs we want to use\n",
    "\n",
    "input_size = 5\n",
    "hidden_size = 32\n",
    "num_layers = 1\n",
    "output_dim = 5\n",
    "\n",
    "sequence_dim = TIMESTEP\n",
    "\n",
    "# instatiate model\n",
    "model = LSTMModel(input_size, hidden_size, num_layers, output_dim, print_info=False)\n",
    "# moving model to the GPU \n",
    "model = model.to(device)\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "739d9c98",
   "metadata": {},
   "outputs": [],
   "source": [
    "# testing if model works/\n",
    "# model(train_dataset.data[:2,:,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "5ff7885e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimizer\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.0001)\n",
    "# loss\n",
    "criterion = torch.nn.MSELoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09904b66",
   "metadata": {},
   "source": [
    "## 4. Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "88f2c340",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of Train Loader 1560 batches of 32\n"
     ]
    }
   ],
   "source": [
    "print(f\"Length of Train Loader {len(train_loader)} batches of {BATCH_SIZE}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "e75ff20c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_step(model, train_loader, criterion, optimizer, device):\n",
    "    # training mode\n",
    "    model.train()\n",
    "    \n",
    "    train_loss = 0\n",
    "    # we track the batch index \n",
    "    for batch_idx, (x_batch, y_batch) in enumerate(train_loader):\n",
    "\n",
    "        x_batch = x_batch.to(device)\n",
    "        y_batch = y_batch.to(device)\n",
    "        \n",
    "        yhat = model(x_batch)\n",
    "\n",
    "        loss = criterion(y_batch, yhat)\n",
    "        train_loss += loss.item()\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "        \n",
    "    train_loss = train_loss/ len(train_loader)\n",
    "    return train_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "e6cb2e1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_step(model, test_loader, criterion, device):\n",
    "    model.eval()\n",
    "    \n",
    "    # set test loss\n",
    "    test_loss = 0\n",
    "    # turn inference\n",
    "    with torch.inference_mode():\n",
    "        for batch_idx, (x_batch, y_batch) in enumerate(test_loader):\n",
    "\n",
    "            x_batch = x_batch.to(device)\n",
    "            y_batch = y_batch.to(device)\n",
    "\n",
    "            yhat = model(x_batch)\n",
    "\n",
    "            loss = criterion(y_batch, yhat)\n",
    "            test_loss += loss.item()\n",
    "        \n",
    "        test_loss = test_loss/ len(test_loader)\n",
    "        return test_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "f1008e2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, train_loader, test_loader, optimizer, criterion, epochs, device):\n",
    "    results = {\"train_loss\": [],\n",
    "              \"test_loss\": []}\n",
    "    \n",
    "    for epoch in tqdm(range(epochs)):\n",
    "        train_loss = train_step(model, train_loader, criterion, optimizer, device)\n",
    "        \n",
    "        test_loss = test_step(model, test_loader, criterion, device)\n",
    "        \n",
    "        print(f\"Epoch: {epoch} | Train loss: {train_loss} | Test loss: {test_loss}\")\n",
    "        \n",
    "        # update results dicts\n",
    "        results['train_loss'].append(train_loss)\n",
    "        results['test_loss'].append(test_loss)\n",
    "        \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "ee785473",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_loss_curves(results):\n",
    "    loss = results['train_loss']\n",
    "    test_loss = results['test_loss']\n",
    "    \n",
    "    epochs = range(len(results['train_loss']))\n",
    "    \n",
    "    plt.figure()\n",
    "    plt.plot(epochs, loss)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ae3f44b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83072471",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df0aeb4f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50cadf38",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}