3.0-tl-mnist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from torch.optim.lr_scheduler import StepLR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task 1\n",
    "(3 points)\n",
    "\n",
    "Implement the training loop for one training epoch.\n",
    "An epoch trains on the whole training dataset once."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, use_cuda, train_loader, optimizer, epoch, log_interval):\n",
    "    \"\"\"\n",
    "    Train one epoch\n",
    "    \n",
    "    model -- the neural network\n",
    "    use_cuda -- true if GPU should be used\n",
    "    train_loader -- data loader\n",
    "    optimizer -- network optimizer\n",
    "    epoch -- number of current epoch\n",
    "    log_interval -- number of training steps between logs\n",
    "    \"\"\"\n",
    "    # TODO: set the model to train mode\n",
    "    \n",
    "    \n",
    "    # TODO: enumerate over the dataloader to get mini batches\n",
    "    #       of images and ground truth labels\n",
    "    # HINT: the builtin python function enumerate() also gives you indices\n",
    "    \n",
    "        \n",
    "        # TODO: set the optimizers gradients to zero\n",
    "        \n",
    "        \n",
    "        # TODO: run the network\n",
    "        \n",
    "        \n",
    "        # TODO: compute negative log likelihood loss\n",
    "        \n",
    "        \n",
    "        # TODO: do backpropagation\n",
    "       \n",
    "        \n",
    "        # TODO: optimize\n",
    "       \n",
    "        \n",
    "        # TODO: print current loss for every nth (\"log_interval\"th) iteration\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We already implemented the validation function for you (this is essentially validate() from the last exercise)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate(model, use_cuda, test_loader):\n",
    "    \"\"\"\n",
    "    Compute test metrics\n",
    "    \n",
    "    model -- the neural network\n",
    "    use_cuda -- true if GPU should be used\n",
    "    test_loader -- data loader\n",
    "    \"\"\"\n",
    "    # create a 10x10 grid of subplots\n",
    "    _, axis = plt.subplots(10, 10)\n",
    "    \n",
    "    # set model to evaluation mode\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    plotted = 0\n",
    "    \n",
    "    # disable gradients globally\n",
    "    with torch.no_grad():\n",
    "        for batch_idx, (data, target) in enumerate(test_loader):\n",
    "            # for each batch\n",
    "            if use_cuda:\n",
    "                # transfer to GPU\n",
    "                data = data.cuda()\n",
    "                target = target.cuda()\n",
    "            \n",
    "            # run network and compute metrics\n",
    "            output = model(data)\n",
    "            test_loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss\n",
    "            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability\n",
    "            \n",
    "            img_correct = pred.eq(target.view_as(pred))\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "            \n",
    "            # plot the first 100 images\n",
    "            img_idx = 0\n",
    "            data = data.cpu().numpy()\n",
    "            \n",
    "            while plotted < 100 and img_idx < data.shape[0]:\n",
    "                # compute position of ith image in the grid\n",
    "                y = plotted % 10\n",
    "                x = plotted // 10\n",
    "                \n",
    "                # convert image tensor to numpy array and normalize to [0, 1]\n",
    "                img = data[img_idx, 0]\n",
    "                img = (img - np.min(img)) / (np.max(img) - np.min(img))\n",
    "                \n",
    "                # make wrongly predicted images red\n",
    "                img = np.stack([img] * 3, 2)\n",
    "                if img_correct[img_idx] == 0:\n",
    "                    img[:, :, 1:] = 0.0\n",
    "                \n",
    "                # disable axis and show image\n",
    "                axis[y][x].axis('off')\n",
    "                axis[y][x].imshow(img)\n",
    "                \n",
    "                # show the predicted class next to each image\n",
    "                axis[y][x].text(30, 25, pred[img_idx].item())\n",
    "                \n",
    "                plotted += 1\n",
    "                img_idx += 1\n",
    "            \n",
    "    test_loss /= len(test_loader.dataset)\n",
    "\n",
    "    # show results\n",
    "    print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\\n'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task 2\n",
    "(4 points)\n",
    "\n",
    "Implement a five-layer fully connected neural network.\n",
    "The dimensions (without batch size) should change like this: 784->200->100->60->30->10\n",
    "Use log softmax to compute the class predictions.\n",
    "\n",
    "Run the code at the end of the notebook to train and validate your implementation.\n",
    "\n",
    "### Task 2.1\n",
    "* sigmoid non-linear activation function\n",
    "* note that the last layer does not need an activation function!\n",
    "\n",
    "### Task 2.2\n",
    "* add a new class \"FCNet2\"\n",
    "* replace sigmoid with ReLU\n",
    "\n",
    "### Task 2.3\n",
    "* add a new class \"FCNet2\"\n",
    "* add batch normalization to the first and third layers (note the difference between 1D/2D/3D versions)\n",
    "\n",
    "\n",
    "**NOTE:** The perfomance should improve slightly with each step. However, due to the random weight initialization applied by PyTorch, your results may vary a bit between trainings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FCNet1(nn.Module):\n",
    "    \"\"\"\n",
    "    Fully Connected Neural Network\n",
    "    \n",
    "    Five fully connected layers with sigmoid non-linearity\n",
    "    \n",
    "    Dimensions\n",
    "    784->200->100->60->30->10\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        super(FCNet1, self).__init__()\n",
    "        \n",
    "        # TODO: initialize network layers\n",
    "        # HINT: take a look at \"torch.nn\" (imported as \"nn\")\n",
    "       \n",
    "\n",
    "    def forward(self, x):\n",
    "        # TODO: reshape batch of images to batch of 1D vectors\n",
    "        \n",
    "        \n",
    "        # TODO: run network layers\n",
    "        \n",
    "        \n",
    "        # TODO: compute log softmax over the output\n",
    "        # HINT: take a look at \"torch.nn.functional\" (imported as \"F\")\n",
    "        \n",
    "        \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FCNet2(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(FCNet2, self).__init__()\n",
    "\n",
    "    def forward(self, x):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FCNet3(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(FCNet3, self).__init__()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task 3\n",
    "(3 points)\n",
    "\n",
    "Implement a convolutional neural network, consisting of two convolutional and two fully connected layers.\n",
    "This time, the dimensions (without batch size) should change like this: 1x28x28->32x26x26->64x12x12->128->10\n",
    "\n",
    "### Task 3.1\n",
    "* two convolutional layers (kernel size 3)\n",
    "* two fully-connected layers\n",
    "* ReLU activation function\n",
    "\n",
    "### Task 3.2\n",
    "* add batch normalization to first convolutional and first fully connected layer\n",
    "\n",
    "### Task 3.3\n",
    "* use max pooling instead of stride to reduce the dimensions to 64x12x12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNet1(nn.Module):\n",
    "    \"\"\"\n",
    "    Convolutional Neural Network\n",
    "    \n",
    "    Two convolutional layers and two fully connected layers\n",
    "    \n",
    "    Dimensions:\n",
    "    1x28x28->32x26x26->64x12x12->128->10\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        super(ConvNet1, self).__init__()\n",
    "        \n",
    "        # TODO: initialize network layers\n",
    "        \n",
    "\n",
    "    def forward(self, x):\n",
    "        # TODO: run convolutional layers\n",
    "        \n",
    "        \n",
    "        # TODO: reshape batch of images to batch of 1D vectors\n",
    "        \n",
    "        \n",
    "        # TODO: run fully connected layers\n",
    "       \n",
    "        \n",
    "        # TODO: compute log softmax over the output\n",
    "        \n",
    "        \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNet2(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ConvNet2, self).__init__()\n",
    "\n",
    "    def forward(self, x):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNet3(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ConvNet3, self).__init__()\n",
    "\n",
    "    def forward(self, x):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyper parameters\n",
    "batch_size = 64\n",
    "test_batch_size = 1000\n",
    "epochs = 10\n",
    "lr = 1.0\n",
    "gamma = 0.7\n",
    "log_interval = 100\n",
    "\n",
    "# use GPU if available\n",
    "use_cuda = torch.cuda.is_available()\n",
    "kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}\n",
    "\n",
    "# initialize data loaders\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('../data', train=True, download=True,\n",
    "    transform=transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.1307,), (0.3081,))\n",
    "    ])), batch_size=batch_size, shuffle=True, **kwargs)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('../data', train=False, \n",
    "    transform=transforms.Compose([\n",
    "        transforms.ToTensor(), \n",
    "        transforms.Normalize((0.1307,), (0.3081,))\n",
    "    ])),\n",
    "    batch_size=test_batch_size, shuffle=True, **kwargs)\n",
    "\n",
    "model = FCNet2()\n",
    "if use_cuda:\n",
    "    model = model.cuda()\n",
    "\n",
    "# initialize optimizer and scheduler\n",
    "optimizer = optim.Adadelta(model.parameters(), lr=lr)\n",
    "scheduler = StepLR(optimizer, step_size=1, gamma=gamma)\n",
    "\n",
    "for epoch in range(1, epochs + 1):\n",
    "    # train one epoch\n",
    "    train(model, use_cuda, train_loader, optimizer, epoch, log_interval)\n",
    "    \n",
    "    # run on test dataset\n",
    "    validate(model, use_cuda, test_loader)\n",
    "    scheduler.step()\n",
    "    \n",
    "    \n",
    "    torch.save(model.state_dict(), \"models/mnist/checkpoint.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}