pytorch_cheatsheet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.init as init\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torchvision.datasets as datasets\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "import torchvision.models as models\n",
    "import torch.backends.cudnn as cudnn\n",
    "import torchvision\n",
    "import torch.autograd as autograd\n",
    "\n",
    "import imp\n",
    "import os\n",
    "import sys\n",
    "import math\n",
    "import time\n",
    "import random\n",
    "import shutil\n",
    "\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import matplotlib as mpl\n",
    "mpl.use('Agg')\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('bmh')\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 1\n",
      " 1\n",
      " 1\n",
      " 1\n",
      " 1\n",
      "[torch.FloatTensor of size 5]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "##Initialize tensor\n",
    "a = torch.ones(5)\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.  1.  1.  1.  1.]\n"
     ]
    }
   ],
   "source": [
    "#Convert tensor to numpy array\n",
    "b = a.numpy()\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 1\n",
      " 1\n",
      " 1\n",
      " 1\n",
      " 1\n",
      "[torch.DoubleTensor of size 5]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Convert numpy array to tensor\n",
    "a = np.ones(5)\n",
    "b = torch.from_numpy(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 loop, best of 3: 6.96 µs per loop\n"
     ]
    }
   ],
   "source": [
    "# let us run this cell only if CUDA is available\n",
    "x = torch.ones(5)\n",
    "y = torch.ones(5)\n",
    "if torch.cuda.is_available():\n",
    "    x = x.cuda()\n",
    "    y = y.cuda()\n",
    "    x + y\n",
    "    \n",
    "def add():\n",
    "    x + y\n",
    "%timeit -n 1 add()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A microsecond µs is 1 millionth of a second"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensor\n",
    "\n",
    "http://pytorch.org/docs/tensors.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 3])\n",
      "\n",
      " 1  2  3\n",
      " 4  5  6\n",
      "[torch.FloatTensor of size 2x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#2D Tensor\n",
    "x = torch.FloatTensor(\n",
    "    [[1, 2, 3], \n",
    "     [4, 5, 6]]\n",
    ")\n",
    "print(x.size())\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 2, 3])\n",
      "\n",
      "(0 ,.,.) = \n",
      "  1  1  1\n",
      "  2  2  2\n",
      "\n",
      "(1 ,.,.) = \n",
      "  3  3  3\n",
      "  4  4  4\n",
      "[torch.FloatTensor of size 2x2x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#3D Tensor\n",
    "x = torch.FloatTensor([\n",
    "    [[1, 1, 1], \n",
    "     [2, 2, 2]],\n",
    "    [[3, 3, 3], \n",
    "     [4, 4, 4]]\n",
    "])\n",
    "print(x.size())\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "(0 ,.,.) = \n",
       " -0.8209  0.0792 -0.0169\n",
       " -0.2988 -0.6930  0.7690\n",
       "\n",
       "(1 ,.,.) = \n",
       "  0.8349 -1.4692  0.0718\n",
       " -1.7422  1.2717 -2.4279\n",
       "[torch.FloatTensor of size 2x2x3]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn(2,2,3)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Autograd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "from torch import Tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 1  1\n",
      " 1  1\n",
      "[torch.FloatTensor of size 2x2]\n",
      "\n",
      "\n",
      " 1  1\n",
      " 1  1\n",
      "[torch.FloatTensor of size 2x2]\n",
      "\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "x = Variable(torch.ones(2,2), requires_grad=True)\n",
    "print (x)\n",
    "print (x.data)\n",
    "print (x.creator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 3  3\n",
      " 3  3\n",
      "[torch.FloatTensor of size 2x2]\n",
      "\n",
      "<torch.autograd._functions.basic_ops.AddConstant object at 0x7ff051618668>\n"
     ]
    }
   ],
   "source": [
    "y = x + 2\n",
    "print (y)\n",
    "print (y.creator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 27\n",
       "[torch.FloatTensor of size 1]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z = y*y*3\n",
    "out = z.mean()\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "out.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 4.5000  4.5000\n",
      " 4.5000  4.5000\n",
      "[torch.FloatTensor of size 2x2]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class NeuralNetwork(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(NeuralNetwork, self).__init__()\n",
    "        # 1 input image channel, 6 output channels, 5x5 square convolution kernel\n",
    "        self.conv1 = nn.Conv2d(1,6,5)\n",
    "        self.conv2 = nn.Conv2d(6,16,5)\n",
    "        self.fc1 = nn.Linear(16*5*5, 120) # an affine operation: y = Wx + b\n",
    "        self.fc2 = nn.Linear(120,84)\n",
    "        self.fc3 = nn.Linear(84,10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.max_pool2d(F.relu(self.conv1(x)), (2,2)) #2x2 pool window\n",
    "        x = F.max_pool2d(F.relu(self.conv2(x)), 2) #if shape is square, you only need to define one number\n",
    "        x = x.view(-1, self.num_flat_features(x)) #Flatten()\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "    \n",
    "    ## Backward() function is automatically defined for you!!\n",
    "    \n",
    "    def num_flat_features(self, x):\n",
    "        #basically counting parameters and flattening \n",
    "        size = x.size()[1:] # all dimensions except the batch dimension\n",
    "        num_features = 1\n",
    "        for s in size:\n",
    "            num_features *= s\n",
    "        return num_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NeuralNetwork (\n",
      "  (conv1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (fc1): Linear (400 -> 120)\n",
      "  (fc2): Linear (120 -> 84)\n",
      "  (fc3): Linear (84 -> 10)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "net = NeuralNetwork()\n",
    "print (net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "-0.0078  0.0949  0.0414  0.0722  0.0060  0.0250 -0.1048 -0.0385 -0.1151 -0.1062\n",
       "[torch.FloatTensor of size 1x10]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = Variable(torch.randn(1,1,32,32)) #nSamples x nChannels x Height x Width\n",
    "yHat = net(X)\n",
    "yHat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loss Function\n",
    "\n",
    "http://pytorch.org/docs/nn.html#loss-functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 38.9785\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "yHat = net(X)\n",
    "target = Variable(torch.range(1, 10))  # a dummy target, for example\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "loss = criterion(yHat, target)\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<torch.nn._functions.thnn.auto.MSELoss object at 0x7ff0907fdac8>\n",
      "<torch.nn._functions.linear.Linear object at 0x7ff0907fd908>\n",
      "<torch.nn._functions.thnn.auto.Threshold object at 0x7ff0907fd3c8>\n"
     ]
    }
   ],
   "source": [
    "print(loss.creator)  # MSELoss\n",
    "print(loss.creator.previous_functions[0][0])  # Linear\n",
    "print(loss.creator.previous_functions[0][0].previous_functions[0][0])  # ReLU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Backprop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv1.bias.grad before backward\n",
      "None\n",
      "conv1.bias.grad after backward\n",
      "Variable containing:\n",
      "1.00000e-02 *\n",
      " -4.9376\n",
      " -3.0339\n",
      " -4.2199\n",
      "  3.5306\n",
      "  6.0344\n",
      "  8.6081\n",
      "[torch.FloatTensor of size 6]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#First clear the existing gradients!! \n",
    "net.zero_grad()     # zeroes the gradient buffers of all parameters\n",
    "\n",
    "print('conv1.bias.grad before backward')\n",
    "print(net.conv1.bias.grad)\n",
    "\n",
    "#Backprop as simple as..\n",
    "loss.backward()\n",
    "\n",
    "print('conv1.bias.grad after backward')\n",
    "print(net.conv1.bias.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Update Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#SGD\n",
    "lr = 0.01\n",
    "for f in net.parameters():\n",
    "    f.data.sub_(f.grad.data * lr)  #subtract gradient from weights..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Other optimizers\n",
    "import torch.optim as optim\n",
    "\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.01)\n",
    "criterion = nn.MSELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#using the optimizers\n",
    "#Zero the gradient\n",
    "optimizer.zero_grad()\n",
    "yHat = net(X)\n",
    "loss = criterion(yHat,target)\n",
    "loss.backward()\n",
    "optimizer.step() #update the weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNNs\n",
    "\n",
    "* https://chsasank.github.io/pytorch-tutorials/beginner/blitz/cifar10_tutorial.html\n",
    "* http://cs231n.github.io/convolutional-networks/\n",
    "\n",
    "Accepts a volume of size \n",
    "\n",
    "W1×H1×D1\n",
    "\n",
    "Requires four hyperparameters:\n",
    "\n",
    "K - Number of filters K\n",
    "F - their spatial extent F\n",
    "S - the stride S\n",
    "P - the amount of zero padding\n",
    "\n",
    "Produces a volume of size W2×H2×D2 where:\n",
    "\n",
    "W2 = (W1−F+2P)/S+1\n",
    "H2 = (H1−F+2P)/S+1 (i.e. width and height are computed equally by symmetry)\n",
    "D2 = K\n",
    "\n",
    "With parameter sharing, it introduces F⋅F⋅D1 weights per filter, for a total of (F⋅F⋅D1)⋅K weights and K biases.\n",
    "\n",
    "In the output volume, the d-th depth slice (of size W2×H2) is the result of performing a valid convolution of the d-th filter over the input volume with a stride of S and then offset by d-th bias."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Concat\n",
    "\n",
    "http://pytorch.org/docs/torch.html#torch.cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Does this work?\n",
    "torch.manual_seed??\n",
    "torch.cuda.manual_seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 0.2895  0.0358 -0.3661\n",
       "-1.4977 -0.0709 -0.2322\n",
       "[torch.FloatTensor of size 2x3]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn(2,3)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 0.2895  0.0358 -0.3661\n",
       "-1.4977 -0.0709 -0.2322\n",
       " 0.2895  0.0358 -0.3661\n",
       "-1.4977 -0.0709 -0.2322\n",
       " 0.2895  0.0358 -0.3661\n",
       "-1.4977 -0.0709 -0.2322\n",
       "[torch.FloatTensor of size 6x3]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2D - combine new tensors as new rows stacked on stop of each other\n",
    "torch.cat([x,x,x], 0) #0 = row = first dimension of input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       " 0.2895  0.0358 -0.3661  0.2895  0.0358 -0.3661  0.2895  0.0358 -0.3661\n",
       "-1.4977 -0.0709 -0.2322 -1.4977 -0.0709 -0.2322 -1.4977 -0.0709 -0.2322\n",
       "[torch.FloatTensor of size 2x9]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2D - combine new tensors as new columns stacked next to each other\n",
    "torch.cat([x,x,x], 1) #1 = column = 2nd dimension of input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "(0 ,.,.) = \n",
       "  0.2895  0.0358 -0.3661\n",
       " -1.4977 -0.0709 -0.2322\n",
       "\n",
       "(1 ,.,.) = \n",
       "  0.2895  0.0358 -0.3661\n",
       " -1.4977 -0.0709 -0.2322\n",
       "\n",
       "(2 ,.,.) = \n",
       "  0.2895  0.0358 -0.3661\n",
       " -1.4977 -0.0709 -0.2322\n",
       "[torch.FloatTensor of size 3x2x3]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2D - Stack - combine tensors along new dimension - 3D now\n",
    "torch.stack([x,x,x],0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 2, 3])\n",
      "\n",
      "(0 ,.,.) = \n",
      "  1  1  1\n",
      "  2  2  2\n",
      "\n",
      "(1 ,.,.) = \n",
      "  3  3  3\n",
      "  4  4  4\n",
      "[torch.FloatTensor of size 2x2x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#3D Tensor\n",
    "x = torch.FloatTensor([\n",
    "    [[1, 1, 1], \n",
    "     [2, 2, 2]],\n",
    "    [[3, 3, 3], \n",
    "     [4, 4, 4]]\n",
    "])\n",
    "print(x.size())\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 2, 3])\n",
      "\n",
      "(0 ,.,.) = \n",
      "  1  1  1\n",
      "  2  2  2\n",
      "\n",
      "(1 ,.,.) = \n",
      "  3  3  3\n",
      "  4  4  4\n",
      "\n",
      "(2 ,.,.) = \n",
      "  1  1  1\n",
      "  2  2  2\n",
      "\n",
      "(3 ,.,.) = \n",
      "  3  3  3\n",
      "  4  4  4\n",
      "[torch.FloatTensor of size 4x2x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "out = torch.cat([x,x],0)\n",
    "print(out.size())\n",
    "print(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 2, 3])\n",
      "torch.Size([2, 4, 3])\n",
      "\n",
      "(0 ,.,.) = \n",
      "  1  1  1\n",
      "  2  2  2\n",
      "  1  1  1\n",
      "  2  2  2\n",
      "\n",
      "(1 ,.,.) = \n",
      "  3  3  3\n",
      "  4  4  4\n",
      "  3  3  3\n",
      "  4  4  4\n",
      "[torch.FloatTensor of size 2x4x3]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "out = torch.cat([x,x],1)\n",
    "print(x.size())\n",
    "print(out.size())\n",
    "print(out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Conv Transpose\n",
    "\n",
    "* http://deeplearning.net/software/theano/tutorial/conv_arithmetic.html\n",
    "* no padding = o' = i' + (k - 1)\n",
    "* w padding = o' = i' + (k - 1) - 2p\n",
    "* w stride = o' = s (i' - 1) + k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 80, 23, 31])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer = nn.ConvTranspose2d(in_channels=80, \n",
    "       out_channels=80, kernel_size=3, stride=2, padding=0, bias=True)\n",
    "input = torch.randn(1, 80, 11, 15)\n",
    "layer(Variable(input)).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'dim_size' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-8bb2f46064ee>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m80\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m11\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m15\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;31m#get num dimensions and subtract 2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mmin_sizes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mdim_size\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlayer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0md\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m#This just gets the dimension sizes of output after transpose\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;31m#So it can't go any smaller, only adds padding - minimum output size is the output of the normal transpose\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;31m#min_size = output of normal transpose\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-39-8bb2f46064ee>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m80\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m11\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m15\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;31m#get num dimensions and subtract 2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mmin_sizes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mdim_size\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlayer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0md\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m#This just gets the dimension sizes of output after transpose\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;31m#So it can't go any smaller, only adds padding - minimum output size is the output of the normal transpose\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;31m#min_size = output of normal transpose\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'dim_size' is not defined"
     ]
    }
   ],
   "source": [
    "layer = nn.ConvTranspose2d(in_channels=80, \n",
    "       out_channels=80, kernel_size=3, stride=2, padding=0, bias=True) #outputdim = s(i-1) + k\n",
    "input = torch.randn(1, 80, 11, 15)\n",
    "k = input.dim() - 2 #get num dimensions and subtract 2\n",
    "min_sizes = [dim_size(layer, input, d) for d in range(k)] #This just gets the dimension sizes of output after transpose\n",
    "#So it can't go any smaller, only adds padding - minimum output size is the output of the normal transpose\n",
    "#min_size = output of normal transpose\n",
    "#max_size = \n",
    "[min_sizes[d] + layer.stride[d] - 1 for d in range(k)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Center Crop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#https://github.com/pytorch/pytorch/blob/master/torch/nn/_functions/conv.py#L59\n",
    "#https://github.com/Lasagne/Lasagne/blob/master/lasagne/layers/merge.py#L162\n",
    "#Author does a center crop which crops both inputs (skip and upsample) to size of minimum dimension on both w/h\n",
    "#But does this get us back to the output image size then?\n",
    "def center_crop(layer, max_height, max_width):\n",
    "    print(\"maxwidth\", max_width)\n",
    "    print(\"maxheigth\", max_height)\n",
    "\n",
    "    batch_size, n_channels, layer_height, layer_width = layer.size()\n",
    "    print('layer', batch_size, n_channels, layer_height, layer_width)\n",
    "    xy1 = (layer_width - max_width) // 2\n",
    "    xy2 = (layer_height - max_height) // 2\n",
    "    print('xy1', xy1)\n",
    "    print('xy2', xy2)\n",
    "    return layer[:, :, xy2:(xy2 + max_height), xy1:(xy1 + max_width)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t1 = Variable(torch.randn(1,80,11,15))\n",
    "skip = Variable(torch.randn(1, 448, 22, 30))\n",
    "layer = nn.ConvTranspose2d(in_channels=80, \n",
    "       out_channels=80, kernel_size=3, stride=2, padding=0, bias=True) #outputdim = s(i-1) + k\n",
    "out = layer(t1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#add example architectures here..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_PATH='data/'\n",
    "RESULTS_PATH='results/'\n",
    "WEIGHTS_PATH='models/'\n",
    "IMAGE_PATH=DATA_PATH+'GET_SMALL_IMAGE_SAMPLE_HERE/'\n",
    "PROJECT_NAME='cheatsheet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_crop_size = (224, 224) # None for full size\n",
    "n_classes = 12 #11 + background\n",
    "# Training\n",
    "seed = 0\n",
    "\n",
    "train_file = RESULTS_PATH+PROJECT_NAME+'-train.csv'\n",
    "test_file = RESULTS_PATH+PROJECT_NAME+'-test.csv'\n",
    "existing_weights_fpath=None#WEIGHTS_PATH+'latest.pth'\n",
    "existing_optimizer_fpath=None #WEIGHTS_PATH+'latest-optim.pth'\n",
    "\n",
    "#Finetune overrides\n",
    "n_epochs=50\n",
    "learning_rate=.00001\n",
    "lr_sched_decay = 0.995 # Applied each epoch exponential - α=αe^−kt0,k are hyperparameters and \n",
    "weight_decay = 0.0001\n",
    "num_epochs = 750\n",
    "max_patience = 150"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adaptive Learning Rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def adjust_learning_rate(lr, optimizer, cur_epoch, decay=.995, n_epochs=1):\n",
    "    \"\"\"Sets the optimizers learning rate to be the original `lr` \n",
    "    multiplied by `decay` every `n_epochs`\n",
    "    \n",
    "    # Arguments\n",
    "        lr: global learning rate defined at start of training\n",
    "        optimizer: your optimizer\n",
    "        cur_epoch: current epochs in training\n",
    "        decay: (.995)\n",
    "        n_epochs: number of epochs per new application of `decay`\n",
    "        \n",
    "    # Output\n",
    "        None: Updates optimizer in-place\n",
    "    \"\"\"\n",
    "    new_lr = lr * (decay ** (cur_epoch // n_epochs))\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = new_lr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 2\n",
    "seed = 1\n",
    "torch.cuda.manual_seed(seed)\n",
    "traindir = os.path.join(IMAGE_PATH, 'train')\n",
    "valdir = os.path.join(IMAGE_PATH, 'val')\n",
    "testdir = os.path.join(IMAGE_PATH, 'test')\n",
    "IMAGE_MEAN=1.2\n",
    "IMAGE_STD=1.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "normalize = transforms.Normalize(mean=IMAGE_MEAN, std=IMAGE_STD)\n",
    "train_dset = camvid.CamVid(CAMVID_PATH, 'train',\n",
    "      transform=transforms.Compose([\n",
    "          transforms.RandomCrop(224),\n",
    "          transforms.RandomHorizontalFlip()\n",
    "          transforms.ToTensor(),\n",
    "          normalize,\n",
    "    ]))\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    train_dset, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# print(train_loader.dataset.classes)\n",
    "# print(train_loader.dataset.class_weight)\n",
    "# print(train_loader.dataset.imgs[:3])\n",
    "# print(train_loader.dataset.mean)\n",
    "# print(train_loader.dataset.std)\n",
    "print(\"TrainImages: %d\" %len(train_loader.dataset.imgs))\n",
    "print(\"ValImages: %d\" %len(val_loader.dataset.imgs))\n",
    "print(\"TestImages: %d\" %len(test_loader.dataset.imgs))\n",
    "print(\"NumClasses: %d\" % len(train_loader.dataset.classes))\n",
    "\n",
    "example_inputs, example_targets = next(iter(train_loader))\n",
    "print(\"InputsBatchSize: \", example_inputs.size())\n",
    "print(\"TargetsBatchSize: \", example_targets.size())\n",
    "\n",
    "#Inputs are tensors of normalized pixel values\n",
    "print (\"\\nInput (size, max, min) ---\")\n",
    "i = example_inputs[0]\n",
    "print (i.size())\n",
    "print(i.max())\n",
    "print(i.min())\n",
    "\n",
    "\n",
    "#Targets are tensors of class labels from 0-11 (0 means background)\n",
    "print (\"Target (size, max, min) ---\")\n",
    "t = example_targets[0]\n",
    "print(t.size())\n",
    "print(t.max())\n",
    "print(t.min())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Preexisting Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
    "                                        download=True, transform=transform)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=4,\n",
    "                                          shuffle=True, num_workers=2)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
    "                                       download=True, transform=transform)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=4,\n",
    "                                         shuffle=False, num_workers=2)\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat',\n",
    "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visdom Visualizations\n",
    "\n",
    "* https://github.com/facebookresearch/visdom\n",
    "* https://github.com/facebookresearch/visdom/blob/master/example/demo.py\n",
    "* http://73.223.178.63:8097"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import visdom\n",
    "viz = visdom.Visdom()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def viz_plot_tst_trn(window, epoch, tst_val, trn_val, name='loss', env='main'):\n",
    "    if window is None:\n",
    "        return viz.line(\n",
    "            X=np.array([epoch]),\n",
    "            Y=np.array([[tst_val, trn_val]]),\n",
    "            opts=dict(\n",
    "                xlabel='epoch',\n",
    "                ylabel=name,\n",
    "                title=env+' '+name,\n",
    "                legend=['Validation', 'Train']\n",
    "            ),\n",
    "            env=env\n",
    "        )\n",
    "    return viz.line(\n",
    "        X=np.ones((1, 2)) * epoch,\n",
    "        Y=np.expand_dims([tst_val, trn_val],0),\n",
    "        win=window,\n",
    "        update='append',\n",
    "        env=env\n",
    "    )\n",
    "\n",
    "def viz_plot_img(window, tensor, env='main', title='Image'):\n",
    "    '''\n",
    "    This function draws an img on your Visdom web app. \n",
    "    It takes as input an `CxHxW` tensor `img`\n",
    "    The array values can be float in [0,1] or uint8 in [0, 255]'''\n",
    "    np_img = decode_image(tensor)\n",
    "    np_img = np.rollaxis(np_img, 2, 0)\n",
    "    viz.image(\n",
    "        np_img,\n",
    "        opts=dict(title=title, caption='Silly image'),\n",
    "        win=window,\n",
    "        env=env\n",
    "    )\n",
    "    \n",
    "def viz_plot_text(window, text, env='main'):\n",
    "    if window is None:\n",
    "        return viz.text(\n",
    "            text,\n",
    "            env=env\n",
    "        )\n",
    "    return viz.text(\n",
    "        text,\n",
    "        win=window,\n",
    "        env=env\n",
    "    )\n",
    "\n",
    "def viz_plot_summary(window, epoch, tst_loss, trn_loss,\n",
    "                       tst_err, trn_err, env='main'):\n",
    "    txt = (\"\"\"Epoch: %d\n",
    "        Train - Loss: %.3f Err: %.3f\n",
    "        Test - Loss: %.3f Err: %.3f\"\"\" % (epoch, \n",
    "        trn_loss, trn_err, tst_loss, tst_err))\n",
    "    return viz_plot_text(window, txt, env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Should plot one chart and update it\n",
    "txt_chart = viz_plot_summary(None, 1, 2, 3, 4, 5)\n",
    "txt_chart = viz_plot_summary(txt_chart, 5, 2, 3, 4, 5)\n",
    "txt_chart = viz_plot_summary(txt_chart, 5, 3, 8, 7, 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Should plot one chart and update it\n",
    "sum_chart = viz_plot_text(None, 'Hello, world3!')\n",
    "sum_chart = viz_plot_text(sum_chart, 'Hello, world4!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Should plot one chart and update it\n",
    "#window, epoch, tst_val, trn_val, name='loss', env='main'\n",
    "loss_chart = viz_plot_tst_trn(None, 9, 14, 27, 'loss')\n",
    "loss_chart = viz_plot_tst_trn(loss_chart, 10, 18, 30, 'loss')\n",
    "loss_chart = viz_plot_tst_trn(loss_chart, 11, 19, 32, 'loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Should plot one chart and update it\n",
    "#window, epoch, tst_val, trn_val, name='loss', env='main'\n",
    "err_chart = viz_plot_tst_trn(None, 9, 14, 27, 'error')\n",
    "err_chart = viz_plot_tst_trn(err_chart, 10, 18, 30, 'error')\n",
    "err_chart = viz_plot_tst_trn(err_chart, 11, 19, 32, 'error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train_loader' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-40-284ece1c142a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m img_chart = viz.image(\n\u001b[1;32m      3\u001b[0m     \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m360\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m480\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mopts\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"Image\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaption\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Silly random'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m )\n",
      "\u001b[0;31mNameError\u001b[0m: name 'train_loader' is not defined"
     ]
    }
   ],
   "source": [
    "inputs, targets = next(iter(train_loader))\n",
    "img_chart = viz.image(\n",
    "    np.random.rand(3,360,480),\n",
    "    opts=dict(title=\"Image\", caption='Silly random'),\n",
    ")\n",
    "viz_plot_img(img_chart, inputs[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Image Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#normalization\n",
    "\n",
    "#denormalization\n",
    "\n",
    "#get mean/std of numpy array\n",
    "\n",
    "#plot tensor image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Image Visualizations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Sky = [128,128,128]\n",
    "Building = [128,0,0]\n",
    "Pole = [192,192,128]\n",
    "#Road_marking = [255,69,0] ???\n",
    "Road = [128,64,128]\n",
    "Pavement = [60,40,222]\n",
    "Tree = [128,128,0]\n",
    "SignSymbol = [192,128,128]\n",
    "Fence = [64,64,128]\n",
    "Car = [64,0,128]\n",
    "Pedestrian = [64,64,0]\n",
    "Bicyclist = [0,128,192]\n",
    "Unlabelled = [0,0,0]\n",
    "\n",
    "label_colours = np.array([Sky, Building, Pole, Road, Pavement,\n",
    "      Tree, SignSymbol, Fence, Car, Pedestrian, Bicyclist, Unlabelled])\n",
    "\n",
    "def view_annotated(tensor, plot=True):\n",
    "    temp = tensor.numpy()\n",
    "    r = temp.copy()\n",
    "    g = temp.copy()\n",
    "    b = temp.copy()\n",
    "    for l in range(0,11):\n",
    "        r[temp==l]=label_colours[l,0]\n",
    "        g[temp==l]=label_colours[l,1]\n",
    "        b[temp==l]=label_colours[l,2]\n",
    "\n",
    "    rgb = np.zeros((temp.shape[0], temp.shape[1], 3))\n",
    "    rgb[:,:,0] = (r/255.0)#[:,:,0]\n",
    "    rgb[:,:,1] = (g/255.0)#[:,:,1]\n",
    "    rgb[:,:,2] = (b/255.0)#[:,:,2]\n",
    "    if plot:\n",
    "        plt.imshow(rgb)\n",
    "        plt.show()\n",
    "    else:\n",
    "        return rgb\n",
    "\n",
    "def decode_image(tensor):\n",
    "    inp = tensor.numpy().transpose((1, 2, 0))\n",
    "    mean = np.array(camvid.mean)\n",
    "    std = np.array(camvid.std)\n",
    "    inp = std * inp + mean\n",
    "    return inp\n",
    "\n",
    "def view_image(tensor):\n",
    "    \"\"\"Imshow for Tensor.\"\"\"\n",
    "    inp = decode_image(tensor)\n",
    "    plt.imshow(inp)\n",
    "    plt.show()\n",
    "    \n",
    "def plot_img_array(img_arr, shape=(16,16)):\n",
    "    # Plot numpy array image with shape (inches?)\n",
    "    plt.figure(figsize=shape)\n",
    "    plt.imshow(img_arr.astype('uint8'))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get a batch of training data\n",
    "inputs, targets = next(iter(train_loader))\n",
    "#inputs, targets = next(iter(val_loader))\n",
    "#inputs, targets = next(iter(test_loader))\n",
    "\n",
    "# Plot Single Image\n",
    "view_image(inputs[0])\n",
    "\n",
    "# Plot Target Image\n",
    "view_annotated(targets[0])\n",
    "\n",
    "# Plot Grid of images\n",
    "out = torchvision.utils.make_grid(inputs, nrow=3)\n",
    "view_image(out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train(model, train_loader, optimizer, criterion, trainF, epoch, projectName):\n",
    "    model.train()\n",
    "    for batch_idx, (inputs, targets) in enumerate(train_loader):\n",
    "        inputs, targets = Variable(inputs.cuda()), Variable(targets.cuda())\n",
    "        optimizer.zero_grad()\n",
    "        output = model(inputs)\n",
    "        loss = criterion(output, targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        pred = get_predictions(output)\n",
    "        train_err = error(pred, targets.data.cpu())\n",
    "        partialEpoch = epoch + batch_idx / len(train_loader) - 1\n",
    "        trainF.write('{},{},{}\\n'.format(partialEpoch, loss.data[0], train_err))\n",
    "        trainF.flush()\n",
    "    print('Epoch {:d}: Train - Loss: {:.4f}\\tErr: {:.4f}'.format(epoch, loss.data[0], train_err))\n",
    "    return loss.data[0], train_err\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test(model, test_loader, criterion, testF=None, epoch=1):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    test_error = 0\n",
    "    for data, target in test_loader:\n",
    "        data, target = Variable(data.cuda(), volatile=True), Variable(target.cuda())\n",
    "        output = model(data)\n",
    "        test_loss += criterion(output, target).data[0]\n",
    "        pred = get_predictions(output)\n",
    "        test_error += error(pred, target.data.cpu())\n",
    "    test_loss /= len(test_loader) #n_batches\n",
    "    test_error /= len(test_loader)\n",
    "    print('Test - Loss: {:.4f}, Error: {:.4f}'.format(\n",
    "        test_loss, test_error))\n",
    "    if testF:\n",
    "        testF.write('{},{},{}\\n'.format(int(epoch), test_loss, test_error))\n",
    "        testF.flush()\n",
    "    return test_loss, test_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "criterion = nn.NLLLoss2d(weight=camvid.class_weight.cuda()).cuda()\n",
    "test(model, test_loader, criterion)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Serialization (Save/Load Weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_weights(model, epoch, loss, err, sessionName, isBest=False):\n",
    "    weights_fname = sessionName+'-%d-%.3f-%.3f.pth' % (epoch, loss, err)\n",
    "    weights_fpath = os.path.join(WEIGHTS_PATH, weights_fname)\n",
    "    torch.save({\n",
    "            'startEpoch': epoch+1,\n",
    "            'loss':loss,\n",
    "            'error': err,\n",
    "            'sessionName': sessionName,\n",
    "            'state_dict': model.state_dict()\n",
    "        }, weights_fpath )\n",
    "    shutil.copyfile(weights_fpath, WEIGHTS_PATH+'latest.pth')\n",
    "    if isBest:\n",
    "        shutil.copyfile(weights_fpath, WEIGHTS_PATH+'best.pth')\n",
    "\n",
    "def load_weights(model, fpath):\n",
    "    print(\"loading weights '{}'\".format(fpath))\n",
    "    state = torch.load(fpath)\n",
    "    start_epoch = state['startEpoch']\n",
    "    model.load_state_dict(state['state_dict'])\n",
    "    print(\"loaded weights from session {} (lastEpoch {}, loss {}, error {})\"\n",
    "          .format(state['sessionName'], start_epoch-1, state['loss'],\n",
    "                  state['error']))\n",
    "    return state\n",
    "\n",
    "def save_optimizer(optimizer, epoch, sessionName):\n",
    "    optim_fname = sessionName+'-optim-%d.pth' % (epoch)\n",
    "    optim_fpath = os.path.join(WEIGHTS_PATH, optim_fname)\n",
    "    torch.save({\n",
    "            'lastEpoch': epoch,\n",
    "            'sessionName': sessionName,\n",
    "            'state_dict': optimizer.state_dict()\n",
    "        }, optim_fpath )\n",
    "    shutil.copyfile(optim_fpath, WEIGHTS_PATH+'latest-optim.pth')\n",
    "\n",
    "def load_optimizer(optimizer, fpath):\n",
    "    print(\"loading optimizer '{}'\".format(fpath))\n",
    "    optim = torch.load(fpath)\n",
    "    optimizer.load_state_dict(optim['state_dict'])\n",
    "    print(\"loaded optimizer from session {}, lastEpoch {}\"\n",
    "          .format(optim['sessionName'], optim['lastEpoch']))\n",
    "    \n",
    "# Add patience function\n",
    "# https://github.com/SimJeg/FC-DenseNet/blob/master/train.py#L176"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initialize Model\n",
    "model = FCDenseNet(in_channels=3, n_blocks=5, layers_per_block=5, growth_rate=16, \n",
    "                 out_chans_first_conv=48, n_classes=n_classes)\n",
    "model = model.cuda()\n",
    "print('  + Number of params: {}'.format(\n",
    "    sum([p.data.nelement() for p in model.parameters()])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if existing_weights_fpath:\n",
    "    state = load_weights(model, existing_weights_fpath)\n",
    "    start_epoch = state['startEpoch']\n",
    "    endEpoch = state['startEpoch'] + n_epochs\n",
    "    print ('Resume training at epoch: {}'.format(state['startEpoch']))\n",
    "    if os.path.exists(train_file): #assume test.csv exists\n",
    "        append_write = 'a' # append if already exists\n",
    "    else:\n",
    "        append_write = 'w' # make a new file if not\n",
    "    trainF = open(os.path.join(train_file), append_write)\n",
    "    testF = open(os.path.join(test_file), append_write)\n",
    "else:\n",
    "    print (\"Training new model from scratch\")\n",
    "    model.apply(weights_init)\n",
    "    start_epoch = 1\n",
    "    endEpoch = n_epochs\n",
    "    trainF = open(os.path.join(train_file), 'w')\n",
    "    testF = open(os.path.join(test_file), 'w')\n",
    "\n",
    "cudnn.benchmark = True # ????\n",
    "optimizer = optim.RMSprop(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n",
    "if existing_optimizer_fpath:\n",
    "    print(\"Loading existing optimizer: \", existing_optimizer_fpath)\n",
    "    load_optimizer(optimizer, existing_optimizer_fpath)\n",
    "criterion = nn.NLLLoss2d(weight=camvid.class_weight.cuda()).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss_chart, err_chart, txt_chart = None, None, None\n",
    "for epoch in range(start_epoch, endEpoch+1):\n",
    "    since = time.time()\n",
    "    trn_loss, trn_err = train(model, train_loader, optimizer, criterion, trainF, epoch, PROJECT_NAME)\n",
    "    tst_loss, tst_err = test(model, val_loader, criterion, testF, epoch)\n",
    "    time_elapsed = time.time() - since  \n",
    "    print('Time {:.0f}m {:.0f}s\\n'.format(\n",
    "        time_elapsed // 60, time_elapsed % 60))\n",
    "    save_weights(model, epoch, tst_loss, tst_err, PROJECT_NAME)\n",
    "    save_optimizer(optimizer, epoch, PROJECT_NAME)\n",
    "    if visdom_enabled:\n",
    "        loss_chart = viz_plot_tst_trn(loss_chart, epoch, tst_loss, trn_loss, 'loss', PROJECT_NAME)\n",
    "        err_chart = viz_plot_tst_trn(err_chart, epoch, tst_err, trn_err, 'error', PROJECT_NAME)\n",
    "        txt_chart = viz_plot_summary(txt_chart, epoch, tst_loss, trn_loss, tst_err, trn_err, PROJECT_NAME)\n",
    "\n",
    "trainF.close()\n",
    "testF.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def predict(model, input_loader, n_batches=1):\n",
    "    input_loader.batch_size = 1\n",
    "    #Takes input_loader and returns array of prediction tensors\n",
    "    predictions = []\n",
    "    model.eval()\n",
    "    for input, target in input_loader:\n",
    "        data, label = Variable(input.cuda(), volatile=True), Variable(target.cuda())\n",
    "        output = model(data)\n",
    "        pred = get_predictions(output)\n",
    "        predictions.append([input,target,pred])\n",
    "    return predictions\n",
    "\n",
    "#predictions = predict_all(model, test_loader, 1)\n",
    "# for out in predictions[:1]:\n",
    "#     view_image(out[0][0])\n",
    "#     view_annotated(out[1][0])\n",
    "#     view_annotated(out[2][0])\n",
    "\n",
    "def view_sample_predictions(n):\n",
    "    #torch.cuda.manual_seed(random.randint(0,10**7))\n",
    "    test_loader = torch.utils.data.DataLoader(test_dset, batch_size=n, shuffle=True)\n",
    "    inputs, targets = next(iter(test_loader))\n",
    "    data, label = Variable(inputs.cuda(), volatile=True), Variable(targets.cuda())\n",
    "    output = model(data)\n",
    "    pred = get_predictions(output)\n",
    "    batch_size = inputs.size(0)\n",
    "    for i in range(batch_size):\n",
    "        view_image(inputs[i])\n",
    "        view_annotated(targets[i])\n",
    "        view_annotated(pred[i])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "view_sample_predictions(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## References\n",
    "\n",
    "* https://github.com/desimone/pytorch-cat-vs-dogs/blob/master/main.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t1 = torch."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "192px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}