04.mlp.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multi Layer Perceptron (MLP) \n",
    "(c) Deniz Yuret, 2018"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Objectives: See the effect of nonlinearities, learn about regularization and dropout to combat overfitting.\n",
    "* Prerequisites: Linear models (03.lin.ipynb), AutoGrad, Param, KnetArray, gpu, nll, zeroone\n",
    "* Knet: xavier, param, param0, relu, dropout (defined and explained)\n",
    "* Knet: dir, gpu, minibatch, KnetArray (used by mnist.jl)\n",
    "* Knet: SGD, train!, load, save (used by trainresults)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Pkg\n",
    "for p in (\"Knet\",\"AutoGrad\",\"Plots\",\"ProgressMeter\")\n",
    "    haskey(Pkg.installed(),p) || Pkg.add(p)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data (see 02.mnist.ipynb)\n",
    "using Knet: Knet, KnetArray, gpu, minibatch\n",
    "include(Knet.dir(\"data\",\"mnist.jl\"))  # Load data\n",
    "dtrn,dtst = mnistdata(xsize=(784,:)); # dtrn and dtst = [ (x1,y1), (x2,y2), ... ] where xi,yi are minibatches of 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For running experiments\n",
    "using Knet: SGD, train!, nll, zeroone\n",
    "import ProgressMeter\n",
    "    \n",
    "function trainresults(file,model; o...)\n",
    "    if (print(\"Train from scratch? \");readline()[1]=='y')\n",
    "        results = Float64[]; updates = 0; prog = ProgressMeter.Progress(60000)\n",
    "        function callback(J)\n",
    "            if updates % 600 == 0\n",
    "                push!(results, nll(model,dtrn), nll(model,dtst), zeroone(model,dtrn), zeroone(model,dtst))\n",
    "                ProgressMeter.update!(prog, updates)\n",
    "            end\n",
    "            return (updates += 1) <= 60000\n",
    "        end\n",
    "        train!(model, dtrn; callback=callback, optimizer=SGD(lr=0.1), o...)\n",
    "        Knet.save(file,\"results\",reshape(results, (4,:)))\n",
    "    end\n",
    "    isfile(file) || download(\"http://people.csail.mit.edu/deniz/models/tutorial/$file\",file)\n",
    "    results = Knet.load(file,\"results\")\n",
    "    println(minimum(results,dims=2))\n",
    "    return results\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "using AutoGrad: Param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Redefine Linear model (See 03.lin.ipynb):\n",
    "struct Linear; w; b; end\n",
    "(f::Linear)(x) = (f.w * x .+ f.b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Linear(inputsize,outputsize) constructs a randomly initalized Linear model:\n",
    "Linear(inputsize::Int,outputsize::Int) = Linear(param(outputsize,inputsize),param0(outputsize))\n",
    "param(d...; init=xavier, atype=atype())=Param(atype(init(d...)))\n",
    "param0(d...; atype=atype())=param(d...; init=zeros, atype=atype)\n",
    "xavier(o,i) = (s = sqrt(2/(i+o)); 2s .* rand(o,i) .- s)\n",
    "atype()=(gpu() >= 0 ? KnetArray{Float32} : Array{Float32})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MLP is a bunch of linear layers stuck together:\n",
    "struct MLP; layers; end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MLP(x,h1,h2,...,hn,y): constructor for a n hidden layer MLP:\n",
    "MLP(h::Int...)=MLP(Linear.(h[1:end-1], h[2:end]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "# Here is an example model\n",
    "model=MLP(784,64,10)\n",
    "summary.(l.w for l in model.layers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Multiple linear layers do not improve over linear model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Plots; default(fmt=:png,ls=:auto)\n",
    "ENV[\"COLUMNS\"]=92"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let us try to just chain multiple linear layers\n",
    "function (m::MLP)(x)\n",
    "    for layer in m.layers\n",
    "        x = layer(x)\n",
    "    end\n",
    "    return x\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train a linear model\n",
    "lin1 = trainresults(\"lin1.jld2\", Linear(784,10)); # 113s [0.241861, 0.266827, 0.0668667, 0.0747]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# Train a multi-linear model\n",
    "mlp1 = trainresults(\"mlp1.jld2\", MLP(784,64,10)); # 110s [0.242123, 0.285064, 0.0697, 0.0799]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# multilinear converges to a similar solution, not identical because problem is non-convex\n",
    "plot([lin1[1,:], lin1[2,:], mlp1[1,:], mlp1[2,:]], ylim=(0.0,0.4),\n",
    "    labels=[:trnLin :tstLin :trnMulti :tstMulti],xlabel=\"Epochs\",ylabel=\"Loss\")  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# error results also close to the linear model\n",
    "plot([lin1[3,:], lin1[4,:], mlp1[3,:], mlp1[4,:]], ylim=(0.0,0.1),\n",
    "    labels=[:trnLin,:tstLin,:trnMulti,:tstMulti], xlabel=\"Epochs\", ylabel=\"Error\")  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Multiple linear layers are useless because they are equivalent to a single linear layer\n",
    "If we write down what is being computed and do some algebra, we can show that what is being computed is still an affine function of the input, i.e. stacking multiple linear layers does not increase the representational capacity of the model:\n",
    "\n",
    "\\begin{align*}\n",
    "\\hat{p} &= \\mbox{soft}(W_2 (W_1 x + b_1) + b_2) \\\\\n",
    "&= \\mbox{soft}((W_2 W_1)\\, x + W_2 b_1 + b_2) \\\\\n",
    "&= \\mbox{soft}(W x + b)\n",
    "\\end{align*}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "Knet.gc() # to save gpu memory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Multi Layer Perceptron (MLP) adds non-linearities between layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using nonlinearities between layers results in a model with higher capacity and helps underfitting\n",
    "# relu(x)=max(0,x) is a popular function used for this purpose, it replaces all negative values with zeros.\n",
    "using Knet: relu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "function (m::MLP)(x)\n",
    "    for layer in m.layers\n",
    "        x = layer(x)\n",
    "        x = (layer == m.layers[end] ? x : relu.(x))  # <-- This one line makes a big difference\n",
    "    end\n",
    "    return x\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp2 = trainresults(\"mlp2.jld2\", MLP(784,64,10));  # 124s [0.00632429, 0.0888991, 0.00055, 0.0259]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "source": [
    "## MLP solves underfitting but still has an overfitting problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# MLP solves the underfitting problem!  A more serious overfitting problem remains.\n",
    "plot([lin1[1,:], lin1[2,:], mlp2[1,:], mlp2[2,:]], ylim=(0.0,0.4),\n",
    "     labels=[:trnLin :tstLin :trnMLP :tstMLP],xlabel=\"Epochs\",ylabel=\"Loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# Test error improves from 7.5% to 2.3%!\n",
    "plot([lin1[3,:], lin1[4,:], mlp2[3,:], mlp2[4,:]], ylim=(0.0,0.1),\n",
    "    labels=[:trnLin,:tstLin,:trnMLP,:tstMLP], xlabel=\"Epochs\", ylabel=\"Error\")  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "Knet.gc() # to save gpu memory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "source": [
    "## MLP with L1 regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# Define new loss functions for L1 and L2 regularization\n",
    "using Knet: nll\n",
    "nll1(m::MLP,x,y; λ=0, o...) = nll(m,x,y; o...) + λ * sum(sum(abs, l.w) for l in m.layers)\n",
    "nll2(m::MLP,x,y; λ=0, o...) = nll(m,x,y; o...) + λ * sum(sum(abs2,l.w) for l in m.layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp3 = trainresults(\"mlp3.jld2\", MLP(784,64,10); loss=nll1, λ=4f-5); # 133s [0.0262292, 0.0792939, 0.0066, 0.0237]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# overfitting less, test loss improves from 0.0808 to 0.0759\n",
    "plot([mlp2[1,:], mlp2[2,:], mlp3[1,:], mlp3[2,:]], ylim=(0.0,0.15),\n",
    "     labels=[:trnMLP :tstMLP :trnL1 :tstL1],xlabel=\"Epochs\",ylabel=\"Loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# however test error does not change significantly: 0.0235 -> 0.0220\n",
    "plot([mlp2[3,:], mlp2[4,:], mlp3[3,:], mlp3[4,:]], ylim=(0.0,0.04),\n",
    "     labels=[:trnMLP :tstMLP :trnL1 :tstL1],xlabel=\"Epochs\",ylabel=\"Error\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "Knet.gc() # to save gpu memory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## MLP with dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Knet: dropout\n",
    "@doc dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "# Dropout is another way to address overfitting\n",
    "function (m::MLP)(x; pdrop=0)\n",
    "    for (i,layer) in enumerate(m.layers)\n",
    "        p = (i <= length(pdrop) ? pdrop[i] : pdrop[end])\n",
    "        x = dropout(x, p)     # <-- This one line helps generalization\n",
    "        x = layer(x)\n",
    "        x = (layer == m.layers[end] ? x : relu.(x))\n",
    "    end\n",
    "    return x\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "mlp4 = trainresults(\"mlp4.jld2\", MLP(784,64,10); pdrop=(0.2,0.0));  # 119s [0.0126026, 0.0648639, 0.0033, 0.0189]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# overfitting less, loss results improve 0.0808 -> 0.0639\n",
    "plot([mlp2[1,:], mlp2[2,:], mlp4[1,:], mlp4[2,:]], ylim=(0.0,0.15),\n",
    "     labels=[:trnMLP :tstMLP :trnDrop :tstDrop],xlabel=\"Epochs\",ylabel=\"Loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# this time error also improves 0.0235 -> 0.0188\n",
    "plot([mlp2[3,:], mlp2[4,:], mlp4[3,:], mlp4[4,:]], ylim=(0.0,0.04),\n",
    "     labels=[:trnMLP :tstMLP :trnDrop :tstDrop],xlabel=\"Epochs\",ylabel=\"Error\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "println([:mlperr,minimum(mlp2[4,:]),:L1err,minimum(mlp3[4,:]),:dropouterr,minimum(mlp4[4,:])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "println([:mlploss,minimum(mlp2[2,:]),:L1loss,minimum(mlp3[2,:]),:dropoutloss,minimum(mlp4[2,:])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "Knet.gc() # to save gpu memory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## MLP with larger hidden layer and dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "# The current trend is to use models with higher capacity tempered with dropout\n",
    "mlp = trainresults(\"mlp.jld2\", MLP(784,256,10); pdrop=(0.2,0.0));  # 123s [0.00365709, 0.0473298, 0.000283333, 0.0141]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "plot([mlp4[1,:], mlp4[2,:], mlp[1,:], mlp[2,:]],ylim=(0,0.15),\n",
    "    labels=[:trn64 :tst64 :trn256 :tst256],xlabel=\"Epochs\",ylabel=\"Loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# We are down to 0.015 error.\n",
    "plot([mlp4[3,:], mlp4[4,:], mlp[3,:], mlp[4,:]],ylim=(0,0.04),\n",
    "    labels=[:trn64 :tst64 :trn256 :tst256],xlabel=\"Epochs\",ylabel=\"Error\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "julia.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Julia 1.0.3",
   "language": "julia",
   "name": "julia-1.0"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.0.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}