lstm_gru.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implmentation of LSTM and GRU\n",
    "\n",
    "The maximum accuracy achived on MNIST for both LSTM and GRU is 97%. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "loss_gru = np.loadtxt(\"loss_gru.txt\")\n",
    "loss_lstm = np.loadtxt(\"loss_lstm.txt\")\n",
    "loss_gru = loss_gru.astype(np.float)\n",
    "loss_lstm = loss_lstm.astype(np.float)\n",
    "two_loss = np.vstack((loss_gru, loss_lstm))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Learning curve for GRU and LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7ffa2ccb8c18>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzsnXd4VEXXwH8nHaQ3RToKqEgVUKxYEXtDxYblFbFhf1/1tWB77fVTsXdARVRQUUQEGwLSi3QIEOm9hPTz/XFvNpvtSXaTbHJ+z7PP3jt3Zu6Zzc09M3POnBFVxTAMwzAAEipaAMMwDKPyYErBMAzD8GBKwTAMw/BgSsEwDMPwYErBMAzD8GBKwTAMw/BgSsGodIjI5SLyY0XLUZ6IyOMiskVENlS0LJUdEVERObii5aiqmFIoJ0QkXURyRKSRT/oc9yFv7Z5/4J738spzsIio1/lkEfmX1/n9IrJKRPaISIaIfOamL3TT9ohIvohkeZ3fH0DGoSLySfRbXzJUdbiqnlbRcpQXItICuAs4TFUPiFKdAV+cIpIiIs+7z8ke97l50b22x+tTICL7vM4vd58PFZEhPnXe7qYPjYbsZUVEThWRSSKyW0S2uv9j/xGRNPf6UBHJddu1Q0SmiEhvr/JXi8jvAepNF5FTyrMtFYEphfJlFTCg8EREOgE1AuTbBjweSYUiMhC4EjhFVWsBPYCJAKraUVVruem/AbcUnqvq/8rWlNIhIkkVcd9oIg7R/N9pBWxV1U2lkKWkv+d9OM9IL6A2cCIwG8Dr2agFrAHO9kob7pZfCgz0qfMqN73CEZH+wBfACKCVqjYELgGaAy28sn7mtrMRMAkYVd6yVlZMKZQvH+P8AxUyEPgoQL4Pgc4ickIEdfYExqvqCgBV3aCqb5VZUh9E5EARGS0im93e5RCva71E5E+317VeRF4VkRSv6yoiN4vIMmCZV9pgEVkmIttF5DUREfdasZ5amLyJbs93iyvXLW7+gC9LEWkhIl+67dgqIq+66cVGSSLS2rsed3T2hIj8AWQC94vIDJ+67xCRse5xqog8JyJrRGSjiLwhIn4dALfnOQE40O25fuCmn+OO9Ha49z7Uq0y62/OdB+wtoWLoCXylquvUIV1VAz2DwfgLqCkiHV1ZOuJ0bP4KVkBEDhKRn93fe4uIDBeRej7tuVtE5onIThH5rLBX716/x32u1onItSHuI8ALwKOq+raqbgNQ1SWqequqLvMto6p5wHCgmYg0LsHvUGUxpVC+TAXqiMihIpKI04MJNF2TCfwPeCLCOq9y/3F6uPVGFbdX/A0wF2gGnAzcLiJ93Sz5wB04va7e7vWbfKo5DzgSOMwr7Sycl1QX4GKgL8EJlvd6oB/QFeju3idYOxKBb4HVQGu3LZ+GuKcvVwKDcHrY/wd0EJF2Xtcvw+mhAjwNtHflOti910O+FarqT67869we+dUi0h4YCdwONAbGAd94K1qcEeeZQD33xRYpU4E7ReQmEelUqFxLiHfnJljHxhsBngQOBA7F6bEP9clzMXA60AboDFwNICKnA3cDpwLtgFDTNx1wRgSjI2qFU38KTlu2AtsjLVeVMaVQ/hT+Q50KLAb+CZLvTaCliPQLVZmqfgLcivOS/AXYJCL3Rk9cwHkZN1bVR1U1R1VXAm8Dl7oyzFTVqaqap6rpruy+o5wnVXWbqu7zSntKVXeo6hqcIXzXEDIEy3sx8LKqZqjqduCpEHX0wnkx3aOqe1U1S1X95o5D8IGqLnTbuRMYgzsd6CqHQ4Cx7ov2euAOt827cZT8pRHe5xLgO1WdoKq5wHM4vfGjvfK8oqprfX7PSHgSR2FdDswA/nGnIEvCJ8AAEUnGaVNIO5SqLnfbkq2qm3F6877Pxyvu6GUbTgfE++/7vqouUNW9+CsTbwrtdR5jvYh86o62MkXkSq+8F4vIDmAfzt/qohIq1yqLKYXy52OcHuXVhOhhqWo28Jj7Cdmbcw2zpwD1gMHAo169+GjQCmd6Y0fhB7gf2B9ARNqLyLciskFEduG8ABv51LE2QL3enjaZQK0QMgTLe6BP3YHuU0gLYHUZ/vl96x5BkY3oMuBrVc3E6d3XBGZ6/V4/uOmRcCDOaAYAVS1w790shCwRoar5qvqaqh6D87w8AbznPT0VQR1rgOU4f+dlqhpSFhFp4r6c/3Gfj0/wfz4i/fuuJjhb3e+mXrJeqqr1gFmA9yj6czd9f2ABcITXtTwgOUD9yUBuiPtXCUwplDOquhrH4HwG8GWY7O8DdYHzI6w7V1VHAfOAw8sipw9rgVWqWs/rU1tVz3CvD8MZ9bRT1To4CsNXkcUqHO96nCmDQloEy4jTjpZB5uD34rzICwnkBeTbhh+BRiLSFUc5FE4dbcHpgXb0+r3quobNSFiHo4gBz1x5C4qPKsv8e6rqPlV9DWfa5LBw+X34CMdjKhJ7xJM48nZ2n48rCNPR8WI9xf+mLUPkLRx5XxBh3ajqFuAGYKiIFCqTNTjPiUdGEakJNCG0UqoSmFKoGK4DTnKHw0Fxe7RDgf8Ey+MaZc8UkdoikuBON3UEppVStgQRSfP6pALTgV2ucbOGa9w9XER6umVqA7uAPSJyCHBjKe9dGj4HbhORZq7xMuhvhdOO9cBTIrKf275j3GtzgONFpKWI1MXx0gmJ+/f5AngWaIBjMC7s2b8NvCgiTQBc+SIdvX0OnCkiJ7tTNHcB2cCUCMsXkuLzt0wUx320j/t3THKnjmrjeiCVgM+A01xZw1Eb2APsEJFmwD0luM/nwNUicpj7Yn44WEZ19gG4C3hYRK4Xkfri0A53VBuk3GJgPPBvN2kakAXc6/5u++FMS87AlIIRC1R1harOCJ8TcAyO60Nc34XTM18D7ACeAW4s4Vy5NwNwermFnxWqmg+cjTPPuwqnJ/wOzigGHEPgZcBunJfhZ6W8d2l4G6fHPg/nxTYOZ/if75vRqx0H4/xeGTjz96jqBBy55wEzcQzSkTACx/g5ymda6j84UyxT3SmTn3AMoWFR1SU4ven/w/mtz8ZxD82JUKZCFlL8b3mN+/08znTNFuBm4ELXThQx7ijjpwhtGo/gOAHsBL4j/AjZ+z7fAy8BP+P8nj+Hyf8Zjh3iCpyR4RYcxfIWod1OnwUGiUgTd+r2TKAPzjOyEmca62JX8VRppBq00ahGuCOlN1S1VdjMhmH4YSMFI65xp0HOcKdCmuFML3xV0XIZRrxiIwUjrnHnmX/BcQfdhzM9cZuq7qpQwQwjTjGlYBiGYXiw6SPDMAzDQ9wFJ2vUqJG2bt26osUwDMOIK2bOnLlFVcMuoIw7pdC6dWtmzIjUm9MwDMMAEJGI1ljY9JFhGIbhwZSCYRiG4cGUgmEYhuEh7mwKhmEYkZKbm0tGRgZZWVkVLUq5kZaWRvPmzUlODhToNTymFAzDqLJkZGRQu3ZtWrduTen2E4ovVJWtW7eSkZFBmzZtSlWHTR8ZhlFlycrKomHDhtVCIQCICA0bNizTyMiUgmEYVZrqohAKKWt7q5VSWLJhN7uycvliZgaZObbznmEYhi/VRimMmfMPfV/6lc5Df+TRUX9Q838NGXD/M+zOcnbXW7F5D0s37q5gKQ3DqGps3LiRyy67jLZt23LEEUfQu3dvvvrqKyZPnkzdunXp1q0bhxxyCHfffbenzNChQ3nuueeK1dO6dWu2bNkSc3mrjaG5VYMapKddVixtZMoT9H2kNk1lK5MLugHw8XW9OKptQ5ITY6gvCwpg2hvQ/SpIjXSHRsMw4g1V5bzzzmPgwIGMGOHs1rp69WrGjh1L/fr1Oe644/j222/Zt28f3bp14/zzz+eYY44JU2tsqTZKoeuy1wKmj0+9F4BMTeWo7Fe58t3pAFzYvTkDj25F5+b1oi/M0u9h/H2wdRmc9WL06zcMo1Lw888/k5KSwuDBgz1prVq14tZbb2Xy5MmetBo1atC1a1f++eefALWUL9VGKdC8R8jLNSWbeWnX80be2XyVfwyjZ8HoWRkAzHjgFBrVSo2eLLnuDoZZO6NXp2EYIXnkm4X8vS6622wcdmAdHj67Y9DrCxcupHv37mHr2b59O8uWLeP444+PpnilotrYFOjQD26fHzbb4KRvPKOHQno8/lOspDIMoxpx880306VLF3r27AnAb7/9RufOnTnggAM466yzOOCAA4DgHkTl4UlVfUYKAPVawsM74OfHofPFMOM9Z24/AOlpl3Fo1nvsI62chTQMIxaE6tHHio4dOzJ69GjP+WuvvcaWLVvo0cOZuSi0KSxdupRjjz2W888/n65du9KwYUPWr19frK7du3dTr14MprN9qD4jhUJE4OQHoXEH6Pc0nOFa+K/90S/rorRrPcddH/0R26XOMIyScNJJJ5GVlcWwYcM8aZmZmX752rdvz3333cfTTz8NwPHHH8/YsWPZvdvxiPzyyy/p0qULiYmJMZe5+ikFX3pdD0N3Qssjoe///C43ZjsAOzJzOfbpSezcl1veEhqGEaeICF9//TW//PILbdq0oVevXgwcONDz8vdm8ODB/Prrr6xatYrOnTtzyy23cOyxx9K1a1feeOMN3nnnnfKROd56vz169NCYbrKTvQeebFYsqXXWiGLn6U+dWbZ7zP8CRl8Hh18IF71XtroMwwjKokWLOPTQQytajHInULtFZKaqhva4wUYK/qTWgoe2FUtKoKDY+c3DZ5WnRIZhGOWGKYVAJBSft0umeEiM7+YXNwAZhmFUFUwpBGNo0RqCUxJsZGAYRvXAlEIEvJbySkWLYBiGUS6YUiglBQXxZaA3DMOIBFMKobhqrOfwoS57il366M/08pXFMAyjHDClEIq2J3gOr02/u9ilod/8Xd7SGIYRh9Sq5R8JecmSJfTp04euXbty6KGHMmjQIMaPH0/Xrl3p2rUrtWrVokOHDnTt2pWrrrqKyZMnIyK8++67njpmz56NiPiF2C4r1SvMRVnIjm4gLQDibI2IYRjRYciQIdxxxx2ce+65AMyfP59OnTrRt29fAPr06cNzzz3nCYcxefJkOnXqxGeffcZ1110HwKeffkqXLl2iLpuNFAzDMMqZ9evX07x5c895p06dwpZp2bIlWVlZbNy4EVXlhx9+oF+/flGXzUYKJSA97TK/1c1loprtHWsYFcr398KG8JGSS8QBnaDfUyUudscdd3DSSSdx9NFHc9ppp3HNNddEFOzuoosuYtSoUXTr1o3u3buTmhrFkP4uNlIoIUemruYQWVPRYhiGEcdcc801LFq0iP79+zN58mSOOuoosrOzw5a7+OKLGTVqFCNHjmTAgAExkc1GCuHoNQimv+U5/Uzug1QnHtLHf6ZzZe/WFSaaYRgloBQ9+lhy4IEHcu2113Lttddy+OGHs2DBAo444oiQZQ444ACSk5OZMGECL7/8MlOmTIm6XDZSCEdyzaCXHhyzkNVb95ajMIZhVAV++OEHcnOdiMsbNmxg69atNGvWLEwph0cffZSnn346ZmG0baQQjuPvhj9eCnp5R2YurRqWozyGYcQVmZmZxYzKd955JxkZGdx2222kpTmbeD377LOeXdfCcfTRR8dEzkJMKYQjtXbIy2VyKjWXVMOo8hQUFARMf+GFF4KWmTx5crHzPn360KdPH798Q4cOLYNkgYnZ9JGItBCRSSKySEQWishtAfKIiLwiIstFZJ6IhN/h2jAMw4gZsRwp5AF3qeosEakNzBSRCarqvRS4H9DO/RwJDHO/qwfmkmoYRiUjZiMFVV2vqrPc493AIsDXknIu8JE6TAXqiUjTWMlkGEb1I952lywrZW1vuXgfiUhroBswzedSM2Ct13kG/ooDERkkIjNEZMbmzZtjJWZw2vbxS2ouFSCHYRglIi0tja1bt1YbxaCqbN261WPALg0xNzSLSC1gNHC7qvoGEAo0f+L311PVt4C3wNmjOepChkP8decFCb/xSv4FfDQlna6XdC13kQzDCE/z5s3JyMigQjqTFURaWloxb6eSElOlICLJOAphuKp+GSBLBtDC67w5sC6WMpWKAL2MO5O/4JuC3nw5G14wpWAYlZLk5GTatGlT0WLEFbH0PhLgXWCRqgbzvRoLXOV6IR0F7FTVyrcBcoPAD9Wk1LuAMmy4U02GtIZhxA+xtCkcA1wJnCQic9zPGSIyWEQGu3nGASuB5cDbwE0xlKf09H0y6KUWspEvZmWUozCGYRixI2bTR6r6O4FtBt55FLg5VjJEjeTgRpvfUu/g3W0/UXwWzDAMIz6x2EeR0vH8oJfScneUrk5bp2AYRiXDlEKk9P8g6KWe6W8FvWYYhhFPmFIoCd2uDJictvefchbEMAwjNphSKAl9nwiYnLy38jlMGYZhlAZTCiUhrW7A5KayrXT1mUuqYRiVDFMK0cJe8IZhVAFMKUSLWR9WtASGYRhlJqxSEJEEEekmImeKyEkisn95CBZ3fOO3XYRhGEbcEXTxmogcBPwHOAVYBmwG0oD2IpIJvAl8qKqBtxUywmPrFAzDqGSEWtH8OM6mNzeoT9xZEWkCXIYTxqJ6zZsk1YC8fRUthWEYRkwIOn2kqgNU9VdfheBe26SqL6lq9VIIAIN/gzp+Wz4YhmFUCSKxKfR3t9NERB4QkS+r9V7KjdpB18sqWgrDMIyYEIn30YOqultEjgX64kwXDYutWJWcg06KTj3mxmoYRiUjEqWQ736fCQxT1TFASuxEigNaHV3REhiGYcSESJTCPyLyJnAxME5EUiMsV/2wnr9hGHFOJC/3i4HxwOmqugNoANwTU6nildH/Kll+c0k1DKOSEYlSaAp8p6rLRKQP0B+YHlOp4pUFX8D8LypaCsMwjFITiVIYDeSLyME4ey63AUbEVKp4ZvR1FS2BYRhGqYlEKRSoah5wAfCSqt6BM3owDMMwqhiRKIVcERkAXAV866Ylx06kaoQZpg3DqGREohSuAXoDT6jqKhFpA3wSW7HigFMfg3NerWgpDMMwokpYpaCqfwN3A/NF5HAgQ1WfirlklZ1jhkD3wNtzGoZhxCuhAuIB4HocfQikAwK0EJGBqvprbEUzDMMwypuwSgF4HjhNVZcAiEh7YCRwRCwFqxbYOgXDMCoZkdgUkgsVAoCqLsUMzYZhGFWSSEYKM0TkXeBj9/xyYGbsRDIMwzAqikhGCjcCC4EhwG3A38ANsRQqnlh85ujSFzaXVMMwKhlhRwqqmg284H4AEJHPgEtiKFfcsLeJmVYMw6g6lDbaae+oSmEYhmFUCiwEtmEYhuEh6PRRiC03BfM+CsnCVevo2ObA8BnNJdUwjEpGKJvC8yGuLY62IPFK8/o1/NI6fngoDJkNDdpWgESGYRilJ6hSUNUTy1OQeGX/OmmBL2xdYUrBMIy4I6xNQURmiMhNIlK/PASKR07Nfib4xc1LYO/WwNfMJdUwjEpGJIbmS4FmwF8i8qmI9BUJPxkuIu+JyCYRWRDkeh8R2Skic9zPQyWUvdKwTJv7J+5Y7Xy/1sv5GIZhxAGRREldrqr/Bdrj7Lj2HrBGRB4RkQYhin4AnB6m+t9Utav7eTRSoeOC7+6CHWuc48wtFSuLYRhGhETkkioinXEMz8/ibM95EbAL+DlYGTeK6rYoyFjpue7YNoEv7NlcvoIYhmGUkUhCZ88EduDsz3yvu8IZYJqIHFPG+/cWkbnAOuBuVV0YRIZBwCCAli1blvGW0adHq/owI8AF8zg1DCPOiCQgXn9VXRnogqpeUIZ7zwJaqeoeETkD+BpoF+Q+bwFvAfTo0aPSWWeDmVgWb9zLIaELxkQewzCM0hLJ9NFOEXlFRGaJyEwReVlEGpb1xqq6S1X3uMfjgGQRaVTWeiuChCDv9u9Gf1i+ghiGYZSRSJTCp8Bm4EIcW8Jm4LOy3lhEDij0YhKRXq4sQXw3Kzf1aqYETL8r+Yti5zNXb+eP5V5GZ3NJNQyjkhHJ9FEDVX3M6/xxETkvXCERGQn0ARqJSAbwMG54DFV9A0fB3CgiecA+4FLV+HxL9moTygmriAuHTQEg/fJYSmMYhlF6IlEKk0TkUuBz9/wi4LtwhVR1QJjrrwKvRnB/wzAMo5yIZProBpz1CTnu51PgThHZLSK7YilcPDGr4OCKFsEwDKPMRLJ4rbaqJqhqkvtJcNNqq2qd8hAyHrgi5/6KFsEwDKPMRDJ9hIicAxzvnk5W1W9jJ1J8kkmQwHiGYRhxRCQB8Z6iaG/mv4Hb3DSjrNg6BcMwKhmRjBTOALqqagGAiHwIzAbujaVg8ch2rUV92RN5gfh0tjIMowoT6Xac9byO68ZCkKrABg3umhqn3raGYVQzIhkpPAnMFpFJONF8jgfui6lUcUp+CB2bsX2f51hxfsjCb0/5AiUzJ4/aabbbqWEYFUPIkYK74vh34CjgS/fTW1U/LQfZ4o68ED/ncc9M8hzPWbsDgJWbi081/fer+XQa+iP5BTaqMAyjYgg5UlBVFZGvVfUIYGw5yRTHRGY43ronB4A92XnF0kfPygCcEUNisIBKhmEYMSQSm8JUEekZc0mqANfn3BVRvrFz/gHwGxGIq1QKzP5gGEYFEYlSOBH4U0RWiMg8EZkvIvNiLVg8spl6DMm5OeL86juysMGBYRgVTCSG5n4xl6IKUZZFbIUzRjZQMAyjoohkpPC4qq72/gCPx1qw6kHg6SMliFbYuxWWTYi1UIZhVGMiUQodvU9EJBE4IjbixC+PnXc4ABLshe5FsBwSbqTwyQUw/CLIySy5gIZhGBEQVCmIyH0ishvoLCK73M9uYBMwptwkjBOa16sBwC8FXQJebyvrqIP/auexc9exLycfgAQJY2jeutz51vwySmsYhhGYoEpBVZ9U1drAs6pax/3UVtWGqmqL13zo06ExADkEXnj2c+rdzEsbRDdZ5knbvjeXISNn8+i3C4EiO3NBQUxFNQzDCEokobPvE5FmInK0iBxf+CkP4eIJiTC43VepD3uOc/Odt/+GnVkAJCc5f46cfNMKhmFUDGG9j9yIqJfiREgtnLdQ4NcYylWlKVQfhQblQoWS5Lof5QZQCh//mc7lBQURB6syDMMoDZG4pJ4PdFDV7FgLE+/0at2A6enbIs6f5777Z6/ZDngZmgPkfXDMQi5MzaemrWUwDCOGRNLxXAlBJsqNYnxwbWQLvwtf+gnu0fbMXLLz8j2GZt+IqoXnBba6zTCMGBPJSCETmCMiEwHPaEFVh8RMqjilcJ3BCdkv8EvqnUHz1ZW97lHRy7/DAz94joM5H0Xi7moYhlEWIlEKY7FgeBFROP2zWg8Ime/x5PcBOCtxGm/lrWCeHlTsen6B8tT3i8nNL+DBsw4LoCRsxGAYRmwIqhREpI6q7lLVDwNcaxlbseKT0oSn+CrlIQ7KHl4s7ZFvFjJpyWYAHjzrsGiIZhiGERGhbAqTCw/cqSNvvo6JNHFOabZcDuRPVKgQAFrf+x0fT13tk8PRPj8v3sjM1dtLflPDMIwghJo+8n7F+e4zafMXAUhLTixxmUgGF/8btyhg+rUfzAAg/akzS3xfwzCMQIQaKWiQ40DnhsuQk9uVKL+SwJiUB/hX4nclKGU62TCM2BBqpNBERO7EeQMVHuOeN465ZHHKnae2585T28PQyPIr0CVhJV0SVvJOvvX4DcOoWEKNFN4GagO1vI4Lz9+JvWjVgzTJDZsn3LDs23nroiOMYRjVnqAjBVV9pDwFqXK0Ow2W/RidusJohVtGzOaszgdG516GYVRrwq5oFpFnRKSOiCSLyEQR2SIiV5SHcHFNi15RqyoaAfL+St/GoQ/+wI7MnChIZBhGVSWSMBenqeou4CwgA2gP3BNTqaoCR99W0RIU49Wfl7MvN5/Za3dUtCiGYVRiIlEKhXGPzgBGqmrkEd+qM0kpJS5yfsJvXJgQm+Cz5i5mGEYkRBLm4hsRWQzsA24SkcZAVmzFqp68mDIMgNFZxberqMNeWsnGqNzDnFkNwwhFWKWgqveKyNPALlXNF5G9wLmxF80o5KOUJ+masJK9mgo4+zCU9OXuG3nVMAwjEJEYmvsDea5CeAD4BAjr6iIi74nIJhFZEOS6iMgrIrJcROaJSPcSS19N6Jqwstj52LnmgmoYRmyIxKbwoKruFpFjgb7Ah8CwCMp9AJwe4no/oJ37GRRhnQZF23cahmFEm0iUQuEWnGcCw1R1DBDWiqqqvwKhjNLnAh+pw1Sgnog0jUCe+KHHdVGtrnDKKDdf+XJWRunqKE3UPsMwqg2RKIV/RORN4GJgnIikRlguHM2AtV7nGW6aHyIySERmiMiMzZs3B8pSOen7P7j8i1IVvTLxRw6T9IDXVJU7P59bBsEMwzACE8nL/WJgPHC6qu7AiZgajXUKgbqsAa2hqvqWqvZQ1R6NG8dR2KXkNGh3aqmKPpb8AeNS7w94rcBsxoZhxIiwSkFVM4EVQF8RuQVooqrRiN+QAbTwOm8OmAUVaELRHgmJntm7ou0488vgSWSTR4ZhhCIS76PbgOFAE/fziYjcGoV7jwWucr2QjgJ2qur6KNQb9zyd/JbnOJGiEBeFSqGgFEMF80g1DCMSIlm8dh1wpKruBXDXLPwJ/F+oQiIyEugDNBKRDOBh3NXRqvoGMA5nlfRyIBO4pnRNqHokBRgdlJqlP8LaqTiOY6Unv0A56P5xDD37MK4+pk3ZZDIMo9ISiVIQ8HpLOcdhZyFUdUCY6wrcHMH9qx3hFMHBO/4ADiqeOHs47F4Hx/uYe0b0d76bl00p5LpB+f73/WJTCoZRhYnE0Pw+ME1EhorIUGAq8G5MparmeGvcHglL/NLPW/EgddhTvNCYm+Dnx4PWqa6iKbNHqk1DGUaVJpIwFy+IyGTgWJz30jWqOjvWglVnvEcKw1OeDJgnudjgLfbY8gbDqB6EVAoikgDMU9XDgVnlI5JRWzIDpqd67dJWUe9otaGCYVRpQk4fqWoBMFdEWpaTPFWTOxeXKHunhPSweUprgBZzSjUMIwSR2BSaAgvdXdfGFn5iLViVok5TuCHa+ySUTClEyyXVXFsNo2oTifeR7dUcDWo2rGgJAB/bQH4uTH0djhwMSakhyxUqA9MJhlG1CaoURORgYH9V/cUn/Xjgn1gLVuX9Zww/AAAgAElEQVSoEzCsU6mJyiTQjPdhwkOQlwMnRBa5xPZlMIyqTajpo5eA3QHSM91rRkkQgbYnRq+6aEwf5ewp/h1JPSW6q2EY8UYopdBaVef5JqrqDKB1zCSqykTRr9NbKXSSlSFy+pbzxl7xhmEUJ5RNIS3EtRrRFqRaUDvshnUR4/1y/yb1Ac/x3uw8EhOEy96eSm6+8k1ElUWurGz2yDCqNqGUwl8icr2qvu2dKCLXATNjK1YV5YxnYM4nUamqcKTQUVYVS+/48PjiGUOpds8bPrxSMGVgGNWDUErhduArEbmcIiXQA2fXtfNjLViVJGW/qFVVqBTuSIpsE5/Ai87cNFuubBiGS1CloKobgaNF5ETgcDf5O1X9uVwkM0IiQulMAqV8/9tKZsOoHoRySa2lqntUdRIwKVSemElnhEB5Mfk1TkksQxgqe88bhuFDKO+jMSLyvIgcLyKeeQ8RaSsi14nIeOD02ItoBOKaxPGcn/hHxPnX7cgKcdWmjwzDcAiqFFT1ZGAicANOmItdIrIV+AQ4ABioqqXblb4685906Hxpmau5Lun7EuVfs80JsvfH8i0s+GcnKzfvoSQ2BTM0G0b1IGSYC1Udh7NDmhEtatSHC96EowbDtlXwRfluOPfapBW8NmkFAOmnR+59ZBhG9SCSgHhGLDiwGxx+QdSrPSZhPpEbCyLv/ttAwTCqB6YUqhjDU57kgoTfSlbIXFINw3AxpVAFaSZbAEgiL3TGAIaC1yYtZ8ryLZ7z2Wu2M3ftDguEZxjVhLBKQUQOEpFU97iPiAwRkXqxF80oKy8mvx702iWJk+CXp9yzopHCs+OXcNk70zzn578+hXNfi9zLyTCM+CaSkcJoIN8Npf0u0AYYEVOpjKhwduLUoNeeTvaKXhKj6aOte7JthGEYcUYkSqFAVfNwQlu8pKp34OzGZlRSGsnOEpaIwCW1hDWu2LyHIx7/iQ+mpJewZMn4e90uWt/7HVNXbo3pfQyjuhCJUsgVkQHAQOBbNy05diIZZWVg0gSEgojzZ+bmk5WbH1UZVm/dC8CvSzdHtV5fpqxw7B8/LtwY0/sYRnUhEqVwDdAbeEJVV4lIG5wFbEY0OOy8mFS7Ku2KYuf7sS9o3mGTV9D/jT9D1lfZZ4EsNpNhRIewSkFV/1bVIao6UkTqA7VV9alw5YxIKZ+X2cK060JIIMz/ZydsWsyAxInlIo9hGJWTSLyPJotIHRFpAMwF3heRF2IvWjUhofLMxOmw3jyZ/K7nfF+O15SSq7veTX4WXj86fF3l3HEXW5VtGFEhkumjuqq6C7gAeF9VjwBOia1Y1Yh+T8NRN1WoCOq+UEWL2yEWrvM3WJ+cOBs2LYy4brGFcYYRV0SiFJJEpClwMUWGZiNa7NcITn+yXG/5e+qQsHn6vvgryzeVf1T0yUs20fre71i0fle539swjMiUwqPAeGCFqv4lIm2BZbEVy4glzWVLsfNAMz1LNu7m3i/ne86nriofl88JfzteRDNWby+X+xmGUZyQUVIBVHUUMMrrfCVwYSyFMsqb8FM8N3xcum25y2vxmnkfGUZ0iMTQ3FxEvhKRTSKyUURGi0jz8hCu2tGpf4XcVktjpF0e2kupvEwJZrMwjOgSyfTR+8BY4ECgGfCNm2ZEkwc2wflvFZ03O6L49caHlPkW+7GPhACL2lrIRrwnkc5OmEIiYRazbVtZ7HTJht088d3fnpFBiQcI/8yCmR+WsJBhGNEmEqXQWFXfV9U89/MB0DiSykXkdBFZIiLLReTeANevFpHNIjLH/fyrhPJXHZJSISEBBv/uLGjrflXx64eeXeZbLEy7jieT3vFLvyxpEtcm/uA5/7+UVxmc+E2J6r78nam8/dsqtu7NKZYecU/+7RPhm/AGcF8stpJhRJewNgVgi4hcAYx0zwcAYa2OIpIIvAacCmQAf4nIWFX92yfrZ6p6Swlkrtoc0Aku/hAyZhRPl+hEOb8kaXLA9J4Ji4udNxF/Q++BeBmofV7GBVr4XY4v6RGX0pHDgW62TsEwokQkb5prcdxRNwDrgYtwQl+EoxewXFVXqmoO8ClwbmkFrXY07wF3er2oty6P6e3CvVI7SjpT0rx78sVf/oXlez0xkU+nr3HTCqiRH0O31qXf03vps7GrvxKyMzOXnftyK1oMowoTSZiLNap6jqo2VtUmqnoezkK2cDQD1nqdZ7hpvlwoIvNE5AsRaRGZ2NWEOl7BaDNj6xLqa2vwNT63lXXFzj+dvppdWUUvJ+9po3u/nE9WbgH/SfqM19aex6/zltH14e/IzC4+tQSwYWeWxw21LJTZ+2jPJsjeXeriM1dvJy8/8iCEpaXLoz/S5ZEfY34fo/pS2jmJOyPIE6jz6fuf+w3QWlU7Az8BAS2NIjJIRGaIyIzNm2MbdbPSklIrptWLz5/mxIQ5rgE6MEs27OLzv9YGvQ5wZoKzn8N7E2YzRy4ja8zdfnkuHDaF6z+a4Zde7jzXDl7tWaqiC/7ZyYXDpvDsj0uiLJRhlD+lVQqRTOBmAN49/+ZAse6mqm5V1Wz39G3Ax+XGk+8tVe2hqj0aN47Ixl316HZlTKv3VQqtEjYxKeUuapAFlNxt1TtERmHd9f/+yC/fPzsCR2998OsFrNqyt0T3LDO715eq2OY9ziO8eH3pRxqGUVkorVKIZKz+F9BORNqISApwKY5rqwc3fEYh5wCLSilP1eWuJfDfjXDQSTG9TWIAV9UkKWBR2rU0Zruf0qhBDsl5e+n40A+0vvc7v7KvT17hOfZMTZXQWD5p8aaA6elb9rI3O8z+0+WJz39DfoFy7NM/8+28dYHzG0YlJuh/qYjsFpFdAT67cdYshMTdre0WnBAZi4DPVXWhiDwqIue42YaIyEIRmQsMAa4uc4uqGrUPgOQ0SEqJ6W26JqwIeu3+5BF+44R/J3/GwF+OZW9O+M15Cjf8yVMJqECClgsyOOnz3GSuem+65/y/SZ8gGt1NgsrCnqw8Mrbv436vMCGGES8EdUlV1dplrVxVxwHjfNIe8jq+D7ivrPepNpz8MEx8JCZV15fgXkLnJ/5BGv5G4uIopybMZIvW5avUhxmQ819aJDj2nwQKF7QpjYh8q9BQE1YzV2+HNOf4+qRxfLinH9A54rqDsm875OdCrSaRlzFvWKMKEck6BaOycNSNMVMK4ahDZsD09LTLGJffizH5x/BmyotsV8cgfmLCHE+ewqmnFMlnRtqNsK03NGgTsL7h09YUlStJCIuyrI/Yvrro+Ok2gMLQku5z7SWKxWEy4pjorIgyyocK3JCnkQR/SZ6ROJ03U14EikYcJyTM9Vz3C62xo+glXJ9d9E34K2C9xXSCKuQFH62MX7iBU174xf/Cns2QE1iheZjuFV4k3As9Zy8MrQtThwW+npuFZO0IXYdhVGJMKcQTiRU3sOuQkFHq/JlZxRdbZWVls3VPNk99v5h3U57zKBSAF5Jf9yiJYuOEKf8HjzeGzG1B7xlw/4fnDob3T/dLvu6Dv/jsrzX++cOx13WJ/vP1ojRvPfJeX+q+0q7k9eZlQ0Hs1zkYRjhs+siIOe0yZ4HXIOffo+cxdq/jPfSv1OIeRhck/s4Fib/TOmsEu7Ly+HbeOjbvzuaauW6UlV2BPXp8vaPI2QsFrofS+rl++Scu3sTExZu4pGfLErYmzJTWeq9ps5JMfz3eBA6/EC56r4TyGEZ0MaVgxJxDE1YXO9+xL5c2sp5JqXeFLPfs+KLFYNe0LHzBKmlkszgtTKSVFw6DmE7jhLcblDhY34LRphSMCsemj4yYE6i//EzymyWrxGtf6Ib4b9Xpd49gCmH5RBhat3hwv5L06EPkDVnNujmOLWKjbzzICNi9Iei0WUGBct0HfzF1ZfnsjGdEmfXznOdi5eSKlsSDKYV45b5//NMu/rj85YgAX0Pz3Umf0zNhacgygxPHsh8BVjsH6X37TR/5stj1jJ79CQA9it2/bF5OC/7Z6XepETu5mAlFCYvcdZtLIl+n4eH5DvBMYG+tbZk5TFy8iZuGzyp5vYGYMwLS/4hOXUZ4Vru/9dLxFSuHF6YU4pVUn1hI/Z6NWnjtaOP7yu2csCpsmXuTP+XepJH0TlhI34SihWrMGUGC+BtkC5VC7m8vwZIf/K7z6QA3Y5E01yV+B1PfYG9OGANvxkz48GzH+2mm//5Sz0/wV3BvpLzIA7zNMyNdWSTR+Q5gTN6+N9waEIesXJ8Fevt2eOwmUVsq8fWN8MEZ0arNCEcl3A/EbApVhSOuhuUTwmarCM5NnFKqclcm/cSV/FQ8cdowrkr0f2mdnjCd25K+JHnisojrfzB5OPwwnI/zzmZwoP+EPZvIH3EJeZuWk5q3i6V/z6b97y/6ZUsjm6ayDZGiuFwN3UV638/L4N8DKFLY6q8Uuj02gfS08PIe8qCjYPZjH6z8BT46h/06XQHYSzz+qTwrICtn19IoOSJFvdEqTgPxDzx3WdIkjkgIrRB+WbqZzbsL4y8W9dAC9tW2p8Osj0hcN4vUPMeGMWSk1xSNVw/vreQXHKN5gF6fJ8WjFJze/va9OXQaOt5ZmV1CXkp+DT5yIsWkLfrCqd59p2zdk83uLNtvIf6oPCMGUwrxxtkvw1VjA1wQaHcqHHNbuYtU3lyY+Fupyg18bzp/RmqQHeXv3VTMbrGraB3G8YlOjKP9CkJESS1UCvm5sGkRf6VvY3dWHsMmB4g5VVAA2cHDjhyaEGh9hbBuxz6OePwn+jw7ObgcXkxftY3VW8NEot20GHKzIqovIuZ+CpOejF59RtQxpRBvHHE1tD3BOT7fy4NHEiAhEU59tELEigd+TbnNE8Pp6IQib6aAYcHXzYIN84olBTJm53ptrPNaxoWe4yTXuD4k6aviBf54CV4/isOn3UN62mWBBf3pIXiymbPWIhyuSCLQ96VfAfz2yQ7GxW/+yQmhFMi+7fD6kewZNZjsvPABB3dl5fLwmAX+tg9vvroBfnkqIvnikfELN9D63u+KQsJ/fAHMHl6xQpUQUwrxTJdLi45L4lZZTWmZsJnu7hTTpV57VTeTLYEL7C2e3jvB3530/T8CG80LgwFekPi7k+Dz5zlwTeFoL8C0wdxPne9IlIJbXoDdWRGGE1eNzMDp3n/Xkl+4Z9S8MJnhpQnL+PDP1Xw+I/DmS/kF5TNFkpmTx4XDprB4g7/rckm4e9Rcbvt0donKjJrhjCAXuh5prJgIY26KoGTl+f81pRDv1GjgfAdTCml1y0+WOKCR+L8oghrCVxd3zXww+ZNi5+e//gebdmVTFuYuWkodSr+ZUOGLdnumz+ggL9vpofq+/PPz4JF68NNQ/8p2bwh4DwF+W7aZnftyOep/E8l9qi2MucUvX57rWVUQ4OW/bsc+Drp/nF96LJi2ahszV2/nyXGLw2cOwRczMxgzJ9Z7YlQeW0IhphTinesnwjmvBr/eqH35yVLNmL1mB+/8Ht69NhR/pd3E1NTiL9gC90WuW5aydebXIcvnuS/g3Hyfl8vkp5we6iIf+1OBa4Se9oZ/Zc938Eko6mgo8OeKLWzYlUVy1laY7b8mRj1TWf4dlBWbg9tISsTGhcWj2lY6Kt9LvqSYUoh3GrSF7iG26qzXqvxkqYYkE37KZtVD7dmXFXxEUVOKrqUv+NNjE5APzqThNwM91xqzneMT/OM4BWSWu9151i527tzBLfc/wDu/rUQDuMQGZGFxZZSZk8+tI0NPpdTPXssdSaP4edFGPyUQcLZqxxpn5BKOgvyiCLnDjoaXw+ybEYP3cvqWvVzxzjQycyrRjn8xwpRCVeaS4Y63khEjlIeS/Ped9qVNwkbWrEmPqMbWX5xO4wBTXABfpz7ERylP09zLBpImQdxPM10vqw3zKXihI6+m/B/fjRvLf7+a70ou1HT33/5l6WbmZfiEBRk10DG2u+TkFZCSnxncOA5cufIebkv6iuXLFnHy88XDmPu9pzcsgJc6wWMNg9bn4fOr4PHGzEj3CvUx4WHY6czf78vJL9P2rFv3ZDNt5VYvd2V/nvp+Mb8v38LkJZs9aYM/nsmPCwNPucUzphSqIoN+gUtHwKFn+a98NqLG28nPc3nixIjyRqPz2kwCu9Puxz46SnrgQtPf9OxxUVOy+Hq2Ex5F8vbxd9q1jE35LwPfm845rwYIbfHZFQA0lW0MS36RNrLeL8uOzev59xdzOeapn0kqKLJrPJb0nhPTx8UvOOAbxwRrpj+LvwXgojf+LEr74yV4sSObdmXR64mf6PiwV5gIgQ6yhg/XnAprpxOO/m/8ySVvTaXnEz+xaZe/++30Vdv4wX35L5/wDmxZzoC3pvLDwg0M+ngmANnp09iwwfltb0wcy6RfJvLL0s1+dcUDphSqIgd2hUPOLHGxHSkHxECYqsupibNIkMhe9zU2zoyZHAvTruO71Pt5OflVDpeV/J0aOILs8JQn6ZdQ/CUZScgRgH6J/hshzRv/PvVeO4RlMyexYcce6udu9Fy7MslZiV7onbVz3TI+Sg6yPkGV7AmP8dGY7/08lN4NY7O5ZcRsdheOErYsg6F1qb1lbtEmT742FXDWgbzeG/4eA8DKLUWG/i17/N15f/1zCsclON5XQ3Y9B8OO9lvvkvrBadQZ1o13Vp3Cf5I/5clNNzHQax/xQR/NAJxFi840nnraDgR2FMnPhdx9jhfc/C9C/g7RxJRCNeSB3MAvjTc6jixnSaoPrXJXxvwe5yZO4dvUB4rZKHx5PsXfwNxTFnOQBAiw6MO3qQ8UO1/0m2N3+DTlMW5MDLSgEoZ/+yMFL3ej+aTbPIv8fMnYsJHUP57jqtmX8uOsojhSOzJzeOzbIjfgfgnT/Mqen/F00b7fy34EoOmq0R4T+aZAU0K5mbDpb/ja31X07/W7+GP6X57Fg21kPXcvvZyPU7zWVuRn0wT/leihfvcf/97I5CWb6PbYBJ78bgHzVvr83puXOAb0nx6B6W87ae+dDk8cgI4cAKOvY9fm8H+jaGBKoTrwn3Q460WotT+5nS7lrptvLbp27B2ew7zENK7K+U/5y2dUKKNSH2Vi6j0lLpfi2jNSJY8Dg0xt/ZT6bxK2rwwZgqT5m0VeT8dNuoj8AuWjP9NZt6P4VM6wFH/72ICkSbyQ7OyCl7Xa6Y03W17Uufly9jqmLN9SbJHh+p1uvTl7mDbiUfonTua2xNHOPb4YxzHjTmHzG2eSSk7QPT+mp91MbTJJIxs+uTBgHm/ay1qGvD8ZgOeS36DLxx35fdmWov28l09wDOi/vwDj7mb0zAz4x2lP7tZ0AJ7+oRRh10uBBcSrDtSoDz2uhR7XkgzU3+v+Ax98Kpz8MLhB3prXr8E7BV0Yc97fnPv1YRUnrxEXnJ9YZIfo5vXSf7ake2V4UWvvGj6dsZaHxiykrayjNvXCljk+cT7v8Cxpi4u8o9onOAvo2ksGl73jjDCe69+Fi45ozikvTGahG4DwyKXPc6S7K+Cr+ed5lGPj7XP89xb3oSZZdEpYBct/Cnj9jISpnuMfU53O1sFZH3l+tyvencbX3XbQNUDZu0bN5UJXxpR9zu6Ej63oDwXbICG2fXlTCtWR/RrCwG+haZdic5lX9W5Nq4b70adDYwjtHm8YxWgqRZ5BRyeWrUfbduYT7Mep/Jx6N4sLWkRU5pTE4u6yB4mz6OykxDncUPAN9yWPZPGYFuya0ZJErg5Yx9zU64udP5ccYC1HCXg95RW/tM9TisLQnJMwhazdmRHXl4CyeuNWWjVtHD5zGbDpo+pKm+MgrU6xpIQE4cRDmjiLj7oFWPtw1ku82OTxchLQiCfqSelXZfvSa+OnvJ7sTBUdkhA4ZEZJuC95pKeuOuv/oE+QtR61pPh01ZmJxY3yHtuFy7S0W+ib4G+AD0X3hOWe41dSXuWojMDbr/6UcnfA9FEzY29XMKVgQP0Au3qd7mVYa9DW+W5yGAtqHhVRlf9oBP7nhhGEExLDx1oKRdeE4Ib9V1JCRAAIwYy0G/3S+if9Wqq6wnFwQuDwGokReruVBVMKBgz+De70iROTsh8ceg4M/AZqui94Ea7o7bNCuuMFAausk1o0M7lZ6wTMYxhGyUgoh7h5phQMSK0NdZoWTxOBSz6GNscXSz6xQxOo5a5nuH0+XPguJKb6VbnfGUVzpwlVIB6MYVQGsnJjH2bDlIIRngQff4S7FsMDm6FeS8cTYmCRj/qm7rczp/V1JHTo60l7PPeK8pLUMKo0U5bHfpW0eR8Z4bngbSeqZrMezrkIJKUUXW/eE3r+C44eQpP6rWhSmH7ENdC4A921H91+PJHZ9/SE+aOcSJeFAdsMw4iYgoIIAxqWARspGOGp1wL6PhHcPzohEc58Hur72BvOfgmOupEre7dm9sN9oWYDOPIGOOcVaHYEAPtOeBiAvxqHXwAUSx7OHRg+k2FUMLWI3IW1tJhSMCqGy7+Aq7+jxol3wtCd9Lz5PTjOxw1v6M7AZYPwfO5FJcq/tqDI33thgYUYNyo/I/deHz5TGTGlYFQMNRtA62OLp3UK/lJvneXuc9v6OLh2PK2zRvjlWd/0JL7P7xmxCCfmPM9tOTcxr6ANC9TfLfeOHH8XRMOo6phNwag8NDnUGR1sXuIJSKa1myK71/PhtUdC6/WQlAoJiXwxeBvvLfqBUb/O5vvU+6DzJTx3/mWsmtsMvj7XKdu+HxtSWlJ3+RhqZvnHvc8jiTEFxzImx1FOswsOppu7uGhFQVO+KjiOFxnmV+7vglYcllCZd/8yjNJjIwWj8tG4AzR3bA5yywy4ZwUntG8MKTUd+wXQo3UDru3Xm9GPDGLf/dvggrdAhDZd+3iqkQEjaXrRM9Q8YQgAqy//nbfaOXF5NMXZZ2LQ8W1Z+ng/rjiqJefnPErHrHcBeCP/bMAZLdyTO4h1J77oqffsnMf5Kj/4fgDTCg6hddYIfss/PEo/iGE4vJ13RszvEdORgoicDrwMJALvqOpTPtdTgY+AI4CtwCWqmh5LmYw4I7VWyI2CaqYEeISbHOaERi6M69T7Zuh9M62AQe06AZciQLpXkcfP68TJh+4PwNrG6xn1zCSGXd4d6I4IHHh4U5jkRJR97YqeDP4kkTtyb2ZW6iAaSPGtJ1/a/3+wNovNIYK5/VXQnp4JS4Nej5RNWo8mUnzXtGty7uH9lGdLdb8f8ntyeoC9E2LNt/lHcVbi1PAZjZgTs5GCiCQCrwH9gMOAASLiG3rzOmC7qh4MvAg8HSt5jGrEwG+cTwk5sUMTTuzQhBYNapL+1Jn069SUfp2acvrh7sK+PvfBwG85/fCmPHSW8ygPrveWc61Oc7Lbn0PHrHc5u0c7+nbcn4dyr4YznqOg9XEAZHe+grnnT2b9ravY3H8s2wdOhm5eaziOuwv+9TMM3cm6luegCSls6PkfOPG/zvXLRsFRN3uy/3TAIBLuWUrGoEWMzDuRR3Kv5LTsp6nRsR//zr2epfWP59cLZ/Fj/yU0vHUSuw67HE56gKxG/iOYPZpGh6wP+G/SXXx3RtHLOUMbFWW64J1iZX7N78TDuQPZeUHp9uHonlUUcO6W3Ft5J+UKfjljIln/3cZfV6/i9/yOAcs9nDuQVQX7F/0O+d0ASC/Yn8/zTvCkf5l/LHMKDvIrH2kIlmJtryTsl5oY83uI3zZ50apYpDcwVFX7uuf3Aajqk155xrt5/hSRJGAD0FhDCNWjRw+dMWNGTGQ2jGiRk1dAZk4e9Wq66zl2rHEW+wVi9nBodxrUChH9sqCgyCV4/hfOjl2d+3su78vJJys3n/r7OfebtHgTx7ZrRHJi4H5fQYEydd5Cenc+DNF8+r78Ow1r12DE9W5sq+UT2SV1WJ58MN1b1i8quDODzOR6jJqzhUXrd9G8fg1uOa454x89i76J7v/l7Qtgy1L45AJyznmDlE7nOavel/8EI/qjafUpSKlFwW3zSN6dwbbMXH7ZmMr53ZoXk3FfZiZJE+4n+dSHWLtpOwcuepfEY27his/XsmDdTibccDhr9woL1+1h5rj32P+YK3nzt1VMrnkvrQvWsPymf2hUK4V6q76DUVd76v380Fd4cHYdlqQ5aZu7307jWS8BMDr/OC5M/I28Wk3h9oWc8uD7TE69izUFjWmZULRwbG39I2lxSE/4syiO0hU59/FJSvHd5S7OfpBHeicyf/okTurQkEYrvgScaaDvG17Flzsv9fvb/JLf2RP7aUz+0ZybOIWfT/6GkyaezbbLfqBB+94B/6bhEJGZqtojbL4YKoWLgNNV9V/u+ZXAkap6i1eeBW6eDPd8hZtni09dg4BBAC1btjxi9Woz8hlGZeLXpZupK3vpsn8K1DnQSdy7Bfbz6W1n7fKLzhsN1u3YR9O6afy+fAudGkC9/C2O40IhuzewNWMZP+9uQf9ercnLLyA3O4sayQLJNRwlm5vJvE25HH5gXRLcIEN7s/N4eeIyBh3fFs3NpvGuhY7nXGN3YyBVZ9vMwsWc+bkw8VHoMoAZ+w6gU/O6pCZ59e4zZsIBhzsOEwDb08net5eUtJrk1DqQ1AWfw2Hn8NPKLBZv2MXFPVrQpE5aVH6jyqAU+gN9fZRCL1W91SvPQjePt1LopaqBt3HCRgqGYRilIVKlEEvvowzAe4eM5oBvPFhPHnf6qC6wDcMwDKNCiKVS+AtoJyJtRCQFuBTw3d17LFAYX+Ai4OdQ9gTDMAwjtsTMJVVV80TkFmA8jkvqe6q6UEQeBWao6ljgXeBjEVmOM0Lwt7oYhmEY5UZM1ymo6jhgnE/aQ17HWUB/33KGYRhGxWArmg3DMAwPphQMwzAMD6YUDMMwDA+mFAzDMAwPMVu8FitEZDNQ2iXNjYAtYXPFB9aWyklVaUtVaQdYWwpppaohYqk4xJ1SKAsiMiOSFX3xgLWlclJV2lJV2gHWlpJi00eGYRiGB1MKhmEYhofqphTeqmgBooi1pW/hQdMAAAZkSURBVHJSVdpSVdoB1pYSUa1sCoZhGEZoqttIwTAMwwiBKQXDMAzDQ7VRCiJyuogsEZHlInJvRcsTCBF5T0Q2uTvSFaY1EJEJIrLM/a7vpouIvOK2Z56IdPcqM9DNv0xEBga6V4zb0UJEJonIIhFZKCK3xXFb0kRkuojMddvyiJveRkSmuXJ95oaHR0RS3fPl7vXWXnXd56YvEZG+5d0WV4ZEEZktIt/GeTvSRWS+iMwRkRluWtw9X64M9UTkCxFZ7P7P9K7Qtqhqlf/ghO5eAbQFUoC5wGEVLVcAOY8HugMLvNKeAe51j+8FnnaPzwC+BwQ4CpjmpjcAVrrf9d3j+uXcjqZAd/e4NrAUOCxO2yJALfc4GZjmyvg5cKmb/gZwo3t8E/CGe3wp8Jl7fJj73KUCbdznMbECnrE7gRHAt+55vLYjHWjkkxZ3z5crx4fAv9zjFKBeRbalXBtfUR+gNzDe6/w+4L6KliuIrK0prhSWAE3d46bAEvf4TWCAbz5gAPCmV3qxfBXUpjHAqfHeFqAmMAs4EmdVaZLv84Wzf0hv9zjJzSe+z5x3vnKUvzkwETgJ+NaVK+7a4d43HX+lEHfPF1AHWIXr9FMZ2lJdpo+aAWu9zjPctHhgf1VdD+B+N3HTg7WpUrXVnXbohtPDjsu2uFMuc4BNwASc3vEOVc0LIJdHZvf6TqAhlaMtLwH/Bgrc84bEZzsAFPhRRGaKyCA3LR6fr7bAZuB9d1rvHRHZjwpsS3VRChIgLd59cYO1qdK0VURqAaOB21V1V6isAdIqTVtUNV9Vu+L0tHsBhwbK5n5XyraIyFnAJlWd6Z0cIGulbocXx6hqd6AfcLOIHB8ib2VuSxLOlPEwVe0G7MWZLgpGzNtSXZRCBtDC67w5sK6CZCkpG0WkKYD7vclND9amStFWEUnGUQjDVfVLNzku21KIqu4AJuPM5dYTkcKdC73l8sjsXq+Ls9VsRbflGOAcEUkHPsWZQnqJ+GsHAKq6zv3eBHyFo6zj8fnKADJUdZp7/gWOkqiwtlQXpfAX0M71tEjBMZyNrWCZImUsUOhJMBBnfr4w/SrXG+EoYKc7zBwPnCYi9V2PhdPctHJDRARn/+1FqvqC16V4bEtjEannHtcATgEWAZOAi9xsvm0pbONFwM/qTPKOBS51vXraAO2A6eXTClDV+1S1uaq2xnn+f1bVy4mzdgCIyH4iUrvwGOe5WEAcPl+qugFYKyId3KSTgb+pyLaUt4Gooj44VvulOPPB/61oeYLIOBJYD+TiaP7rcOZxJwLL3O8Gbl4BXnPbMx/o4VXPtcBy93NNBbTjWJyh6zxgjvs5I07b0hmY7bZlAfCQm94W52W4HBgFpLrpae75cvd6W6+6/uu2cQnQrwKfsz4UeR/FXTtcmee6n4WF/8/x+Hy5MnQFZrjP2Nc43kMV1hYLc2EYhmF4qC7TR4ZhGEYEmFIwDMMwPJhSMAzDMDyYUjAMwzA8mFIwDMMwPJhSMKotIrLH/W4tIpdFue77fc6nRLN+w4gVphQMwwlCWCKlICKJYbIUUwqqenQJZTKMCsGUgmHAU8Bxbmz+O9wAeM+KyF9uzPobAESkjzj7RIzAWTiEiHztBmVbWBiYTUSeAmq49Q130wpHJeLWvUCc/QAu8ap7sldc/eHuynDDKFeSwmcxjCrPvcDdqnoWgPty36mqPUUkFfhDRH508/YCDlfVVe75taq6zQ2B8ZeIjFbVe0XkFnWC6PlyAc4K1i5AI7fMr+61bkBHnJg1f+DEK/o9+s01jODYSMEw/DkNJ77MHJyQ3w1xYvwATPdSCABDRGQuMBUnIFk7QnMsMFKdyKsbgV+Anl51Z6hqAU5okNZRaY1hlAAbKRiGPwLcqqrFAoqJSB+c0Mbe56fgbDKTKSKTcWIGhas7GNlex/nY/6dRAdhIwTBgN862oYWMB250w38jIu3daJy+1AW2uwrhEJyQ2oXkFpb34VfgEtdu0RhnC9ZyjTJq/H97d2yCUAxFAfRmQ8dwAvewtXIDW0ew+uCfQvgrCLFIfFiIrc05bUh9wwvcxy9eIjDaKZ9zDHROcswY3Szzs3dLsvty75pk31pbMxpDbx9npyRra23po6L67ZKx9vKe0SR76L0/ZqjA32lJBaAYHwFQhAIARSgAUIQCAEUoAFCEAgBFKABQXkcCeTbl7UM+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ffa2cece470>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.xlabel(\"Iteration\")\n",
    "plt.ylabel(\"Loss (CrossEntropyLoss)\")\n",
    "plt.title(\"MNIST Learning curve for LSTM and GRU\")\n",
    "plt.plot(two_loss.T[:,0],label=\"GRU\")\n",
    "plt.plot(two_loss.T[:,1], label=\"LSTM\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as dsets\n",
    "from torch.autograd import Variable\n",
    "from torch.nn import Parameter\n",
    "from torch import Tensor\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import math\n",
    "\n",
    "\n",
    "cuda = True if torch.cuda.is_available() else False\n",
    "    \n",
    "Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor    \n",
    "\n",
    "torch.manual_seed(125)\n",
    "if torch.cuda.is_available():\n",
    "    torch.cuda.manual_seed_all(125)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "STEP 1: LOADING DATASET\n",
    "'''\n",
    "train_dataset = dsets.MNIST(root='./data', \n",
    "                            train=True, \n",
    "                            transform=transforms.ToTensor(),\n",
    "                            download=True)\n",
    " \n",
    "test_dataset = dsets.MNIST(root='./data', \n",
    "                           train=False, \n",
    "                           transform=transforms.ToTensor())\n",
    " \n",
    "batch_size = 100\n",
    "n_iters = 6000\n",
    "num_epochs = n_iters / (len(train_dataset) / batch_size)\n",
    "num_epochs = int(num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n",
    "                                           batch_size=batch_size, \n",
    "                                           shuffle=True)\n",
    " \n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n",
    "                                          batch_size=batch_size, \n",
    "                                          shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GRU cell implmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GRUCell(nn.Module):\n",
    "\n",
    "    \"\"\"\n",
    "    An implementation of GRUCell.\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, input_size, hidden_size, bias=True):\n",
    "        super(GRUCell, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.bias = bias\n",
    "        self.x2h = nn.Linear(input_size, 3 * hidden_size, bias=bias)\n",
    "        self.h2h = nn.Linear(hidden_size, 3 * hidden_size, bias=bias)\n",
    "        self.reset_parameters()\n",
    "\n",
    "\n",
    "\n",
    "    def reset_parameters(self):\n",
    "        std = 1.0 / math.sqrt(self.hidden_size)\n",
    "        for w in self.parameters():\n",
    "            w.data.uniform_(-std, std)\n",
    "    \n",
    "    def forward(self, x, hidden):\n",
    "        \n",
    "        x = x.view(-1, x.size(1))\n",
    "        \n",
    "        gate_x = self.x2h(x) \n",
    "        gate_h = self.h2h(hidden)\n",
    "        \n",
    "        gate_x = gate_x.squeeze()\n",
    "        gate_h = gate_h.squeeze()\n",
    "        \n",
    "        i_r, i_i, i_n = gate_x.chunk(3, 1)\n",
    "        h_r, h_i, h_n = gate_h.chunk(3, 1)\n",
    "        \n",
    "        \n",
    "        resetgate = F.sigmoid(i_r + h_r)\n",
    "        inputgate = F.sigmoid(i_i + h_i)\n",
    "        newgate = F.tanh(i_n + (resetgate * h_n))\n",
    "        \n",
    "        hy = newgate + inputgate * (hidden - newgate)\n",
    "        \n",
    "        \n",
    "        return hy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GRUModel(nn.Module):\n",
    "    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim, bias=True):\n",
    "        super(GRUModel, self).__init__()\n",
    "        # Hidden dimensions\n",
    "        self.hidden_dim = hidden_dim\n",
    "         \n",
    "        # Number of hidden layers\n",
    "        self.layer_dim = layer_dim\n",
    "         \n",
    "       \n",
    "        self.gru_cell = GRUCell(input_dim, hidden_dim, layer_dim)\n",
    "        \n",
    "        \n",
    "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
    "     \n",
    "    \n",
    "    \n",
    "    def forward(self, x):\n",
    "        \n",
    "        # Initialize hidden state with zeros\n",
    "        #######################\n",
    "        #  USE GPU FOR MODEL  #\n",
    "        #######################\n",
    "        #print(x.shape,\"x.shape\")100, 28, 28\n",
    "        if torch.cuda.is_available():\n",
    "            h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).cuda())\n",
    "        else:\n",
    "            h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n",
    "         \n",
    "       \n",
    "        outs = []\n",
    "        \n",
    "        hn = h0[0,:,:]\n",
    "        \n",
    "        for seq in range(x.size(1)):\n",
    "            hn = self.gru_cell(x[:,seq,:], hn) \n",
    "            outs.append(hn)\n",
    "            \n",
    "\n",
    "        out = outs[-1].squeeze()\n",
    "        \n",
    "        out = self.fc(out) \n",
    "        # out.size() --> 100, 10\n",
    "        return out\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM cell implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTMCell(nn.Module):\n",
    "\n",
    "    \"\"\"\n",
    "    An implementation of Hochreiter & Schmidhuber:\n",
    "    'Long-Short Term Memory' cell.\n",
    "    http://www.bioinf.jku.at/publications/older/2604.pdf\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, input_size, hidden_size, bias=True):\n",
    "        super(LSTMCell, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.bias = bias\n",
    "        self.x2h = nn.Linear(input_size, 4 * hidden_size, bias=bias)\n",
    "        self.h2h = nn.Linear(hidden_size, 4 * hidden_size, bias=bias)\n",
    "        self.reset_parameters()\n",
    "\n",
    "\n",
    "\n",
    "    def reset_parameters(self):\n",
    "        std = 1.0 / math.sqrt(self.hidden_size)\n",
    "        for w in self.parameters():\n",
    "            w.data.uniform_(-std, std)\n",
    "    \n",
    "    def forward(self, x, hidden):\n",
    "        \n",
    "        hx, cx = hidden\n",
    "        \n",
    "        x = x.view(-1, x.size(1))\n",
    "        \n",
    "        gates = self.x2h(x) + self.h2h(hx)\n",
    "    \n",
    "        gates = gates.squeeze()\n",
    "        \n",
    "        ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1)\n",
    "        \n",
    "        ingate = F.sigmoid(ingate)\n",
    "        forgetgate = F.sigmoid(forgetgate)\n",
    "        cellgate = F.tanh(cellgate)\n",
    "        outgate = F.sigmoid(outgate)\n",
    "        \n",
    "\n",
    "        cy = torch.mul(cx, forgetgate) +  torch.mul(ingate, cellgate)        \n",
    "\n",
    "        hy = torch.mul(outgate, F.tanh(cy))\n",
    "        \n",
    "        return (hy, cy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "STEP 3: CREATE MODEL CLASS\n",
    "'''\n",
    " \n",
    "class LSTMModel(nn.Module):\n",
    "    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim, bias=True):\n",
    "        super(LSTMModel, self).__init__()\n",
    "        # Hidden dimensions\n",
    "        self.hidden_dim = hidden_dim\n",
    "         \n",
    "        # Number of hidden layers\n",
    "        self.layer_dim = layer_dim\n",
    "               \n",
    "        self.lstm = LSTMCell(input_dim, hidden_dim, layer_dim)  \n",
    "        \n",
    "        self.fc = nn.Linear(hidden_dim, output_dim)\n",
    "     \n",
    "    \n",
    "    \n",
    "    def forward(self, x):\n",
    "        \n",
    "        # Initialize hidden state with zeros\n",
    "        #######################\n",
    "        #  USE GPU FOR MODEL  #\n",
    "        #######################\n",
    "        #print(x.shape,\"x.shape\")100, 28, 28\n",
    "        if torch.cuda.is_available():\n",
    "            h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).cuda())\n",
    "        else:\n",
    "            h0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim))\n",
    "\n",
    "        # Initialize cell state\n",
    "        if torch.cuda.is_available():\n",
    "            c0 = Variable(torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).cuda())\n",
    "        else:\n",
    "            c0 = Variable(torch.zeros(self.layer_dim, x.size(0), hidden_dim))\n",
    "\n",
    "                    \n",
    "       \n",
    "        outs = []\n",
    "        \n",
    "        cn = c0[0,:,:]\n",
    "        hn = h0[0,:,:]\n",
    "\n",
    "        for seq in range(x.size(1)):\n",
    "            hn, cn = self.lstm(x[:,seq,:], (hn,cn)) \n",
    "            outs.append(hn)\n",
    "            \n",
    "    \n",
    "\n",
    "        out = outs[-1].squeeze()\n",
    "        \n",
    "        out = self.fc(out) \n",
    "        # out.size() --> 100, 10\n",
    "        return out\n",
    " \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "STEP 4: INSTANTIATE MODEL CLASS\n",
    "'''\n",
    "input_dim = 28\n",
    "hidden_dim = 128\n",
    "layer_dim = 1  # ONLY CHANGE IS HERE FROM ONE LAYER TO TWO LAYER\n",
    "output_dim = 10\n",
    " \n",
    "# model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)\n",
    "model = GRUModel(input_dim, hidden_dim, layer_dim, output_dim)\n",
    "\n",
    "#######################\n",
    "#  USE GPU FOR MODEL  #\n",
    "#######################\n",
    " \n",
    "if torch.cuda.is_available():\n",
    "    model.cuda()\n",
    "     \n",
    "'''\n",
    "STEP 5: INSTANTIATE LOSS CLASS\n",
    "'''\n",
    "criterion = nn.CrossEntropyLoss()\n",
    " \n",
    "'''\n",
    "STEP 6: INSTANTIATE OPTIMIZER CLASS\n",
    "'''\n",
    "learning_rate = 0.1\n",
    " \n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "STEP 7: TRAIN THE MODEL\n",
    "'''\n",
    " \n",
    "# Number of steps to unroll\n",
    "seq_dim = 28 \n",
    "\n",
    "loss_list = []\n",
    "iter = 0\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        # Load images as Variable\n",
    "        #######################\n",
    "        #  USE GPU FOR MODEL  #\n",
    "        #######################\n",
    "          \n",
    "        if torch.cuda.is_available():\n",
    "            images = Variable(images.view(-1, seq_dim, input_dim).cuda())\n",
    "            labels = Variable(labels.cuda())\n",
    "        else:\n",
    "            images = Variable(images.view(-1, seq_dim, input_dim))\n",
    "            labels = Variable(labels)\n",
    "          \n",
    "        # Clear gradients w.r.t. parameters\n",
    "        optimizer.zero_grad()\n",
    "         \n",
    "        # Forward pass to get output/logits\n",
    "        # outputs.size() --> 100, 10\n",
    "        outputs = model(images)\n",
    "\n",
    "        # Calculate Loss: softmax --> cross entropy loss\n",
    "        loss = criterion(outputs, labels)\n",
    "\n",
    "        if torch.cuda.is_available():\n",
    "            loss.cuda()\n",
    "\n",
    "        # Getting gradients w.r.t. parameters\n",
    "        loss.backward()\n",
    "\n",
    "        # Updating parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "        loss_list.append(loss.item())\n",
    "        iter += 1\n",
    "         \n",
    "        if iter % 500 == 0:\n",
    "            # Calculate Accuracy         \n",
    "            correct = 0\n",
    "            total = 0\n",
    "            # Iterate through test dataset\n",
    "            for images, labels in test_loader:\n",
    "                #######################\n",
    "                #  USE GPU FOR MODEL  #\n",
    "                #######################\n",
    "                if torch.cuda.is_available():\n",
    "                    images = Variable(images.view(-1, seq_dim, input_dim).cuda())\n",
    "                else:\n",
    "                    images = Variable(images.view(-1 , seq_dim, input_dim))\n",
    "                \n",
    "                # Forward pass only to get logits/output\n",
    "                outputs = model(images)\n",
    "                \n",
    "                # Get predictions from the maximum value\n",
    "                _, predicted = torch.max(outputs.data, 1)\n",
    "                 \n",
    "                # Total number of labels\n",
    "                total += labels.size(0)\n",
    "                 \n",
    "                # Total correct predictions\n",
    "                #######################\n",
    "                #  USE GPU FOR MODEL  #\n",
    "                #######################\n",
    "                if torch.cuda.is_available():\n",
    "                    correct += (predicted.cpu() == labels.cpu()).sum()\n",
    "                else:\n",
    "                    correct += (predicted == labels).sum()\n",
    "             \n",
    "            accuracy = 100 * correct / total\n",
    "             \n",
    "            # Print Loss\n",
    "            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}