code/rbm/RBM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# View mnist.pkl.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.cm as cm\n",
    "import matplotlib.pyplot as plt\n",
    "import gzip\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with gzip.open('mnist.pkl.gz', 'rb') as f:\n",
    "    train_set, valid_set, test_set = pickle.load(f) #mnist.pkl is a tuple with length 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_x, train_y = train_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       ..., \n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]], dtype=float32)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, ..., 8, 4, 8])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50000"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50000"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "784"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_x[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "由此可以看出，train_x是$50000\\times 784$的矩阵，表明有50000个样本，每个样本长度为$784=28^2$，是像素尺度为$28\\times 28$的图象。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f76d31e6c50>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAFfCAYAAACfj30KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJztvX2MfF1W1/vd3dVV1b/+veFgniFyI4OjEWIkOihOcGS8\nY4LMH4D/QNBkHI0xiNwYEnVCMnFGxoSIwYwB54YY7yBRSCYBL2iYebggvowIoyjKS3Ay3BkB4Xkc\neGae30vXW3dv/+he9Xxr1dr7nOqu6lNV/f0kO2efU9V19qnu/p511l5r7ZRzhhBCiG446HoAQghx\nl5EICyFEh0iEhRCiQyTCQgjRIRJhIYToEImwEEJ0iERYCCE6RCIshBAdIhEWQogO6XU9gJTS6wB8\nJYBPARh3OxohhFgLQwBfAODFnPNv1964MRFOKf1VAH8dwOsB/FcA/1fO+T8Gb/1KAP9sU+MQQogO\n+XMAvr/2ho24I1JKXw/gOwG8B8AfwqUIv5hS+tzg7Z/axBiEEGIL+FTTGzblE/4WAN+Tc/6+nPMv\nA/hGAKcA/mLwXrkghBD7SqO+rV2EU0pHAN4E4CfsWL4s1fbjAN687vMJIcQuswlL+HMBHAJ42R1/\nGZf+YSGEEFcoRE0IITpkEyL8WwDOAbzgjr8A4KUNnE8IIXaWtYtwznkG4GcBvM2OpZTS1f5Prft8\nQgixy2wqTvjvA/jelNLPAvgYLqMl7gH43g2dTwghdpKNiHDO+UNXMcHfhks3xM8B+Mqc86c3cT4h\nhNhVUtcLfaaU/jAu3RdCCLFvvCnn/J9rb1B0hBBCdIhEWAghOkQiLIQQHSIRFkKIDpEICyFEh0iE\nhRCiQyTCQgjRIRJhIYToEImwEEJ0iERYCCE6RCIshBAdIhEWQogOkQgLIUSHSISFEKJDJMJCCNEh\nEmEhhOgQibAQQnSIRFgIITpEIiyEEB0iERZCiA6RCAshRIdIhIUQokMkwkII0SESYSGE6BCJsBBC\ndIhEWAghOkQiLIQQHSIRFkKIDpEICyFEh0iEhRCiQyTCQgjRIRJhIYToEImwEEJ0iERYCCE6RCIs\nhBAdIhEWQogOkQgLIUSHSISFEKJDel0PQIhdI6XUauv7q3x2m+M55/m2qW+fwWOMxt009lWvJ4LH\n13TMHy9d2y4jERY7y6ZEos15U0o4ODgobq3fVpCj9zVtLy4uWjceV1OLxLom3m0x4cw54+LiYmHf\nH7O+vw5/bB/EWCIsdo6SsN3EAl31/IeHhzg4OKhurV8bfzTukvD54+fn5zg7O1vYRsfOzs5weHiI\nXq83H5ff5z4LMd9cov02sPXa9qZh11Jr9h3sOhJhsVO0tRJvQ4RNvEzAeMui5sdT6/vmBZDb2dkZ\nZrNZ2M7OznBwcIDZbIacM3q9Ho6OjubbUt/G7S16bylbv0RknXpxbRLbs7Oz+TVa35qdw6zhXUYi\nLHaC0uN6za+5KSE2EfbiFYmbCVo09mhbc3H47WQywXQ6xXQ6XehbA14TKhtTv99vbGzRc/PH7AbD\n1ATRW+e1/tnZGabTKWaz2Xw7m82WXDG34X7aNBJhsdXUJqqafJebgq3gfr+/IG6+3+v1ijcLv2Xr\nt+Sr5f3xeDxv/X4f4/EYvV5vbqGaAJ+fn89FeDAYLLThcLi0zy4Ldq1Erhb+niMB5mMmsmbZ1rZ8\nc+n1ephMJvNz5ZwXXBK7ztpFOKX0HgDvcYd/Oef8xes+l9hvShNvbXymmxRhEyIWtn6/Pxcy7h8d\nHS2MMerzsZLVGR0bjUY4PT1Fv9/HaDRa8EGbAJuo2Q2j3+9jOBxiOBzi+PgYx8fH875tvXul5Edm\nK79JgAEsCKxZtqX+ZDKZ31T8eey6/E1gV9mUJfwLAN4GwL6hsw2dR+wpbf+5an7UTXFwcDD3+5rl\naxalCZzt9/v9laINvMVZ2z5//nzB4ubrNgt4Npst3TCGwyHu3bs3b8fHxwv77Bv2Pm+/n1JaEtvS\nvrkYai4UazUBNsue3RO7zKZE+Czn/OkNfba4gzRZvaUJrE1gQmh+X7N82Zq0xiLcxoIvuQF8Ozg4\nmAuwj8JgC9he9yJswntycoL79+/j5ORk3vr9/oLgmij7dnR0tCTCNUE2C7dNOzo6WripeMve+9p3\nmU2J8O9NKf1PAGMA/wHAt+acf21D5xJ7hv/nauOWKAnyJmB3BLseIgtzOBy2dp+klFq5AKyxALMF\nzI/9k8lkYaxmtR8fH88F+MGDB/PtgwcP5iJci6JgfzdTEuScM2azGcbj8dzVwM0fY1+6F2ATYbkj\nyvw0gHcC+O8APg/AewH825TSH8g5P9/A+cQdI/Kj+n4tfOqmsDiyO8KsX7MwT05OlkS46eZRCnUr\n+WkjobLHfp6oK7kjTHwfPnw4b+bL9s0sb26libmob2MajUYYjUZh31w4dm08Ccf+YolwhZzzi7T7\nCymljwH4HwC+DsAH130+cX1u8gdcsk7XcazW9xNYtf6m/kF7vd6C4HLjx/r79++3EmEeb5MAswhz\nnC3H1HJ0gQkf+6pt7DxOaw8ePJiLsBdd3re+n5irpSRbpEMpKYSz5c7OzuZuCXaH3Mbv97bZeIha\nzvnVlNLHAbxx0+cSi7DF6I/5ftNn+M9resT2x5reUxpzJMKrpN9ugl6vtyS+pdbWEgZec3O0cUWY\nO8LcIcPhcB5dYDG3nMhgroaHDx/OBZfHyBOJLHyR1e2F1/pN9Sv8DcPfNPzNgxNPrO1TurKxcRFO\nKd3HpQB/36bPJV6jJHS+H/1c7fNqj/+112v9aEylbS1xwPc3JcKHh4cLk2/czBdsbTAYLH1/tRtT\nKR43ippgn/RwOCwK8MHBAU5OTua+X5uIY781h9T56Aj+bv1N04stN388StaoCbEXYM608+fZZTYR\nJ/z3APwLXLogfheAvw1gBuAH1n0uEdP0j+5ft/dEn8O0tUBrCQZR4zFGfT7GQhuFbnF/kyLMj/e1\nZtERfA21302bLDXrm2AOBoNQgO3zDg8Pl1wPPHloNwufZOKjMfj3BWB+jlpRHi/CXoB96nVkCc9m\nswUBtrYvbMIS/nwA3w/gdQA+DeCjAP5Yzvm3N3Au0UBkcXqLxotVSZzbxq+23bJYNrkwrN8UuhUl\nEqybw8PDpaSMUuv3+wvfY9MTSunmFe2bJTybzTAYDEIL2L4v9ldbny1hu2FwunUkwJElHAlwtF/y\nX5d82d4ajizhfWATE3PfsO7PFKtTEl6/tffyz/m+bSOhq/ksVxHLto/rPoKgNI4ofGpdWIxuU/MR\nBLUtu1vauHIAzH3CJlAsTOzasInEyGft3REWnhY9YbRxRXC5SS/GTe6IKHMuEuB98wurdsQeEllX\nJcvKv5+3/lg0a9+m3xR2VfMPl3ymTeewz90E7ArguNmo9Xqv/YutIsY1F5K9h8PO2AK23y0nVZR8\n2CzCPvuu5F6KLOGoBrDv1ybmSlawLGGxszT5Gn0EQZMA2+NvLYuq7TH/WkloInGOBD1qmxRh/11E\n6by8X/te/bHo9dL7jo6Oqi4I9hlzplzJf+1jdGtPT0DZEi4VYG/jE/YTc+zrZktYIiy2npIARz4+\ne7/f+mPe6ouswNp7Sq9FYlsT4bZiv0kR9j5T72LhY/5na/urcH5+jn6/P58g43FxNbfpdFqtnuYt\nYb5x1CxxYDkqoqlQu3dHRD5h3paiI/ahjrAhEd5Dokfa0qx77VHYf1ZT4H50rM2+t7RKIswWHgtv\nabtJEY6+09Kx6HdTOlZKeoi2R0dHSy4Is8Q5RXk6nc73eUKxdCz6/XPftpH4lsQ4KtZeSzCJhDgS\n4H0QYonwnuKFgvtsrdXEN3JHtCkMHr2n9HNc+NwLb9SvWdb+2KZEmL+T6IYX7belTbytf837gCMr\n098g/Y2Tt03jq403Et9oBY3aqiA1d4RNQMoSFluFt1b84zFHDvjJK/uZNmLC4smWVLTvW+11f6Pw\nwsvHmibCuG1ShNvCsbQA5v3SthZhYLGx9j52QbAg8Q3Xu2v8DcvfkP3Y+Xz+GIAFkWWxjY5ZQaFa\nCUt+3S/XxJ8nn7DohJKbgPv2j1fyx/pj/Bk1K9jcEaUVJJqsK7/lfsn6jfb9pFctm2sbYAEGmt0M\nkQVZE7looc9o4U8ra2kF3r0Pnfd5rDVr3MZbEl7/2ng8xrNnz/D8+fP51trp6SlOT0/nFdWsRRES\n+zY5JxHeEaLJkVIixiougrauCO8TjlpUZatNGFdNhCOf8C4J8aoiwYkN0arJJdEtLZRp/V6vNxfg\npsiStq4QHmubG8dkMlkQXS/AVk3NhLjkltgnAQYkwjtD9Ige9S2bq2ktMZ6E4c/nfsknfJ1t7Vjb\nG4z3fZYEeJtE2FvCNTipobYeW+QnbVrBuOaS8seaRLetCHtLeDKZzAU3alzW0qxgX5RoH2OFJcI7\nBAtuqfV6vYVVHrh8oe8PBoNGC5j7beJiV31PbYKwZg1HIWGRX7NrrmsJ8+SajxYwCzFyBZSs0FUy\nGEuiG/XbuE/sPSbCJrbWIgE2EY6WvFeImuiEyAr2MamWyeWXsCltucyinaPWL1lPbful10qWd7Qf\nXXsto6trVrGEgUWfMEcL2OM5T1x5izOyQm3r/058LDMfazNBGCVi1M5/cXGB6XS6UMC9tp1Op0uh\nbKUwtV1HIrwD1ETICxyLMBdq4WLjtn98fLyyCDclJkTH/Ot+n8/TNI7I+i9Z09vAdSxhdkdw1IBf\nBqgUDhYdLyXrRN9lLf042i+Jr+9bgfnSkka8X3O5cHTEPgixRHiHiB7J/eN+ZAnzygncTk5OFj7X\n+nwsEsDoH7ntseh4dO7StuQP3wdLmC1QdkewOPFjey07zbfSHELULwmtb3bDaCvEtYU+fVSED0mL\nLO19EGBAIrwzeAHyQmyTXbzyL6+ma6sq2AoLVuCbP7upX/rHbfrHbjpmn8/X6vuRENcm8raFVYXC\nuyPYEmYf6unpadVC9a+1+c5sfxVx99Z3bd9EuLTsPffZ91tr+4BEeEcouSN81AGvtMAr6vJCjo8e\nPZoLMn92m35twqzpPbXmr7W03zSmbWNVn3AUHcHWooV2nZ6etvLZ2jb6nkrHbiK0tX1zr/isuGjr\nIyCiyAy5I8St490RPtjei3C0ou7jx4/x+PHjuRCv4o/d5Da61tr34N/T9HldcZPoCJ6YY1fE6ekp\nnj171iqKgUXYqH1/AEIBrUU9RPvRsVKqcnScr4G/xyiLb9eRCO8IkTsisorZJeHdErzY4+PHj+ci\n3NYK3jV26Z+UBdMEzCxH9glzkkOThXhda9GL7U2E2IuwL9wTRUDMZrOd+t3dFInwjhAFzfMf/8HB\nQRjQHk2mRBYGPzab4Oacd1J8I/ifutQvvb/EKtZl28+P3BEla7hJeG8iwm0m2lZtPouv9nd5l5AI\n7xjRHzZbxb72akmQoz94Fo19E+BVHmub9oF6FIm16DusiQzfXH2yBvuF24gwX9t1RbjNRBz7nZv2\nfTq1f/0uCjAgEd4Z2ljCbA17i6P0R89WL7NvAhwJU02ca32gPFFp0R72nuhn/fj8WNlq5EksdklE\nIly73lWIhLQmvFE/2pbSrGUJi50hEmAW4pQuU4vbWMD+D35fRLfEqo/tNXEGlkP2bHtxcbEkxqXx\nlI57dwRbwuaOeP78+cJ1NfVX/a7aCCz/PfnXoz7/rcod8RoS4R3CC4YJL7ebuCNK7LJAR8JUEoqa\ny4J/ngXX0nw53tmORd9bTXxZrEyEo8k5c0fUxhi9tsp3tqrAlm5qNSGWO+ISifCOwP9Q9sdrMZ0m\nxgCubQkDZbEtCcq2Ez2i16y7NhYlizDXWTD4GH9vkcCUjnl3BFvDbUTYf/Z1RLgkpLXj0c/699f8\nxnfVGpYI7xD+DzqlhPPzcwCvCeh1LOFIKHZRdCNKQly7Ofn3+32O1Y4EwydotBFgHhtbwSUBNhH2\n11g7tsp3toq4tvnOSjfC6CZ415AI7wClP2Tvhsg54/DwsNWEHAtKJL67av1G1MTXvqM2wsLfmS0N\n5b87+z5LVl1t3wuxt4IjS7jNZ17n++LxrNJvOtYkzHdRiCXCO4QX4IuLi4XXALS2gL1IROK7L0Ic\n/cP7G1TNrxmJsPV9Kc5SCnUbaxhYjNGNoiM4Tvg6n7/Kd1bb3uRYkwV915AI7wiRkESvHx4etnZH\n2M/xthTfuqvUHoWjJ4WSddbm6YEn5ezcvOUxRfv2+bXoiMgSjq55HbQR9tK5mn62JtZ3DYnwjuHF\nF1gUGh8QX3NL2Oy+fYbfsiBv8h9kFcH3N4u2PxOFSPmbVBsBtqcQL8Lmn7c+R060FTN/c4gy5tga\nFvuBRHiH8K4C+6e1f3YWmNqkzvPnzzEYDOaLbLap+RtZeNyPLOq27brfhR+L3+fvoWlb8gtHYmw+\n4aZlnaKVQ6LMOhZxLmLjbxR3/bF9X5EI7wjeF2nHbGtiHFlRUaD/YDBYWiyzqd/WZ5pzbi3s6xLh\nSJR9Td5ai6zhkhBbUkxbEa6tZMHt4uKiusT7XY6l3WckwjuIFxwW4GhSx1vCttJyae230rHIr+pF\nyvq1hSR7vcs/u3UIcK3P3wEvn+NXchiPx/PokiZXRCTCvHp0SYjbLrDp11YruUvE/iAR3iG8H5L3\nffIGW4G+FGK/30ev18PBwUEoGKVjtWB7/5pfcsn67K5oSu1t+i6ixq/Zd8A3IfapcrPyiU1CXLOE\nm8S46XUAS0va7+viluI1JMI7SumfsSm8yYq/swizOET9o6Ojpcm9pr6d5+joCOfn5zg6OloS4E2I\nMAuovxFxooOtTmH96XS6kk+4JKzRvm/9fj88nlJaKnZeCqMT+4NEeIco/fOZMLAlHLkj7J+fJ4xY\naH0z8WRxjaIKon1e5cNbcJxtZmO/zvdQEkvre3eE+cOfPXs231qbTCathN3GWxNf37fvod/vzxvv\n26RbSmm+xE/NHSH2C4nwjsBixf+I/nhJhPv9Pkaj0cJkEYC5MPO2NCkUhb+V9geDwcLPA69ZwOba\nuI6glCYl/ZazztgdYeL79OlTPH36FE+ePMHTp08XRLgpcaNkCfMNjY8Nh8P5klO24on/jgHg8PAw\nnJiLYpnF/iAR3jFMBPw/oh0viTBbwByVYNbYbDabC7BZZ/4xmIUh6kevsQvCQrvOzs4W3BPX+Q5s\ny8LpRZirkHEd3mfPnuHJkyd48uQJXn31Vbz66qsYj8cr+4S96EYi3O/354uuTiYTHB8fL7ka7Pvp\n9XoS4DuIRHiHaBJg20YiHAnwxcXF3DJregRmEW6zTpgXYF6c1FwdNxGUmtvAW8KRO+Lp06f47Gc/\nO2+j0aiVK8K7I5q2/X4f9+7dw2Qywb179xYEmD/PbqDmjuD3SYj3G4nwjlHzCwMIRdi7IOwfmZMD\n/CNylBzgV8at9dnPaZNwLMDsprju91ATX7OEI3fE06dP5xbwZz7zGbzyyisYjUaNbojIEo7El/v9\nfh/j8RgnJydLFrC/SeWcl9wR0U1R7BcS4R0m+oeM3BHeAo7e1xQKxZNGTW06nc5D5rwAm8vjJoLi\n3RGlkDlvCVskhFnCLMJ+uaAmd0RJdH2/3+/PY39ZTA1+QgCwJMKl2hZif5AI7yF+csqEtibGUaad\nCdhgMMBkMlkKn/Liy8f4MZuz7tglYVsrC+lTev0xuzaz4nlSkEXO+qenp/M44MlkMg/X4/FG4V9N\nzb43ngg9ODiY147w4/buoOiJxHzxr776Kp4+fYpnz57h9PR0nlTinzDE/iAR3jPYkuNJstlsNhfg\nKJoiKhhjAmzhVKXaC9ExAEspy9b8xFWpHKQfr/m9/cRbaUwmwpwdxyFgkdslEty233WpRkSTS8jG\ne3R0NI/asDhmG7+N21vSYveRCO8hkSVcE2AWMpuks8doC28zN0JpIs4LIoCqFcyP65Y4wu4LE1y2\nHHlCkW8cJlBs5U6n0wURY4uSr6HpUb8kxizAXLODnzLs53q9HiaTycJ1sQDbeI+OjhZil238dvOQ\nJbyfSIT3EG/hzmazJQH2j/RsBfu4YetHERCl/TYCfHR0NE/miIr8MLyShV0fj5ndDXbzsJA0E2B2\nR9QmICNBbhJiroTmf4Yt/ejmZ+Pq9XpzC5gtYYnwfiMR3kO8pRUJMAvB0dHRXHzNIoua97/W/LFe\nhCML2HzNFxcX8/dx3+CwvJILxRc+NxE2nzALcWki0ouw9fl79d+xCXD0/dvn8g0limEeDAYYj8fo\n9XpLNS3Mipc7Yn+RCO8h3h1Rc0FYgkBT5levt7iKc6mxCPsiPmz9mmAOBgPk/FrBHy++HL7lhZgF\nmOtDcLEiL8BtRTja8vdrWxuLf93E1yYoSy4IC2Ez3zhXeWNftibm9peVRTil9BYAfwPAmwB8HoCv\nzTn/iHvPtwH4SwAeA/j3AP5KzvkTNx+uaCKaLLLj3k9sIlyrnMb7pToR0T6HXvmQLZvoMzHOOc8r\nrLHA+FUq+Dq8JczCxeuw1XzCJXeEnYe3te/ZH/c1goHlkMDZbLZ00zs8PJzfTKKaxz7LTuwH17GE\nTwD8HIB/DOCH/IsppXcB+GYA7wDwKQB/B8CLKaUvyjlPrz9U0RZ+TD47O1sSYF+svW09YQ5n86Ft\n/jWODfY+Zou4MCvPxuwF2ETMvxbFQpeWhecQNe+OiCbm/PcY9f14TRTZ6uU4aR8FEX2/NjnZlAij\nhI39Y2URzjl/BMBHACDF5a/+GoD35Zz/5dV73gHgZQBfC+BD1x+qaAtbaCYWNmnkoxDarqrBReNL\nrSbCLMDj8Xhe0GY6nS6tGMLj42WV+HoiIS6JMPtVI3dEyQcc7Zde4xA1nqhjSz6afOS+/Y5KLh62\n3MX+sFafcErpDQBeD+An7FjO+UlK6WcAvBkS4VuBLTQvCFEsbm35If9ozVZ1bd/igSMLeDweYzgc\nzq1Tvpf7G4TP4CuF1kUC7KMjOMqglA7cVuD4/aXEEtv6GGIOF+RjfH3R98t9sT+se2Lu9QAyLi1f\n5uWr18QtUIptBZaXFGLL2G/9MZ7xZwsyOsYizAJsFjBPlnEcMAuwuU4if61PS/aWMId4+YSHmiW8\n6vfM2xLxA2Pz8k6rWOVid1F0xB3D/yNziBUnSJjQmQXMLgMvuKV976KI3BfR636izG+b0qG5PKdF\nflj4nX/8bxLCTXznTcfF3WLdIvwSgATgBSxawy8A+C9rPpdYIyxwpQmpkuA2WWz+vb5Fn1eyMk04\nI+H1JTl9NEhUVe62hFiIEmsV4ZzzJ1NKLwF4G4D/BgAppYcAvgzAP1znucRmKFnK3hKuNf95JeH1\nlm+TuBvssmARns1mGA6HSxEcnBjB4WBcy0GIrrhOnPAJgDfi0uIFgC9MKX0JgFdyzr8G4P0A3p1S\n+gQuQ9TeB+DXAfzwWkYsNkKbR+M2Aln6zDauizbi7l0RvkQm+3mjxIhohRFZwqJLrmMJfymAn8Tl\nBFwG8J1Xx/8JgL+Yc/6OlNI9AN+Dy2SNfwfgq7JihLcSFjd/LHpv5C5ocklE720jviWibDxeksl+\nPpq4kwiLbeM6ccL/BkD1GS7n/F4A773ekERX1PzCvn6DbUv+Yz5W8wM3TchFPuHIHRGtWHF+fr4k\nwCV3hL9mIW4LRUcIAOX16/x7SttoIq3JGm7rguDPKbkjOImBK5r5cpychl2qVyzEbSIRFgsC7Lcl\nK5j7TcJdaz4RoST0jK/O5hMYTKTPz88xmUzmS82zT9jShP3EnIRY3DYSYQGgbgmXXmvyA0evl1wQ\ntTA1/uzIEvYTdybQZ2dn8xRpL8L2nsgnLCEWt4lEWMxp44pY5Wfs9baWcNtJP1481AswW8lnZ2cY\njUYLAuwn5vyae4aEWNwWEmERsk4BahLhVaIjvCUcCbCV6Dw/P8doNArdEdGip0J0gURYbAQTWF5i\niVe/GI1Gc0EEsOAPjlwO9jneQuVoiZzzwsrNpRVCOLWZ188rRWaUJgmFWAcSYbERTISjUpNHR0dL\n5RstzZgFz4ei2WRaVHiHa/fafqmovC8qZFl23jov+ayFWCcSYbERcs6hFWyuAE6FjurksgDbGnj9\nfn9pYs+IykTWBNgLcVSs3p+HCxkJsS4kwmIjRJbweDxeqtdQElQfB2wJGVZj2PuOfU1eFuFojTtf\nWrO0ZJNfIkqIdSMRFhshEmEWYH7dRLVUIc1WJD47O5v/rG0j/7C9Fgmwt4Jt0s6vGO1D12y8soTF\nupEIi43AbgarYMbRCFzhrOQDjkpUAnEx9GhVC79YqV9myVvCVm2NV7wwuO6yEOtEIiw2grkYzBJm\nH7AdN3E2K7hUI9hWxPDLBLHgRvur+ITPzs7CYu82XoWziU0hERYbgUPUZrNZeMxWvQAwT77wAuwL\ntfvJNy7A4yfneDn5NtERnMbsk0n4BiDEOpEIi41g7ghvUZoA93q9ebQEgHkUBAuwrcbMKyRbGBrH\nBrMFzAuTRj7hyB1hlnBUvN5cKlGdCSHWgURYbAQWXe5bhAOnF5tgRiJsi4GaCFsyBrDoG/YlLtm1\n0cYSNn+zHzuPUZaw2AQSYbERzIo0q9WsSXYXsNXqxff4+HhJgC0dGcCCxWr7vnZEkwB7n7Dhw9Si\naAkh1oVEWGwEE0cLP2PXAa9bl1KapzJbs2XrR6MRTk9P52Fkg8EAFxcXC75e6xt8nlq0hQk9R13Y\n+9iSjtpsNmusIOf7Ueqz0qEFIBEWt0RNZDiKgoX49PR0XnTHXALHx8dza5aXNbJsOhNfS/6IRHg4\nHM7F124Sh4eHGAwGC6Uv2SUyHo/nW64zUar6xsd8zeRSX0J8N5EIi1ujJDLsL7bMutFotFD1zFwY\ns9lsQST9kkYsusBivLBZweba4FRpmyjs9/uYTCYLAswiPJlM5pZwVAUuqjcRZeP5vupS3F0kwmLj\nsN+W960fFfqxKms+zdmWtT8+Pl7KtuNaE5EoW8qyF2CruGbi690jk8lk7qM2EY7WyCsds/OZ+Nu+\nTwYRdxOJsNgoHELGosdCzKFrZgl7C9h+3uKLSxYwrzfHfmG/KrOvtmYCbZOBJrjW562JsC/0ExUB\nMvG1cc89J+5FAAAgAElEQVRmMxwcHCykYPN3JO4eEmFxK5jQRI/cLFRcbc0LsL0vEmBeddnXJmZ3\nRMl10e/3F1Zmjvocs8zREz6awu/bz/nVPNhqlgjfXSTCYmOw8NYK33hLuFTox1vBwKIla9ast4Tt\n9Ui4vQC3aSzCXPjH73NYHS8sygLsK7WJu4dEWGwUL8CRKPtaEixUZgGbQLMrgQWWJ92iqmy2Hp2N\ngf3E9tm2bepz3HLk6+V9yw70NZTZVaG6FHcbibDYOJFfmGFLOKq0xm4KDj1rCjuz9/m16FiAvc+W\nhbbU90JsPx/tz2azuQDb9fvsQSWC3G0kwuJW8HV//cQcW4S+0prFD5tPN7KALWqBV+lgS9jOawLs\nQ8T85FnUOHvPi3FpfzqdFgXYBFqW8N1GIixuHW8N+2prvtKaTdSZX9fH/Vqhn5IlDLwmwKV14/h8\nkSDXRNpbybw/mUzm1+wFmFd9FncXibDoHF8u0peRtMkt21pK83g8rrbSRKAvg2ljMAvb3ATsM/Zu\nBhZam9iL9i1pBIjLY/IkHt88VklxVpLHbiMRFluBT/m1sC0T5vPzcwAIV2/2NSYs046Lyfs6xNEx\nLihkfuRSkaCoapsvGGT1kksrgfhm56yt9sxb/u6a+mJ7kQiLrcIX9zEBNjGKRNiE+Pnz5wupzuxz\n9VvrA1jwGXsRtuNefDn8zVvObDFzcSEjEmAT4dKqz77PfvXIQm4KCxTbg0RYbAUlS5hfB7Dgd2VL\nmC1g87WyOFrjfWBxTTpvkfZ6vSWrmctyegs4ClPr9/sL11kSYPu8KPbYH/M3Jm+tl9wWYjuRCIvO\niQS49B5vCVudicFgsFDe0lKD/aoaXFfCxy9bGUwAC2FlXoTZqu71etWkDZtsLImvF2IOc+NmJTQ5\n0qLUou9ObC8SYbE1lESYBSZyR/T7fZyeni4IcEoJZ2dncws5qhnBLoloss6O2aSdRWbYtk3aMouw\n/9zIHVGKwIgWII38xvydid1AIiy2AhYSrgfsj3P2momwWbh+GSITQVvI0wuwz6JjEeb9aFKsVryH\n9/2qHCUL2G4InB7tl1ay74OXjeJj/L3VkmPEdiERFp0TpTGzaNnrUclL74JgweLQL19rwizZyB1h\n57Jz23HvMimVr/Thdfb5tagM2/I1lQSYRd6Ol743sf1IhMVWEAkGLzNvrSTCNcHyKcyc+MEibESP\n/TxG7wYohZOZ+ySyhKN19iwLcDweL1n1XoA53dm7b3yEicR4u5EIi60gCq8CsLRlES5VJ4ssU/YB\nWxgZW5N2Dhbj0vhs3wt0dCy6kZTC3fwK1DwWL8Dsu7br4yZ2B4mw2Bp8qJUnpbQQouark/FnROUu\nTYCn0+l80VDvE+ZzNfXbJElEIsw3DbaEOfKCrVi7HvNx23X78/mbl8R4N5AIi62kFGrFMbilQukc\nKdCU1WZFgIBlV0FtQs1oEjpfKMi7Mfx5oygI23oLnyuxWeyyzzCsha6J7UAiLHYOtgy5poN/jI8s\nS/t5L2TD4XDJGi31vTjXttaPVn0uLfIZRU1EN5NSTLG1yGpvY72L20UiLHaKSEAjAWY/cJMAWxgb\nW8rWj7ZRIge7DyJB9iLsl2gq+Yu9q4LHUqv05r8L23rXhR3TBF53SITFzsGxsiak0+l06XUTMj4e\nWdHT6RTD4XDunvDuCjt2cXExz7gruS5KAhxFZngR9qLrBZibibDVUbbY4tJNpzSRyN+N6AaJsNgp\n2DdqQur9tPYasDyR5sXbBGw4HM5rT1idYsu0sxoQkcvAW9wm+v69LMKlqA2eZCz5stmnbSF6k8mk\nVWIHC3Lt+xW3i0RY7BzenWB4cfaC7YWXs+6sDKZtTZgt9ZljjVl0LZnDXitFKrCYsgvAC23J6vUC\nbDeLpjhpX6vY4or9BKbEtztWFuGU0lsA/A0AbwLweQC+Nuf8I/T6BwH8efdjH8k5v/0mAxUCWLaE\nvaVrYmsWZ8kH7OtPHB8fYzgcYjKZ4Pj4eKEgj0/2YEs1gsPHgEVLmMXOC7D5crlAUE2ER6PRUnSI\nfQ8+eoLFF8BSerN8wt1xHUv4BMDPAfjHAH6o8J4PA3gnAPsPmVzjPEKEmNjyPicycNhWyQfMVdgG\ngwHG4zHu3buH6XS6JMDmIvDWqhctnuiKws9YnL11bBXf2A/sxdf7qmvheX7S0s7JySmcms3uFXG7\nrCzCOeePAPgIAKRykOQk5/zpmwxMiAh+hPbHLFaWa/OWXBFmAQ8Gg7kI25JEHLnAVq+JoLd0DS9k\nfgKPfb42PiuFyTeRKEqjJsIlATZXhVns0fcm8e2eTfmE35pSehnAZwD8KwDvzjm/sqFziTtE5Mv0\nUQrWN0H14suTbrxaM6cy+6iFw8PD+SSd+ZsNH17mIw/YbWGCzpNknIARhaKVIjZ4EjCygH1WIX+H\nPFZl13XLJkT4wwB+EMAnAfweAN8O4EdTSm/Out2KNcCP0UA5UcIe802QbCLLogo4GmI8Hs9F2PuA\nbWVns5RtySIfK2ylJHmCzl73PlufPGEiGlm+0YRcv9+vuiCm0+nCpJ0/l22jsDpxu6xdhHPOH6Ld\nX0wp/TyAXwHwVgA/ue7zibtJm1Rcvy4bL2tvNSRMqLjkZC08zCxQcyOwW+Hi4mIh+gHAkh+Yz+GP\nsyi2CVXzKczs9+ZMQu/nPjg4WIgqsXN7d4W4HTYeopZz/mRK6bcAvBESYdEB/pHfhIgtP5ussyLx\nHPbFn3F+fr7wnsha5T77bZu2RhS6FiVccKZcKQPPPsdn1HHfVv/g+Gpxe2xchFNKnw/gdQB+c9Pn\nEsLDwsVhbf7R+/DwMKxPbJ/Bj/u8mof31Ub7vkSl7wOvxR9HFrAXYXuPWbs+ASNKk55MJgurdnB2\nnV2jX8VZ3A7XiRM+waVVa3/FX5hS+hIAr1y19+DSJ/zS1fv+LoCPA3hxHQMWYhVYgM2XHMUXHxwc\nzAuq+7Av729lEWbfsm/9fj8MM2O3glESYBt39F4Oo4smEtkiH4/H84gQKxofCbC4fa5jCX8pLt0K\n+ap959XxfwLgmwD8QQDvAPAYwG/gUnz/Vs55duPRCrECLMAmWgAW+va+g4PLpYV8Ld9owsvqTPDE\nHvejrW/edVCyZGtFfkqRHBxjbGMYjUYYjUahlR8Vihe3x3XihP8NgNpv609ffzhCrJ9IjL2FDGDB\nAgYQCrDFFnOIW9s2GAzCGGRO/PDWcGQB1wSa/ci+Dob5qP312bVJhLtBtSPEXuPjin2YmIWVAai6\nIHxyBwux9UvHBoPBwoKjLJxeTG0cOeelyAoTZrNcIwH2FrAlovh4YbOALYPQfNfyCd8+EmGx1/iJ\nOTtm8bycPMGvRUke3qr1QsvNkkI4PIx9t2bNWmZbyeXAx3wmoL0W+YA5CcVqJZcs4CihQ9weEmGx\nt9gjP7si+LgJMddUKGWd+Qk3FlyrvOb3fRq0waJZCy3zgsyTdLyiso+EsPH5SAi7draALSJEPuHu\nkAiLvcZnh0VZYuYn9gLsIxlY6IbD4UL5S+7bQqJWCpOtVnYbzGaz+TJH0cScTy/2zV7zFjBXiYvi\ngFmAR6ORRLhjJMJir2HBsq3BffO1np2dhbG8vGURtmalMM3yHA6H86QIPwnHgunjfDkyw8bPW+57\nAbbz2U2E+94CHo/HGI1G8zA6X49Y3B4SYXEniMSM4XRejpLgrfV7vd7Co35keXImWyTA9h4fLWG0\nEUO+WZyfn+Po6Gh+Tt5yCU+bYByNRgtREzwxx24bsXkkwkIElArsmGvAl8f04W32Mznn0FfLAuyz\n4VYhSvKwyAqOsCitzBFNLFpdCZ8m7d0hYj1IhIW4gt0WAEK3gF+lw+oyRAJs4hoJsFmnPmpiHUJs\nn8chbr62BYswi7G5UXj83PdbcXMkwkIQLIIsMmYFW5+FuCTAJrAscLY6slnCPjrCj6FGSXy9OEYx\nxDVrmCMxfOEju37FE68PibAQjpK4cFhYtNIzZ+DxApsswsfHx/PYYU7guC6REPvXrdRmJL5RuB2n\nRNuNwq9eLQFeHxJhIRxNAuPLYZpVyCLMK3BYBIVVMmuyhG0Mbd0SUYKHP16zgr0QW4U2f418/Yqi\nWB8SYSGIkjvCv2ZCy6+x5Wi+4ouLi7kFbAuL1nzCq8AiWxNgFuE2Qmzui9lsFk40mrtCLon1IBEW\n4oqmiTnej1Z7tlAxm7CzuhDHx8fzUpIW1saF2EuTXG2sYZ9RxxmA/FpbAR4Oh0sZe/46/XFxMyTC\nQgTULDyuQcEWMCd0cLGde/fuLYhwbUWM62CCyJNznJxiYXI+s85XWTMhjsTfX6dYHxJhIYjICo5e\ns31Og44SPM7PzzEajZZEuI0l3IQXSvZNs+uAY5Xb+IS9CHsBliW8XiTCQgTUQsZYfP1ySb5/dna2\nJMJeiGsC3OSS4HN5lwlvVwlR88kZJsCcyi0RXh8SYSGuQdvJtJzzUoozr3x8XUvY+675WORK8MWI\nSpbxYDBYig9mHzdP9on1IBEWew+LVNM2atFrbT/v6OgIjx8/xqNHj/Dw4UPcv38f9+7dm1ddswy2\nkqiVjkcFfbjvj3G8Lxf2sRsD15Uo+a7Zf62oiPUhERZ7TUlYS80X7yn1m4TbWr/fx+d8zufg0aNH\nePDgAU5OTpZE+CZlJCPXQ3TMCzAvec+WuoXRect9HT5sESMRFnuNnzSLJtCsz5ENpTKW3Np8rhdh\ns4SPj48X1n2LRLjtI39TkR0fw8ylLlmEmwRYlvBmkAiLvcVbqyUR5ZKQvFBm6Zj/nOjzrLE7omYJ\nR5EOTXgx9ALMrckSNgH2CSUsxOwnlgivD4mw2GsiAS5ZvDxpVWscJRBZybzf7/fx8OHDeWOfsBVU\nt1q+Nt5VYbFlS9Uf95aw9wmzGLNYR5awWB8SYbHXeDeBXzGDm48WiCIIvBBHn8X7R0dHczdEkyV8\n3YiDkvXLZSfb+ITZHeGLw0fRHBLj9SARFntNyRIuCTBnknGfj3G4V5Mb4+joCCcnJwutrU+4DZEv\nOKr9GwlxTYSjKmryCW8GibDYW9gnHAkwW7VHR0fz9F1O5Y36ZhFHi4D6/tHREY6Pj3F8fDwXX1uP\nruYTXoUmK7iNALMQ+1U/fE1huSTWi0RY7DU+/MwLsLVSbV2/9I8Jsv95dlX447wSs/9cXt9tVZom\n5rx4loR4NpstTcqVVtKQJbx+JMJir4lC0LwQ+4wxv5KyX9KexbO2tb5fRsi7OW663HxNfL0Q+9WY\nvTtiPB7P6yBHny0BXj8SYbG31PzBpToKZqGay4DdB9Y3EW7bmqxm747wxYOa9kvREW0s4cgdUapn\nIfHdDBJhsTV4ISr127YousGLLvfN4jXRrVnENcvXb73gclyxj4rwAgvExeV9v8kn7GN8axauBPh2\nkQiLraBUp8HvR9ltpeZr55YEmEXY+4OjY+YTLvmB2eKNIiZ8goddI7Ca2DW5DCI/7nXcChLgzSIR\nFltBUz0HL8BReFgt7tdbvaWQND8BF/V9vHCtXxPfyBJelShlueSSWMUa5s8Xm0UiLDonSi8ubU3c\natEIJX9vTYB9kfNaK1m4TTeHyIK/qQgbbSbn2govf6bYPBJhsRVEk2hRLYZVXAwloa1ta6Jt++zb\nLQltTXi5MLp3R7SlJqhN7oko1EwWcHdIhMVWUCop6UWMoxj88jw1y7UpEy5KVS5NvFlcb61mRBuf\n9TrcEUZbS9hbxPaz0VbcDhJh0TlRZlsU1+tFuJZYYQVySuIbpSKvUsCnZKmXLPmam2UVEY4s1lV8\nwqu6JMTmkQiLraBmBfvJtiipgmN5OZSsJrxR0kTJt+v3m/zXvh/tr+qOKLkMVo2OiKzh0jnE5pEI\ni62gKbGiJMK+LoOvz1CacPPHvZ+3ZImzFczjbtOi99uxJpomz9YZJyxuF4mw6JwoOqKWYuxF2CqT\n2db6NT+xb7bOW1vXgh+zvw7etn3tutQs4XXFCovNIREWW0HkjogqnkUifO/ePdy/f39eKtL67Bdu\nK8JNrgNf42Edk2pNREJZEtMmt0RpYk6C3B0SYXEjIhFqk/nG+z6utxah4C3fyApmv3BTJISv37CK\na6ENJQEtHWsbYpZzXlp+qNZ/9dVX8fTpUzx79gzPnz/HaDRaqJpm9SIkxLePRFisTJvH7WjCqtT3\nlcxq9R6Gw+FcbFl0uVi6r/EQFdNhP+9NhbYNLG610LBShhvX8rXGS9JzWUreZxH+7Gc/Oxfi09PT\nuRDb+y4uLtZ6zaIdEmGxEiVfqBfh2iSX3zZlsPExnozjCbioYPpgMChWLSsV0vE3lnUKcunxn/f9\nSha++hnv+3XifOPjT58+xZMnT/DkyZMla5iLuIvbRyIsVqbNrH8U2RDVVGA/b5tWqvcbVT6rFdgp\nCbG/Fr7m67KK79YvsOnXhON9v0inX7iTX3v+/PncCjZLeDweL6wpJxHuBomwaM0qPtNSjQdf28Gv\nauGz3/yxaKWL0ioY9vmlWN9S5lpJfNdhETdFMnjXQsmyjWoB19rz589xenq6sI3cEfIJ3z4riXBK\n6VsB/BkAvx/ACMBPAXhXzvnj7n3fBuAvAXgM4N8D+Cs550+sZcRiK6hFEng3Q5taviVxjcQ3so6j\nfW/xRkV1anG7q8TxNtEmmYJFOBJSs1r9MT5ufX9sNBphPB5jNBotNLkjumdVS/gtAL4LwH+6+tlv\nB/BjKaUvyjmPACCl9C4A3wzgHQA+BeDvAHjx6j3TdQ1cdEMUshVtfUhZk4uhVLvX96Moh1Lfajw0\n1XmI0ofXLcC2LcXw8uSbiTCv+2ZLD/ljvkU/1+Zn5I7ojpVEOOf8dt5PKb0TwP8C8CYAH706/NcA\nvC/n/C+v3vMOAC8D+FoAH7rheMUW4H2nUdQDuyPaWLvRWm7RMa7zEIWw8WtNSRe36YpoK8Ds7zXh\ntWbWrDUvzF6keb/mM5Y7oltu6hN+DCADeAUAUkpvAPB6AD9hb8g5P0kp/QyAN0MivNOUIge8sJml\nGSVX2PptbSfY/D77eZu2UT2HaL8kwtF134SSH9gvSc+WMLsQLKzMWiTIpb6JOws9H5M7ojuuLcLp\n8i/z/QA+mnP+pavDr8elKL/s3v7y1Wtix4nEK7I4IxE2AfaLZ5YW1PThaCbCTXUdvL+Xx13r8zWu\nkygqIor/rYkwT6pZi6zjqI1Go1ZxxxLhbriJJfwBAF8M4MvXNBaxI5R8wl6ArZnvl63bqOAOt9Jx\n9vN6F0hUNrI0/i5YxRKezWahCHNji7hpW0sQ4a24fa4lwiml7wbwdgBvyTn/Jr30EoAE4AUsWsMv\nAPgv1x2kuDltBKnmF+XY36blfHq93pLYthHYyP3gIyK4xkOpbOQ66vQ2HavF/PpjpWSLaBl6DiHz\nfb8fWb0+WsIEXWwvK4vwlQB/DYCvyDn/Kr+Wc/5kSuklAG8D8N+u3v8QwJcB+Ic3H65YhTYC6x/L\nIx+ptSjWtxQD7AW15GqIJuA4CmLVFON1TKDVttaPVqwoHYv8sVF/NpvNrV7z/7L7gfejSTebYGM3\ngyzc7WfVOOEPAPgGAF8N4HlK6YWrl17NOY+v+u8H8O6U0idwGaL2PgC/DuCH1zJi0YqS/zN6rY1F\naatatFnXzdKLo0iHaGKuFBvsQ82abhLrFOBalTHvx+WJrajvM938PrsgeCKuqe/jgf1Emwry7Aar\nWsLfiMuJt3/tjv8FAN8HADnn70gp3QPwPbiMnvh3AL5KMcK3RyRQfr80sRZNtNmxprXc2mS2lWJ/\nS4XXm1KM102tilmUXuyjDUoWbqnAjt+vRTj4Y5y0IUt4d1k1Tvig+V1Azvm9AN57jfGINVES3JIA\nNyUzHB4erpQyvEoacs2q5nCzUlzvuq3hUunIKLOtVLWM931MbpSGbH2fcOGz4LgfpTbzDUACvBuo\ndsSeE4ku7zct1c77kQ+31EpLCUVrvJWSLnzpyZo7wl/zdShltvmtr2BWKqRT2q+91vSZfKxU6EeW\n8G4hEd5TSlav37aNdjg8PAzLSJaiHmp1I2o1fnlbKrhj1+e363BRRFZwFF9rFm5koUb9Wl0H3rLP\n2FvX/hj7f9n9IRHeLSTCe0gkUKVYWr98UKnf6/XmQtumRdXSSv0mC9y7I0rXeFNqcbzceDKNU4Rr\nvtzpdLrQr9Vw8IJa8jVHE4F8TBNzu4FEeE9pMwHnBbht2BkvLVRqtoR8TditH2XclZpdG1+nv+7r\n4EPQmjLb2B3BSRVRFIOfWCttp9NpUVD9sdINIgqfE9uNRHjPaRLjaCHN0npvLMD379+vtiixo9Rq\nrpM2E2/rjJQo+YO9KLI7gtODfTxvm7RitpyjAj+1bVMT249EeE9ZJQzNxLDJd+st4fv37+PBgwdh\ni1wJtTKSNmZ/DVF/U7Sp8cBuAbaEuciOL6AeWceR5TyZTObnbtp6gW3aF9uLRHjLaYrx9ZECkdBF\nghgJr9V5KC28+eDBg7mlyysd+/XdhsNho1vB+6fb0ja9GMDC43gt3dje61OLfZqx7U+n06WCOl54\nefUKtoZLlrH5hdtes9gfJMJbQpMV2EZUr7ON3A7c530W35OTk7nocl2HyPKNXAk3jWZok14cPcLX\n+k1JF1zr15eXLG1NdHl5eU6qaJpAkwDvPxLhjokeuf02qkxWimCwia42pR5NhGuRC3yMLV9eXp4T\nLvxEW8m/exPaFs7h4jlNk11tFtb0IlybjONtKb3YIhxKJSQlwHcDifAWUAu7MhHj2rylDDO/rlpJ\niLlfi4zwURKlmr9+Tbc2ArzOzLZSlpuJay3My1u5tSy4qNZvm4LqPumijSUsAb47SIQ7pPR47gXL\nRNivyRbVabDQsDaxt23ig7lfKsjDBXdKKcbrTKqoiS73fe0GL6ZREkSbVY45PjiK+/WJGtHn1JIq\nJMB3C4lwx5REykc2mDXqRdCXi+z3+9VwsKYsuahv21JNCF5e3hdd30RGm21rqcVc36FpfbXrbFdJ\nSfbujZIIS3zvJhLhLaIUI8uRDCbCvDIF94fD4Uoxuqu0phWO/dpuNUv4JqJcckNE6cVsuUZpwr4g\nTklQfWuynrmVst+U2SYAiXDntElS8D5hs3otSoGjFo6Pj6uWbpN1XOrX/Me+/kPNCl6XRQwsF1f3\nmW3eh+tXIY5Si2v1HVisS8VzvNXrV9DwE4PKbBMS4Q6JfMKRIEfuCEuYsNhdS5K4d+9eo/j6Gr1R\nqFsprritkNcmG/21r0qtyI5fNLOU2Rb12Y/LUQ3e32vpxdESRdHxUooxN3F3kQhvAdGjui+yYxNz\nJsJmCd+/fx8PHz7Ew4cP8ejRI5ycnDT6enl/lboNpYy7aL+W7bYu33BNgKNFM31mG/dL1rG3ns0S\n9uIaia0da4rikDvibiMR7phaFhwLG7sjzP9rFvCjR4/w6NEjPH78eJ4y3DTJxlZrk7Ba38brt+u0\ncpto8gV7K7TN8vGW2RaFmUX7s9msMTqD93nsTX1x95AIr0DJuquFmtWOtbU6B4PBQtEc7wfmZu6I\nkuXr9yPLu7RdhVWExVuCtSy4Wsyv708mk1apxVxop03FMxbhpmQRIZqQCLegzSx/JGa1/iqt3+/P\nXQ7WTIQtIsLH6tZ8t37yb1MJFW2pPaL747xwZlP872QyaUwrZreEn4Tj6IYonCy6eUh4xapIhFtQ\nshYj0W2KMPCvRZNgfr/f7y9Mvln//v37C+nDJsJtoh02nVDRBrYe20xeRbG/tXhe9gHXUoutglkU\n5+uXDGojvhJisQoS4QYi/2yped9rqb6D7ZeiEfyxqHgOW8K2hDyLcC3SoSTAJfFdtyBHKbq12g6+\nmLoXzCisjFeyaOPrjdZy82FmkQhLfMVNkQi3wAtxFN7FQhutoRYdq1nOvO33+0uFc3jfuyNWjXTg\na2wjytclEik/wVZbN83CzaKlgUrRDKX3l9KLo3XcuNBOqTymv0YJsmiLRLiBkgBHk1xc3yFqftXh\nNu4Ks4Sjwjl+a0V0mqIcIkuYr3UTrohagRq2hH3RdJ8E4a3ZWp3eyMVQSjWuJWBElnB0DUJcB4lw\nC1iIffaYt4C5mE6t+Qm0ksDb53OxHO7zPlvCNd81H7Pr4+v0/ZvSVCHMLyNUSwlm0W1qXL+3qThP\nm2LuUY0HCbG4KRLhlrAFWUrdtZTiNq2p0I6P8W2yrLmwelPkQy0SYtMxv6VHd5/lFhXMmc1mS+u3\nWd83E+Gm+sCR37dpcc3SdZT2haghEW7AC5oXSvbzcjZbUxsMBo0xvP480eoX/hiLsB9/ze1Q6t+U\ntskKtUw3noCL1nHjxsc4vTgqnOP3fRU27weuuSOifSHaIBFuQeQTjibhWIR54oybRTSYCPtMtjZZ\nbk37NuY2W9+vHVsXUXgXZ7yxJcyTZ+PxeCHJ4vnz53j27Nm82b5tp9NpaM2Wlo5vilcuhacJcRMk\nwi2IJueixTCttgOvSBxVOjs5OcFwOAwFuCTKtegGfywaf+m6botSZIRtvQCXVrEYjUYLAvz06dP5\nlvsswiyspTXm2kY6SHTFupEIN1CblGMR9v5gczt4K9gEeRURborrZd/vqjQ9Ul9XdJosSRY4H/FQ\nWpF4PB4vuB7M6i216XTaahwSVtElEuEWsLXpBdhHKfjIBV6BwhdDj9wJJUu3zeRaRFNomG1LAnld\noWqbAXdxcbGUWFHbnp6ezsXXr2ZsFrRWqxC7hES4gZI/mC1gv+RQJMQc0cAiXApNK8XytqX2WO1F\nNvKHliaj/GeXiOrqlmrw+qy3WvMRERwPzJEOtZuJxFlsExLhFkTuiJII+3jgKKzMLwUUZctFVrCN\npa0wl0SX+6tYrE3nYdqsOuGLrpfSkXk/ihG2tGO/dlskwk3jFuK2kQg34CflSu6IyBJmV0TkjijV\ndKhZwlFomRfjKPqgZBXWIgf8sSbR4tfbrlocxQLX+lEtCG8JR6FkUd+PWYgukAi3wFvCfkKutAx8\nSSIE/MYAAAnpSURBVIhZhEsRDlF6sY3Ftk2+4DaNoxKaluzxn186L4AwXbi0bltJoEtC7ms/+NTj\nyBL2Y5b4im1BItxAFJrG7oimibmSAPuoh9L2ppltJeFlS7ipODr7WaPPj/Z5ss0X2PH9Uk3gUp3g\nUulKrv3rrd9orEJsAxLhFpR8wlGqsq/tUEo3Pjo6KoptKRqCx8PbiJL/1/f9UkAl8Ts7O1v4bH8u\nv2+TaKXFNXnf+4j9DaCNb7lU+9ePUdaw2DYkwg2UfMJcz6HmjiiFp5kI1ybdIgvY96N9o+aC8Ouy\nsZhF7ezsrCpgfj9aTNPXfbD9NinFbV0mbQrtRPtCdIVEuAWRJcwCHE3M1cTYmn2237ZxNbSNDY5C\nz5oEOJoUs3XVSufw5+fMtqjWAx+LCujUirv7bLdSa+M+EaJrJMIN+Iw5Xy+izcRcFB1hhXb8uaLz\nX4cmC5gn5SIhjmrx2ufyOUp9E9mopoOv9eCLpjcJayn0LNoXYtuRCLegVnvB+2sj0YtWCN5U3YZV\nYn/Pz88b43J5v8mvyn2fXlxrUQUzb73bVoh9QyLcAE9e2SP6eDxeiHAwQfUrP/ii67zt9Tbz1ddc\nD9Gxpvhc3udzNPW966GU2VYqFanIBnFXkAg3YOLgq3pZxpsJcM55vvKDjxH28cKDwWCjIhwJcanf\nNqliNpvNP9+fL+r7ibmmzLamBAsh9hWJcAM+oWE6nS6kHPN7ptNpddULblb3dxPU6j94ES7F4kbb\nNqFphl9WnmODp9PpQuyxt4B9E2KfkQg3EFnC7ILghIfJZLK0qrKPiODiPZses/dPR/tt4nF9nDCf\np7TvEzNq7ggJr7jLSIQbYEt4NpvNXRAppQUBns1m8zXe/PJD0bGo+Po6x9zUOFmjtMilj9/154jO\na9RSlUs+4SjKQaIs9p2VRDil9K0A/gyA3w9gBOCnALwr5/xxes8HAfx596MfyTm//YZj7QyzhHnl\nCi/A5qYoFWb3/U2JcE3IvKhxlIRPdoiO+XOUzg2g0cdcsoT92JvOKcSus6ol/BYA3wXgP1397LcD\n+LGU0hflnEf0vg8DeCcAi8Oa3HCcneF9wgAW9nmyrrZ6clS2cpNjrsXOcp+TI3z4WpvkB39eo00J\nS/+5mpQTd5GVRNhbsymldwL4XwDeBOCj9NIk5/zpG49uCzCL13zALFKz2az1WnBRicpNjrntNprE\nK/XbnhdoX9Q9ukmUrkOIfeSmPuHHADKAV9zxt6aUXgbwGQD/CsC7c87+PTsBW4smyCbAUT1gX4Cn\n1N/0IpulR3nfX6Wtes5VsuCiz4j2hdg3ri3C6VJF3g/goznnX6KXPgzgBwF8EsDvwaXL4kdTSm/O\nO/gfxVZi0zpvTXV/21Q/2+R11I6XtrWfbfr8mpjL7yvEJTexhD8A4IsBfDkfzDl/iHZ/MaX08wB+\nBcBbAfzkDc7XGZqhF0JsimvNDqWUvhvA2wG8Nef8m7X35pw/CeC3ALzxOucSQoh9ZmVL+EqAvwbA\nV+Scf7XF+z8fwOsAVMVaCCHuIitZwimlDwD4cwD+LIDnKaUXrtrw6vWTlNJ3pJS+LKX0u1NKbwPw\n/wL4OIAX1z14IYTYdVZ1R3wjgIcA/jWA36D2dVevnwP4gwB+GMB/B/CPAPxHAH8i5zxbw3iFEGKv\nWDVOuCraOecxgD99oxEJIcQdYnNpW0IIIRqRCAshRIdIhIUQokMkwkII0SESYSGE6BCJsBBCdIhE\nWAghOkQiLIQQHSIRFkKIDpEICyFEh0iEhRCiQyTCQgjRIRJhIYToEImwEEJ0iERYCCE6RCIshBAd\nIhEWQogOkQgLIUSHSISFEKJDJMJCCNEhEmEhhOiQbRDhYdcDEEKIDdGob9sgwl/Q9QCEEGJDfEHT\nG1LO+RbGURlASq8D8JUAPgVg3OlghBBiPQxxKcAv5px/u/bGzkVYCCHuMtvgjhBCiDuLRFgIITpE\nIiyEEB0iERZCiA7ZShFOKf3VlNInU0qjlNJPp5T+SNdjWgcppfeklC5c+6Wux3UdUkpvSSn9SErp\nf15dx1cH7/m2lNJvpJROU0r/X0rpjV2M9To0XV9K6YPB7/JHuxpvW1JK35pS+lhK6UlK6eWU0j9P\nKf2+4H07+btrc33b9rvbOhFOKX09gO8E8B4AfwjAfwXwYkrpczsd2Pr4BQAvAHj9Vfvj3Q7n2pwA\n+DkA3wRgKcQmpfQuAN8M4C8D+KMAnuPy99i/zUHegOr1XfFhLP4uv+F2hnYj3gLguwB8GYA/BeAI\nwI+llI7tDTv+u2u8viu253eXc96qBuCnAfwD2k8Afh3A3+x6bGu4tvcA+M9dj2MD13UB4Kvdsd8A\n8C20/xDACMDXdT3eNV3fBwH8UNdjW8O1fe7V9f3xPf3dRde3Vb+7rbKEU0pHAN4E4CfsWL781n4c\nwJu7Gtea+b1Xj7i/klL6pyml/6PrAa2blNIbcGld8O/xCYCfwf78HgHgrVePvL+cUvpASul3dD2g\na/AYl5b+K8Be/u4Wro/Ymt/dVokwLu9ahwBedsdfxuUfxq7z0wDeicsMwW8E8AYA/zaldNLloDbA\n63H5h7+vv0fg8nH2HQD+TwB/E8BXAPjRlFLqdFQrcDXW9wP4aM7Z5ib25ndXuD5gy353vS5OelfJ\nOb9Iu7+QUvoYgP8B4Otw+YgkdoSc84do9xdTSj8P4FcAvBXAT3YyqNX5AIAvBvDlXQ9kQ4TXt22/\nu22zhH8LwDkuHebMCwBeuv3hbJac86sAPg5gJ2aeV+AlXPry78TvEQByzp/E5d/vTvwuU0rfDeDt\nAN6ac/5NemkvfneV61ui69/dVolwznkG4GcBvM2OXT0ivA3AT3U1rk2RUrqPy1989Y9k17j6o34J\ni7/Hh7icsd673yMApJQ+H8DrsAO/yyuB+hoAfzLn/Kv82j787mrXV3h/p7+7bXRH/H0A35tS+lkA\nHwPwLQDuAfjeLge1DlJKfw/Av8ClC+J3AfjbAGYAfqDLcV2HKz/2G3FpNQHAF6aUvgTAKznnX8Ol\nL+7dKaVP4LJC3vtwGeXywx0Md2Vq13fV3gPgB3EpWG8E8Hdx+VTz4vKnbQ8ppQ/gMhzrqwE8TymZ\nxftqztmqGO7s767p+q5+r9v1u+s6PKMQVvJNuPzljwD8BwBf2vWY1nRdP4DLP+YRgF8F8P0A3tD1\nuK55LV+By9Cfc9f+H3rPe3EZ7nSKyz/wN3Y97nVcHy7LFH4El//EYwD/P4D/G8Dv7HrcLa4ruqZz\nAO9w79vJ313T9W3j706lLIUQokO2yicshBB3DYmwEEJ0iERYCCE6RCIshBAdIhEWQogOkQgLIUSH\nSISFEKJDJMJCCNEhEmEhhOgQibAQQnSIRFgIITpEIiyEEB3yvwHGFZtYaWxMwgAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f76e03dd6d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(train_x[0].reshape((28, 28)), cmap = cm.Greys_r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# rbm code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import timeit\n",
    "\n",
    "try:\n",
    "    import PIL.Image as Image\n",
    "except ImportError:\n",
    "    import Image\n",
    "\n",
    "import numpy\n",
    "\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "import os\n",
    "\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
    "\n",
    "from utils import tile_raster_images\n",
    "from logistic_sgd import load_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class RBM(object):\n",
    "    \"\"\"Restricted Boltzmann Machine (RBM)  \"\"\"\n",
    "    def __init__(\n",
    "        self,\n",
    "        input=None,\n",
    "        n_visible=784,\n",
    "        n_hidden=500,\n",
    "        W=None,\n",
    "        hbias=None,\n",
    "        vbias=None,\n",
    "        numpy_rng=None,\n",
    "        theano_rng=None\n",
    "    ):\n",
    "        \"\"\"\n",
    "        RBM constructor. Defines the parameters of the model along with\n",
    "        basic operations for inferring hidden from visible (and vice-versa),\n",
    "        as well as for performing CD updates.\n",
    "\n",
    "        :param input: None for standalone RBMs or symbolic variable if RBM is\n",
    "        part of a larger graph.\n",
    "\n",
    "        :param n_visible: number of visible units\n",
    "\n",
    "        :param n_hidden: number of hidden units\n",
    "\n",
    "        :param W: None for standalone RBMs or symbolic variable pointing to a\n",
    "        shared weight matrix in case RBM is part of a DBN network; in a DBN,\n",
    "        the weights are shared between RBMs and layers of a MLP\n",
    "\n",
    "        :param hbias: None for standalone RBMs or symbolic variable pointing\n",
    "        to a shared hidden units bias vector in case RBM is part of a\n",
    "        different network\n",
    "\n",
    "        :param vbias: None for standalone RBMs or a symbolic variable\n",
    "        pointing to a shared visible units bias\n",
    "        \"\"\"\n",
    "\n",
    "        self.n_visible = n_visible\n",
    "        self.n_hidden = n_hidden\n",
    "\n",
    "        if numpy_rng is None:\n",
    "            # create a number generator\n",
    "            numpy_rng = numpy.random.RandomState(1234)\n",
    "\n",
    "        if theano_rng is None:\n",
    "            theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))\n",
    "\n",
    "        if W is None:\n",
    "            # W is initialized with `initial_W` which is uniformely\n",
    "            # sampled from -4*sqrt(6./(n_visible+n_hidden)) and\n",
    "            # 4*sqrt(6./(n_hidden+n_visible)) the output of uniform if\n",
    "            # converted using asarray to dtype theano.config.floatX so\n",
    "            # that the code is runable on GPU\n",
    "            initial_W = numpy.asarray(\n",
    "                numpy_rng.uniform(\n",
    "                    low=-4 * numpy.sqrt(6. / (n_hidden + n_visible)),\n",
    "                    high=4 * numpy.sqrt(6. / (n_hidden + n_visible)),\n",
    "                    size=(n_visible, n_hidden)\n",
    "                ),\n",
    "                dtype=theano.config.floatX\n",
    "            )\n",
    "            # theano shared variables for weights and biases\n",
    "            W = theano.shared(value=initial_W, name='W', borrow=True)\n",
    "\n",
    "        if hbias is None:\n",
    "            # create shared variable for hidden units bias\n",
    "            hbias = theano.shared(\n",
    "                value=numpy.zeros(\n",
    "                    n_hidden,\n",
    "                    dtype=theano.config.floatX\n",
    "                ),\n",
    "                name='hbias',\n",
    "                borrow=True\n",
    "            )\n",
    "\n",
    "        if vbias is None:\n",
    "            # create shared variable for visible units bias\n",
    "            vbias = theano.shared(\n",
    "                value=numpy.zeros(\n",
    "                    n_visible,\n",
    "                    dtype=theano.config.floatX\n",
    "                ),\n",
    "                name='vbias',\n",
    "                borrow=True\n",
    "            )\n",
    "\n",
    "        # initialize input layer for standalone RBM or layer0 of DBN\n",
    "        self.input = input\n",
    "        if not input:\n",
    "            self.input = T.matrix('input')\n",
    "\n",
    "        self.W = W\n",
    "        self.hbias = hbias\n",
    "        self.vbias = vbias\n",
    "        self.theano_rng = theano_rng\n",
    "        # **** WARNING: It is not a good idea to put things in this list\n",
    "        # other than shared variables created in this function.\n",
    "        self.params = [self.W, self.hbias, self.vbias]\n",
    "        # end-snippet-1\n",
    "\n",
    "    def free_energy(self, v_sample):\n",
    "        ''' Function to compute the free energy '''\n",
    "        wx_b = T.dot(v_sample, self.W) + self.hbias\n",
    "        vbias_term = T.dot(v_sample, self.vbias)\n",
    "        hidden_term = T.sum(T.log(1 + T.exp(wx_b)), axis=1)\n",
    "        return -hidden_term - vbias_term\n",
    "\n",
    "    def propup(self, vis):\n",
    "        '''This function propagates the visible units activation upwards to\n",
    "        the hidden units\n",
    "\n",
    "        Note that we return also the pre-sigmoid activation of the\n",
    "        layer. As it will turn out later, due to how Theano deals with\n",
    "        optimizations, this symbolic variable will be needed to write\n",
    "        down a more stable computational graph (see details in the\n",
    "        reconstruction cost function)\n",
    "\n",
    "        '''\n",
    "        pre_sigmoid_activation = T.dot(vis, self.W) + self.hbias\n",
    "        return [pre_sigmoid_activation, T.nnet.sigmoid(pre_sigmoid_activation)]\n",
    "\n",
    "    def sample_h_given_v(self, v0_sample):\n",
    "        ''' This function infers state of hidden units given visible units '''\n",
    "        # compute the activation of the hidden units given a sample of\n",
    "        # the visibles\n",
    "        pre_sigmoid_h1, h1_mean = self.propup(v0_sample)\n",
    "        # get a sample of the hiddens given their activation\n",
    "        # Note that theano_rng.binomial returns a symbolic sample of dtype\n",
    "        # int64 by default. If we want to keep our computations in floatX\n",
    "        # for the GPU we need to specify to return the dtype floatX\n",
    "        h1_sample = self.theano_rng.binomial(size=h1_mean.shape,\n",
    "                                             n=1, p=h1_mean,\n",
    "                                             dtype=theano.config.floatX)\n",
    "        return [pre_sigmoid_h1, h1_mean, h1_sample]\n",
    "\n",
    "    def propdown(self, hid):\n",
    "        '''This function propagates the hidden units activation downwards to\n",
    "        the visible units\n",
    "\n",
    "        Note that we return also the pre_sigmoid_activation of the\n",
    "        layer. As it will turn out later, due to how Theano deals with\n",
    "        optimizations, this symbolic variable will be needed to write\n",
    "        down a more stable computational graph (see details in the\n",
    "        reconstruction cost function)\n",
    "\n",
    "        '''\n",
    "        pre_sigmoid_activation = T.dot(hid, self.W.T) + self.vbias\n",
    "        return [pre_sigmoid_activation, T.nnet.sigmoid(pre_sigmoid_activation)]\n",
    "\n",
    "    def sample_v_given_h(self, h0_sample):\n",
    "        ''' This function infers state of visible units given hidden units '''\n",
    "        # compute the activation of the visible given the hidden sample\n",
    "        pre_sigmoid_v1, v1_mean = self.propdown(h0_sample)\n",
    "        # get a sample of the visible given their activation\n",
    "        # Note that theano_rng.binomial returns a symbolic sample of dtype\n",
    "        # int64 by default. If we want to keep our computations in floatX\n",
    "        # for the GPU we need to specify to return the dtype floatX\n",
    "        v1_sample = self.theano_rng.binomial(size=v1_mean.shape,\n",
    "                                             n=1, p=v1_mean,\n",
    "                                             dtype=theano.config.floatX)\n",
    "        return [pre_sigmoid_v1, v1_mean, v1_sample]\n",
    "\n",
    "    def gibbs_hvh(self, h0_sample):\n",
    "        ''' This function implements one step of Gibbs sampling,\n",
    "            starting from the hidden state'''\n",
    "        pre_sigmoid_v1, v1_mean, v1_sample = self.sample_v_given_h(h0_sample)\n",
    "        pre_sigmoid_h1, h1_mean, h1_sample = self.sample_h_given_v(v1_sample)\n",
    "        return [pre_sigmoid_v1, v1_mean, v1_sample,\n",
    "                pre_sigmoid_h1, h1_mean, h1_sample]\n",
    "\n",
    "    def gibbs_vhv(self, v0_sample):\n",
    "        ''' This function implements one step of Gibbs sampling,\n",
    "            starting from the visible state'''\n",
    "        pre_sigmoid_h1, h1_mean, h1_sample = self.sample_h_given_v(v0_sample)\n",
    "        pre_sigmoid_v1, v1_mean, v1_sample = self.sample_v_given_h(h1_sample)\n",
    "        return [pre_sigmoid_h1, h1_mean, h1_sample,\n",
    "                pre_sigmoid_v1, v1_mean, v1_sample]\n",
    "\n",
    "    # start-snippet-2\n",
    "    def get_cost_updates(self, lr=0.1, persistent=None, k=1):\n",
    "        \"\"\"This functions implements one step of CD-k or PCD-k\n",
    "\n",
    "        :param lr: learning rate used to train the RBM\n",
    "\n",
    "        :param persistent: None for CD. For PCD, shared variable\n",
    "            containing old state of Gibbs chain. This must be a shared\n",
    "            variable of size (batch size, number of hidden units).\n",
    "\n",
    "        :param k: number of Gibbs steps to do in CD-k/PCD-k\n",
    "\n",
    "        Returns a proxy for the cost and the updates dictionary. The\n",
    "        dictionary contains the update rules for weights and biases but\n",
    "        also an update of the shared variable used to store the persistent\n",
    "        chain, if one is used.\n",
    "\n",
    "        \"\"\"\n",
    "\n",
    "        # compute positive phase\n",
    "        pre_sigmoid_ph, ph_mean, ph_sample = self.sample_h_given_v(self.input)\n",
    "\n",
    "        # decide how to initialize persistent chain:\n",
    "        # for CD, we use the newly generate hidden sample\n",
    "        # for PCD, we initialize from the old state of the chain\n",
    "        if persistent is None:\n",
    "            chain_start = ph_sample\n",
    "        else:\n",
    "            chain_start = persistent\n",
    "        # end-snippet-2\n",
    "        # perform actual negative phase\n",
    "        # in order to implement CD-k/PCD-k we need to scan over the\n",
    "        # function that implements one gibbs step k times.\n",
    "        # Read Theano tutorial on scan for more information :\n",
    "        # http://deeplearning.net/software/theano/library/scan.html\n",
    "        # the scan will return the entire Gibbs chain\n",
    "        (\n",
    "            [\n",
    "                pre_sigmoid_nvs,\n",
    "                nv_means,\n",
    "                nv_samples,\n",
    "                pre_sigmoid_nhs,\n",
    "                nh_means,\n",
    "                nh_samples\n",
    "            ],\n",
    "            updates\n",
    "        ) = theano.scan(\n",
    "            self.gibbs_hvh,\n",
    "            # the None are place holders, saying that\n",
    "            # chain_start is the initial state corresponding to the\n",
    "            # 6th output\n",
    "            outputs_info=[None, None, None, None, None, chain_start],\n",
    "            n_steps=k,\n",
    "            name=\"gibbs_hvh\"\n",
    "        )\n",
    "        # start-snippet-3\n",
    "        # determine gradients on RBM parameters\n",
    "        # note that we only need the sample at the end of the chain\n",
    "        chain_end = nv_samples[-1]\n",
    "\n",
    "        cost = T.mean(self.free_energy(self.input)) - T.mean(\n",
    "            self.free_energy(chain_end))\n",
    "        # We must not compute the gradient through the gibbs sampling\n",
    "        gparams = T.grad(cost, self.params, consider_constant=[chain_end])\n",
    "        # end-snippet-3 start-snippet-4\n",
    "        # constructs the update dictionary\n",
    "        for gparam, param in zip(gparams, self.params):\n",
    "            # make sure that the learning rate is of the right dtype\n",
    "            updates[param] = param - gparam * T.cast(\n",
    "                lr,\n",
    "                dtype=theano.config.floatX\n",
    "            )\n",
    "        if persistent:\n",
    "            # Note that this works only if persistent is a shared variable\n",
    "            updates[persistent] = nh_samples[-1]\n",
    "            # pseudo-likelihood is a better proxy for PCD\n",
    "            monitoring_cost = self.get_pseudo_likelihood_cost(updates)\n",
    "        else:\n",
    "            # reconstruction cross-entropy is a better proxy for CD\n",
    "            monitoring_cost = self.get_reconstruction_cost(updates,\n",
    "                                                           pre_sigmoid_nvs[-1])\n",
    "\n",
    "        return monitoring_cost, updates\n",
    "        # end-snippet-4\n",
    "\n",
    "    def get_pseudo_likelihood_cost(self, updates):\n",
    "        \"\"\"Stochastic approximation to the pseudo-likelihood\"\"\"\n",
    "\n",
    "        # index of bit i in expression p(x_i | x_{\\i})\n",
    "        bit_i_idx = theano.shared(value=0, name='bit_i_idx')\n",
    "\n",
    "        # binarize the input image by rounding to nearest integer\n",
    "        xi = T.round(self.input)\n",
    "\n",
    "        # calculate free energy for the given bit configuration\n",
    "        fe_xi = self.free_energy(xi)\n",
    "\n",
    "        # flip bit x_i of matrix xi and preserve all other bits x_{\\i}\n",
    "        # Equivalent to xi[:,bit_i_idx] = 1-xi[:, bit_i_idx], but assigns\n",
    "        # the result to xi_flip, instead of working in place on xi.\n",
    "        xi_flip = T.set_subtensor(xi[:, bit_i_idx], 1 - xi[:, bit_i_idx])\n",
    "\n",
    "        # calculate free energy with bit flipped\n",
    "        fe_xi_flip = self.free_energy(xi_flip)\n",
    "\n",
    "        # equivalent to e^(-FE(x_i)) / (e^(-FE(x_i)) + e^(-FE(x_{\\i})))\n",
    "        cost = T.mean(self.n_visible * T.log(T.nnet.sigmoid(fe_xi_flip -\n",
    "                                                            fe_xi)))\n",
    "\n",
    "        # increment bit_i_idx % number as part of updates\n",
    "        updates[bit_i_idx] = (bit_i_idx + 1) % self.n_visible\n",
    "\n",
    "        return cost\n",
    "\n",
    "    def get_reconstruction_cost(self, updates, pre_sigmoid_nv):\n",
    "        \"\"\"Approximation to the reconstruction error\n",
    "\n",
    "        Note that this function requires the pre-sigmoid activation as\n",
    "        input.  To understand why this is so you need to understand a\n",
    "        bit about how Theano works. Whenever you compile a Theano\n",
    "        function, the computational graph that you pass as input gets\n",
    "        optimized for speed and stability.  This is done by changing\n",
    "        several parts of the subgraphs with others.  One such\n",
    "        optimization expresses terms of the form log(sigmoid(x)) in\n",
    "        terms of softplus.  We need this optimization for the\n",
    "        cross-entropy since sigmoid of numbers larger than 30. (or\n",
    "        even less then that) turn to 1. and numbers smaller than\n",
    "        -30. turn to 0 which in terms will force theano to compute\n",
    "        log(0) and therefore we will get either -inf or NaN as\n",
    "        cost. If the value is expressed in terms of softplus we do not\n",
    "        get this undesirable behaviour. This optimization usually\n",
    "        works fine, but here we have a special case. The sigmoid is\n",
    "        applied inside the scan op, while the log is\n",
    "        outside. Therefore Theano will only see log(scan(..)) instead\n",
    "        of log(sigmoid(..)) and will not apply the wanted\n",
    "        optimization. We can not go and replace the sigmoid in scan\n",
    "        with something else also, because this only needs to be done\n",
    "        on the last step. Therefore the easiest and more efficient way\n",
    "        is to get also the pre-sigmoid activation as an output of\n",
    "        scan, and apply both the log and sigmoid outside scan such\n",
    "        that Theano can catch and optimize the expression.\n",
    "\n",
    "        \"\"\"\n",
    "\n",
    "        cross_entropy = T.mean(\n",
    "            T.sum(\n",
    "                self.input * T.log(T.nnet.sigmoid(pre_sigmoid_nv)) +\n",
    "                (1 - self.input) * T.log(1 - T.nnet.sigmoid(pre_sigmoid_nv)),\n",
    "                axis=1\n",
    "            )\n",
    "        )\n",
    "\n",
    "        return cross_entropy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_rbm(learning_rate=0.1, training_epochs=15,\n",
    "             dataset='mnist.pkl.gz', batch_size=20,\n",
    "             n_chains=20, n_samples=10, output_folder='rbm_plots',\n",
    "             n_hidden=500):\n",
    "    \"\"\"\n",
    "    Demonstrate how to train and afterwards sample from it using Theano.\n",
    "\n",
    "    This is demonstrated on MNIST.\n",
    "\n",
    "    :param learning_rate: learning rate used for training the RBM\n",
    "\n",
    "    :param training_epochs: number of epochs used for training\n",
    "\n",
    "    :param dataset: path the the pickled dataset\n",
    "\n",
    "    :param batch_size: size of a batch used to train the RBM\n",
    "\n",
    "    :param n_chains: number of parallel Gibbs chains to be used for sampling\n",
    "\n",
    "    :param n_samples: number of samples to plot for each chain\n",
    "\n",
    "    \"\"\"\n",
    "    datasets = load_data(dataset)\n",
    "\n",
    "    train_set_x, train_set_y = datasets[0]\n",
    "    test_set_x, test_set_y = datasets[2]\n",
    "\n",
    "    # compute number of minibatches for training, validation and testing\n",
    "    n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size\n",
    "\n",
    "    # allocate symbolic variables for the data\n",
    "    index = T.lscalar()    # index to a [mini]batch\n",
    "    x = T.matrix('x')  # the data is presented as rasterized images\n",
    "\n",
    "    rng = numpy.random.RandomState(123)\n",
    "    theano_rng = RandomStreams(rng.randint(2 ** 30))\n",
    "\n",
    "    # initialize storage for the persistent chain (state = hidden\n",
    "    # layer of chain)\n",
    "    persistent_chain = theano.shared(numpy.zeros((batch_size, n_hidden),\n",
    "                                                 dtype=theano.config.floatX),\n",
    "                                     borrow=True)\n",
    "\n",
    "    # construct the RBM class\n",
    "    rbm = RBM(input=x, n_visible=28 * 28,\n",
    "              n_hidden=n_hidden, numpy_rng=rng, theano_rng=theano_rng)\n",
    "\n",
    "    # get the cost and the gradient corresponding to one step of CD-15\n",
    "    cost, updates = rbm.get_cost_updates(lr=learning_rate,\n",
    "                                         persistent=persistent_chain, k=15)\n",
    "\n",
    "    #################################\n",
    "    #     Training the RBM          #\n",
    "    #################################\n",
    "    if not os.path.isdir(output_folder):\n",
    "        os.makedirs(output_folder)\n",
    "    os.chdir(output_folder)\n",
    "\n",
    "    # start-snippet-5\n",
    "    # it is ok for a theano function to have no output\n",
    "    # the purpose of train_rbm is solely to update the RBM parameters\n",
    "    train_rbm = theano.function(\n",
    "        [index],\n",
    "        cost,\n",
    "        updates=updates,\n",
    "        givens={\n",
    "            x: train_set_x[index * batch_size: (index + 1) * batch_size]\n",
    "        },\n",
    "        name='train_rbm'\n",
    "    )\n",
    "\n",
    "    plotting_time = 0.\n",
    "    start_time = timeit.default_timer()\n",
    "\n",
    "    # go through training epochs\n",
    "    for epoch in range(training_epochs):\n",
    "\n",
    "        # go through the training set\n",
    "        mean_cost = []\n",
    "        for batch_index in range(n_train_batches):\n",
    "            mean_cost += [train_rbm(batch_index)]\n",
    "\n",
    "        print('Training epoch %d, cost is ' % epoch, numpy.mean(mean_cost))\n",
    "\n",
    "        # Plot filters after each training epoch\n",
    "        plotting_start = timeit.default_timer()\n",
    "        # Construct image from the weight matrix\n",
    "        image = Image.fromarray(\n",
    "            tile_raster_images(\n",
    "                X=rbm.W.get_value(borrow=True).T,\n",
    "                img_shape=(28, 28),\n",
    "                tile_shape=(10, 10),\n",
    "                tile_spacing=(1, 1)\n",
    "            )\n",
    "        )\n",
    "        image.save('filters_at_epoch_%i.png' % epoch)\n",
    "        plotting_stop = timeit.default_timer()\n",
    "        plotting_time += (plotting_stop - plotting_start)\n",
    "\n",
    "    end_time = timeit.default_timer()\n",
    "\n",
    "    pretraining_time = (end_time - start_time) - plotting_time\n",
    "\n",
    "    print ('Training took %f minutes' % (pretraining_time / 60.))\n",
    "    # end-snippet-5 start-snippet-6\n",
    "    #################################\n",
    "    #     Sampling from the RBM     #\n",
    "    #################################\n",
    "    # find out the number of test samples\n",
    "    number_of_test_samples = test_set_x.get_value(borrow=True).shape[0]\n",
    "\n",
    "    # pick random test examples, with which to initialize the persistent chain\n",
    "    test_idx = rng.randint(number_of_test_samples - n_chains)\n",
    "    persistent_vis_chain = theano.shared(\n",
    "        numpy.asarray(\n",
    "            test_set_x.get_value(borrow=True)[test_idx:test_idx + n_chains],\n",
    "            dtype=theano.config.floatX\n",
    "        )\n",
    "    )\n",
    "    # end-snippet-6 start-snippet-7\n",
    "    plot_every = 1000\n",
    "    # define one step of Gibbs sampling (mf = mean-field) define a\n",
    "    # function that does `plot_every` steps before returning the\n",
    "    # sample for plotting\n",
    "    (\n",
    "        [\n",
    "            presig_hids,\n",
    "            hid_mfs,\n",
    "            hid_samples,\n",
    "            presig_vis,\n",
    "            vis_mfs,\n",
    "            vis_samples\n",
    "        ],\n",
    "        updates\n",
    "    ) = theano.scan(\n",
    "        rbm.gibbs_vhv,\n",
    "        outputs_info=[None, None, None, None, None, persistent_vis_chain],\n",
    "        n_steps=plot_every,\n",
    "        name=\"gibbs_vhv\"\n",
    "    )\n",
    "\n",
    "    # add to updates the shared variable that takes care of our persistent\n",
    "    # chain :.\n",
    "    updates.update({persistent_vis_chain: vis_samples[-1]})\n",
    "    # construct the function that implements our persistent chain.\n",
    "    # we generate the \"mean field\" activations for plotting and the actual\n",
    "    # samples for reinitializing the state of our persistent chain\n",
    "    sample_fn = theano.function(\n",
    "        [],\n",
    "        [\n",
    "            vis_mfs[-1],\n",
    "            vis_samples[-1]\n",
    "        ],\n",
    "        updates=updates,\n",
    "        name='sample_fn'\n",
    "    )\n",
    "\n",
    "    # create a space to store the image for plotting ( we need to leave\n",
    "    # room for the tile_spacing as well)\n",
    "    image_data = numpy.zeros(\n",
    "        (29 * n_samples + 1, 29 * n_chains - 1),\n",
    "        dtype='uint8'\n",
    "    )\n",
    "    for idx in range(n_samples):\n",
    "        # generate `plot_every` intermediate samples that we discard,\n",
    "        # because successive samples in the chain are too correlated\n",
    "        vis_mf, vis_sample = sample_fn()\n",
    "        print(' ... plotting sample %d' % idx)\n",
    "        image_data[29 * idx:29 * idx + 28, :] = tile_raster_images(\n",
    "            X=vis_mf,\n",
    "            img_shape=(28, 28),\n",
    "            tile_shape=(1, n_chains),\n",
    "            tile_spacing=(1, 1)\n",
    "        )\n",
    "\n",
    "    # construct image\n",
    "    image = Image.fromarray(image_data)\n",
    "    image.save('samples.png')\n",
    "    # end-snippet-7\n",
    "    os.chdir('../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... loading data\n",
      "Training epoch 0, cost is  -90.590108121\n",
      "Training epoch 1, cost is  -80.306465234\n",
      "Training epoch 2, cost is  -74.3871721077\n",
      "Training epoch 3, cost is  -72.2911275348\n",
      "Training epoch 4, cost is  -68.7493255098\n",
      "Training epoch 5, cost is  -63.3319654523\n",
      "Training epoch 6, cost is  -65.6836096852\n",
      "Training epoch 7, cost is  -67.5699326992\n",
      "Training epoch 8, cost is  -68.1104919343\n",
      "Training epoch 9, cost is  -64.2462694866\n",
      "Training epoch 10, cost is  -61.3355699009\n",
      "Training epoch 11, cost is  -62.0303931597\n",
      "Training epoch 12, cost is  -64.0581166301\n",
      "Training epoch 13, cost is  -63.4414152005\n",
      "Training epoch 14, cost is  -62.7701948813\n",
      "Training took 71.232035 minutes\n",
      " ... plotting sample 0\n",
      " ... plotting sample 1\n",
      " ... plotting sample 2\n",
      " ... plotting sample 3\n",
      " ... plotting sample 4\n",
      " ... plotting sample 5\n",
      " ... plotting sample 6\n",
      " ... plotting sample 7\n",
      " ... plotting sample 8\n",
      " ... plotting sample 9\n"
     ]
    }
   ],
   "source": [
    "test_rbm()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](rbm_plots/samples.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Basic about theano"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "numpy_rng = numpy.random.RandomState(1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on class MRG_RandomStreams in module theano.sandbox.rng_mrg:\n",
      "\n",
      "class MRG_RandomStreams(__builtin__.object)\n",
      " |  Module component with similar interface to numpy.random\n",
      " |  (numpy.random.RandomState).\n",
      " |  \n",
      " |  Parameters\n",
      " |  ----------\n",
      " |  seed : int or list of 6 int\n",
      " |      A default seed to initialize the random state.\n",
      " |      If a single int is given, it will be replicated 6 times.\n",
      " |      The first 3 values of the seed must all be less than M1 = 2147483647,\n",
      " |      and not all 0; and the last 3 values must all be less than\n",
      " |      M2 = 2147462579, and not all 0.\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  __init__(self, seed=12345, use_cuda=None)\n",
      " |  \n",
      " |  binomial(self, size=None, n=1, p=0.5, ndim=None, dtype='int64', nstreams=None)\n",
      " |  \n",
      " |  get_substream_rstates(self, n_streams, dtype, inc_rstate=True)\n",
      " |      Initialize a matrix in which each row is a MRG stream state,\n",
      " |      and they are spaced by 2**72 samples.\n",
      " |  \n",
      " |  inc_rstate(self)\n",
      " |      Update self.rstate to be skipped 2^134 steps forward to the next stream\n",
      " |      start.\n",
      " |  \n",
      " |  multinomial(self, size=None, n=1, pvals=None, ndim=None, dtype='int64', nstreams=None)\n",
      " |      Sample `n` (`n` needs to be >= 1, default 1) times from a multinomial\n",
      " |      distribution defined by probabilities pvals.\n",
      " |      \n",
      " |      Example : pvals = [[.98, .01, .01], [.01, .49, .50]] and n=1 will\n",
      " |      probably result in [[1,0,0],[0,0,1]]. When setting n=2, this\n",
      " |      will probably result in [[2,0,0],[0,1,1]].\n",
      " |      \n",
      " |      Notes\n",
      " |      -----\n",
      " |      -`size` and `ndim` are only there keep the same signature as other\n",
      " |      uniform, binomial, normal, etc.\n",
      " |      TODO : adapt multinomial to take that into account\n",
      " |      \n",
      " |      -Does not do any value checking on pvals, i.e. there is no\n",
      " |      check that the elements are non-negative, less than 1, or\n",
      " |      sum to 1. passing pvals = [[-2., 2.]] will result in\n",
      " |      sampling [[0, 0]]\n",
      " |  \n",
      " |  multinomial_wo_replacement(self, size=None, n=1, pvals=None, ndim=None, dtype='int64', nstreams=None)\n",
      " |      Sample `n` times *WITHOUT replacement* from a multinomial distribution\n",
      " |      defined by probabilities pvals, and returns the indices of the sampled\n",
      " |      elements.\n",
      " |      `n` needs to be in [1, m], where m is the number of elements to select\n",
      " |      from, i.e. m == pvals.shape[1]. By default n = 1.\n",
      " |      \n",
      " |      Example : pvals = [[.98, .01, .01], [.01, .49, .50]] and n=1 will\n",
      " |      probably result in [[0],[2]]. When setting n=2, this\n",
      " |      will probably result in [[0,1],[2,1]].\n",
      " |      \n",
      " |      Notes\n",
      " |      -----\n",
      " |      -`size` and `ndim` are only there keep the same signature as other\n",
      " |      uniform, binomial, normal, etc.\n",
      " |      TODO : adapt multinomial to take that into account\n",
      " |      \n",
      " |      -Does not do any value checking on pvals, i.e. there is no\n",
      " |      check that the elements are non-negative, less than 1, or\n",
      " |      sum to 1. passing pvals = [[-2., 2.]] will result in\n",
      " |      sampling [[0, 0]]\n",
      " |  \n",
      " |  n_streams(self, size)\n",
      " |  \n",
      " |  normal(self, size, avg=0.0, std=1.0, ndim=None, dtype=None, nstreams=None)\n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      size\n",
      " |          Can be a list of integers or Theano variables (ex: the shape\n",
      " |          of another Theano Variable).\n",
      " |      dtype\n",
      " |          The output data type. If dtype is not specified, it will be\n",
      " |          inferred from the dtype of low and high, but will be at\n",
      " |          least as precise as floatX.\n",
      " |      nstreams\n",
      " |          Number of streams.\n",
      " |  \n",
      " |  pretty_return(self, node_rstate, new_rstate, sample, size, nstreams)\n",
      " |  \n",
      " |  seed(self, seed=None)\n",
      " |      Re-initialize each random stream.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      seed : None or integer in range 0 to 2**30\n",
      " |          Each random stream will be assigned a unique state that depends\n",
      " |          deterministically on this value.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      None\n",
      " |  \n",
      " |  set_rstate(self, seed)\n",
      " |  \n",
      " |  uniform(self, size, low=0.0, high=1.0, ndim=None, dtype=None, nstreams=None)\n",
      " |      Sample a tensor of given size whose element from a uniform\n",
      " |      distribution between low and high.\n",
      " |      \n",
      " |      If the size argument is ambiguous on the number of dimensions,\n",
      " |      ndim may be a plain integer to supplement the missing information.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      low\n",
      " |          Lower bound of the interval on which values are sampled.\n",
      " |          If the ``dtype`` arg is provided, ``low`` will be cast into\n",
      " |          dtype. This bound is excluded.\n",
      " |      high\n",
      " |          Higher bound of the interval on which values are sampled.\n",
      " |          If the ``dtype`` arg is provided, ``high`` will be cast into\n",
      " |          dtype. This bound is excluded.\n",
      " |      size\n",
      " |        Can be a list of integer or Theano variable (ex: the shape\n",
      " |        of other Theano Variable).\n",
      " |      dtype\n",
      " |          The output data type. If dtype is not specified, it will be\n",
      " |          inferred from the dtype of low and high, but will be at\n",
      " |          least as precise as floatX.\n",
      " |  \n",
      " |  updates(self)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors defined here:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(RandomStreams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<theano.sandbox.rng_mrg.MRG_RandomStreams at 0x7fef50f5f610>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theano_rng"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Elemwise{Cast{int64}}.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theano_rng.binomial(size=(100,10))# size is tupe of int or theano variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test = theano_rng.binomial(size=(100,10), p=0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elemwise{Cast{int64}}.0\n"
     ]
    }
   ],
   "source": [
    "print(test)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}