spark_ml/data_processing(ch03).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pyspark import SparkConf, SparkContext\n",
    "conf = SparkConf().setMaster(\"local\").setAppName(\"My App\")\n",
    "sc = SparkContext(conf = conf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'1|24|M|technician|85711'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_data = sc.textFile(\"./ml-100k/u.user\")\n",
    "user_data.first()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Users: 943, genders: 2, occupations: 21, ZIP codes: 795\n"
     ]
    }
   ],
   "source": [
    "user_fields = user_data.map(lambda line: line.split(\"|\"))\n",
    "\n",
    "num_users = user_fields.map(lambda fields: fields[0]).count()\n",
    "num_genders = user_fields.map(lambda fields: fields[2]).distinct().count()\n",
    "num_occupations = user_fields.map(lambda fields: fields[3]).distinct().count()\n",
    "num_zipcodes = user_fields.map(lambda fields: fields[4]).distinct().count()\n",
    "print (\"Users: %d, genders: %d, occupations: %d, ZIP codes: %d\" % (num_users, num_genders, num_occupations, num_zipcodes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/python2/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([ 0.00064269,  0.00192808,  0.00449886,  0.0279572 ,  0.02956393,\n",
       "         0.03374144,  0.04563129,  0.02538642,  0.02088756,  0.01863813,\n",
       "         0.02088756,  0.01606735,  0.0170314 ,  0.01863813,  0.00674829,\n",
       "         0.00482021,  0.0054629 ,  0.00192808,  0.00128539,  0.00128539]),\n",
       " array([  7. ,  10.3,  13.6,  16.9,  20.2,  23.5,  26.8,  30.1,  33.4,\n",
       "         36.7,  40. ,  43.3,  46.6,  49.9,  53.2,  56.5,  59.8,  63.1,\n",
       "         66.4,  69.7,  73. ]),\n",
       " <a list of 20 Patch objects>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEACAYAAABfxaZOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEghJREFUeJzt3W+sXPV95/H3x+XPhqR1u21DFxzftkBwlm0DqWrcolVu\nmm6xiVRrFWkXb3ezQdqtpeIm2koVNH2AebZ9ULVBVAW2JCpRs9CWZvFKbmoicqV0o3WcBhcCvsFs\nEvwHcIM2pBtWpYR8+2CO6TBc7p0Z33vnmN/7JY18z5nfmfl47vXnHv/OmTOpKiRJbdgw6wCSpPVj\n6UtSQyx9SWqIpS9JDbH0Jakhlr4kNWSs0k+yPclikieS3PQ6Y25LcjTJ4SRXDa3/epK/TvJwki+s\nVnBJ0uTOWWlAkg3A7cB7gaeBQ0keqKrFoTE7gEuq6rIkVwO/D2zr7v4uMF9V31z19JKkiYyzp78V\nOFpVT1XVS8C9wM6RMTuBewCq6iCwMcmF3X0Z83kkSWtsnDK+GDg+tHyiW7fcmJNDYwp4MMmhJP95\n2qCSpDO34vTOKrimqp5J8sMMyv9IVf3lOjyvJGnEOKV/Etg8tLypWzc65m1LjamqZ7o/v5HkUwym\ni15T+km8CJAkTaiqMsn4caZ3DgGXJplLch5wPbBvZMw+4AMASbYBz1fVqSQXJHlLt/7NwC8AX14m\nfK9vt9xyy8wzmNOc5jTn6ds0VtzTr6qXk+wBDjD4JXF3VR1Jsntwd91VVfuTXJfkSeAF4IZu8wuB\nT3V78ecAf1RVB6ZKKkk6Y2PN6VfVp4HLR9bdObK8Z4ntvgZceSYBJUmrx1MpJzA/Pz/rCGMx5+oy\n5+oy52xl2nmh1Zak+pJFks4GSag1OJArSXqDsPQlqSGWviQ1xNKXpIZY+pLUEEtfkhpi6UtSQyx9\nSWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX6ti89wcSSa+bZ6bm3V0qSleWlmrIgn3Lz498Xbv\n33LR1B/7JrXOSytLkpZl6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMs\nfUlqiKUvSQ2x9CWpIZa+JDXE0pekhlj6ktQQS1+SGmLpS1JDLH1JashYpZ9ke5LFJE8kuel1xtyW\n5GiSw0muHLlvQ5IvJdm3GqElSdNZsfSTbABuB64FrgB2JdkyMmYHcElVXQbsBu4YeZgPA4+vSmJJ\n0tTG2dPfChytqqeq6iXgXmDnyJidwD0AVXUQ2JjkQoAkm4DrgD9YtdSSpKmMU/oXA8eHlk9065Yb\nc3JozO8Avw7UlBklSavknLV88CTvA05V1eEk80CWG793795Xvp6fn2d+fn4t40nSWWVhYYGFhYUz\neoxULb8DnmQbsLeqtnfLNwNVVb81NOYO4LNVdV+3vAi8m8Fc/r8HvgO8Cfhe4M+q6gNLPE+tlEX9\nlYT7F5+eeLv3b7kIv+/SdJJQVcvuTI8aZ3rnEHBpkrkk5wHXA6Nn4ewDPtCF2AY8X1WnquojVbW5\nqn682+6hpQpfkrQ+VpzeqaqXk+wBDjD4JXF3VR1Jsntwd91VVfuTXJfkSeAF4Ia1jS1JmsaK0zvr\nxemds5vTO9L6W6vpHUnSG4SlL0kNsfQlqSGWviQ1xNKXpIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9J\nDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDXE0pekhlj6ktQQ\nS1+SGmLp61U2z82RZOKbpLPDObMOoH45fuwY9y8+PfF2799y0RqkkbTa3NOXpIZY+pLUEEtfkhpi\n6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1JCxSj/J9iSLSZ5IctPrjLktydEkh5Nc2a07\nP8nBJA8neTTJLasZXpI0mRVLP8kG4HbgWuAKYFeSLSNjdgCXVNVlwG7gDoCqehF4T1VdBVwJ7Eiy\ndXX/CpKkcY2zp78VOFpVT1XVS8C9wM6RMTuBewCq6iCwMcmF3fL/78acz+ACb7UawSVJkxun9C8G\njg8tn+jWLTfm5OkxSTYkeRh4Fniwqg5NH1eSdCbW/EBuVX23m97ZBFyd5J+v9XNKkpY2zvX0TwKb\nh5Y3detGx7xtuTFV9bdJPgtsBx5f6on27t37ytfz8/PMz8+PEU+S2rCwsMDCwsIZPUaqlp9iT/I9\nwFeA9wLPAF8AdlXVkaEx1wE3VtX7kmwDfreqtiX5IeClqvpWkjcBfwH816rav8Tz1EpZtPaSTP0h\nKtNu5/ddmk4Sqmqij65bcU+/ql5Osgc4wGA66O6qOpJk9+Duuquq9ie5LsmTwAvADd3m/wz4w+4M\noA3AfUsVvlbf5rk5jh87NusYknpmrI9LrKpPA5ePrLtzZHnPEts9CrzrTAJqOn7soaSl+I5cSWqI\npS9JDbH0Jakhlr4kNcTSl6SGWPo9t3lujiQT3yRpKWOdsqnZ8dRLSavJPX1JaoilL0kNsfQlqSGW\nviQ1xNKXpIZY+pLUEEtfkhpi6UtSQyx9SWqIpa+ZOve88ye+xMTmublZx5bOWl6GQTP10t+/OPFl\nJrzEhDQ99/QlqSGWviQ1xNKXpIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SG\nWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDVkrNJPsj3JYpInktz0OmNuS3I0yeEkV3br\nNiV5KMljSR5N8qHVDC9JmsyKpZ9kA3A7cC1wBbAryZaRMTuAS6rqMmA3cEd313eAX6uqK4CfAW4c\n3Vaa1DQfpn4mH6i+eW5uXZ9PWkvjfDD6VuBoVT0FkOReYCewODRmJ3APQFUdTLIxyYVV9SzwbLf+\n20mOABePbCtNZJoPU4fpP1D9+LFj6/p80loaZ3rnYuD40PKJbt1yY06Ojknyo8CVwMFJQ0qSVse6\nHMhN8hbgT4EPV9W31+M5JUmvNc70zklg89Dypm7d6Ji3LTUmyTkMCv8TVfXAck+0d+/eV76en59n\nfn5+jHiS1IaFhQUWFhbO6DHGKf1DwKVJ5oBngOuBXSNj9gE3Avcl2QY8X1Wnuvs+BjxeVR9d6YmG\nS1+S9GqjO8O33nrrxI+xYulX1ctJ9gAHGEwH3V1VR5LsHtxdd1XV/iTXJXkSeAH4IECSa4BfAh5N\n8jBQwEeq6tMTJ5UknbFx9vTpSvrykXV3jizvWWK7/wV8z5kElCStHt+RK0kNsfQlqSGWvpox7Tt5\nzwa+a1jjGmtOX3ojWO938q4n3zWscbmnL0kNsfQlqSGWviQ1xNKXpIZY+pLUEEtf6plpTr+UxuUp\nm1LPTHP6padealzu6UtSQyx9SWqI0zvSGjl92QepTyx9aY28kS/7oLOX0zuS1BBLX5IaYulLUkMs\nfUlqiKUvSQ2x9CWpIZa+JDXE0pekhlj6ktQQS1+SGmLpS1JDLH1JaoilL0kNsfQlqSGWviQ1xNKX\npIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9JDRmr9JNsT7KY5IkkN73OmNuSHE1yOMlVQ+vvTnIqySOr\nFVqSNJ0VSz/JBuB24FrgCmBXki0jY3YAl1TVZcBu4PeH7v54t60kacbG2dPfChytqqeq6iXgXmDn\nyJidwD0AVXUQ2Jjkwm75L4Fvrl5kSdK0xin9i4HjQ8snunXLjTm5xBhJ0oydM+sAw/bu3fvK1/Pz\n88zPz88siyT1zcLCAgsLC2f0GOOU/klg89Dypm7d6Ji3rTBmRcOlL0l6tdGd4VtvvXXixxhneucQ\ncGmSuSTnAdcD+0bG7AM+AJBkG/B8VZ0auj/dTZI0QyuWflW9DOwBDgCPAfdW1ZEku5P8cjdmP/C1\nJE8CdwK/cnr7JJ8EPg+8PcmxJDeswd9DkjSGseb0q+rTwOUj6+4cWd7zOtv+u6nTSZJWle/IlaSG\nWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDXE0pekhlj6ktQQS19q2LnnnU+SiW+b5+Zm\nHV1T6tX19CWtr5f+/kXuX3x64u3ev+WiNUij9eCeviQ1xNKXpIZY+pLUEEtf0rrZPDfngeMZ80Cu\npHVz/NgxDxzPmHv6ktQQ9/QlTez0+f06+1j6kibm+f1nL6d31sm0B7AkaTW5p79OPIAlqQ/c05ek\nhlj6ktQQS1+SGmLpS1JDLH1JaoilL0kNsfQlqSGWviQ1xNKXpIZY+lOY5pIKkqY3zQe4ew3+pXkZ\nhilMc0kFL6cgTW+aC7z5b25p7ulLUkMsfUlvSNNMCSXhTRdc8Ib+SMexpneSbAd+l8Evibur6reW\nGHMbsAN4AfhgVR0ed1tJWm1ncs3/N/IVcVcs/SQbgNuB9wJPA4eSPFBVi0NjdgCXVNVlSa4G7gC2\njbPtrHzuc5/j3j/+44m2OXniBO94xzvWKNHq+fLBz886wljOlpxniy8f/Dz/4uqfnXWMFZlztsaZ\n3tkKHK2qp6rqJeBeYOfImJ3APQBVdRDYmOTCMbediTvuuotHv36Sv/u+t459e+q55/mDj31s1tFX\n9NgXzo4yPVtyni3OltfTnLM1zvTOxcDxoeUTDMp8pTEXj7ntzLzzmnfznn/9b8Ye/+1vPc//e+5v\neO7UqTVMJUlrZ61O2ez9iennnnsef37Pf+OLD+4fe5vjX32Sb37jb9YwlSStrVTV8gOSbcDeqtre\nLd8M1PAB2SR3AJ+tqvu65UXg3cCPrbTt0GMsH0SS9BpVNdFO9jh7+oeAS5PMAc8A1wO7RsbsA24E\n7ut+STxfVaeSPDfGtlMFlyRNbsXSr6qXk+wBDvCPp10eSbJ7cHfdVVX7k1yX5EkGp2zesNy2a/a3\nkSQta8XpHUnSG8fM35GbZHuSxSRPJLlp1nlOS3J3klNJHhla9wNJDiT5SpK/SLJxlhm7TJuSPJTk\nsSSPJvlQ37ImOT/JwSQPdxlv6VvGYUk2JPlSkn3dcu9yJvl6kr/uXtMv9DjnxiR/kuRI9zN6dd9y\nJnl79zp+qfvzW0k+1LecXdb/kuTLSR5J8kdJzps050xLf+jNW9cCVwC7kmyZZaYhH2eQa9jNwGeq\n6nLgIeA31j3Va30H+LWqugL4GeDG7jXsTdaqehF4T1VdBVwJ7EiytU8ZR3wYeHxouY85vwvMV9VV\nVXX6NOg+5vwosL+q3gG8E1ikZzmr6onudXwX8FMMpqg/Rc9yJrkI+FXgXVX1kwym53cxac6qmtkN\n2Ab8+dDyzcBNs8w0km8OeGRoeRG4sPv6R4DFWWdcIvP/AH6+r1mBC4AvAj/dx4zAJuBBYB7Y19fv\nO/A14AdH1vUqJ/B9wP9ZYn2vco5k+wXgc33MCVwEPAX8QFf4+6b5tz7r6Z3Xe1NXX721qk4BVNWz\nwFtnnOdVkvwogz3p/83gh6A3Wbspk4eBZ4EHq+pQ3zJ2fgf4dWD4YFcfcxbwYJJDSf5Tt65vOX8M\neC7Jx7upk7uSXED/cg77t8Anu697lbOqngZ+GzgGnAS+VVWfYcKcsy79s11vjoIneQvwp8CHq+rb\nvDbbTLNW1XdrML2zCdia5IolMs00Y5L3AadqcLHA5U4h7sP3/ZoaTEdcx2BK71/Ss9eTwd7ou4Df\n67K+wOB/833LCUCSc4FfBP6kW9WrnEm+n8FlbOYY7PW/OckvLZFr2ZyzLv2TwOah5U3dur461V1T\niCQ/AvTi7blJzmFQ+J+oqge61b3MWlV/CywA2+lfxmuAX0zyVeC/Az+X5BPAsz3LSVU90/35DQZT\nelvp3+t5AjheVV/slu9n8EugbzlP2wH8VVU91y33LefPA1+tqv9bVS8zOO7ws0yYc9al/8obv5Kc\nx+DNW/tmnGlYePUe3z7gg93X/xF4YHSDGfkY8HhVfXRoXW+yJvmh02cUJHkT8K+AI/QoI0BVfaSq\nNlfVjzP4WXyoqv4D8D/pUc4kF3T/syPJmxnMQz9K/17PU8DxJG/vVr0XeIye5Ryyi8Ev+9P6lvMY\ng6sX/5MkYfB6Ps6kOXtw4GQ78BXgKHDzrPMM5fokg8tBv9i92DcwOIDymS7vAeD7e5DzGuBl4DDw\nMPCl7jX9p33JCvxEl+sw8Ajwm9363mRcIvO7+ccDub3KyWCu/PT3+9HT/276lrPL9E4GO3eHgT8D\nNvY05wXAN4DvHVrXx5y3MNhhegT4Q+DcSXP65ixJasisp3ckSevI0pekhlj6ktQQS1+SGmLpS1JD\nLH1JaoilL0kNsfQlqSH/AAJT7GPwno9yAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f27715cc390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%pylab inline\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "ages = user_fields.map(lambda x: int(x[1])).collect()\n",
    "plt.hist(ages, bins=20, color='lightblue', normed=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6IAAAJxCAYAAABYJQ+yAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xm4ZFdZL/7vm4mEgBCGJCQhzUwCCIiQMNNMYRASJDKI\nyDwZVBQHBoFERBlUkFEQUNFLhEgYghODiqhRAoIyBvn9vNgtUfRevd6rohfIun+sddKVQ4d0+pyz\nqqvz+TxPP121q86pt3bt2nt911p7n2qtBQAAAGY5YNkFAAAAcOUiiAIAADCVIAoAAMBUgigAAABT\nCaIAAABMJYgCAAAw1eUG0ao6rqr+oKo+U1WfqqofHMuPqKr3V9Xnq+p9VXWNhZ95TlV9oao+V1Wn\nbOUbAAAAYLXU5f0d0ao6OsnRrbW/rKqrJfmLJKcleXyS/9lae1lVPSvJEa21Z1fVLZK8NckdkhyX\n5INJbtr8wVIAAACyByOirbV/aK395bj9b0k+lx4wT0vylvG0tyR5yLh9apK3tda+1lr7YpIvJDlp\nk+sGAABgRV2hc0Sr6gZJbpvkz5Mc1Vr7ctLDapIjx9OOTbJz4ce+NJYBAABADtrTJ45pue9I8ozW\n2r9V1fqptldo6u1ufh4AAID9SGutdrd8j0ZEq+qg9BD6662194zFX66qo8bjRyf5x7H8S0muv/Dj\nx41luytq0/+deeaZW/J7r6x1rlKtq1LnKtW6KnWuUq2rUucq1boqda5SratS5yrVuip1rlKtq1Ln\nKtW6KnWuUq2rUudW1frN7OnU3F9O8tnW2isXlp2X5HHj9mOTvGdh+SOr6pCqumGSmyS5YA9fBwAA\ngP3c5U7Nraq7JPmeJJ+qqk+kT8F9bpKXJjmnqp6Q5G+TPDxJWmufrapzknw2yVeTnNEuLw4DAABw\npXG5QbS19qdJDryMh+9zGT/z4iQv3kBde2379u3LeNkrbFXqTFan1lWpM1mdWlelzmR1al2VOpPV\nqXVV6kxWp9ZVqTNZnVpXpc5kdWpdlTqT1al1VepMVqfWVakzmV/r5f4d0S174SoDpQAAAPupqkrb\nyMWKAAAAYLMIogAAAEwliAIAADCVIAoAAMBUgigAAABTXe6fbwEAAGDfcfy2bdm5Y8eyy9gQf74F\nAABghVRVzr3womWXcblOP+EYf74FAACAfYMgCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBU\ngigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJ\nogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWI\nAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAK\nAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigA\nAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJogAA\nAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAA\nMJUgCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADA\nVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABT\nCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwl\niAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUg\nCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIo\nAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIA\nAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFNdbhCtqjdX1Zer\n6pMLy86sqr+rqo+Pf/dfeOw5VfWFqvpcVZ2yVYUDAACwmvZkRPRXktxvN8tf3lq73fj3e0lSVScm\neXiSE5M8IMnrqqo2rVoAAABW3uUG0dbanyT5l908tLuAeVqSt7XWvtZa+2KSLyQ5aUMVAgAAsF/Z\nyDmi319Vf1lVb6qqa4xlxybZufCcL41lAAAAkCQ5aC9/7nVJXthaa1X1oiQ/n+RJV/SXnHXWWZfc\n3r59e7Zv376X5QAAALBMn/7I+fnMBefv0XOrtXb5T6raluS9rbVbf7PHqurZSVpr7aXjsd9LcmZr\n7SO7+bm2J68NAADALlWVcy+8aNllXK7TTzgmrbXdXjNoT6fmVhbOCa2qoxcee2iST4/b5yV5ZFUd\nUlU3THKTJBdc8ZIBAADYX13u1NyqOjvJ9iTXrqodSc5Mcs+qum2Si5N8MclTk6S19tmqOifJZ5N8\nNckZhj0BAABYtEdTc7fkhU3NBQAAuMKuTFNzAQAAYFMIogAAAEwliAIAADCVIAoAAMBUgigAAABT\nCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwl\niAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUg\nCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIo\nAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIA\nAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwliAIA\nADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAA\nwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAA\nUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABM\nJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwliAIAADCV\nIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSC\nKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmi\nAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgC\nAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFQHLbsAAACAZTt+27bs3LFj2WVcaQii\nAADAld7OHTty7oUXLbuMPXL6Cccsu4QNMzUXAACAqQRRAAAAphJEAQAAmEoQBQAAYCpBFAAAgKkE\nUQAAAKYSRAEAAJhKEAUAAGCqyw2iVfXmqvpyVX1yYdkRVfX+qvp8Vb2vqq6x8NhzquoLVfW5qjpl\nqwoHAABgNe3JiOivJLnfumXPTvLB1trNk/xBkuckSVXdIsnDk5yY5AFJXldVtXnlAgAAsOouN4i2\n1v4kyb+sW3xakreM229J8pBx+9Qkb2utfa219sUkX0hy0uaUCgAAwP5gb88RPbK19uUkaa39Q5Ij\nx/Jjk+xceN6XxjIAAABIkhy0Sb+n7c0PnXXWWZfc3r59e7Zv375J5QAAADDTpz9yfj5zwfl79Ny9\nDaJfrqqjWmtfrqqjk/zjWP6lJNdfeN5xY9luLQZRAAAAVtetTr5zbnXynS+5f85rX36Zz93Tqbk1\n/q05L8njxu3HJnnPwvJHVtUhVXXDJDdJcsEevgYAAABXApc7IlpVZyfZnuTaVbUjyZlJXpLkN6vq\nCUn+Nv1KuWmtfbaqzkny2SRfTXJGa22vpu0CAACwf7rcINpae9RlPHSfy3j+i5O8eCNFAQAAsP/a\n26vmAgAAwF4RRAEAAJhKEAUAAGAqQRQAAICpBFEAAACmEkQBAACYShAFAABgKkEUAACAqQRRAAAA\npjpo2QUAAAD7r+O3bcvOHTuWXQb7GEEUAADYMjt37Mi5F1607DIu1+knHLPsEq5UTM0FAABgKkEU\nAACAqQRRAAAAphJEAQAAmEoQBQAAYCpBFAAAgKkEUQAAAKYSRAEAAJhKEAUAAGAqQRQAAICpBFEA\nAACmEkQBAACYShAFAABgKkEUAACAqQRRAAAAphJEAQAAmEoQBQAAYCpBFAAAgKkEUQAAAKYSRAEA\nAJhKEAUAAGAqQRQAAICpBFEAAACmEkQBAACYShAFAABgKkEUAACAqQRRAAAAphJEAQAAmEoQBQAA\nYCpBFAAAgKkEUQAAAKYSRAEAAJhKEAUAAGAqQRQAAICpBFEAAACmEkQBAACYShAFAABgKkEUAACA\nqQRRAAAAphJEAQAAmEoQBQAAYCpBFAAAgKkEUQAAAKY6aNkFAAAAV8zx27Zl544dyy4D9pogCgAA\nK2bnjh0598KLll3GHjn9hGOWXQL7IFNzAQAAmEoQBQAAYCpBFAAAgKkEUQAAAKYSRAEAAJhKEAUA\nAGAqQRQAAICpBFEAAACmEkQBAACYShAFAABgKkEUAACAqQRRAAAAphJEAQAAmEoQBQAAYCpBFAAA\ngKkEUQAAAKYSRAEAAJhKEAUAAGAqQRQAAICpBFEAAACmEkQBAACYShAFAABgKkEUAACAqQRRAAAA\nphJEAQAAmEoQBQAAYCpBFAAAgKkEUQAAAKYSRAEAAJhKEAUAAGAqQRQAAICpDlp2AQAAsK84ftu2\n7NyxY9llwH5PEAUAgGHnjh0598KLll3G5Tr9hGOWXQJsiKm5AAAATGVEFACALWW6K7CeIAoAwJZa\nlemuiSmvMIupuQAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAA\nUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABM\nJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwliAIAADCV\nIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSC\nKAAAAFMJogAAAEwliAIAADCVIAoAAMBUB23kh6vqi0n+NcnFSb7aWjupqo5I8vYk25J8McnDW2v/\nusE6AQAA2E9sdET04iTbW2vf1lo7aSx7dpIPttZunuQPkjxng68BAMBuHL9tW6pqn/8HsN6GRkST\nVL4xzJ6W5B7j9luSfCg9nAIAsIl27tiRcy+8aNllXK7TTzhm2SUA+5iNjoi2JB+oqo9W1ZPGsqNa\na19OktbaPyQ5coOvAQAAwH5koyOid2mt/X1VXTfJ+6vq8+nhdNH6+5c466yzLrm9ffv2bN++fYPl\nAAAAsAyf/sj5+cwF5+/RczcURFtrfz/+/6eqeneSk5J8uaqOaq19uaqOTvKPl/Xzi0EUAACA1XWr\nk++cW51850vun/Pal1/mc/d6am5VXbWqrjZuH57klCSfSnJekseNpz02yXv29jUAAADY/2xkRPSo\nJO+qqjZ+z1tba++vqo8lOaeqnpDkb5M8fBPqBAAAYD+x10G0tfbfk9x2N8v/Ocl9NlIUAAAA+6+N\nXjUXAAAArhBBFAAAgKkEUQAAAKYSRAEAAJhKEAUAAGAqQRQAAICpBFEAAACmEkQBAACYShAFAABg\nqoOWXQAAwL7k+G3bsnPHjmWXAbBfE0QBABbs3LEj51540bLL2COnn3DMsksA2Cum5gIAADCVIAoA\nAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAA\nAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAA\nTCWIAgAAMJUgCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAw\nlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFQHLbsAAGD/d/y2bdm5Y8eyywBgHyGIAgBb\nbueOHTn3wouWXcYeOf2EY5ZdAsB+z9RcAAAApjIiCgArzJRXAFaRIAoAK2xVprya7grAIlNzAQAA\nmEoQBQAAYCpBFAAAgKkEUQAAAKYSRAEAAJhKEAUAAGAqf74FANbxtzkBYGsJogCwzqr8bc7E3+cE\nYDWZmgsAAMBURkQBmMaUVwAgEUQBmGhVprya7goAW8vUXAAAAKYyIgqw4kx3BQBWjSAKsOJWZbpr\nYsorANCZmgsAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUg\nCgAAwFSCKAAAAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAwlSAKAADAVIIo\nAAAAUwmiAAAATCWIAgAAMNVByy4AYF91/LZt2bljx7LLAADY7wiiAJdh544dOffCi5ZdxuU6/YRj\nll0CAMAVYmouAAAAUwmiAAAATCWIAgAAMJUgCgAAwFQuVgRM5Uq0AAAIosBUq3Il2sTVaAEAtoqp\nuQAAAEwliAIAADCVIAoAAMBUzhGF/YSLAAEAsCoEUdhPrMpFgFwACAAAU3MBAACYShAFAABgKkEU\nAACAqQRRAAAApnKxIpZiVa7weuhhh+U/v/KVZZcBAAD7FUGUpVilK7yuQp2Jq9ECALA6TM0FAABg\nKkEUAACAqQRRAAAAphJEAQAAmEoQBQAAYCpBFAAAgKn8+Zb9yKr8bU4AAODKTRDdj6zK3+ZM/M1L\nAAC4MjM1FwAAgKkEUQAAAKYSRAEAAJhKEAUAAGCqpV6sqKqW+fJ77NDDDst/fuUryy4DAABgv7DU\nILpKV3hdhVpdiRYAAFgFpuYCAAAwlSAKAADAVIIoAAAAUwmiAAAATCWIAgAAMJUgCgAAwFSCKAAA\nAFMJogAAAEwliAIAADCVIAoAAMBUgigAAABTCaIAAABMJYgCAAAw1ZYF0aq6f1VdWFV/XVXP2qrX\nWe/THzl/1kttyKrUmaxOratSZ7I6ta5Kncnq1LoqdSarU+uq1JmsTq2rUmeyOrWuSp3J6tS6KnUm\nq1PrqtSZrE6tq1JnMr/WLQmiVXVAktckuV+SWyb57qo6YStea73PXLAaH/aq1JmsTq2rUmeyOrWu\nSp3J6tS6KnUmq1PrqtSZrE6tq1Jnsjq1rkqdyerUuip1JqtT66rUmaxOratSZzK/1q0aET0pyRda\na3/bWvtqkrclOW2LXgsAAIAVslVB9NgkOxfu/91YBgAAwJVctdY2/5dWnZ7kfq21p4z7j05yUmvt\nBxees/kvDAAAwD6jtVa7W37QFr3el5Icv3D/uLHscgsCAABg/7ZVU3M/muQmVbWtqg5J8sgk523R\nawEAALBCtmREtLX29ar6/iTvTw+7b26tfW4rXgsAAIDVsiXniAIAAMBl2aqpuQAAAHwTVXWlvW7O\nlS6IVrffvu+qOnDZNbA5Fj/Lff1zrap7VdV9l10Hu1TVocuuIUlWZX+77IZAVX3LMl9/M1TVVZdd\nw6rb1/f1zDfarbV+2bLq2QzL3s4Xj0trt5exTtc+27aC01Or6tCqOnyjv2clGgibZe3Dbq1dXFXX\nr6qTll3TZhvn5x5YVQ9cdi2rZNk7xd1prX09SarqUUl+Yl9t5I26bpnkB1YldFyWfXE72BtVdfsk\nTx63b11VBy+hhgOTZOxvD6+qb51dw55Y22aX2RCoqmOSPHvcvnVVXX9ZteytqrpNkueN2ydV1WFL\nLmmPVNUB+0KjfuH7srbfv/pyK9pzV4IO/gdU1bYlvfYBo93aqurEqrpnstz91UYs7G+/XlWHjf3G\ndK21i0c9T0ny6LFs+jpd+GxvW1UvGMeCVXF6kruO4/uD9/aX7Lc7jt1Z28jG3zV9e5JvX25Fm6+q\n7prkvUkeVFUHbeUBduxENtwbskyLB/+qunpVPbSqjltSLZcczEdnwraqem/6dvqu1tp/LKOuy7K2\nbY26PpTk35I8boklbdhCI/D0qrrhsuvZgL9M8vSqOj/Jjyc5enYBC+vy+5K8I8mzqmp6Hd/M6Jxc\na5CcVlWPqqrrrD024fXX9j8XJbldVX0mya8mudlWv/ZmWXgPf5XkiVX1sSQvSLLPh+nRyL94NAQP\nX/vMlxFMF74vt6+qP0uyT3bcrFdVBy508N90hLajll3XZqiqE6rqJUkeleQ/J7/22vH14jHy9J1J\n3pxkpTtLF/a3d0vywSR3nLSvrXX/376qnpXkXkl+d6tff10tl4zAjnb0U9KP058ax4J91rqOux1J\nXp/kI+mZo/bms9yvg+j6ns5x/2FJnpvkta21X1xedRu3fvSmqk5OcmaSi1prZ7TWvrZVPTxV9bz0\nqyK/rKpuvRWvMcPCwf9uSf4kyZOSvLmq7jKWT/mOLPR6XlxVh7XWvt5a+9sk90/y1dbap6r/KaR9\nxkLHzkuT/FySayR52Co1QtbvOKvqPlX1R0num+Rnx3a+z9vNdnrDJP83yXVba49ure3c6oP9+t9f\nVdetqt9K8m3pQfQuSbbvS9vxCCBHVdXL0xsCpyZ5z9pjW/W6i6MC4/5xST6e5HpJHt1a+/2teu2N\nWmjILXbiXaX6KO4FSW7SWntQa+2vlxHoroiF0fo3JvnlJD8zlm/5yMhix+PCsl9J8pokL2itnb/V\nNWyG8fkfUFWPTPJ76Z2Rb62qay23so0Z9Z+dPtvnKa21L0963d3N0Hh+khcmeU1r7YMz6tgsl7Gd\n/2h6Z9Wvt9besNXft7XOknWLt6fPGjqvtfZPW/n6C3WsfbYXj/9bkmsn+eEkF7fW3jWjjr2x9jmu\nddyNxf87yZeTXNBae+ra6O4V/d37dRBd6Om8XVXdPck1k7wvfbTgDsutbuPGAeDIqvruqrpxa+0j\nSf44ybE1pvVsRUOgqp6Z5MjW2t3SR1peVlXHbvbrbIXFneJCg+oXk/xokh9qrT0wye8keUmya4ex\n1RZ6CV+Y5OyqekxVXSXJY5N893jO/51Ry2VZ37Ezlp2U5BZJHpLklUkuSvKMJZR3hdWlpzzdoPr0\n1RPTA8kzk1w3yUm1hGmtV9TC9vPoqnpIkv8vyclJLqyqM8bTtqwnfW1djttr28hhSa7aWntKa+3N\nSX4+yd2S3HSr6rg8u+m8u1mSn05yy9baXVprjxzLH7dFr3/j5FKf10Oq6qNJ7tFae26SX0rvKE1V\nbcmfV9sE10suFaKfkuQPkxzTWntwkr+oqheN5+6r7yHJJedRvynJZ1trj0jytKp6zvrtZAted/E0\noatX1Y3GQ7+R3jb59HjePjf6VZc+t66qn9t8XpIHJbnnWI9/k+Rnl1TiXhvv5xlVdafW2j+nd0wc\nnuTIWZ0qC/uG+1bVE6vqiCQvS/I/khywL24T38zCdn69qrrdWPxbSS5OcpUZ72e0lQ+uqhck+emq\nukNr7eeS/FWSq83a1y58tqdW1Tur6hFJvpLkrCSH15iNsy9a+ByPrqo3VtXT09t7pya5flVtH/vT\nKzyAs18F0erTGS8Zfq8+NfXFSd6aPrXiz9JHbc5J/wLceTx3JdbD2hd24T0+KX1U8vgk76mqU9Kn\nGHw+yUOTzevZrT41ZG1H/NUkf11Vr0pyVJKfaa19aTNeZyvVpUcdr5dee5L8dpJjknx9POeV6Tv8\nJ46f2/Id5dhJvi5JJfnJJE9I7xU/O8n/qaofGM9bSsNuNJzWOnZuXlVHjodum+Tg1tpXknw4fVTh\n22ofPR9w0dgODqn+N4/fkOSOSe6d5GlJ/ijJu1trp7bWvrrMOvdEVd2oqn4tyWOSPD3J2eMzeU2S\nJ4/P72ub2UG1+DvGurxaVb0iyRlVdYP0Edmdtetc/F9Ncs8kD6wlTOkf62AtPN1r3P/rJB9N0qrq\ntuOpL0lfZ0de1u/ay9e/efr05LWOsMcl+bH00Za3jqedleT2VXWv8Xldczx3nzhGVT8P6BkL91+R\n5MFJHj86QpPkB9LPF79qa+2rC+9haaOj61+7qr6t+iyYqyX5RJIvVtVvpnfkvnVtO9kqC502z0vy\n+0meXVVPaa29P8kH0reLfU71kaW1xvRNkhzaWvvf6R0R18+ujocz088du/1yKr3ixvbwF+kdZfes\nqre11t6R5D+S3D2T2stVddWqent6Z+ih6aeRHZM+OnvfJPv8SPNaO2Whrbo2g+5ZVfWe9FG089I7\ntTb93NuqutbiMWaE+Q+lt/n+JclZVfVd6Z0l35neht7sGi6xbvnz0kdifzG9Y/a16ZnksCSn7Cv7\n+mS3+80nJ3lRkn9On+H0zPRR0d9J8tjW2l5NX99n3vBGVb8wwqlJbjUW3TDJ1dOnhd2xtfa09B38\nk5N8Jn1u86kjeEwZ9dpba70kCwfHtRGa49O/RL+RPnpzyySfSx/xPbnGOW4baQBUn3L1S+M1njYW\nXz+9sfs3rbW7ttY+XFW3WlZI2lOjsXzQaDydl+SFVXX/1tpvpfdA3zHJVcbTfyrJz1XVIVvZKKmq\nY8cOsZIo4CFnAAAgAElEQVQckj6F8YwkX0/fOSXJU0YtB7fWvrZVtVxGfWvT79oIGm9JD5vPrD4y\n/v7+tLrN2An9S5IjkzxnZp17Yv0Ovqqum94Jcdckj2ut/XGSLyY5ubV2h9baK8bznrXZoWQj1neM\nVNUdkrw4vUPglPSRiROr6j6ttfelb9u/UVWvTvL4ZOMdVFV1xyRPXLh/5/SRpUPS971vb639Q5J/\nTx9VvkZr7d+SfDZ91PmaG3n9K1DnLarqXskl2/ApVfXh9DD1kqq6X5J3JrkwfZ95QGvtvekH12dt\nwusfUH1mQ1prn2+tPSXJd4yH/yO9R/6I6uemPy19vbwgyQuqT9P8zao6dJnHqLX6k2Ssm+dX1X3G\nov+VPh3zGlX1oKr6ztbaZ5P8WpJ3VNWbMvYFm9UpekXUOE1gN6/90CRPTT+v/eFJfijJfxsdTztq\nXEBls8JzVV1nBJ3FZdvTZxadlD6ycEZV3WLU9biquv4YyVn21UWvWVUnJpeMLB1dVeekHwd+eoTN\nV6efQ3nbqjqotfb3SdaOFavixPQBi59On7a5tt2/Ocl3JbnR7n9s713GZ3tskj9vrT0gPXQelz59\n81fG/fsve5u4LKONcLeMWRPp+4Uj00f5v2OMlu9IX6dvTA/YJy3uYzahhiPTP6+bVtXdxvZ5kySf\nb609vbX2s+nTnH8iycdGPY+pTbzKfI3TT9ZmXC0sPzzJtyR5ZJJbJ7lzkt8dz3lj+kWTbrxZdeyt\ntQC9rvbj0kfmd7bWnpXkF9I7nh6R3uF97ap6dVVdlH5K2R5b+SBau+ZcfyVJS/Kr1S/Qcb/0k/3/\nR5IjxtN/Kn0D/Vr6iMdxSfbpK+dW1anpO/S1+69L8oSqOiF9g319kv+W3hvxitHY+3D6BnKP5Io3\nABZ6sU5Nb0T8S/pG96jqUwl+Mz3Mf3Y87+npo8632Pt3uvnqG88RPjS9kffPrbU7JLlqeqD69vSG\n/P2S3GJ8AX8v/SqWBy7+jg3Ws7536YD0zpMT0j+vY9PPT/vT1tq9W2t/VVUntn6u0NszcVutdVdw\nHB6c5KOttbskuU6S09JnGPx+klePzqBT0mcevH9WrXtifKZrPfk3H42lf0ryp+m9pGsHoRclObSq\nfqiqnlBVf5rewTO1A+CbGY3BqqrvqqrrtdY+muRTSQ6pqm2ttf9KH318xtjmvj+9B/PvWmuv2qQy\n/iM93B5SffT7ZUn+fRzofzTJtUYHy8+ldw6eXf0CLP89/eJbWz5iPho3N0jy1+P+Uenf8aenn5Oz\n1qO7Npp/QvrIR9JH9d6zCWV8R3qjIyMEH54e0G6T5M/Tg+dj07/bd0q/dsE56Qf2zyR56N72Mm+i\np1Q/tWWt0+PEJO8f+6+PJHlY+nt4WJLHV9WPpAf9P05v/G040O+N6lNGf34EpwOq6hG160/k/HJ6\no/7r6dcG+ET6NrA2yvszVXX4JnTYHDs+839L8sbqF/J5ffUpitdN74R4VZL7JPmR1tpnW782wG8k\nOTf5hn3wMjw6Y/9Y/bzJn0zy2621u6dvtz+U3sZ6XZLvTT+OJX2f8Orp1e6h0Tnw1IVt4p7p6/2c\nJOe21r5zHDfeld62fFBt8mjVwgyN7xqh6aD0Y+uPjWPP8enT9v9kdEL/cvpsqSMu85cu19eT3Ca9\ns+qP09sMd0pyUGttR5K01n4gfYDoBukXK3pwNmFEciEL/GOSm6d/jr+cfuz+Svpo49px/pPpMwdv\nl96Besv0EckNq6prpH8XHjPun1FVL6mqo1tr/57k9unh91pJ7tNae0tV3aC1dm6S/5O+Xpamdl18\nrFW/+vmzqk9l/rskL03ywCRprV2QPvB1u/TP78np+9Hvbq39zhV60dbayv5LcuDC7YPSezj+Mckb\nxrLrpF9B9h5rz03ytvQGyOFJjlv2e9iD93jz9CD6+HH/velf6qQ3lF6/8NwbJvnJcfsGm/Dab09v\nLN1o3L/32PAOTO8FOSfJH6QPy99q2evqm7yPWya5xrh93fTeunPTA9Qrkrx0PPbSsa4P3+J67pzk\n5uP2k9OngCb9Tx/8XJJDxv2fTO8FvcrEdXXjdffvmH4BklsmeXn6FNZ3j/W3beF5rx7b4/uSHLvs\nz/wy3tsNx/fnD8Znfav03sk/TO/BO3g87+7poxJvTfLgZde9m/dxWpLzx7b7/vTG3wnp5+g+auF5\nn0zyA+P2gRt8zYPSGxjHjfuHp3c6vH7cf076+Y03HPfvn2THws8/LH0a9wFjH3zbLVw/B667f88k\n3z9uf0v6aNin0kc835R+TvDB6Y2HH9no9y19dsPa8ea66aOGH0m/KEbSO8PeO24fvPBzD03yc2u/\nY8nbWK3VkD5j4Ktjm3vJWPae9AunJMnVF37uzCRPHLcPWFLtV124fUB6Q6nSR25fkt4QPDK9kXrY\nuP2KsU/7aPrIxBGbtA5fkbFPTT/efCXJ94z790+frXDWws/cdOE79MQlf/6L7aubJrnfuH2d9H3n\nH6cfC16T5IfHY+eln+N88Oyar+D7u1N6B8SH0jvtvn3sz76Q0VYYz3tWejC47lZsz+mjrH869ok/\nOfZHN0pvUz1v4XkPTB9sSJLjl73+1r2H9fvbF6V3VL56YdnfJ7nDwv03pB9PDktypw2+/gHrb6fP\nIvtkkocvPPbrSV43bl9t7MOuv4nr4fC1GtI7b9459iVvTR8s+vX0dtRDkpy/8HP3S2/nHZnR9pv8\n+VV6m/5VC+/hkPTj4kfT20IfTO8AuXZ6+2ltW7xpetvvBzdUw7I34k1YiVcdG/VL0uea32dsAGs7\n/zPGhnBW+nzwj6RfSXLptV/G+zlg3f2D00dx35V+AH3lwpftgekH159KP6fkk2M97NUOM72h/uok\nP5g+2nq99F7iO2RXw+rsJK9cWPc3W/Y6W/ceFg+eh6U3BP7/9J7Ou43l35vkxeP2w9Kn5T1i7Jzu\nuYm1XNKYG/dvmX6Q/vH0Rt1V0g/q7xzr+obpQfT3xw7g7IxOgLXft8Xr7ubpYeKAUfvzRh0PGI8/\nOn1E69SFn9mecWDMQoN02f/yjQfHI9N39qenh6qPpR8krpE+mnN2+sVWll77N3sfY9nz0xuC10yf\n2vnL4/P6wfSLa9x2PO8u2aQOovHd+K70Cw69OP3q0g9On5FxSnqP6FvTQ/JaoP+LJL8wbh+QPsPg\nE1loYG3B+lr8vt13/P+48dl/67j/iiR3H7dfNfYPN0nf5x22ibXcMH00+reSfGTdY59M8rBx+9Zj\nH/BnGY39JW9z649BDxrf+7cvLDsqffryTcf9tT+B8IH0Cz8tq/ZbJPmxcfvm6QHi40m+dyz7vvTO\n1WPGtnjSWH7g2IZvvAk1LHZEVPrx85bpF537+yR3Ho/dPr1D7BXjeU9P7+h94LK3gYX38m3pIzs/\nP77rJ4zlL8quQP3SsU6/Pb1hurTPfw/ez3enB74Xpp+CkXH7zPR96uvSOySeNt7TW5Ncc+Hn9zqM\nZvf78odlHE/T23i/mx5ET0u/4NP3pAeYTyW51+I2tux1uZv3cq/x/9Hp+/rnpF9BO2M/+LH0Y8WP\npbfDN7XDeny2b07vPD9krNtXZwTgsS/4XHob+i/HY4dtxrocv/snxu3D08/9/OlcOnD+Qvox+tj0\nY/Z549+fJTltiZ/b2sDW76ZfxTjp7aKXpp/eeN8kf5fRyZ0epD+68HO32sj3orUVC6Lr32z69IQ/\nTB+pudZYdu30KSE/v/C8k9OD6IuyDzWWL+e93ju9J/dq6YHv1Um+lB5StmU0mNKnxnzfePwWG3i9\nk9ID2fPGuvrQeJ1npgf9I8bzbpY+1e06y15H6+qvdfcfmR7Unz3uvyA9pB+XPu3uL8byF6b3iJ6y\nhbWtbZuPza4RhVePHdVZY3td7IW9SZLbLdzfspGF9IBwlXXLvmP8/13pF5w5cty/ffqB+sXju/eq\n9Ebevjwa/qgkjxm3rzvew5+nTy9/XfqFYjL2I2ds5bq+gnUvBqrrpgf+Q9Kn8/x2+sj9BUmevPC8\nG6YHrsdv4raxWMePJ/mvJB8c9w9J7y19ffoB/anpjerbjMevlUs34k5Pcu0tWFeXCpDpDZEPjn9H\npI9onJnkzPH4x9I7nu401tdzssGe8Vy6A6zSr8L92iRnjGWfzsLo+nj9T6Tv2++a5Pv2sW3uWumd\nZmv7gusl+dcsjMaM/cAfjdtPTj/Pelm1L67/j47v+LvGZ/+Aseyq4/FXpc+e+uskP7rJdSyOztwr\nvUH6geyaqfTI9GsrrD3n+PRRsHdlyTOLsmsEvNIb089IP9f3lPSG6ZuSPGk85zeza3TpV9LbYLdb\nRt17+N6OGt/z30sPA4sj03cf+63T0zv+H5keVu692dvmwnaxdkw9K70z8c+TvGjd8x6Q3v567rLX\n3/rtZN2+4qHpHevvSG/LnDy269eubS/jeU8a6/mt2cCMvfH6i9+z66SPOJ6b3ll+dvro3cFjm108\nRl4//Vi6oVHY3X2243vwqfQ2+n3H5/z57DoePji7LlBU6e28hy3781yo//bpU4NPTB8keceo/+3Z\nNYPv6mO9/lb6nxnbnNde9pu/AivpoN0s+/Ykb97N8pPTpwg+N73Bf5Nl138F3udxYwP4cHrj6bfH\n8nukD4n/Qfqo1e+m96ocucHXW+sVfEqSZ47blR7WfmNsdB9IDyVrvbybNmqwCetrfefEHdKnK35g\n7BTWpml/a3qv7lPSG83vTm8Ivi1b0DheqOfB6b1vN0k/oK9Nvbtm+hTdC9IPivfYk/e3BfWdml3T\nLE5Jb4D8V3b1Ir4zu6YuH5jeOfFr6Y2mVyW52rK3gct4X8eOneX7xw72gPSR0JdnjDikj4h+KH1a\n622yCaMhm1D3+u35zukN5g+kT4E7fOzTPrrwnErvgT44vXdyM3p4FxsaNxvr78bpI/Y/m10dYd82\n1unjRm3vTm9AHZBdMze2bJpe+lTbX8gIvON1fykLU5TH8vunN0pOTJ+q+yvp0/A21BDINzbKvmV8\nDhdnjAaP5Y/PN46Kvj/Jg5a9ze3mPT0kvTH1mrGO1k4LeUX6hTUWn/v3GY2UfeFf+kjnnyfZmV3H\nqwPSOxtfPu5fI/14+r+SPGQLajg+PUC8e+xbvnWsz7V96sfTj69XTT8HOBmdlUtaZ99wjEk/Xv1X\nRsfpWPaI9DbHLdJH7T6UPsL00mxw6v+E97g2G2Nt1O7pSS5cePyH0o8HJ+7J+tnD17zXuvv3Th8J\n/HD6NUquk37MPT9jhsZ43uOzMI11YfnS13EuHQCPT29r/3T6seHW6e2Zd4994GPTjw1PTe8EOHpv\n1+XCa67f125L72y8z1j2wvTw9M70WQj3SD8erE3D3pIBlPSA+/b0i45tH8uulT6w87qF570ivQNv\nqQM569bj9dPbF69In6nzu2P569P/Isba8+6ePpvg4GzygN5SN+o9WFlHrH2o4/6N0hthTx87+Jtl\n4Tykhecdkj6d6H1Z4pD3Hry/3R0ATkvyhHH7uekNmkeP+8/JmHc/1s1en0My1t156X9v8LnpIyzv\nG48dnH4gOi89tD0jffrvN3QGLHn9re9pvF/GVOVx/z7pF2lZ6+l9TPro7snpPT7btuqzTG+gXzO9\nZ+xfxvq72/ii32DheXdPbyQ9ebNq2YNaL+lRHDvL9eewPT+7OkBunR6kb7bud+wTATTrekcXln9H\n+lUw1y//k/Q/i5P0TogXZx+ZSrbu4PCdGb3h6VMMr54eAp+fPkL6N2M/+OT00Z43ZBPObc43jsT+\nWnqj6R1jW75R+gHrBxae96jx+DFJrjdpXa0frb3XqPcaSf42Y4QzuwLzddJHKd88vvubei5OeofB\n+ek98/cfn83n1z3nQ2Pd/WV6z/1S96e7++6kHzc/ll3nFp+ePvp1s3H/n9I7SP84vdPj0GW+h4W6\nD0sPeyelzyL61YwR8PH4Lcd6/9aFZZv6fRn3T05vjF+chbZH+pT5tU7R247nfDpjxHxf+Jfe9vjh\nhc/6zIxzmRee88r0UbzDxvdtQx3hW/x+npk+xfbY9Jlyv5rellk79n08/QJRa9vHQ3LpEa6NTMO9\nSvosrKund+A+Pv0Ye5fx+Dmjvtumj87/UfoMrt8Z+4lt32w7W8K6XAygh6SPKv92+qy9q6e3rT45\n1u/Z6R3+B433/bGMc/Q3sZ7npnc4rbWND0ofgX1lerB6U5KfHY/dJGOEdAvWyx3T9/tvSQ+dP5j+\np9MyPvfbph8bv3MsO2H9Z7vsz3R8Rm8et2+ePvPy/ult2DeN+l+dvv982pbUsuyV8U1W0trVHl+f\n3hg+IX1k4Ceya+rozdJ7OxZ7ns/Mbnq19qV/63dw6XPZT86uOdfb0kc+X5w+TeR/juW3SA/Xt9/g\n639v+mjhfcdO5GfGet258IW5e3bTkN/X/qUfYF451uG1xhfmdRk9Nulheq0BcMx4n7fe5O10/ed5\n0/QG59qO8DfHdnxqeuPj2eue/7LsGnXY6hHQxQPtgWOn/aEkF6x73uI5bC9L8oFlf9a7eS/fk0tf\nnOdhWbjQR/oI+KHpB8610ZFvT++R/nyS5y/7PezmPR076n5f+oH1H8b+74D0kd3fGtvXbTL+5ERG\nb/AmvPb67fj52XWRn98eNR051vsvjRruPfYjd1v3s1vScMq6EciF5R9Nb3QenB7Yz1j3M4ekNwqe\nmE0OT+nh7Y/GtnXn9Cu132nsTx+98Lxt6dO/v3sf2M4WG5a3Sm+EXy+7rrnwSwuPvzHjvN70htdL\nMs4bX0Ld60egj86u00ZenOSccfuk9CmP1x33D0vvDHzZZtWx7v7aeZ+npZ8q87Ekr1p4/Frpow4P\nH/dvkORb9pHP/+j06ZLvSz9/7wPpwezg9NN1Tll47r3TR0CXNoK7B+/tW9I7ej6YfjrWh9Jnazwy\nfYRu7bPant5hsGkBJQudS9nVhj0kfWTu0wuf/x3S2yZ3Hc97ano75kmbVcsmvZ+j1tbXuH+9sX28\nLclRY9mh6e2utWsTvGVsN2vXadlQh1t2DSSs/f+I9KnsR62r8yML988Zdd51C9fN0emjv6emZ5Hf\nSA/in8yu62ocPfYHb1zy53jAuv9vk3H64tjuFjuWvzfJZ8btI8Z36UezhYMPS9/QL2OlrTUaj01v\nBH9/xiX31zbIsext6Tv4T6Q39D+e3oO/T4zW7OZ9rT+IHjg2go/n0udcbF/bcNOD4tcXNpoNTyNN\nbzD9V3ZNy713eu/dOeknTr9h7DTXGqH7ynlza1+itR3S9vSpY2dm1zkXD00PgYs7goszLrKw2dvo\nwv0bpQfgR6cfeK4ydlLPTR9VOCO99/Cfxnq+xsL7+Hg2+Vylb1Zveu/hWemhfG262Cdy6YsQPSK9\nB+yQ9NGkxyz781+obe1iOI9Ln6J+n/Rw+eH0oLZ2yfizszDdNrumHh+ZfaAhtZtt6LD0XtWLFpa9\nO8lTFx7/kawbpdhgDTdIDxqHjvv3HfuCq47v1fPTG3K/njHa+f/aO/Nwq8ry739uRGanyExFxCkN\nTURlEATMWfFXTj8jcGrUHKEUU3PCMZwwB1QEBI3ULENUfJPM9IddKb7mmIm9mUO+OaapRQP3+8f3\nXu9aZ3NQ4Oy91zrwfK9rX2evtdc5517PXs/z3OP3Rg6d7yDP6bUUWEobPF7FtXNHlBqepVnvFd/9\nZ5FRdW88B9sihfTEBsrViXzDXhDjdTGqiXq17Oes+LyhCPoaheOzYv0Zi7JjtkLe8KuBveO6HVD0\nYWjZ8hfed0BOtKnkRESGIo1fiuNJKMJ0cax1DXlO0fr+JjJCByFjpx9q2dKrcN1YlMZYeoplyNMN\n7ZfbAKPi3ElxL5ciQ/RwFMmrHDlOK/eTlTVlTL6ZvjCFSB9GTpRTCCcA4dRv6/3VfqeFvz+PPEPr\nu/EsZsyk56Io7RLkmWU/I+S6yYZIR50Qz8EWyLj/BTlJYQ/Uym8vpEtOQ3wC69VRns2BwfH+HHIn\nf8fC9/wQcszOjZ8NLbWJ9edJ8o4MuyJ97xSkP5+GdMI+Vfgua86NQM6nEfG9LSx81h+1vDmvaTKW\nOUAfM3hDkMfovhiwm2g9dXRNpCgPoYEtAep8b+uhDfQs4JQ4tzZKI90a1VjdjDaJS9HmdWCdZbgY\n+FG874gM/QtResg+1JHWug6yLi398lxy8oReKGJsSJE+o7BQHoE8eXXZTGMxvr8g20lIAR0di/At\nBZnui+d0bJwbS9RkxnFnGpiexZLOjz7I2XAmSsl4gFBEaL2GrTKp7Uj5HEzLtjE3onqQjNHtQOCJ\neH8RcvCcVFhHKsGYXfOdjCbfZDdDDoAsGr0HMqr6xvGW8Tyv1tqcWAE5NkYG+wlxfCY5i+MU5KDY\nsXD9f5Nv/Ju29f+vgLxd4/ucg5yU88mzOG5Ayl5nxKA4I+ZlXcibPkauzeJ7yojJ3kJ19c9QU69a\n4jO3LoXoOcomORcpksMQUcXIuG4cMqh6xLX7UUJrgaXcx8nkWS7j4x4yMo3DkFLcLebIJKRE1zUN\nGjkfDiHvh/hHtPYPQdkW26D9fV7Z4xXy1mY7fCnmxw/ieA2kc1yO2LYfKKxBj1CoYaziC6VE3oL2\ntJORUZI5WbdAzpYtkNPyCiJzpgFybIf2nIfieFtkrGyMnGTXkDtO1keOx2K7odINfpY0qm9FuumJ\ncZy18NiLlk7h6SgSuncb/ner8xQ59d6N98fEd9yz+HvIyXwETSJ+Qzr7FbR04D+K9NBx8V2X0sou\n1r5hhWNDBvL1cbwWSle+MY7vjbVyN7Tun0MT2bubPkDLOIiZAj8SKV4/Qx7NF4GD4prhBNVwlV/k\nSlsW5T0RKVAXxgSfS55OeA65sf0VpGSd3yC51kNe7qw32ICQrSk1Xsso4+o1x32RMfGteL8zisr8\nFLGRvRMLUX/kuBjZQNmeII8Y7EXObPooUkL3j8+GIkbWJSIjNHHTQZGOXyAHxHFIWb87xi7rDXl/\nLEaVqGEryN6ZXCn6ZtzDHcjo2JGIghTm2h3AqfE+SyVruDGyDPfRl5YU/Jsjpr8HkdE/Mc4fjZSZ\nbJO/CUX/6vJ90DItb90YozsR6+6t5EbvkFiDDkKG1jXxfG9V+P1WnUR1krN2/q+GlP+3gR3i3LFI\nGdgMKZoPkLdpqouxvoyybh5j1RspnY8hZaRhRGjLKFdtP8iuSNHYNtbJ22PM7qLgyEWprTfRQDbx\nZZG95ngNpDDNKnz/28X9ZL1LV0c9DLNU4oZElpDiOzbm6SeQfnIdisY/GWuUob7mDTF6llHOJTI/\nkML+O1q24+kNLCgcP4b22k9SkTrgj7nPKcAfCt/NNWifK6Zt3xXv634/5ERtc5ET/F1y/WAiMCPe\nfwsZzJVrFVZzP6uj/fUrKJNrOuItyCK9x8f6sEHhd9qUiRjP4BXkBF570HK/XIBSmLekZbnAUSFf\nU+uVyVsuTUOZEOuh9PZSydvIbY3LUNndBBSd7Y1abm0Yn28da1ZW1nYc0hEb1l5tqTKXOWAfMZC9\nYzHPUoj2QdGNq1F6wHWoxrGuBdB1voel1TItJk/BHRIb1mGFz/9G3m6ioRtATOBnyx6rpcg2BqWR\nZR75vZASPAp5x55CXu8tCQYy5LW/Id73a7B8fYAXCseHIC/sujGuvyPaoiDFNKufaFHv0KSxPDzk\nOQCljL0Vi/qYwjXdkYf2WCpQw1Yj/zC0ee8Zz8E/kcGUtWO4FNUFZikyn0VpcZVhy0bG9L7kRv/e\nSKnO+tn2RspLL+RpnU5OpNGHOtHN18j0VaRw7oOyLu5HkcZiutPBsfbejxSqhjeqj/lyFlHrjxyS\nWT175xi38+KznshIzyK6FyKPeVMjC7EWnR7z6rdElLYqL+RkGBjjd3G8Vovvf2zhus8XxnLjEuVt\nrefiAOCnrZw/ApXknIJq1a6ijTwKyyHn95ATZxRykq2L9qb5Mb5NSVtfimwHANfG+14oirRbHH8J\n+FPh2k7IiLo61p5bafAe2sZ72xU5TbM07E8jDpGMFXc/lHWW3W8PCoR7bVkfaD07a0MUUc7WzaMI\n/SDWqMfR3tWTmrrF1v5ek8dye7TvZ8zjQ1AmxyTyIMmYGM/Bhd97EBnWbXL2kOtEXRCB02Uxdy5E\n+3rWi3QA2iM7IQf0LciBO486l14th+xdyHtFP0mU0ZQkS2322yEoxfZmcofMZMIhE8c3ob0922tL\nyXopZcCWYUDXYcmQ94J4QHdDCl1lUkc/5l4GxmTKIo9jgJcLn58QEy7z8O5G8zbRzsjYa8FAWYUX\nweRFnl5zQiyYu8azcCE5uVNPpHw+SSHttQkyTgXGF+TLWrPsjup6R9VcX8oYoxqvf6G01k/EAl4k\nczkfGauVeQZoyezbAyl5VyFDejzy8m0an68fm9He5N7AI5CHsjJsg3G8S8y5LrEJXEKuAFwJzI73\no5CHvc006a1sUGsjpf0x8rrPQShy/27I8SOU5rhxfF7sc9uoKJMV5Lsl5vSkmNezgJ/H53sig3mr\nOD4Y1V33qx3vEr7vXanpy1uCDLU9TS9AKXPHxVrfDxlP26H96A5kUF2AjKhKMLki59gZKEPgU6gO\nfwERhSmsD92QAX0nYUQ3Wc5rUXbRtDjenibt4TVyZPNnLDKIh8ZcvgQ58C+L7/foWBtnABMKz8km\ncf3JZX/3H3GPq6EU/Ptjrv2WnDvkW8AvC9dej7ISuhTO1W0/QA7xfdBaPoC8N3mn+PlHcqfO8WU8\nm8t4H5+KNTerXR0PnF74LHOeXhnjeThycgyjjnWYyJifgLI09oz/PTPW9ywotQCYHu+7UB0G2vVp\ngpN2GWUZgroDfBdlNUwpfNYB1YBn/Ak/RA6qUlKI/79cZQ/aUgYyIyOaGhN8XRTy/i7VpgsvKnuG\nFKlHUf78zURPHqRYZYvn1ijN4Ov1XCTb24slmxRbcVNEitI/kEK6fZxbExkpRyJSirqx4S6jzN1R\ndPxc+U8AABVPSURBVDGrF5oZE/tO6tjst06yXkaksiNnx4so0rUAKf2VmVfUMPvGz37IKDkixvsm\n5KzIPv8O8uhXpga0dj6T0/jPRBGKfZGyNLBwzWIiHZI69OutmVPr1cjxf4Bt4twnkIJ6M4qI7h3P\nRd/i32rUGsWSdUm7o7TlqYVzL5GXZpwPXF34rO4R4/b2qn3mkHK0NXIqFOupuiKlPUsXHBjr60WU\n1N+OJR02/VEUaXx81zfFfDmbAus4KifJ2DlLIXhB0bjvox6CpTkhyFPuRsbP1cnbO2WlIvsiJ+4B\nyBHxW1q2E6sEMeFS7u90RBCzGzJCDkWtmv6AUuM7IYdkpjPUpcyolXm1AyqjejjmVlZS8SiF9hbI\n0P8jFUxtbmW9HR7ybomI/t5BUfGpKNI8Lp7zCRTYf+v4/w+M7/J7yIiaieoYD0MO6CxQczlylpba\nh7Oqr5gbjwF7FM49X3N8ENJTf9/W77FucpctwEcMaFdkjN6HyAcqw9rZiqytpeB2jQW/J0rJ+ANR\nRI1qG98krwHbvsobQDPHD3lqM4/30Jgwu6Km4P8D9I/P+iCimuFljh1S3rPeriNiodyo8HklnAvI\nu/ibbEGK8dyPipJQICP/UpQOOjzOZQ3VN0SU6dcjz95JyJvfsJrgNtzHcFSHOwYpS2sixTrzOF8R\nm3z2zH+ROrSfqplT3eO5XBCKxC5IkbuEqKWN6/rGevulEsfrKHLF7kIU6ewTx4eQ14Ftj5yTpaRk\nVflF7hE/FhluP43zXcijiJuhyMO4suWtkT0rs/gyygrohhT+61C94gBUZnBJ3ONs5NQpO/OhCyVG\nRBCT9V8Lx9ehqFIPlD2QZetk7Y2+Hcc3UEJN2HLeW39kFF1HOFRQ6n72XP+QvG3PF8jLHbIIcVv6\ngRYdeV1QecBi8l7U/VE680GIoOoZcjK1s5BzdExrf6+ksaw1AAfGzzXiWTg6jndA5SKdkaPqx8gR\nWe/+y5kj9Grg4Hg/GGVnHEfO8D8HOU6PpsS2R1V/oSj1Daic5UDkeLod6SBroch893p/j22Wu2wB\nlmFgNypzgf8Y2Wq9uF9AaTEbx/EsRKYzi5zVL9to51Ho3bMqvqiJfKGo3QIUIc68umcguvBNUU3b\nUzHRHgdOq8A9dAD+CmzxUc9GFV5UtCa4lXk0EHn1xqO0nPuQ82YNZJxkTNNfQ8X1M8peI2LDLDog\nOqEMjvko+jABKUyrIYfFNGRM9UfR/GENkqs7qgv5XmxE01DPS0N1l9cDI+Labqj2pnPh9xtFRFQb\nLV4LZRLMJqKbyOiYiaKzmQH1FLmhWnoLnqq9yD3iWVR9I5Ry3adwTRYZ37fw3ZfuMEOZLTfG+6+h\nCOOvCQU1zq+OnJAjgf8qW+YqvWIuZwbmeKL3M1JIp5D3z/w2eTu4NmddNPB+suf0PKRHZamjHZCT\n8sg4PhVlSzUkIyrW9otQqcSAWD/nxGc9ENnLTJRVsiXKzhkT6+lUSiavCTk3rTkejnStJ1GkeW3k\n8J9Jy76hY6iJ9NZJnkHZOMb/uIcg5ozx/k7sB1nt4k5EJlx6feS49oy5MRuxiU9BhujdyOaYQ6F1\nYFVepQvQXl+0jDh0QHnz85Gi9xBKYzglNoesZmAgSi/sQUWN62aNXc34bYLaRmRexhnI+NgJpZfN\nJGdN3Qh5H0tlo6y5n6yHaZs9sA2Ws3I1wa2NFSIb2hR5ZOfFhnVPbFC7oJTRLLJbCe8oMiq/gQy5\nQ1BK5ND47AjkKX+APCV/fCgpRh2M0Fbm1MiYU0NRFLZXbEazkGGSRWTPRTVutZ7yhj0ftf8rzu0E\n3NnK+dNRBDlTSLakgpHvqrxo6RHfDxl3M1Ad6MaIPfEpCv0tS5CxtoY162G5N3l2yb7xrO4ex10Q\nwVIlMziq8EJOp7cJ4xI56b5G3q/8GWS0LSSM+KrsAzX3sRZyQP4p9vs+SLE+Mj7vGs/xNJS+P4Oa\nVhOt7SsrKEsWKTwVRermojTsD8j5KzaP8b0ojjsiJ9+jKIpbamou0jsvy8Yl5tEvQ+6tUAlU1prl\n+0hv3YicgLGuTlLEAXMNiiBvHs/lI/F9ZoznhyF9uvRgQ3t70VIPGIEM055UmKm5AwkrBHd3M+ts\nZlPQBBqM2NC+iuoG90WsYu8BPzeza9CitNDd33f3f5Ule9nwgJkNMbP5aKxuAyab2W3IAHkaeXLf\nQClY+5vZZu7+srv/xN3fKu0GauDur8dPj5+Ly5Wodbj7Inef4u6LM1nLhrsvNrNNzOxWMzvDzEa5\n+++ARWheTUfPQQ/gq+7+ANpEX4vff68k0TEhW0P/hBwktyN5n3P3+WZ2OPKYZ9HHg8xsfaTQ/Bo5\nqR5qoxwdCnOqm5ltiwgIuqGMgp1QpOxJdx+N5trxZvZppOxPdvf/FP9mI54PM7P42/8xs45mdqKZ\njTazrkix62xmveLa1eLXbkT1wf3MbHV3/727311v2VYiPItqqL6JFNChSLH/N6pr64uU9lfKEjB7\n1sysL1J4TzSz6eiZfSkuexw5oK4yswlornwC9YNMaAXu/gEy0CbHqUtRLXBn5Lx7AUWUd3X3OfE7\nldgHMpjZEag12nZIjzrd3V9EkZzBZtbb3f+O9oW5SE/4urvfU/w7ddyDu8XPOYjzoxNaU89ABhzu\n/gLKdrk2ru2OdJgT3f0od/9HnWRZLmR7k7s/ApxsZnvGuCxEqbcvuvtzSE/dwsy2Q+O6C3L4zXb3\nYW3dn1pBR1Resz/af25GTrP5wPVm9n0UtZ/o7hfU+X+vCuhgZpua2VS0Bjzl7m+5+5/LFmxpsIqt\nQ5VFKHuLCz+/iNILOiIP89mo1uJHZrYbSi0Yjyb9MOAzwEx3f7+cO6gWYowmojTLeXFuNPA5dz/V\nzPZH3rlzkeK8XSyoCe0Y2fwpHPdHitOVqAbkPqRAr4d6Ax5qZhsiJ053YL9QuCqD2MCPQh7e19AG\nusDMVkf3dZ+7/8TMxqHat1nuPqnOMnRAdTU7AC+jFlELwhAei1Kw7kIK1amojvAyd7+3nnIsRbYN\nETPwM3G8HUoZugfVtv0dRYu3Ry0PbozrjkDpWZsBv6va915VmJllBoaZDUeOkHGIZbwUpbgIMxuE\n1v6/oqjd88gAnYPm+DeAh939LTPbHj0XT7n7b0oSud0g1oGXEEv/M+HYfdPdjzGzDSqtjEr2y4Hb\n3f2h2BuuQ3rU06jU4R13P7+1322EA9jM1kT1dX2QUXxrnN8AEREd5u631fv/tgXF+V84txUy8D/n\n7s+b2VzgV+5+UTgkv4EilSch59X8RjnUzawHimKvjVrFvR7nN0fZUJsBPwsHRMIKIJx8eyIn86Ky\n5fk4dCxbgPaA4iJXmJxfRqlOB7v7q+HBP9DM7nD3X5jZYYgs4vgwtOaVInx18U/k9e4cRv3qSGFa\nJz7vhyjx33D3f6LUjYR2CjPbE9VHz4rjQaFYboqcOItQHc4sd3/WzN4EtjOzSYgifyJwS9nGSMER\nZcg7PgoRDE1DRABjgZFm9pa7/9HMXgK+ZmaHoojOUe7+eJ1lGo4MjceQ4Xs28BkzexKxOg5F9T9/\nQkb/DOCLEVloBtYCLjWzC5G3fTEy1n9sZg+hSM2vUMujU2MTHYgMlfvdfUGT5FxZ0MHMNkZpzf1Q\nuuu/UVS0VJjZOihV/VjkgLgTEc3ca2Z3ozTyHYHTzOyCiNylKOgyItam/yZ6PqLI3dbxWeWMUDP7\nDHKK3e3ub5jZMNSaBWQ4zUHO6n3M7DGgr5mt4e5/K/wNa2AW0vsoPfWxghF6OcrUGuLujzXo/y43\nwpD3ghPqCwQbrrs/FXvpBLRnnQ9caWYz3P01M3scOQU7NyAC2gLu/r6ZzUMOpt5m9kmUqvsU+q7n\nNPL/rwpw92dRdky7QIqILiPMbBOULrAQ1Vp9iJjcTnL3R8ysNyIEecHdJ8bxRu4+vzShKwwz64nS\niDZH0ZpPIw9ZH6SovoxYht8oS8aE+iGUownI2XAeUo5uRKl5o1Bj6isinbUbegbWQymtz7r7w2XI\nnaE1j7uZfQoZoN2Bvd19UUT6vwD8wt3vDINgExQtvT6cKvWW7QBUK9XH3V8ys1OR8TfV3ReGcncr\nqhcxd39+affUCJhZd6RUrolaLvRGa+mHyEiaEdd1Qc69AcC/3f2WRsu2sqKqHnEzWxfV3D2K0vF+\njhwnXRGb6xR3/99mtqu737/UP5TwkTCzhxHBzJNly7I0mNnZiMjtKVQDfBtK0T/T3QfFNceilNiz\n3P32kuT8NNLttkCtBJ9GtYuvxOdLRCCbjeJabmYdkXP0myjb4LPISfoCGusT3P3nprKyddz94Gbf\ng5l1RvXrn0fjOtndb2jW/0+oFpIh2gpaSR8ciAhzbkKRu12QV/cgcpa0D1Bt2OGo1cy7TRa73aEm\nhWwXRLBxGfLKvViiaAl1RkQQryLvWTgfRWzeQ1HEI919rpmtDfwAeLCKG1NE749FDIr3IcXkRODi\nLHIXytNOKAo5ERlb/2n9L9ZNrtnA7919vJlthIz+B4Afu/uHZjbA3R+NazNio6bUMkeK8mmIgGRH\nNC5HI5bERwppxT+uUoQhof6oScsblTkaTXXNZyNDdG55Eq4cMLPVGr3mrCjMbARqZ3cWKmf6i5nt\njGos90B1is+hqONg1O+wEyJ6+0+zHGg1Mhtymndz9yeyc2UaoGbWMTIdsuPOqA3KWijb4Nvu/qaZ\nXQr8zd3PjnKHY9x9UARLhpTp8Asn5aLifSSsekhkRQWY2Z5mNrrgWRoUH/VD0YzzEeHDT1Aa4ZUo\ntXCfWJDuRZtrMkKXDcWi6kuAZ9z9tWSErnyI+XE9sC3qd/c+qg/riAym8WZ2U7x/uWwj1My6RtS+\neG4c8HVkVPVCacQPI1KNoRHtASlSVwOHuvukJimEZwL7mNln3P1llKY7BCklZEZovPdmKnLu/i93\nPwelYU5G3/uDwMVm9l0UHeuNaq4SVmLEvJ+H6sE3NrO+ZvYAYsn8ejJC64MqGqFm1s/MLkH9oLui\nkotOAO7+PygFdg/kzP8typI6FLXxWZjdU7ON0Pif7u4L3f0JEzqUbIT2RuUOB8bxXqi12ZtILx2B\nMo1A5RmbmdnIyD7pYGbD3f2lsrNO3P2DZIQmJEO0JdYCzjCzbczsZ8API81tZ2CvuKYjYmvrHu/v\nQoyOHWNSNavuqt0jNpYuKF1kqAdJScLKifAkX4kioCCj823gRRQNmYxac5xegni12AMYbmZ9zOzg\nSBtdA7FQ7gLsjtJyQc2+B8QLd//Q3X/dzNTCGNvZqD0UaCxPcffXmiXDMuAqRKbUFznzfgW8g8ob\nRrv722UKl9A0TEM9DE9CWUY3u/vJ6ftfOWFi8l4Dsd/3cPdj3H0hWq8mFC5dhIiVXg9d4DyUKXEI\ncqxVAs125BURkVmA11HN/86RijsE1YPOdvcHUdbOV8ysW2Tq/AFxmHRHLZAeLEH8hIRWkQzRlrgd\nFcpPR4pcf6R8Po+K5A9wtV3ZAHjP3f/u7le4+/eSV2fF4O7PRtSoMnVMCQ3FxcAmZrZXzJlHENPs\n8+7+sLu/WpZgZtahsNG/gRib5yEnyT/QZv8MovTfx90nm1oK/RIxVZa9nl4NvG0ihHF3/2vhfkpH\nGMWXoFYSPwF+6e7XxfglrCJwtZG6DvW4HFR29kNCYxDr6QVoDd0Ntd3qaXnLq9OBXcxsnInIbGfg\nL/G7hroSPOfug71kjoCqoBCF7YnS23sjp+gkxF/Sy8y6uPtdqJY1a39yLXB1CpYkVBGpRrQGZtYP\nKcfD3f03QT4yFJGNbISIdYYC17r7VWXUKyQktGeY2VGox1rfsmWBFv0tvXBuE9Ri5EVgrIvpbxhi\n+Fw/rvk8IoE4DZGUJWfUMsDMBgBPeAOImxISEsqHicn7JMR4/BAipdkWZY0c6O6/jusGIN2qP3CO\nF/qDl12DWRXU1vuayOkmof1pb5TmfjwiKBqMWnc9bWY7IGKw3VK2QUKVkQzRVhB1DBu6+5cj7eEo\nVBP2EODA01GHlZCQsJwIUoXDUYsBr4qyYWa9EFnWY4iM6HXUEPp/Afe6+7sZ0yDQA3mkL3L3n5Uk\ncqtIzrGEhIQyYTmT9zaudlwbAaOBMagWfP/W1v0qkyw1G7UOUjNbx93fMbMrUZ/PW8xsMGJp/zPq\nuXoNatsx1d3fM7OuKQKaUHWUnUpWVbSWPvh/gcfdfW4yQhMSVhyRmjfF3ReXZYSa+v5m783MTkJt\nAp5DhuYpgKF0shEE6Q9qqn4McFWkjFXKCIVyyDwSEhISMrj7HcA9wFfi1GuIU+MK4HPIEdkC4UBL\nRmggalHdzIaY2vFMMrG2v4RY2UG11m+hVk3rI+O/O+rJTDJCE9oDkiHaCtz9L6hO9PI4fjRqQatE\n/JGQkLCCyBQeM9sKGZwT4/yZKFL7KurD9iOUBXGemb0K7BxkGneVInhCQkJC+8DpwMFmtnM49HdC\nrK6Dg721BZIDbUmY2X6IDfd4VOd5I9AZWGxmI2JcXwc+iTo23Ovu57n7B2XJnJCwvEipuUtBVdMH\nExIS2o5ozTQReB+4GVHdb+DuA4JMY3fUOuAHqC78i8BL7v6bkkROSEhIaFcws8kodXQe6sF+gru/\nGZ+lEoKPgZl1QhHOkcA4RKL3CiofOQ51bdgTONvdZ5clZ0JCW5AM0YSEhFUKwSp7PqqnWYTSmaYh\nco0z3H26mW2AasM7ejXaySQkJCS0K5jZesAs4IfuPi3OJRKi5YCZbYYY0Ue7+9tm9iZqe/MKItCc\n7an3ekI7RseyBUhISEhoMjoCGwL7A/sBd7v7JDNbCFwPTHf3P5vZFHd/pUxBExISEtor3P0vZnYb\niuZNS0boCsFQG8EeZrYmYsldE5ibakATVgYkQzQhIWFVw9+Bf6OWAv/l7m/E+edQH85D3f3mZIQm\nJCQktBk3oprGDqjePmH58GdE/PRTpLOfE2RQCQkrBVJqbkJCwioHM/sWsD2ivP8QEUHMBya6+ztl\nypaQkJCQkFCEme2K2rYsKluWhIR6IhmiCQkJqxyCjOxIFBXdApjs7jeUKlRCQkJCQkJCwiqEZIgm\nJCSssjCz7sCioMFPSEhISEhISEhoEpIhmpCQkJCQkJCQkJCQkNBUdChbgISEhISEhISEhISEhIRV\nC8kQTUhISEhISEhISEhISGgqkiGakJCQkJCQkJCQkJCQ0FQkQzQhISEhISEhISEhISGhqUiGaEJC\nQkJCQkJCQkJCQkJTkQzRhISEhISEhISEhISEhKYiGaIJCQkJCQkJCQkJCQkJTcX/AzSLZt9Cwbzj\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f276e0c4e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "count_by_occupation = user_fields.map(lambda fields: (fields[3], 1)).reduceByKey(lambda x, y: x + y).collect()\n",
    "x_axis1 = np.array([c[0] for c in count_by_occupation])\n",
    "y_axis1 = np.array([c[1] for c in count_by_occupation])\n",
    "x_axis = x_axis1[np.argsort(y_axis1)]\n",
    "y_axis = y_axis1[np.argsort(y_axis1)]\n",
    "\n",
    "pos = np.arange(len(x_axis))\n",
    "width = 1.0\n",
    "\n",
    "ax = plt.axes()\n",
    "ax.set_xticks(pos + (width / 2))\n",
    "ax.set_xticklabels(x_axis)\n",
    "\n",
    "plt.bar(pos, y_axis, width, color='lightblue')\n",
    "plt.xticks(rotation=30)\n",
    "fig = matplotlib.pyplot.gcf()\n",
    "fig.set_size_inches(16, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rating_data_raw = sc.textFile(\"./ml-100k/u.data\")\n",
    "num_ratings = rating_data_raw.count()\n",
    "\n",
    "movie_data = sc.textFile(\"./ml-100k/u.item\")\n",
    "num_movies = movie_data.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f276f789e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def convert_year(x):\n",
    "    try:\n",
    "        return int(x[-4:])\n",
    "    except:\n",
    "        return 1900 # there is a 'bad' data point with a blank year, which we set to 1900 and will filter out later\n",
    "\n",
    "movie_fields = movie_data.map(lambda lines: lines.split(\"|\"))\n",
    "years = movie_fields.map(lambda fields: fields[2]).map(lambda x: convert_year(x))\n",
    "years_filtered = years.filter(lambda x: x != 1900)\n",
    "movie_ages = years_filtered.map(lambda yr: 1998-yr).countByValue()\n",
    "values = movie_ages.values()\n",
    "bins = movie_ages.keys()\n",
    "# plt.hist(values, bins=bins, color='lightblue', normed=True)\n",
    "fig = matplotlib.pyplot.gcf()\n",
    "fig.set_size_inches(16,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rating_data = rating_data_raw.map(lambda line: line.split(\"\\t\"))\n",
    "ratings = rating_data.map(lambda fields: int(fields[2]))\n",
    "max_rating = ratings.reduce(lambda x, y: max(x, y))\n",
    "min_rating = ratings.reduce(lambda x, y: min(x, y))\n",
    "mean_rating = ratings.reduce(lambda x, y: x + y) / float(num_ratings)\n",
    "median_rating = np.median(ratings.collect())\n",
    "ratings_per_user = num_ratings / num_users\n",
    "ratings_per_movie = num_ratings / num_movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(count: 100000, mean: 3.52986, stdev: 1.12566797076, max: 5.0, min: 1.0)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.stats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6UAAAJWCAYAAACgUCFFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAH09JREFUeJzt3X+w5Xdd3/HXe7OgBqktLaYmYTOF2KwSJdhOQhNHr4OF\nSEfDNFNLtDrFkmaUqFPHmXTq2FmUdlSUsRiVxKadcQab0GYi0NE0aF2naUx2oZgK3SVRIT+3FBTR\nCMiavPvHPQkny83u2ewm73vvPh4zO3u+3+/ne8/nZr57cp/n+z3fW90dAAAAmLBjegIAAACcukQp\nAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY1aK0qq6tKoOVtU9VXXNBtu/rarurqoPVNW+qrpk\nadtHl7edzMkDAACwtdWxfk9pVe1Ick+SVyZ5OMn+JK/r7oNLY07v7k8vHn9Nknd291ctlv8wyd/p\n7k8+M98CAAAAW9UqZ0ovTHJvd9/X3YeT3JjksuUBjwfpwpcmeWxpuVZ8HgAAAE4xq8TiWUkeWFp+\ncLHuSarqtVV1IMl7knzP0qZO8t6q2l9VV57IZAEAANheTtoZzO7+1cUlu69N8ualTZd099cleU2S\nN1bV15+s5wQAAGBr27nCmIeS7FpaPnuxbkPdfXtVvbiqXtDdf9zdhxbrP15Vt2T9cuDbj9yvqo7+\n4VYAAAC2rO6ujdavEqX7k5xbVeckOZTkdUmuWB5QVS/p7j9YPP66JM/t7j+uqtOT7OjuR6rqeUle\nleRNR5nkSt8MT8+ePXuyZ8+e6WnACXEcs104ltkuHMtsB47jZ17Vhj2aZIUo7e5Hq+rqJLdl/XLf\nG7r7QFVdtb65r09yeVV9d5LPJflMkm9f7H5GklsWZ0F3JnlHd992Qt8NAAAA28YqZ0rT3bcmOe+I\nddctPf6pJD+1wX4fSXLBCc4RAACAbcqvajmFrK2tTU8BTpjjmO3Cscx24VhmO3Acz6rN8jnOqurN\nMhcAAABOnqp6yhsdOVMKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAmJ3T\nEwAAeCbsOuecPHD//dPTgBP2ol27cv99901PA54x1d3Tc0iSVFVvlrkAAFtfVeXmgw9PTwNO2OW7\nz4yfk9nqqirdXRttc/kuAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0Qp\nAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAA\nY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0Qp\nAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAA\nY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0Qp\nAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAA\nY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY0QpAAAAY1aK0qq6tKoOVtU9VXXNBtu/rarurqoPVNW+\nqrpk1X0BAAA4dR0zSqtqR5Jrk7w6yUuTXFFVu48Y9hvd/bLufnmSf5bk3x/HvgAAAJyiVjlTemGS\ne7v7vu4+nOTGJJctD+juTy8tfmmSx1bdFwAAgFPXKlF6VpIHlpYfXKx7kqp6bVUdSPKeJN9zPPsC\nAABwajppNzrq7l/t7q9K8tokbz5ZXxcAAIDta+cKYx5Ksmtp+ezFug119+1V9eKqesHx7rtnz54n\nHq+trWVtbW2F6QEAALCZ7N27N3v37l1pbHX30QdUnZbkw0lemeRQkn1JrujuA0tjXtLdf7B4/HVJ\n3tXdL1pl36Wv0ceaCwDAqqoqNx98eHoacMIu331m/JzMVldV6e7aaNsxz5R296NVdXWS27J+ue8N\n3X2gqq5a39zXJ7m8qr47yeeSfCbJtx9t35PyXQEAALDlHfNM6bPFmVIA4GRyppTtwplStoOjnSk9\naTc6AgAAgOMlSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgj\nSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEA\nABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgj\nSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEAABgjSgEA\nABgjSgEAABgjSgEAABgjSgEAABizc3oCAGw+u845Jw/cf//0NACAU4AoBeALPHD//bn54MPT04AT\ncvnuM6enAMAKXL4LAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADA\nGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEK\nAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADA\nGFEKAADAmJWitKouraqDVXVPVV2zwfbvqKq7F39ur6qvXdr20cX6D1TVvpM5eQAAALa2nccaUFU7\nklyb5JVJHk6yv6re1d0Hl4b9YZJv6O5PVdWlSa5P8orFtseSrHX3J0/u1AEAANjqVjlTemGSe7v7\nvu4+nOTGJJctD+juO7v7U4vFO5OctbS5VnweAAAATjGrxOJZSR5YWn4wT47OI70hya8vLXeS91bV\n/qq68vinCAAAwHZ1zMt3j0dVfVOS1yf5+qXVl3T3oap6Ydbj9EB3334ynxcAAICtaZUofSjJrqXl\nsxfrnmRxc6Prk1y6/PnR7j60+PvjVXVL1i8H3jBK9+zZ88TjtbW1rK2trTA9AAAANpO9e/dm7969\nK42t7j76gKrTknw46zc6OpRkX5IruvvA0phdSX4zyXd1951L609PsqO7H6mq5yW5Lcmbuvu2DZ6n\njzUXAJ4dVZWbDz48PQ04IZfvPtNxzLZw+e4z4+dktrqqSnfXRtuOeaa0ux+tqquzHpQ7ktzQ3Qeq\n6qr1zX19kh9N8oIkv1BVleRwd1+Y5Iwkt1RVL57rHRsFKQAAAKemlT5T2t23JjnviHXXLT2+MskX\n3MSouz+S5IITnCMAAADblF/VAgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAA\nwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhR\nCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAA\nwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhR\nCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAA\nwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhRCgAAwBhR\nCgAAwBhRCgAAwJid0xMAAACe2nOe+0WpqulpwDNGlAIAwCZ2+HN/kZsPPjw9DTghl+8+8ym3uXwX\nAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACA\nMaIUAACAMaIUAACAMStFaVVdWlUHq+qeqrpmg+3fUVV3L/7cXlVfu+q+AAAAnLqOGaVVtSPJtUle\nneSlSa6oqt1HDPvDJN/Q3S9L8uYk1x/HvgAAAJyiVjlTemGSe7v7vu4+nOTGJJctD+juO7v7U4vF\nO5Octeq+AAAAnLpWidKzkjywtPxgPh+dG3lDkl9/mvsCAABwCtl5Mr9YVX1Tktcn+fqT+XUBAADY\nnlaJ0oeS7FpaPnux7kkWNze6Psml3f3J49n3cXv27Hni8draWtbW1laYHgAAAJvJB++6Ix/ad8dK\nY6u7jz6g6rQkH07yyiSHkuxLckV3H1gasyvJbyb5ru6+83j2XRrbx5oLAM+OqsrNBx+engackMt3\nn+k4ZltwLLMdXL77zHR3bbTtmGdKu/vRqro6yW1Z/wzqDd19oKquWt/c1yf50SQvSPILVVVJDnf3\nhU+170n6vgAAANjiVvpMaXffmuS8I9Zdt/T4yiRXrrovAAAAJKvdfRcAAACeEaIUAACAMaIUAACA\nMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIU\nAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACA\nMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIU\nAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACA\nMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIU\nAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACA\nMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIU\nAACAMaIUAACAMStFaVVdWlUHq+qeqrpmg+3nVdUdVfXZqvqhI7Z9tKrurqoPVNW+kzVxAAAAtr6d\nxxpQVTuSXJvklUkeTrK/qt7V3QeXhv1Rku9P8toNvsRjSda6+5MnYb4AAABsI6ucKb0wyb3dfV93\nH05yY5LLlgd09ye6+/1J/nKD/WvF5wEAAOAUs0osnpXkgaXlBxfrVtVJ3ltV+6vqyuOZHAAAANvb\nMS/fPQku6e5DVfXCrMfpge6+/Vl4XgAAADa5VaL0oSS7lpbPXqxbSXcfWvz98aq6JeuXA28YpXv2\n7Hni8draWtbW1lZ9GgAAADaJD951Rz60746Vxq4SpfuTnFtV5yQ5lOR1Sa44yvh64kHV6Ul2dPcj\nVfW8JK9K8qan2nE5SgEAANiazr/o4px/0cVPLL/z59/6lGOPGaXd/WhVXZ3ktqx/BvWG7j5QVVet\nb+7rq+qMJO9L8vwkj1XVDyb56iQvTHJLVfXiud7R3bc9/W8NAACA7WSlz5R2961Jzjti3XVLjz+W\n5EUb7PpIkgtOZIIAAABsX35VCwAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAA\nAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNE\nKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAA\nAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNE\nKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAA\nAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGNEKQAAAGN2\nTk8AtpNd55yTB+6/f3oaAACwZYhSOIkeuP/+3Hzw4elpwAm7fPeZ01MAAE4RLt8FAABgjCgFAABg\njCgFAABgjCgFAABgjCgFAABgjCgFAABgjCgFAABgjCgFAABgjCgFAABgjCgFAABgjCgFAABgjCgF\nAABgjCgFAABgjCgFAABgjCgFAABgzEpRWlWXVtXBqrqnqq7ZYPt5VXVHVX22qn7oePYFAADg1HXM\nKK2qHUmuTfLqJC9NckVV7T5i2B8l+f4kb3ka+wIAAHCKWuVM6YVJ7u3u+7r7cJIbk1y2PKC7P9Hd\n70/yl8e7LwAAAKeuVaL0rCQPLC0/uFi3ihPZFwAAgG3OjY4AAAAYs3OFMQ8l2bW0fPZi3SqOa989\ne/Y88XhtbS1ra2srPg0AAACbxQfvuiMf2nfHSmNXidL9Sc6tqnOSHEryuiRXHGV8Pd19l6MUAACA\nren8iy7O+Rdd/MTyO3/+rU859phR2t2PVtXVSW7L+uW+N3T3gaq6an1zX19VZyR5X5LnJ3msqn4w\nyVd39yMb7XsC3xsAAADbyCpnStPdtyY574h11y09/liSF626LwAAACRudAQAAMAgUQoAAMAYUQoA\nAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAY\nUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoA\nAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAY\nUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoA\nAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAY\nUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMAYUQoAAMCYndMTWFZV01MA\nAADgWbSpovTmgw9PTwFOyOW7z5yeAgAAbCku3wUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCM\nKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGDMSlFaVZdW1cGquqeqrnmKMW+rqnur6ner6uVL\n6z9aVXdX1Qeqat/JmjgAAABb385jDaiqHUmuTfLKJA8n2V9V7+rug0tjviXJS7r7K6vqoiS/mOQV\ni82PJVnr7k+e9NkDAACwpa1ypvTCJPd2933dfTjJjUkuO2LMZUl+OUm6+64kX1ZVZyy21YrPAwAA\nwClmlVg8K8kDS8sPLtYdbcxDS2M6yXuran9VXfl0JwoAAMD2c8zLd0+CS7r7UFW9MOtxeqC7b38W\nnhcAAIBNbpUofSjJrqXlsxfrjhzzoo3GdPehxd8fr6pbsn458IZRetPP/fQTj1964cU5/6KLV5ge\nAAAAm8kH77ojH9p3x0pjV4nS/UnOrapzkhxK8rokVxwx5t1J3pjkpqp6RZI/6e6PVdXpSXZ09yNV\n9bwkr0rypqd6on/8/T+80qQBAADYvM6/6MknGd/58299yrHHjNLufrSqrk5yW9Y/g3pDdx+oqqvW\nN/f13f1rVfWaqvr9JH+e5PWL3c9IcktV9eK53tHdtz3dbwwAAIDtZaXPlHb3rUnOO2LddUcsX73B\nfh9JcsGJTBAAAIDty69qAQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwo\nBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAA\nYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwo\nBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAA\nYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwo\nBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAA\nYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwo\nBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYIwoBQAAYMxKUVpVl1bVwaq6p6queYox\nb6uqe6vqd6vqguPZFwAAgFPTMaO0qnYkuTbJq5O8NMkVVbX7iDHfkuQl3f2VSa5K8vZV9+XZ88G7\n7pieApwwxzHbhWOZ7cKxzHbgOJ61ypnSC5Pc2933dffhJDcmueyIMZcl+eUk6e67knxZVZ2x4r48\nSz60zz82tj7HMduFY5ntwrHMduA4nrVKlJ6V5IGl5QcX61YZs8q+AAAAnKKeqRsd1TP0dQEAANhG\nqruPPqDqFUn2dPeli+V/maS7+yeXxrw9yW91902L5YNJvjHJ3zrWvktf4+gTAQAAYMvq7g1PXu5c\nYd/9Sc6tqnOSHEryuiRXHDHm3UnemOSmRcT+SXd/rKo+scK+R50gAAAA29cxo7S7H62qq5PclvXL\nfW/o7gNVddX65r6+u3+tql5TVb+f5M+TvP5o+z5j3w0AAABbyjEv3wUAAIBnyjN1oyO2gKpyyTTA\nJuE1me3Cscx24Dh+donSU0xVnVtVr0nWr72eng88XVX1/Ok5wInymsx24VhmO3AczxGlp4iqOr2q\n3pLkPyf54un5wNO1dCz/aFWdPj0feDq8JrNdOJbZDhzH81a5+y5b3OKM0k1Jzujul0/PB56uqvrO\nJD+e9f9p/Nvu/vTwlOC4eU1mu3Assx04jjcHZ0pPDZ9N8o4k/ztZ/92zVfXqqvrKxbLjgE2vqr4o\nyblJ7uvua7r7U1X1V5a2nzY3OzguXpPZLhzLbAeO403Af+RtqKqedAa8uw8n+a0kh6vqY0nekuSb\nk/x2VV3Q3Y/5MDebzZHHZHf/RZJfSXJvVf1IVf1Skp+rqrdX1XO6+9GRicIxHPmGiddktqqqeu7y\nsmOZrWj5De3EcbxZuHx3G1nE6E8keU5Vvae7f2Np86EsfqDv7rcsxn8qyU8n+WYf5mYzWXpXsqtq\nR3c/tlj+aJLbk/xMkp9K8l+S/FySf5fk+44YC6MWMfrjSU6rqt/o7vcubfaazJaxdCz/9ap6T5L3\nLt4oTBzLbCFV9cYkb6iqN3T3+6vqtMWb2o7jYc6UbhOLd3DeluRvJtmX5JqqeuPiksfH7yD2O4//\nQ1v4lSR/VlVf8qxPGJ5CVb0+yYNJ3nTktsW7mf81ycu7+y3d/ZEkVya5vKpeIEjZLKrqG5O8P8lf\nS3JPkn9TVd+w2FZek9kqquqbs35Z419N8ptZf0PwpYttjmW2hKUznc9P8ukk/zxJuvtRx/HmIEq3\nj+cnuSDJ93b3O7L+zs7fTvKPkif+x/H4u5qpqguT/Ick+7v7MwPzhS9QVV+a5LIkP5nkH1TVuYvL\nZp54reruP+7uB5d2e1mS9yT502d3tnBUjyX5me7+3u6+IcnvJLl0eYDXZLaIB5K8sbu/r7vfmeT3\nknxJ8vlfmeFYZrPr7l78LHFGkrdn/Ufj71xs3rkY4zge5PLdbaK7/7SqPprkn2b9csb/meQrklxc\nVf+9ux9Okqp6XpLvTfJPkvxkd/+nmRnDF+ruR6rqB7r7/qr6iiQ/luQ7Hj8D+vi7mVX1nCTnJPnX\nSc5L8hPd/ZdzM4cv8P4k+5YuDbszyRfc1dFrMptdd384yYcXn8O7KZ8/S/pFSd7X3X+6WHYss2k9\n/vGeqvpEkj/P+mdIv7Wq/keSP0vyycU4x/EQZ0q3l1uSXFBVX9Hdj2T9cpvPJvkbSVJVlyT5TJJ3\nd/cF/qGxGXX3/YuHP5vk3Kp6VbL+maald+UPJ/m7SX6vuy/q7ltmZgsb6+5Pd/dfLN2A69VZP+P0\nxNklr8lsJYv4fHd370pyc5JvS/KKxLHM5rf08Z6vSfLfktya5KuzfhJnd+I4niZKt5fbk3wi62dL\n093/K8mFSb64qi7K+hml53T3PWMzhBV19/9NckOSH1ksP1pV51XVv6iqc5LcdMRnP2DTqarTli4Z\n+7XFuvOq6luTvDhek9kCHv88Xnf/4uLvdyZ5SZKdVfX3sv5DvWOZreDuJL+QZG/WP/ZzMMn/qaqv\njZ+TR7l8dxvp7kNV9a4kP1FVv59kf5LPJTnc3R9IctfoBOE4LC61ua6q/n5VvS3rx/LeJLd0932z\ns4OVPZbkuVl/w/Bli2P5wST/qrv/aHRmsKIj7zxaVS9O8sVJ/l93vy/rn5mGrWBHki9P8gPd/dtV\n9ZYkr+/un83i95Qyo9zhePupqm/J+g2OLk5ybXdfOzwleFqq6vR8/hKbH+vutw1PCY5bVb0iyR2L\nP/9xceMj2FIWZ/zPSvLmJOcneXt3/9LsrOD4VNWXPH7josUVAF/e3R8bnhYRpdvW4kYw7eYvbGVV\n9cNJzk5yzfJd8WArqaqzk3xXkrc6jtnKqurLk/zDrL+54lhmy6qqnX5G3lxEKbBpPX63vOl5AADw\nzBGlAAAAjHH3XQAAAMaIUgAAAMaIUgAAAMaIUgAAAMaIUgAAAMaIUgAAAMaIUgAAAMb8f0xKlWdJ\nUuSrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f276ef23bd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "# create plot of counts by rating value\n",
    "count_by_rating = ratings.countByValue()\n",
    "x_axis = np.array(count_by_rating.keys())\n",
    "y_axis = np.array([float(c) for c in count_by_rating.values()])\n",
    "\n",
    "# we normalize the y-axis here to percentages\n",
    "y_axis_normed = y_axis / y_axis.sum()\n",
    "\n",
    "pos = np.arange(size(x_axis))\n",
    "width = 1.0\n",
    "\n",
    "ax = plt.axes()\n",
    "ax.set_xticks(pos + (width / 2))\n",
    "\n",
    "plt.bar(pos, y_axis_normed, width, color='lightblue')\n",
    "plt.xticks(rotation=30)\n",
    "fig = matplotlib.pyplot.gcf()\n",
    "fig.set_size_inches(16, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, 272),\n",
       " (2, 62),\n",
       " (3, 54),\n",
       " (4, 24),\n",
       " (5, 175),\n",
       " (6, 211),\n",
       " (7, 403),\n",
       " (8, 59),\n",
       " (9, 22),\n",
       " (10, 184),\n",
       " (11, 181),\n",
       " (12, 51),\n",
       " (13, 636),\n",
       " (14, 98),\n",
       " (15, 104),\n",
       " (16, 140),\n",
       " (17, 28),\n",
       " (18, 277),\n",
       " (19, 20),\n",
       " (20, 48),\n",
       " (21, 179),\n",
       " (22, 128),\n",
       " (23, 151),\n",
       " (24, 68),\n",
       " (25, 78),\n",
       " (26, 107),\n",
       " (27, 25),\n",
       " (28, 79),\n",
       " (29, 34),\n",
       " (30, 43),\n",
       " (31, 36),\n",
       " (32, 41),\n",
       " (33, 24),\n",
       " (34, 20),\n",
       " (35, 25),\n",
       " (36, 20),\n",
       " (37, 57),\n",
       " (38, 121),\n",
       " (39, 22),\n",
       " (40, 35),\n",
       " (41, 52),\n",
       " (42, 183),\n",
       " (43, 221),\n",
       " (44, 151),\n",
       " (45, 48),\n",
       " (46, 27),\n",
       " (47, 25),\n",
       " (48, 66),\n",
       " (49, 215),\n",
       " (50, 24),\n",
       " (51, 23),\n",
       " (52, 56),\n",
       " (53, 28),\n",
       " (54, 65),\n",
       " (55, 21),\n",
       " (56, 187),\n",
       " (57, 106),\n",
       " (58, 154),\n",
       " (59, 382),\n",
       " (60, 208),\n",
       " (61, 21),\n",
       " (62, 232),\n",
       " (63, 93),\n",
       " (64, 200),\n",
       " (65, 80),\n",
       " (66, 38),\n",
       " (67, 30),\n",
       " (68, 34),\n",
       " (69, 65),\n",
       " (70, 131),\n",
       " (71, 38),\n",
       " (72, 137),\n",
       " (73, 66),\n",
       " (74, 39),\n",
       " (75, 79),\n",
       " (76, 82),\n",
       " (77, 72),\n",
       " (78, 21),\n",
       " (79, 55),\n",
       " (80, 29),\n",
       " (81, 58),\n",
       " (82, 168),\n",
       " (83, 155),\n",
       " (84, 68),\n",
       " (85, 288),\n",
       " (86, 23),\n",
       " (87, 211),\n",
       " (88, 21),\n",
       " (89, 76),\n",
       " (90, 300),\n",
       " (91, 98),\n",
       " (92, 388),\n",
       " (93, 20),\n",
       " (94, 400),\n",
       " (95, 278),\n",
       " (96, 56),\n",
       " (97, 63),\n",
       " (98, 27),\n",
       " (99, 136),\n",
       " (100, 59),\n",
       " (101, 67),\n",
       " (102, 216),\n",
       " (103, 29),\n",
       " (104, 111),\n",
       " (105, 23),\n",
       " (106, 64),\n",
       " (107, 22),\n",
       " (108, 33),\n",
       " (109, 234),\n",
       " (110, 133),\n",
       " (111, 24),\n",
       " (112, 46),\n",
       " (113, 51),\n",
       " (114, 48),\n",
       " (115, 92),\n",
       " (116, 143),\n",
       " (117, 86),\n",
       " (118, 71),\n",
       " (119, 181),\n",
       " (120, 26),\n",
       " (121, 74),\n",
       " (122, 61),\n",
       " (123, 54),\n",
       " (124, 24),\n",
       " (125, 182),\n",
       " (126, 45),\n",
       " (127, 23),\n",
       " (128, 184),\n",
       " (129, 30),\n",
       " (130, 353),\n",
       " (131, 30),\n",
       " (132, 22),\n",
       " (133, 26),\n",
       " (134, 25),\n",
       " (135, 55),\n",
       " (136, 35),\n",
       " (137, 47),\n",
       " (138, 51),\n",
       " (139, 24),\n",
       " (140, 21),\n",
       " (141, 107),\n",
       " (142, 33),\n",
       " (143, 20),\n",
       " (144, 206),\n",
       " (145, 316),\n",
       " (146, 29),\n",
       " (147, 20),\n",
       " (148, 65),\n",
       " (149, 36),\n",
       " (150, 32),\n",
       " (151, 307),\n",
       " (152, 106),\n",
       " (153, 23),\n",
       " (154, 51),\n",
       " (155, 22),\n",
       " (156, 37),\n",
       " (157, 51),\n",
       " (158, 173),\n",
       " (159, 107),\n",
       " (160, 120),\n",
       " (161, 58),\n",
       " (162, 42),\n",
       " (163, 23),\n",
       " (164, 63),\n",
       " (165, 30),\n",
       " (166, 20),\n",
       " (167, 69),\n",
       " (168, 69),\n",
       " (169, 37),\n",
       " (170, 22),\n",
       " (171, 28),\n",
       " (172, 27),\n",
       " (173, 43),\n",
       " (174, 177),\n",
       " (175, 38),\n",
       " (176, 63),\n",
       " (177, 115),\n",
       " (178, 273),\n",
       " (179, 42),\n",
       " (180, 63),\n",
       " (181, 435),\n",
       " (182, 28),\n",
       " (183, 53),\n",
       " (184, 251),\n",
       " (185, 48),\n",
       " (186, 92),\n",
       " (187, 56),\n",
       " (188, 112),\n",
       " (189, 187),\n",
       " (190, 59),\n",
       " (191, 27),\n",
       " (192, 35),\n",
       " (193, 121),\n",
       " (194, 305),\n",
       " (195, 96),\n",
       " (196, 39),\n",
       " (197, 118),\n",
       " (198, 181),\n",
       " (199, 40),\n",
       " (200, 216),\n",
       " (201, 386),\n",
       " (202, 20),\n",
       " (203, 43),\n",
       " (204, 42),\n",
       " (205, 22),\n",
       " (206, 64),\n",
       " (207, 230),\n",
       " (208, 33),\n",
       " (209, 33),\n",
       " (210, 132),\n",
       " (211, 37),\n",
       " (212, 25),\n",
       " (213, 126),\n",
       " (214, 128),\n",
       " (215, 93),\n",
       " (216, 131),\n",
       " (217, 76),\n",
       " (218, 53),\n",
       " (219, 28),\n",
       " (220, 22),\n",
       " (221, 146),\n",
       " (222, 387),\n",
       " (223, 105),\n",
       " (224, 141),\n",
       " (225, 27),\n",
       " (226, 50),\n",
       " (227, 58),\n",
       " (228, 21),\n",
       " (229, 30),\n",
       " (230, 133),\n",
       " (231, 21),\n",
       " (232, 93),\n",
       " (233, 110),\n",
       " (234, 480),\n",
       " (235, 94),\n",
       " (236, 124),\n",
       " (237, 48),\n",
       " (238, 29),\n",
       " (239, 159),\n",
       " (240, 24),\n",
       " (241, 23),\n",
       " (242, 20),\n",
       " (243, 81),\n",
       " (244, 238),\n",
       " (245, 22),\n",
       " (246, 195),\n",
       " (247, 26),\n",
       " (248, 56),\n",
       " (249, 161),\n",
       " (250, 122),\n",
       " (251, 77),\n",
       " (252, 21),\n",
       " (253, 97),\n",
       " (254, 159),\n",
       " (255, 83),\n",
       " (256, 207),\n",
       " (257, 57),\n",
       " (258, 23),\n",
       " (259, 46),\n",
       " (260, 24),\n",
       " (261, 27),\n",
       " (262, 160),\n",
       " (263, 123),\n",
       " (264, 120),\n",
       " (265, 46),\n",
       " (266, 23),\n",
       " (267, 185),\n",
       " (268, 328),\n",
       " (269, 323),\n",
       " (270, 138),\n",
       " (271, 278),\n",
       " (272, 51),\n",
       " (273, 22),\n",
       " (274, 71),\n",
       " (275, 95),\n",
       " (276, 518),\n",
       " (277, 54),\n",
       " (278, 23),\n",
       " (279, 434),\n",
       " (280, 258),\n",
       " (281, 26),\n",
       " (282, 22),\n",
       " (283, 53),\n",
       " (284, 47),\n",
       " (285, 32),\n",
       " (286, 288),\n",
       " (287, 62),\n",
       " (288, 75),\n",
       " (289, 27),\n",
       " (290, 150),\n",
       " (291, 296),\n",
       " (292, 124),\n",
       " (293, 388),\n",
       " (294, 150),\n",
       " (295, 196),\n",
       " (296, 147),\n",
       " (297, 192),\n",
       " (298, 127),\n",
       " (299, 280),\n",
       " (300, 20),\n",
       " (301, 275),\n",
       " (302, 21),\n",
       " (303, 484),\n",
       " (304, 26),\n",
       " (305, 222),\n",
       " (306, 32),\n",
       " (307, 112),\n",
       " (308, 397),\n",
       " (309, 20),\n",
       " (310, 21),\n",
       " (311, 294),\n",
       " (312, 223),\n",
       " (313, 262),\n",
       " (314, 245),\n",
       " (315, 87),\n",
       " (316, 75),\n",
       " (317, 22),\n",
       " (318, 174),\n",
       " (319, 23),\n",
       " (320, 154),\n",
       " (321, 125),\n",
       " (322, 51),\n",
       " (323, 89),\n",
       " (324, 66),\n",
       " (325, 142),\n",
       " (326, 187),\n",
       " (327, 284),\n",
       " (328, 283),\n",
       " (329, 65),\n",
       " (330, 147),\n",
       " (331, 70),\n",
       " (332, 183),\n",
       " (333, 26),\n",
       " (334, 333),\n",
       " (335, 22),\n",
       " (336, 130),\n",
       " (337, 34),\n",
       " (338, 75),\n",
       " (339, 254),\n",
       " (340, 44),\n",
       " (341, 21),\n",
       " (342, 201),\n",
       " (343, 237),\n",
       " (344, 190),\n",
       " (345, 232),\n",
       " (346, 193),\n",
       " (347, 199),\n",
       " (348, 58),\n",
       " (349, 41),\n",
       " (350, 50),\n",
       " (351, 44),\n",
       " (352, 42),\n",
       " (353, 25),\n",
       " (354, 233),\n",
       " (355, 26),\n",
       " (356, 24),\n",
       " (357, 76),\n",
       " (358, 43),\n",
       " (359, 27),\n",
       " (360, 102),\n",
       " (361, 121),\n",
       " (362, 25),\n",
       " (363, 311),\n",
       " (364, 20),\n",
       " (365, 58),\n",
       " (366, 33),\n",
       " (367, 58),\n",
       " (368, 45),\n",
       " (369, 22),\n",
       " (370, 75),\n",
       " (371, 51),\n",
       " (372, 66),\n",
       " (373, 233),\n",
       " (374, 274),\n",
       " (375, 28),\n",
       " (376, 30),\n",
       " (377, 33),\n",
       " (378, 375),\n",
       " (379, 196),\n",
       " (380, 162),\n",
       " (381, 127),\n",
       " (382, 53),\n",
       " (383, 71),\n",
       " (384, 22),\n",
       " (385, 281),\n",
       " (386, 23),\n",
       " (387, 304),\n",
       " (388, 52),\n",
       " (389, 271),\n",
       " (390, 31),\n",
       " (391, 121),\n",
       " (392, 111),\n",
       " (393, 448),\n",
       " (394, 149),\n",
       " (395, 57),\n",
       " (396, 54),\n",
       " (397, 102),\n",
       " (398, 172),\n",
       " (399, 319),\n",
       " (400, 22),\n",
       " (401, 153),\n",
       " (402, 68),\n",
       " (403, 50),\n",
       " (404, 44),\n",
       " (405, 737),\n",
       " (406, 342),\n",
       " (407, 226),\n",
       " (408, 27),\n",
       " (409, 203),\n",
       " (410, 28),\n",
       " (411, 58),\n",
       " (412, 51),\n",
       " (413, 50),\n",
       " (414, 25),\n",
       " (415, 24),\n",
       " (416, 493),\n",
       " (417, 365),\n",
       " (418, 20),\n",
       " (419, 32),\n",
       " (420, 36),\n",
       " (421, 62),\n",
       " (422, 97),\n",
       " (423, 64),\n",
       " (424, 44),\n",
       " (425, 204),\n",
       " (426, 101),\n",
       " (427, 31),\n",
       " (428, 55),\n",
       " (429, 414),\n",
       " (430, 65),\n",
       " (431, 21),\n",
       " (432, 62),\n",
       " (433, 41),\n",
       " (434, 44),\n",
       " (435, 379),\n",
       " (436, 145),\n",
       " (437, 263),\n",
       " (438, 33),\n",
       " (439, 31),\n",
       " (440, 51),\n",
       " (441, 20),\n",
       " (442, 143),\n",
       " (443, 24),\n",
       " (444, 24),\n",
       " (445, 135),\n",
       " (446, 36),\n",
       " (447, 139),\n",
       " (448, 38),\n",
       " (449, 75),\n",
       " (450, 540),\n",
       " (451, 98),\n",
       " (452, 206),\n",
       " (453, 156),\n",
       " (454, 236),\n",
       " (455, 189),\n",
       " (456, 217),\n",
       " (457, 277),\n",
       " (458, 183),\n",
       " (459, 118),\n",
       " (460, 68),\n",
       " (461, 23),\n",
       " (462, 33),\n",
       " (463, 133),\n",
       " (464, 53),\n",
       " (465, 80),\n",
       " (466, 101),\n",
       " (467, 44),\n",
       " (468, 143),\n",
       " (469, 43),\n",
       " (470, 57),\n",
       " (471, 31),\n",
       " (472, 263),\n",
       " (473, 35),\n",
       " (474, 327),\n",
       " (475, 20),\n",
       " (476, 83),\n",
       " (477, 35),\n",
       " (478, 113),\n",
       " (479, 202),\n",
       " (480, 60),\n",
       " (481, 56),\n",
       " (482, 26),\n",
       " (483, 59),\n",
       " (484, 139),\n",
       " (485, 26),\n",
       " (486, 189),\n",
       " (487, 231),\n",
       " (488, 154),\n",
       " (489, 109),\n",
       " (490, 57),\n",
       " (491, 33),\n",
       " (492, 55),\n",
       " (493, 135),\n",
       " (494, 47),\n",
       " (495, 213),\n",
       " (496, 129),\n",
       " (497, 279),\n",
       " (498, 149),\n",
       " (499, 102),\n",
       " (500, 225),\n",
       " (501, 73),\n",
       " (502, 37),\n",
       " (503, 160),\n",
       " (504, 251),\n",
       " (505, 112),\n",
       " (506, 242),\n",
       " (507, 58),\n",
       " (508, 87),\n",
       " (509, 33),\n",
       " (510, 30),\n",
       " (511, 24),\n",
       " (512, 21),\n",
       " (513, 22),\n",
       " (514, 194),\n",
       " (515, 40),\n",
       " (516, 21),\n",
       " (517, 37),\n",
       " (518, 73),\n",
       " (519, 49),\n",
       " (520, 23),\n",
       " (521, 150),\n",
       " (522, 30),\n",
       " (523, 100),\n",
       " (524, 306),\n",
       " (525, 54),\n",
       " (526, 68),\n",
       " (527, 137),\n",
       " (528, 53),\n",
       " (529, 43),\n",
       " (530, 45),\n",
       " (531, 30),\n",
       " (532, 274),\n",
       " (533, 260),\n",
       " (534, 80),\n",
       " (535, 218),\n",
       " (536, 163),\n",
       " (537, 490),\n",
       " (538, 81),\n",
       " (539, 56),\n",
       " (540, 63),\n",
       " (541, 133),\n",
       " (542, 135),\n",
       " (543, 197),\n",
       " (544, 31),\n",
       " (545, 162),\n",
       " (546, 59),\n",
       " (547, 23),\n",
       " (548, 155),\n",
       " (549, 25),\n",
       " (550, 40),\n",
       " (551, 334),\n",
       " (552, 84),\n",
       " (553, 100),\n",
       " (554, 114),\n",
       " (555, 52),\n",
       " (556, 44),\n",
       " (557, 53),\n",
       " (558, 20),\n",
       " (559, 76),\n",
       " (560, 101),\n",
       " (561, 357),\n",
       " (562, 72),\n",
       " (563, 30),\n",
       " (564, 34),\n",
       " (565, 35),\n",
       " (566, 149),\n",
       " (567, 155),\n",
       " (568, 81),\n",
       " (569, 70),\n",
       " (570, 22),\n",
       " (571, 20),\n",
       " (572, 20),\n",
       " (573, 51),\n",
       " (574, 46),\n",
       " (575, 26),\n",
       " (576, 36),\n",
       " (577, 188),\n",
       " (578, 24),\n",
       " (579, 74),\n",
       " (580, 47),\n",
       " (581, 28),\n",
       " (582, 62),\n",
       " (583, 27),\n",
       " (584, 24),\n",
       " (585, 80),\n",
       " (586, 166),\n",
       " (587, 98),\n",
       " (588, 225),\n",
       " (589, 43),\n",
       " (590, 48),\n",
       " (591, 84),\n",
       " (592, 360),\n",
       " (593, 156),\n",
       " (594, 25),\n",
       " (595, 96),\n",
       " (596, 20),\n",
       " (597, 41),\n",
       " (598, 25),\n",
       " (599, 47),\n",
       " (600, 89),\n",
       " (601, 148),\n",
       " (602, 29),\n",
       " (603, 47),\n",
       " (604, 27),\n",
       " (605, 90),\n",
       " (606, 236),\n",
       " (607, 37),\n",
       " (608, 163),\n",
       " (609, 28),\n",
       " (610, 75),\n",
       " (611, 42),\n",
       " (612, 27),\n",
       " (613, 28),\n",
       " (614, 39),\n",
       " (615, 103),\n",
       " (616, 43),\n",
       " (617, 108),\n",
       " (618, 216),\n",
       " (619, 89),\n",
       " (620, 110),\n",
       " (621, 171),\n",
       " (622, 228),\n",
       " (623, 45),\n",
       " (624, 141),\n",
       " (625, 106),\n",
       " (626, 32),\n",
       " (627, 181),\n",
       " (628, 27),\n",
       " (629, 121),\n",
       " (630, 108),\n",
       " (631, 20),\n",
       " (632, 118),\n",
       " (633, 58),\n",
       " (634, 135),\n",
       " (635, 34),\n",
       " (636, 20),\n",
       " (637, 104),\n",
       " (638, 67),\n",
       " (639, 148),\n",
       " (640, 109),\n",
       " (641, 40),\n",
       " (642, 318),\n",
       " (643, 206),\n",
       " (644, 42),\n",
       " (645, 122),\n",
       " (646, 38),\n",
       " (647, 58),\n",
       " (648, 297),\n",
       " (649, 24),\n",
       " (650, 311),\n",
       " (651, 21),\n",
       " (652, 22),\n",
       " (653, 283),\n",
       " (654, 147),\n",
       " (655, 685),\n",
       " (656, 24),\n",
       " (657, 29),\n",
       " (658, 71),\n",
       " (659, 191),\n",
       " (660, 224),\n",
       " (661, 121),\n",
       " (662, 23),\n",
       " (663, 158),\n",
       " (664, 166),\n",
       " (665, 142),\n",
       " (666, 245),\n",
       " (667, 47),\n",
       " (668, 46),\n",
       " (669, 100),\n",
       " (670, 46),\n",
       " (671, 124),\n",
       " (672, 29),\n",
       " (673, 35),\n",
       " (674, 41),\n",
       " (675, 34),\n",
       " (676, 77),\n",
       " (677, 49),\n",
       " (678, 28),\n",
       " (679, 62),\n",
       " (680, 38),\n",
       " (681, 22),\n",
       " (682, 399),\n",
       " (683, 75),\n",
       " (684, 86),\n",
       " (685, 20),\n",
       " (686, 71),\n",
       " (687, 21),\n",
       " (688, 24),\n",
       " (689, 36),\n",
       " (690, 115),\n",
       " (691, 32),\n",
       " (692, 40),\n",
       " (693, 154),\n",
       " (694, 157),\n",
       " (695, 38),\n",
       " (696, 29),\n",
       " (697, 105),\n",
       " (698, 127),\n",
       " (699, 149),\n",
       " (700, 21),\n",
       " (701, 33),\n",
       " (702, 31),\n",
       " (703, 47),\n",
       " (704, 90),\n",
       " (705, 114),\n",
       " (706, 29),\n",
       " (707, 236),\n",
       " (708, 105),\n",
       " (709, 138),\n",
       " (710, 86),\n",
       " (711, 221),\n",
       " (712, 162),\n",
       " (713, 30),\n",
       " (714, 44),\n",
       " (715, 167),\n",
       " (716, 269),\n",
       " (717, 93),\n",
       " (718, 39),\n",
       " (719, 67),\n",
       " (720, 30),\n",
       " (721, 168),\n",
       " (722, 42),\n",
       " (723, 22),\n",
       " (724, 85),\n",
       " (725, 25),\n",
       " (726, 24),\n",
       " (727, 322),\n",
       " (728, 26),\n",
       " (729, 21),\n",
       " (730, 38),\n",
       " (731, 83),\n",
       " (732, 20),\n",
       " (733, 107),\n",
       " (734, 64),\n",
       " (735, 53),\n",
       " (736, 24),\n",
       " (737, 33),\n",
       " (738, 148),\n",
       " (739, 35),\n",
       " (740, 20),\n",
       " (741, 104),\n",
       " (742, 26),\n",
       " (743, 35),\n",
       " (744, 27),\n",
       " (745, 60),\n",
       " (746, 67),\n",
       " (747, 293),\n",
       " (748, 108),\n",
       " (749, 305),\n",
       " (750, 33),\n",
       " (751, 168),\n",
       " (752, 73),\n",
       " (753, 57),\n",
       " (754, 34),\n",
       " (755, 38),\n",
       " (756, 109),\n",
       " (757, 166),\n",
       " (758, 357),\n",
       " (759, 32),\n",
       " (760, 41),\n",
       " (761, 66),\n",
       " (762, 21),\n",
       " (763, 129),\n",
       " (764, 109),\n",
       " (765, 23),\n",
       " (766, 175),\n",
       " (767, 37),\n",
       " (768, 64),\n",
       " (769, 29),\n",
       " (770, 64),\n",
       " (771, 70),\n",
       " (772, 33),\n",
       " (773, 161),\n",
       " (774, 224),\n",
       " (775, 28),\n",
       " (776, 105),\n",
       " (777, 36),\n",
       " (778, 65),\n",
       " (779, 37),\n",
       " (780, 55),\n",
       " (781, 42),\n",
       " (782, 232),\n",
       " (783, 30),\n",
       " (784, 39),\n",
       " (785, 27),\n",
       " (786, 117),\n",
       " (787, 57),\n",
       " (788, 249),\n",
       " (789, 33),\n",
       " (790, 231),\n",
       " (791, 25),\n",
       " (792, 47),\n",
       " (793, 55),\n",
       " (794, 39),\n",
       " (795, 159),\n",
       " (796, 358),\n",
       " (797, 26),\n",
       " (798, 239),\n",
       " (799, 24),\n",
       " (800, 28),\n",
       " (801, 26),\n",
       " (802, 75),\n",
       " (803, 33),\n",
       " (804, 332),\n",
       " (805, 273),\n",
       " (806, 140),\n",
       " (807, 204),\n",
       " (808, 23),\n",
       " (809, 20),\n",
       " (810, 26),\n",
       " (811, 21),\n",
       " (812, 20),\n",
       " (813, 29),\n",
       " (814, 35),\n",
       " (815, 183),\n",
       " (816, 25),\n",
       " (817, 36),\n",
       " (818, 21),\n",
       " (819, 28),\n",
       " (820, 21),\n",
       " (821, 62),\n",
       " (822, 25),\n",
       " (823, 185),\n",
       " (824, 20),\n",
       " (825, 145),\n",
       " (826, 115),\n",
       " (827, 28),\n",
       " (828, 104),\n",
       " (829, 64),\n",
       " (830, 106),\n",
       " (831, 73),\n",
       " (832, 25),\n",
       " (833, 267),\n",
       " (834, 54),\n",
       " (835, 102),\n",
       " (836, 50),\n",
       " (837, 46),\n",
       " (838, 93),\n",
       " (839, 57),\n",
       " (840, 197),\n",
       " (841, 31),\n",
       " (842, 27),\n",
       " (843, 204),\n",
       " (844, 81),\n",
       " (845, 31),\n",
       " (846, 405),\n",
       " (847, 146),\n",
       " (848, 150),\n",
       " (849, 23),\n",
       " (850, 51),\n",
       " (851, 209),\n",
       " (852, 49),\n",
       " (853, 41),\n",
       " (854, 217),\n",
       " (855, 23),\n",
       " (856, 26),\n",
       " (857, 22),\n",
       " (858, 21),\n",
       " (859, 39),\n",
       " (860, 71),\n",
       " (861, 43),\n",
       " (862, 165),\n",
       " (863, 107),\n",
       " (864, 294),\n",
       " (865, 66),\n",
       " (866, 20),\n",
       " (867, 95),\n",
       " (868, 208),\n",
       " (869, 47),\n",
       " (870, 269),\n",
       " (871, 115),\n",
       " (872, 71),\n",
       " (873, 20),\n",
       " (874, 34),\n",
       " (875, 89),\n",
       " (876, 21),\n",
       " (877, 81),\n",
       " (878, 134),\n",
       " (879, 29),\n",
       " (880, 368),\n",
       " (881, 259),\n",
       " (882, 137),\n",
       " (883, 268),\n",
       " (884, 43),\n",
       " (885, 102),\n",
       " (886, 240),\n",
       " (887, 171),\n",
       " (888, 20),\n",
       " (889, 326),\n",
       " (890, 119),\n",
       " (891, 47),\n",
       " (892, 226),\n",
       " (893, 59),\n",
       " (894, 245),\n",
       " (895, 20),\n",
       " (896, 362),\n",
       " (897, 185),\n",
       " (898, 30),\n",
       " (899, 135),\n",
       " (900, 45),\n",
       " (901, 124),\n",
       " (902, 49),\n",
       " (903, 136),\n",
       " (904, 47),\n",
       " (905, 40),\n",
       " (906, 41),\n",
       " (907, 147),\n",
       " (908, 74),\n",
       " (909, 26),\n",
       " (910, 60),\n",
       " (911, 98),\n",
       " (912, 53),\n",
       " (913, 132),\n",
       " (914, 23),\n",
       " (915, 26),\n",
       " (916, 317),\n",
       " (917, 35),\n",
       " (918, 103),\n",
       " (919, 217),\n",
       " (920, 26),\n",
       " (921, 110),\n",
       " (922, 127),\n",
       " (923, 74),\n",
       " (924, 82),\n",
       " (925, 32),\n",
       " (926, 20),\n",
       " (927, 120),\n",
       " (928, 32),\n",
       " (929, 49),\n",
       " (930, 63),\n",
       " (931, 61),\n",
       " (932, 241),\n",
       " (933, 184),\n",
       " (934, 174),\n",
       " (935, 39),\n",
       " (936, 142),\n",
       " (937, 40),\n",
       " (938, 108),\n",
       " (939, 49),\n",
       " (940, 107),\n",
       " (941, 22),\n",
       " (942, 79),\n",
       " (943, 168)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# to compute the distribution of ratings per user, we first group the ratings by user id\n",
    "user_ratings_grouped = rating_data.map(lambda fields: (int(fields[0]), int(fields[2]))).groupByKey() \n",
    "# then, for each key (user id), we find the size of the set of ratings, which gives us the # ratings for that user \n",
    "user_ratings_byuser = user_ratings_grouped.map(lambda (k, v): (k, len(v)))\n",
    "user_ratings_byuser.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Toy Story\n",
      "GoldenEye\n",
      "Four Rooms\n",
      "Get Shorty\n",
      "Copycat\n",
      "Shanghai Triad (Yao a yao yao dao waipo qiao)\n",
      "Twelve Monkeys\n",
      "Babe\n",
      "Dead Man Walking\n",
      "Richard III\n"
     ]
    }
   ],
   "source": [
    "# we define a function to extract just the title from the raw movie title, removing the year of release\n",
    "def extract_title(raw):\n",
    "    import re\n",
    "    grps = re.search(\"\\((\\w+)\\)\", raw)    # this regular expression finds the non-word (numbers) between parentheses\n",
    "    if grps:\n",
    "        return raw[:grps.start()].strip() # we strip the trailing whitespace from the title\n",
    "    else:\n",
    "        return raw\n",
    "\n",
    "# first lets extract the raw movie titles from the movie fields\n",
    "raw_titles = movie_fields.map(lambda fields: fields[1])\n",
    "# next, we strip away the \"year of release\" to leave us with just the title text\n",
    "# let's test our title extraction function on the first 5 titles\n",
    "for raw_title in raw_titles.take(10):\n",
    "    print extract_title(raw_title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[u'Toy', u'Story'], [u'GoldenEye'], [u'Four', u'Rooms'], [u'Get', u'Shorty'], [u'Copycat']]\n"
     ]
    }
   ],
   "source": [
    "# ok that looks good! let's apply it to all the titles\n",
    "movie_titles = raw_titles.map(lambda m: extract_title(m))\n",
    "# next we tokenize the titles into terms. We'll use simple whitespace tokenization\n",
    "title_terms = movie_titles.map(lambda t: t.split(\" \"))\n",
    "print title_terms.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# next we would like to collect all the possible terms, in order to build out dictionary of term <-> index mappings\n",
    "all_terms = title_terms.flatMap(lambda x: x).distinct().collect()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of terms: 2645\n",
      "Index of term 'Dead': 723\n",
      "Index of term 'Rooms': 321\n"
     ]
    }
   ],
   "source": [
    "# create a new dictionary to hold the terms, and assign the \"1-of-k\" indexes\n",
    "idx = 0\n",
    "all_terms_dict = {}\n",
    "for term in all_terms:\n",
    "    all_terms_dict[term] = idx\n",
    "    idx +=1\n",
    "num_terms = len(all_terms_dict)\n",
    "print \"Total number of terms: %d\" % num_terms\n",
    "print \"Index of term 'Dead': %d\" % all_terms_dict['Dead']\n",
    "print \"Index of term 'Rooms': %d\" % all_terms_dict['Rooms']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<1x2645 sparse matrix of type '<type 'numpy.float64'>'\n",
       " \twith 2 stored elements in Compressed Sparse Column format>,\n",
       " <1x2645 sparse matrix of type '<type 'numpy.float64'>'\n",
       " \twith 1 stored elements in Compressed Sparse Column format>,\n",
       " <1x2645 sparse matrix of type '<type 'numpy.float64'>'\n",
       " \twith 2 stored elements in Compressed Sparse Column format>,\n",
       " <1x2645 sparse matrix of type '<type 'numpy.float64'>'\n",
       " \twith 2 stored elements in Compressed Sparse Column format>,\n",
       " <1x2645 sparse matrix of type '<type 'numpy.float64'>'\n",
       " \twith 1 stored elements in Compressed Sparse Column format>]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# this function takes a list of terms and encodes it as a scipy sparse vector using an approach \n",
    "# similar to the 1-of-k encoding\n",
    "def create_vector(terms, term_dict):\n",
    "    from scipy import sparse as sp\n",
    "    x = sp.csc_matrix((1, num_terms))\n",
    "    for t in terms:\n",
    "        if t in term_dict:\n",
    "            idx = term_dict[t]\n",
    "            x[0, idx] = 1\n",
    "    return x\n",
    "all_terms_bcast = sc.broadcast(all_terms_dict)\n",
    "term_vectors = title_terms.map(lambda terms: create_vector(terms, all_terms_bcast.value))\n",
    "term_vectors.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x:\n",
      "[ 0.49671415 -0.1382643   0.64768854  1.52302986 -0.23415337 -0.23413696\n",
      "  1.57921282  0.76743473 -0.46947439  0.54256004]\n",
      "2-Norm of x: 2.5908\n",
      "Normalized x:\n",
      "[ 0.19172213 -0.05336737  0.24999534  0.58786029 -0.09037871 -0.09037237\n",
      "  0.60954584  0.29621508 -0.1812081   0.20941776]\n",
      "2-Norm of normalized_x: 1.0000\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "x = np.random.randn(10)\n",
    "norm_x_2 = np.linalg.norm(x)\n",
    "normalized_x = x / norm_x_2\n",
    "print \"x:\\n%s\" % x\n",
    "print \"2-Norm of x: %2.4f\" % norm_x_2\n",
    "print \"Normalized x:\\n%s\" % normalized_x\n",
    "print \"2-Norm of normalized_x: %2.4f\" % np.linalg.norm(normalized_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x:\n",
      "[ 0.49671415 -0.1382643   0.64768854  1.52302986 -0.23415337 -0.23413696\n",
      "  1.57921282  0.76743473 -0.46947439  0.54256004]\n",
      "2-Norm of x: 2.5908\n",
      "Normalized x MLlib:\n",
      "[ 0.19172213 -0.05336737  0.24999534  0.58786029 -0.09037871 -0.09037237\n",
      "  0.60954584  0.29621508 -0.1812081   0.20941776]\n",
      "2-Norm of normalized_x_mllib: 1.0000\n"
     ]
    }
   ],
   "source": [
    "from pyspark.mllib.feature import Normalizer\n",
    "normalizer = Normalizer()\n",
    "vector = sc.parallelize([x])\n",
    "normalized_x_mllib = normalizer.transform(vector).first().toArray()\n",
    "\n",
    "print \"x:\\n%s\" % x\n",
    "print \"2-Norm of x: %2.4f\" % norm_x_2\n",
    "print \"Normalized x MLlib:\\n%s\" % normalized_x_mllib\n",
    "print \"2-Norm of normalized_x_mllib: %2.4f\" % np.linalg.norm(normalized_x_mllib)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}