tensorflow/linear_regression/01_LineFit.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using tensor flow to fit a straight line\n",
    "\n",
    "We now (mis-)use tensorflow to create something simple as fitting a straight line."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating some data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "N = 30\n",
    "x_data = (np.linspace(0,10,N)).astype('float32')\n",
    "y_data = (2.42 * x_data + 0.42 + np.random.normal(0,1,N)).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30,)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAXMAAAEACAYAAABBDJb9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAGHVJREFUeJzt3Xl0XOWBpvGnLGsrqSTbwtgymxycsHQYE3KSQA+LaEgw\n",
       "MwkkfWYCk3Rngd6mMyQQT2JoptvuzkboLJPJmUz3pIGBJJCVmAQIa+Nkkm7o7gkGg7HBAYNZQ8DW\n",
       "UrIk27rzx1dulWwZq1TLvXXr+Z1TR7XcKr0cpNdX3/3ud0GSJEmSJEmSJEmSJEmSJKnm2oAHgPXA\n",
       "RuBzhecXAHcDjwN3AfNiSSdJmrFs4etc4H7gVOBq4JOF51cBV8WQS5I0C1ngn4HfAjYBiwrPLy48\n",
       "liQl2BzCMMsQYY8cYHvR65l9HkuSEqybMMxyJvuX96u1jyNJmjuL9wwAtwFvBl4iDK+8CPQCv55m\n",
       "+y3A0bMNKEkN6lfAskp/6CFMzlRpB34GnEUYbllVeP5ypj8AGlU6TI2tiTtAmdbEHaAMa+IOUKY1\n",
       "cQco05q4A5RpTdwBylRSd850z7wXuJ4wbj4H+AZwL/Ag8F3gYmAr8N5SvrkkqTJmWuYbgJOmef5V\n",
       "4OzKxZEkzcacuAPUgXVxByjTurgDlGFd3AHKtC7uAGVaF3eAMq2LO0Da1PuYuSTFoaTudM9cklLA\n",
       "MpekFLDMJSkFLHNJSgHLXJJSwDKXpBSwzCUpBSxzSUoBy1ySUsAyl6QUsMwlKQUsc0lKActcklLA\n",
       "MpekFLDMJSkFLHNJSgHLXJJSwDKXpBSwzCUpBSxzSUoBy1ySUmCmZX4EcB/wKPAI8NHC82uAZ4EH\n",
       "C7cVFc4nSaqgxcCJhfudwGbgOGA18PGDvDeqYi5JSquSunPuDLd7sXADGAYeAw4rPM6U8g0l6TW0\n",
       "Q8snIHs85B+AXV8FdscdKq36gKcJe+irga3AQ8A1wLxptnfPXNJMNEHufnjnCPzvCH47D7kfxB0q\n",
       "RiV1Z6l71Z3AOuDTwFrgUODlwmufAnqBi6cJ9JdFj9cVbpJU7GQ48m54shOagJ3AwjHIHw08F3O2\n",
       "Wugv3PZaTZVGPpqBO4FLD/B6H7BhmufdM5c0E6fD8QMQReG2J4L5eWBp3MFiUpXuzAA3AF/e5/ne\n",
       "ovuXATfWKpCk1MlCxzb4b7vg/gj+cBRy62ncKdRV6c5TgQlgPZPTEM8lFPzDhDHztcCiWgWSlEqH\n",
       "Q/etMP8J6L4JmB93oBglrjsTF0iS6kBJ3dmof75IUqpY5pKUApa5JKWAZS5JKWCZS6o3bcDxwMJp\n",
       "XpsD2c9DxyuQewlaPjrNNpolZ7NI2isDvBH4bcIZ5aVaDtmXYckgtI1Cx5VTX26/EpbnYXMED0Zw\n",
       "WB4y7y0/diwS152JCyQpFnOg63vQk4fjdkD218CxpX1E5zNww0Q4Q/S5CA4ZBk6ZfL3nIbg3mjyL\n",
       "9JoI5tfr+i5OTZSUSO+DpefCtixs7Iare6B7urPGD6QZ8ofD+wvrlSwBzskAJ0xuMrEDnip6y5Y9\n",
       "MP5K+dEF7plLaXIGZP4C+BPC2HUJMp+C1UV7zc9G0D5Q2md0vAS3Ft6/PYIlw8CZRRu8FdqH4WO7\n",
       "4KIxaN9OWDeqHiWuOxMXSNJstPxRGCK5Yg/8Tr6wbkprCR/wXjhmGAYKZfzZ3dB9f4kh/i20D8Ly\n",
       "HdA9ArmvTLPNsZD5M+CTTF53oR4lrjsTF0hSyTLQmoeNhSKeiOBtQ8CFpX1G7uvQsTMcwOx4htnt\n",
       "NfcAZ1DyeHvdSVx3Ji6QpJI1wZw9sLNomOT9eeCPZ/FZhxOmFrZUNmLqJK47ExdI0mx03wcfHoPn\n",
       "I7g9gmweOCbuVCmWuO5MXCBJszIfum8LY9ZdW4Gz4g6UconrzsQFkqQ64DxzSWo0lrkkpYBlLkkp\n",
       "YJlLKpYBugtfpSk8ACrVh9+BtgFoGS8sgvWWuAM1uMR1Z+ICSdrPQmgfmlxx8AdRYV2T9riDNbDE\n",
       "dWfiAknaT39Y7yQqOsNzySDhTE3FI3HdmbhAkvbzeugagZcLRf50BK2jhHVQSpElrGJ4Gp6uX66q\n",
       "dOcRwH3Ao8AjwN5LMS0A7gYeB+4C5tUqkKRK6/xcuNjDeYMwLw/tl5X4AYuhcyu8cQCWDULuYaCr\n",
       "CkEbRVW6czFwYuF+J7AZOA64mrDMJMAq4KpaBZJUFW8F3s/k73sJur8DK8cnV1X8T6OQ/UKlAzaQ\n",
       "mnTnWuBsYBOwqPDc4sLjWAJJituCh+Dvi8bcvxXBgtvjTlXHqt6dfcDTQA7YXvR8Zp/HNQskKQly\n",
       "fwsX7IRdUVgqtz8PrVce/H06gJK6c26JH94J/AD4GDA0zTc+0DdfU3R/XeEmKVWGVsLtb4QFJ8Ge\n",
       "DDTdC2NXx52qjvQXblXXDNwJXFr03CbC8ApALw6zSI0uQ7j4RG/cQVKgKqsmZoBrgI3Afy96/kfA\n",
       "Bwv3P0gYS5fUuCLgWeCFuINoeqcCE8B64MHCbQVhauI9ODVRkiotcd2ZuECSVAe8OIUkNRrLXJJS\n",
       "oNSpiZLqV4Zwhuc84F+AV+KNo3rjmLkUvybouhWWDMNbdkD7DmZ1yr5qKHHdmbhAUgP6ALxpGMYK\n",
       "p9r/nwno3hh3KL0mD4BK2ldmKZybnVyV9pwMjB0ZayRVlGUupUeGcOblofu/FD0I3xyB3xB2+P5m\n",
       "N7Q+XNt4qncOs0jVl4Wu+6BjFNrHIHcLUy8OkYGOz0PLWFirvHML4ToFSq7EdWfiAknp0/lVePdO\n",
       "GC+sWHjmCLT/xTQbzgeOBJpqHFClS1x3Ji6QlD49/w/uKlpL/NsR9NwVdyqVxQOgUuPZtRl+sivc\n",
       "j4Dbx2B0ulVMpVlzz1yqjAzhAsvTXSh5EXQ8DScMwLGD0PkY0y98p/qRuO5MXCCpDh0FuSegbQya\n",
       "x6D1T6fZJku4nOOZQGtt46kKEtediQsk1Z+u9fDp3eFCyVsiWJAHTo47laoqcd2ZuEBSnclAZg+M\n",
       "Fh3g/IOdwCVxB1NVeQBUSpkIsq/AzwsPx4Cf7yZc0UeqGffMpfK9HbLDcM4AHDUEXbfgzljaldSd\n",
       "mWqlKBLV6PtIaXcUYQnbl4Gf4o5S2iWuO/2Bk6TSOWYuSY3GMpekFLDMJSkFLHNJSoGZlvm1wEvA\n",
       "hqLn1hDmuT5YuK2oaDIpXc6GeZsg9yLkvg60xR1Ijek04E1MLfPVwMdn8F5ns6jRnQAdebglgo0R\n",
       "vGMEur4RdyglXlVms/xfYPs0zydqDqSUUP8OLm6B84DjgOvaYde799mmGeZcBrlrIfMRvHiESlTu\n",
       "mPklwEPANbjcpnQgedi2a/Lh80DTaNHrc6Drdjj90/CZD8Pbroau7+HOkqqkj6nDLIcSftgywKcJ\n",
       "hT6diDC+vvfWX514Uuy6gNw0z8+Djm3we2PwmQh68tD0oaLXl8Oi4XDJtyiCkQi6RoClNcis5Ohn\n",
       "aldWbYi6j6llPtPXHDNX2rWGtVJaxqF5HHLfBZr32WYBzLkSWr9EWHO82MmwbGByRcSJCBYPAcfX\n",
       "JL2SqmZl3lt0/zLgxloHkpKh4/PhoObOCPIRnJaH9j8v4QPaoeMZWL0L1kewchw6HwfmViux6kJV\n",
       "uvMmwkDfOLANuAi4AXiYMGa+FlhUy0BScvQ8ALcXrTX+/Qh6/r7EDzkC5t0B3c/AvB8Di6uRVHUl\n",
       "cd2ZuEBSZXXdCJ/YNVnmHxmDzr+NO5XqXuK6M3GBpArrhY7n4G0D8OZB6NgKHBJ3KNU91zOXYpAj\n",
       "zEaIgPuAfKxplAaJ6073zCWpdK5nLkmNxjKXpBSwzCUpBSxzSUoBzzCTZmYJ8FbgN8Av8MC+GpA/\n",
       "9Kp3p0P7MPTvgMOHoOuH+Fetqi9x3Zm4QNI0FgJvYdqTfTqfg1sLZ3eORnDcEPC7Nc6nxpO47kxc\n",
       "IKXKHOj4IrSOQGseOr5AyXvNze+D9hFYtgPaRqDpvVNfb9odFtDae7r+n4wCl1bqP0A6gMR1Z+IC\n",
       "KU3aVsK/ycO2CJ6J4I15aC2laBeFIt9QKOr1USh0eiY36VoPn9kdlqZ9OoKFecKlFKVqSlx3Ji6Q\n",
       "0qTnZ7C2aK/55gh61pXwASfDMTsm3x9F8LoB4KSibfqg81fQsTOsV962sqL/CdL0SupOZ7Oozu1+\n",
       "CR6dgPMLQysb9sCuF0v4gKfgmRZ4FPgtworOzzcDTxdtsxWGlxGurjUIu3ZWJLpUZ9wzVzUtg/bt\n",
       "cOEIXDAS7nN0aR+xd8z89QPQnt9/zFyKhasmquEsAd5D+Fn7IfDCLD7jUMLVtJ4CXq5YMmn2Eted\n",
       "7pkrbseHq/f0PADtq3COuOpD4rozcYFUdw4jzOvup/QiPhLaB+CLE3BbBCcOQ/ZLFU8oVV7iujNx\n",
       "gVRXToPsEJw1AH1D0HUb0FTC+z8GHxqdnKnydATtQ9UKK1WQ65krTXLfgm93wj1dsLkTjjod+A8l\n",
       "fEAEu4p+KXYVnpNUKn9xVIbmMRgomgN+yRjwiRI+oBeyr8Kf74GbInjDMGT/qlpppQpKXHcmLpDq\n",
       "Sfc/wep9z748o8QPWQq5G2DBndD8xyRshoB0AInrzsQFUl05AnKboX3Usy/VYKrSndcCLwEbip5b\n",
       "ANwNPA7cBcyrZSA1lAxhNcPWuININVSVA6DXASv2ee5yQpm/Abi38Fgq1TzIXQs9v4Su64D502wT\n",
       "ES4KMVbbaFI69TF1z3wTsKhwf3Hh8XTcM29s7cAfAKsIV+opNhdyD8NFY3B3BBePQW4DrhkkQRW7\n",
       "s4+pZb696H5mn8c1CaTEaw9lfWYeLt0F3XnIXFD0+gnQOxQObkZR+LpkCFgeV2ApQWJZNTE6yDde\n",
       "U3R/XeGm9LsATnwd3JsN/96/by70fw1GvlN4fQ/szsAewo/iBOExe+IKLMWov3Cruj72H2ZZXLjf\n",
       "i8Ms2t9l4ao8e+eI74hgbvG49xzo+jmcPwLfjuDdI5D7BzyZTYIaDrNcTRgHhXDw86paB1LiLYfO\n",
       "EfhpBC9H8IEx6L5jn22y0P7ZMAc8+7nwWBJV6s6bgOeBcWAb8GHC1MR7cGqiXtv50PlCuEZn920c\n",
       "+OdE0lSJ687EBZKkOuBCW5LUaCxzSUoBy1ySUsAyl6QUsMwlKQUsc0lKActcB9MNnA6cgBd1kBqa\n",
       "88zr1/JwybUTdkBPHnLfxEKXaiVx3Zm4QJqprk1w3URYVyUfwbFDlHYxZUmz50lDqpTRPji/sCee\n",
       "Bf59G/D6GANJOgDLXK+hfRNcPxHuvwp8fwx4OM5EkuLjMEt8MsBJwDuAhbN4/zLoeA56B8MFlTu/\n",
       "hGPmUq0krjsTF6hBZMIBy4V5eOsOaB8ETpnF57QAxzG5dr2k2khcdyYuUIN4FywbCgcuowjWRtD5\n",
       "bNyhJM2YB0AFwNFwdvPktR7OAUYW4zCJpFlyzzweZ8CheXi+sGf+lT3QtTHuUJJmLHHdmbhAjaP9\n",
       "SmgZg55h6HiW6kwrzBDG03txr1+qpMR1Z+ICNZh5wFJgbhU+uxW67oCO0XDrugdor8L3kRpR4roz\n",
       "cYFUKdnPwjkjMBaF27tGoPMLcaeSUiJx3Zm4QKqUnnVwc2FMPorg1gh6/jHuVFJKOJtFtTK+CX4y\n",
       "Hn7mIsL98cfiTiWpOtwzT68F0LkZjh2E4wehcwtwSNyhpJRIXHcmLpD+1bHQ9Qg0j4YVEjlhFp/R\n",
       "CvQDZwJtlQwnNbiad+dWwuJLDwL/NM3rlnkytUHHS/C1CdgewTUTkP0NkIs7mCQghu58CljwGq9b\n",
       "5tXVxOymHS6HIwcnD15GERyzAzi5wvkkzU4sB0A9WaT2miD3N9A0Gm65bwDNJbz/VXilGbYXHg4B\n",
       "L7aE5yU1oicJQyz/AvzhNK+7Z14VrSvhzXl4NYLBCE7LQ/avSvuM3P+AvmG4dFdYlCv3d9XJKmkW\n",
       "at6dvYWvC4H1wGn7vB4Ba4pu/bWJlXY998B3i4ZIfhJBz3THLF5LBngnsAo4H//CkuLUz9SujHVH\n",
       "eDWwcp/n3DOvity1sHLXZJn/5W7o/kHcqSRVTE27M8vk7IcO4BeEq9rEFqiB9EL2BThrCFYMFWai\n",
       "LI07lKSKqWl3LiUMrawHHgGuiDtQg5kHvA/4PaAn5iySKitx3Zm4QJJUB1ybRZIajWUuSSlgmVfP\n",
       "2dD1JGQHoPsWoDvuQJJUjkYcMz8Gsnm4LYJtEfz+KHTfFXcoSXUlcd2ZuEA18J/hAyOTc8BHIpiz\n",
       "G/8SkjRzHgBNgEHYsmfy/8WThGVmmYgxkySVpRH3zNugcwO8Iw9/tgd68tDyR/ts0wKtV8L8W6Dt\n",
       "U4QTsCRpr8R1Z+IC1UgW+FPIrGb/9WgykLsTzhqB6yM4byfk7icsZytJkMDuTFygBHgddBeuah9F\n",
       "sDuCw4aAN8cdTFJiOGZeB5qheWLymhJzgNaI2V1kQpJqwj3z/TVBbj1cPAY/jeDS8cLFkFuLtumC\n",
       "7h9Byyh0vAyZC+MKKykWievOxAVKiPnQdQMseBS6vwscOvXl7h/ChaPwSgT3R9Cdx0u6SY0kcd2Z\n",
       "uED1oW0YXii6+MR/3Q1cGXcqSTXjmHk6NA/ApsL9CNgwhtfnlBQj98xn5zzoHIH/MgZn5aHzMcIF\n",
       "QCQ1hpK6sxbXfIxq9H3S6ETgbGA7cBMwEm8cSTWUuO5M8555K9AWdwhJqZS47kxcoApogs5rYO7u\n",
       "cOu6manTCiWpXInrzsQFKl/rSnhLHgYKKyK+fQQ6/zruVJJSJXHdmbhA5VvwE7ixaNrgXRH0/DLu\n",
       "VJJSxamJFXIWtH0FmlYDC6e+NLYVfjE++fgf9sDurTXMJkk1V4d75k0fCMvWXhXBxeOQfQHoKdpg\n",
       "IXQ8A6cMQv8gZH8NHBVTWEnplLjuTFygg8u9AP9YNIxy4Shw2T4bdQK/C/xHYH7NI0pKu5oPs6wg\n",
       "nKr4BLCqAp+XAHvaYXHR48PnQmbfE3aGgZuB7xHmgUtS3WoCtgB9QDOwHjhun23qcc/869Cfh4cj\n",
       "WBtBZ55wAo8k1UpNu/MU4I6ix5cXbsXqsMxpgdxXoes5mLcReHvcgSQ1nJK6s9yLIRwGbCt6/Czw\n",
       "tjI/MwnGYegS4JK4g0jSTJRb5jP9l2NN0f11hZskaVI/+18vuGZOZuowyxXsfxC0HodZJCluNe3O\n",
       "ucCvCAdAW0jNAVBJil3Nu/NcYDNhVssV07xumUtS6RLXnYkLJEl1wLVZJKnRWOaSlAKWuSSlgGUu\n",
       "SSlgmUtSCljmkpQClrkkpYBlLkkpYJlLUgpY5pKUApa5JKWAZS5JKWCZS1IKWOaSlAKWuSSlgGUu\n",
       "SSlgmUtSCljmkpQClrkkpYBlLkkpYJlLUgpY5pKUAuWU+RrgWeDBwm1FJQJJkmprNfDxGWwXVTtI\n",
       "lfXHHaBM/XEHKEN/3AHK1B93gDL1xx2gTP1xByhTSd1Z7jBLpsz314P+uAOUqT/uAGXojztAmfrj\n",
       "DlCm/rgDlKk/7gC1VG6ZXwI8BFwDzCs/jiRpNg5W5ncDG6a5nQf8L2ApcCLwAvDF6sWUJL2WSg2T\n",
       "9AE/Bk6Y5rUtwNEV+j6S1Ch+BSyb6cZzy/hGvYQ9coD3EPbYpzPjMJKk2rsBeJgwZr4WWBRvHEmS\n",
       "JEkH9dfAY4S9+JuB7njjzMgKYBPwBLAq5iylOgK4D3gUeAT4aLxxZq2JcELaj+MOMgvzgO8Tfu43\n",
       "AifHG6dkVxB+fjYANwKt8cZ5TdcCLzF1qHcBYQLH48BdJHu23XT5E9uZb2dy5sxVhVuSNREO3PYB\n",
       "zcB64Lg4A5VoMWGWEUAnsJn6yr/Xx4FvAT+KO8gsXA9cVLg/lwT9Ms5AH/AkkwX+HeCDsaU5uNOA\n",
       "NzG1DK8GPlm4v4pkd850+euiM98DfDPuEAdxCnBH0ePLC7d6tRY4K+4QJTocuAc4k/rbM+8mlGG9\n",
       "WkDYAZhP+Ifox8DZsSY6uD6mluEmJo/jLS48TrI+DjyJZEadGcdCWxcBt8fwfUtxGLCt6PGzhefq\n",
       "UR/hX/0HYs5Rqi8DnwAm4g4yC0uBl4HrgF8CXweysSYqzauE80aeAZ4HdhD+Ya0niwhDFxS+1vME\n",
       "jRl1ZiXL/EAnGL2raJsrgXHCGFyS1ft6Mnt1EsZtPwYMx5ylFO8Efk0YL6/HJSPmAicBXyt8zVNf\n",
       "f9kdDVxK2BFYQvg5en+cgcoUUb+/04nszA8BvwDaYs4xEyczdZjlCurvIGgzcCfhl7LefJbwl9FT\n",
       "hHMZ8oSpsPViMSH7XqcCt8aUZTYuAP6u6PHvA/8zpiwz1cf+wyyLC/d7qc9hlg+RwM5cQTgyfkjc\n",
       "QWZoLuHsqz6ghfo7AJohlN+X4w5SAWdQf2PmAD8D3lC4vwb4fHxRSracMAuqnfCzdD3wkVgTHVwf\n",
       "+x8A3bsDdjkJPYBYpI+p+RPbmU8ATzO59vnX4o0zI+cSDgJtIeyZ15NTCWPN66n/9ebPoD5nsywH\n",
       "/pkETi2boU8yOTXxesJfekl1E2Fsf5zwF92HCQdx76E+pibum/8i6rMzJUmSJEmSJEmSJEmSJEmS\n",
       "JEmSJFXS/we1mg6Ml5wypgAAAABJRU5ErkJggg==\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10f8d5610>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(x_data,y_data)\n",
    "np.shape(x_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building the TensorFlow Graph\n",
    "\n",
    "We are now defining the model for TensorFlow, for finding the optimal values we need to minimize the residual sum of squares: The sum of the quadratic deviation of the data $y_i$ to the model $\\hat{y}_i = a \\cdot x_i + b$ for all $N$ data points $i$\n",
    "\n",
    "$$\n",
    "    \\sum_{i=1}^N(y_i - a*x_i+b)^2\n",
    "$$\n",
    "\n",
    "As a first step we write a TensorFlow-graph, with which we can calculate this residual sum. What do we need?\n",
    "\n",
    "* 2 variables (a,b) `tf.Variable` with the start values e.g. 1.0, 0.01\n",
    "* 2 data vectors, or in TensorFlow parlance one dimensional tensors in which we can feed the data\n",
    "\n",
    "You might ask, why we do the differentiation between variables and place_holders. Variables reflect the parameters of state of the model, while placeholders the data we feed in. Later, for more complex models, we won't feed in all data at once but random batches (minibatches) of the complete data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss 370.624694824 (a=3.0, b=0.42)\n"
     ]
    }
   ],
   "source": [
    "# Defining the graph (construction phase)\n",
    "a = tf.Variable(1.0, name = 'a') #Note that 1.0 is needed\n",
    "b = tf.Variable(0.01, name = 'b')\n",
    "x = tf.placeholder('float32', [N], name='x_data')\n",
    "y = tf.placeholder('float32', [N], name='y_data')\n",
    "loss = tf.reduce_sum(tf.square(a*x + b - y)) #Sum is called reduce_sum \n",
    "\n",
    "init_op = tf.initialize_all_variables() #Initialization ('symbolic'), this creates a new operation in the graph. Which\n",
    "# initializes the graph. We call this later, when we have a concrete realization of the graph\n",
    "\n",
    "# Working with a concrete realization (launching a graph). \n",
    "# A session sess is created  with the with construct \n",
    "# the session is closed, when moving out of the with class\n",
    "with tf.Session() as sess: \n",
    "    sess.run(init_op) #Doing the initialization on the concrete realization of the graph\n",
    "    res = sess.run(loss, feed_dict={x:x_data, y:y_data, a:3.0, b:0.42}) #Evaluation until loss using the provided data\n",
    "    print(\"Loss {} (a=3.0, b=0.42)\".format(res))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimizing the variables a and b\n",
    "\n",
    "We now let TensorFlow find the optimal (lowest loss) values of a, b. We do this with a `GradientDescentOptimizer`, which does a steepest descent with a given learning rate. The opimizes is given the loss to minimize and it builds the necessary gradients (using at the graph) for doing the gradient descent itself."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch0 Loss 2296.54980469 (a=1.30410981178, b=0.056304179132)\n",
      "Epoch1 Loss 1448.64746094 (a=1.54495990276, b=0.0932072401047)\n",
      "Epoch2 Loss 916.657714844 (a=1.73570239544, b=0.122663378716)\n",
      "Epoch3 Loss 582.875671387 (a=1.88675498962, b=0.146220505238)\n",
      "Epoch4 Loss 373.451965332 (a=2.00636935234, b=0.165104717016)\n",
      "Epoch196 Loss 19.0639801025 (a=2.43587446213, b=0.413452327251)\n",
      "Epoch197 Loss 19.0573062897 (a=2.43575453758, b=0.414259552956)\n",
      "Epoch198 Loss 19.0506477356 (a=2.43563485146, b=0.415065526962)\n",
      "Epoch199 Loss 19.0440216064 (a=2.43551516533, b=0.415870279074)\n"
     ]
    }
   ],
   "source": [
    "train_op = tf.train.GradientDescentOptimizer(0.0001).minimize(loss) #A new activity doing one optimization step \n",
    "# when called.\n",
    "# Other optimizers \n",
    "#train_op = tf.train.AdamOptimizer().minimize(loss) \n",
    "#train_op = tf.train.MomentumOptimizer(0.0001, 0.9).minimize(loss)\n",
    "\n",
    "epochs = 200\n",
    "results = np.zeros((epochs, 4))\n",
    "init_op = tf.initialize_all_variables() #Initialization\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init_op) #Running the initialization\n",
    "    for e in range(epochs): #Fitting the data for some epochs\n",
    "        res = sess.run([train_op, loss, a, b], feed_dict={x:x_data, y:y_data})  \n",
    "        results[e] = res\n",
    "        if ((e < 5) | (e > epochs-5)):\n",
    "            print(\"Epoch{} Loss {} (a={}, b={})\".format(e,res[1],res[2],res[3]))\n",
    "    res = sess.run([loss, a, b], feed_dict={x:x_data, y:y_data})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inspection of the results\n",
    "Learning curve and fit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAtUAAAEPCAYAAACN/ji/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcXWV9+PHPnUlCJjOTSQJCkKCD4L5AEBBxYRREoNZK\n",
       "qUsrClhtbX8FLCqbVKK2ilgrLuhLZBGoooKK0lZlkYAiIEJAWcImKEFIEAjJDAlZ5vn98ZwwdyZz\n",
       "J3PmLueecz7v1+u8cu65y3yfy+U733nus4AkSZIkSZIkSZIkSZIkSZIkSZIkSZIkSZIkSZIkSRrj\n",
       "HGA58Luqa/OAy4G7gcuAOVX3nQjcAywFDmhRjJKkic0EbgBuAe4APpNcnyifS5Ia6HXAQkYX1acB\n",
       "xyXnxwOnJucvISbs6UA/cC/Q0ZIoJUlbMiv5dxpwPfBaaudzSVIT9DO6qF4KbJecz09uQ+ylPr7q\n",
       "cT8F9m52cJKkVGYBNwIvpXY+l6RSanVv8HbEISEk/25KyM8GllU9bhmwQwvjkiTV1kH8NnE5cBVw\n",
       "O7XzuSSV0rQMf3ZIjonulyRlbxjYDegDfga8Ycz9W8rnklR4rS6qlxO/JnwE2B5YkVx/CNix6nEL\n",
       "kmtj3Qvs3MwAJamJ7gN2yTqIOjwJ/C/wSmrn87HM25Lyqq1ydj+bT1TcNHb6BDafqDgD2InYiMo4\n",
       "r5eiJyQMQ+hMFW0xLco6gJxZlHUAObMo6wByJo+9udswsrJHF3ANsB+18/lYeWrzoqwDSGlR1gGk\n",
       "sCjrAFJalHUAKSzKOoCUFmUdQAqp8lcze6ovBPYlJuQHgY8Tk+73gL8HHgDekTz2juT6HcAG4J+p\n",
       "PxGvI64msrHO15GkMtseOI84rroDuAC4EljC+PlckkqpmUX139a4vn+N659OjkZZT+z5XtvA15Sk\n",
       "svkdsPs41x+ndj6XpNIp8lrQ64k91WW3OOsAcmZx1gHkzOKsA5AaaHHWAaS0OOsAUlicdQApLc46\n",
       "gBQWZx1ASouzDkBRmjHVj0CY37xQJCm1PI0vbpQytllSMaTKX0XvqZ6RdRCSJEkqviIX1ZsmKkqS\n",
       "JElNVeSi2p5qSZIktUTRi2p7qiVJktR0RS6qHf4hSZKklihyUe3wD0mSJLVE0Ytqe6olSZLUdEUu\n",
       "qh3+IUmSpJYoclHt8A9JkiS1RJGLanuqJUmS1BJFLqrtqZYkSVJLFL2otqdakiRJTVfkotrhH5Ik\n",
       "SWqJIhfVDv+QJElSSxS9qLanWpIkSU1X5KLa4R+SJElqiSIX1Q7/kCRJUksUuai2p1qSJEktUeSi\n",
       "2p5qSZIktUTRi2p7qiVJktR0RS6qHf4hSZKklihyUe3wD0lqjB2Bq4DbgduAo5Pri4BlwJLkODCL\n",
       "4CSpHUzLOoAmcviHJDXGeuBfgVuAHuAm4HIgAP+VHJJUakUuqh3+IUmN8UhyAAwCdwI7JLcrmUQk\n",
       "KW9eCewK3AdcnXEsIvaKTPah74NwbvNCkaTUUuSwttUP/IHYY30K8ABwK3A2MGecxxehzZLq0vUh\n",
       "mDsE7xiEZw9C7xlZRzRJhc5faYrqwyD8d/NCkaTU8p6ge4DfAG9Lbm9L7KmuAP9OLKzHCsSx15uO\n",
       "geaGKKnN9MFWa+GBACHAygDzhoCXZx3YOAYYna/ynrMnlKaofieE7zUvFElKLc8JejrwM+BDNe7v\n",
       "B343zvU8t1lS/XaGbQZjQb3p2GslcEDWgU1CqvxV9NU/HFMtSfWrEHuh7wBOr7q+fdX5IYxfVEsq\n",
       "tz/Cy5+GVcQadTFwWyfw20yjUqqe6rdA+J/mhSJJqeW11/a1wDBx9Y9Ny+cdBJxP/MV4K3AJsN04\n",
       "z81rmyXVLXRC+BisewyOfBQ6N0LX48Abs45skgqdv9IU1W+GcFnzQpGk1AqdoGsoY5ulEpr+QZj9\n",
       "EPSugFmfhpXPhXA1hJ9DWJA8qIt8rRhU6PyVpqh+I4SrmheKJKVW6ARdQxnbLJVMxyEwfwhuCHB7\n",
       "gOPWwuAghONjb3VuFTp/pSmqXwvh2uaFIkmpFTpB11DGNkslM+ciOLNqIuJQgIPuzTqqBnCiYsKJ\n",
       "ipIkSU23/jHYMDxy+4fA9Y/UfHhBuaOiJEmSpih0wgOD0F2BMzfAfR3wlbXw1HFZR1YWJwK3E5df\n",
       "+jawFTAPuBy4G7iMunfmCi+DcHu9gUpSA5VxKEQZ2yyVRHjOyGTEf98TKidBxyeAl2UdWYO0ff7q\n",
       "B35PLKQBvgscDpwGbPqr5njg1HGem6aofiGEu6capCQ1Qdsn6CYoY5ulEgjvgLCiAJMRJ9L2+Wse\n",
       "cBcwlzj85FLgTcBSRtY4nZ/cHitNUb0ThPvrCVSSGqztE3QTlLHNUoGFXgjfjB2XYY+so2myXOSv\n",
       "fwBWAyuAC5JrT1TdXxlze5M0RfUCCA9NMT5JaoZcJOgGK2ObpYIKe0G4B8JZEHqyjqYFUuWvLCYq\n",
       "7gx8iDgM5EngIuCwMY8J1G7IoqrzxckxHlf/kJS1geSQpBwLncAJwNHA/4PKxRkHpMQ7gbOqbr8H\n",
       "OAO4kzjsA2B76h/+MRfCyqmFKElNUcZe2zK2WSqQZyYjXgVhx6yjabG2X6d6KbA3I1tV7g/cQRxb\n",
       "fXjymMOBS+r8OfZUS5IkTVl4B/Ab4CfA/lB5MOOANI7jGFlS7zxi8TsPuILGLak3A8K6egOVpAYq\n",
       "Y69tGdsstbPnAX8JvHyc+w6FubfBc++FX/8ahu+BsGeL42snhc5faYrqSrJdZqV54UhSKoVO0DWU\n",
       "sc1Sm5r+d9D9FLx+JcwZgu5/r7rzAJg7BNcGeCrAxevhuR/JLNT2UOj8lbJxYQMEh4BIaheFTtA1\n",
       "lLHNUjuaBVutgd+F2Om4IsTCmpfGu7e+EK5L7gsBrgww77ZMI85e24+pbiW3KpckSYJtoXvjyGaH\n",
       "zwJesh54bpyM+IuBkfUiAB4DWNPiGNVCaXuqn4Qw3thsScpCGXtty9hmqR3NgK7H4ZKkJ3pJgO4h\n",
       "uOmf486IS/8LelbDycNwWoDeIeDNY17jL4GPE5dCLuouitUKnb/SFtV/hvCs5oQiSakVOkHXUMY2\n",
       "S+3qVbGwnjMEWz8Ft12VbOayaTLiC6HrS9B9JvCa0U/t/iw8ZxBO3AgLB2H2pcRV3Iqs0PkrbVH9\n",
       "Jwg7NCcUSUqt0Am6hjK2WWpn0+HSt8DGeyGcPcmdEefCjKfh0aSXe22AZw8CezU72IwVOn+lLaof\n",
       "gLBTUyKRpPQKnaBrKGObpTYVOiGcFId7hLeneGJ/XBlkuGoi454rgQOaFGi7KHT+SltUL4Xw4uaE\n",
       "IkmpFTpB11DGNkttKOyY7Iy4eAo7I3ZCz33wiQ3wSIDzhqFrJbB1EwJtJ4XOX2mL6lsgLGxOKJKU\n",
       "WqETdA1lbLOUpdfBvJ/B1ldD5Z3xUnh70jt9YuytnpLnQN91MHMQ+u4AylBfFTp/pS2qr4OwT3NC\n",
       "kaTUCp2gayhjm6WsvCqu6PGNAN8LsPMQ3HL1mMmImrxC56+0RfXPIbyxOaFIUmqFTtA1lLHNUkZ6\n",
       "z4XPVY17Hgzwg8cnORlRm3Pzlyprga6sg5CkHNsRuAq4HbgNODq5Pg+4HLgbuAxwTwApe2F0HXgX\n",
       "8P4HoTKYVUBqX2l7qr8P4dDmhCJJqeWx13Y+sFty3kP8Lf1i4DTguOT68cCpNZ6fxzZLOfX3B8Mv\n",
       "NsKyAD8I8KwhqLwr66hyrND5K21R/S0IhzUnFElKrQgJ+hJgf2ApsF1ybX5yezxFaLOUhTnAtkx6\n",
       "g5VNkxF/cwY860dxsmLHIc0MsAQKnb/SFtVnQfhAc0KRpNTynqD7gT8AvcATVdcrY25Xy3ubpVbr\n",
       "gN5vwox1MGstzP4lMLv2w0MPhHOcjNgUqfLXtGZF0SbWAjOzDkKSCqAH+D5wDLB6zH2BiX/5LKo6\n",
       "X5wcksY17YPw/L+BxdPjtLAj9oBLvwyrDt/8sWEv4FvANcBCx07XbSA5SiFtT/XnIBy35cdJUkvk\n",
       "tdd2OvAz4ENV15YSh30AbI/DP6QG6bsQvl61gsf1AebeM/oxoTNZc3p5yp0RlY6rf1RZgz3VklSP\n",
       "CnA2cAdwetX1HwObes4OJ461llS3NXfD5WtH6rkrN0K4b+T+sCNwJfBmYA+oXNT6GFUEaXuqT4Lw\n",
       "meaEIkmp5bHX9rXAMHALsCQ5DiQuqXcFW15SL49tlrLUA723wgtXwV5PwqzlwE7xrobsjKjJK3T+\n",
       "SltU/yuELzQnFElKrdAJuoYytlmq1wxgP+AgoA9Cr5MRM+Hwjypu/iJJkvJmHXGIx08gvBC4mVjg\n",
       "LYTKvdD3PZjzB5hzFbBzloEqv9L2VB8B4ZvNCESSpqCMvbZlbLPUAKEzGca6omoyYgVm/wbe/zQs\n",
       "CXDaRpi1AujLMtICK3T+SltUvwvCd5sTiiSlVugEXUMZ2yzVKewIYXFy7Fh1x/bQswY2Vq0OsseT\n",
       "xEmLajyHf1RZg8M/JElSboS3AzcRl7HcDyoPVt35NKzvgE3LUQ8DT1SI9Y6UStqe6gMgXNacUCQp\n",
       "tTL22paxzdIUTHZnxN5zYNdB+HKAv1gDvTdS/M38slLo/JW2qH49hF80JxRJSq3QCbqGMrZZSins\n",
       "mRTT58TiekIdUHk/9J4LncfjN/LNVOj8lbao3hPCjc0JRZJSK3SCrqGMbZYmow9m7wEP/8eYyYhq\n",
       "H4XOX2mL6pdDuK05oUhSaoVO0DWUsc3SluwHLxiEGzbANRvhDR/LOiCNq9D5K21RvQuEe5sTiiSl\n",
       "VugEXUMZ2yxNZAYc9hQ8naze8fsAvU8BL8w6MG2m0PkrbVG9AMJDzQlFklIrdIKuoYxtlmoIPfD4\n",
       "d+G+4ZEl8UKAfVcCb806Om3GJfWqrAFmZh2EJEkqu7AnsAR618CrhuCXyfUHgSXTgXuyi01llLan\n",
       "ehaEp5oTiiSlVsZe2zK2WaoSOiGcOGYy4puhaxBeuBJmrYFZx2YaomopdP5KW1R3QBiGUGlOOJKU\n",
       "SqETdA1lbLOUeGZnxKshPGfMnVsDewMLWh+XJqnQ+WsKjQtPQ9iq8aFIUmqFTtA1lLHNKpa5wHwg\n",
       "ZQdd+BsIyyGcFHurlUOFzl9TKaqfhDCn8aFIUmqFTtA1lLHNKoaOuMHKVuugZw3M/jUwiXoi9EA4\n",
       "O64+FvZqepRqpkLnr6kU1Y9AmN/4UCQptUIn6BrK2GYVQscHYeEQrAqwIcCRa6HvwomfM2pnxN7W\n",
       "xKkmykX+mgNcDNwJ3AG8CpgHXA7cDVzG+H8NTqWovh/CTlMNVJIaKBcJusHK2GblTxebDe+YfQF8\n",
       "tWrZu98EmPP78Z8eOiGc4M6IhZOLJfW+CPwf8GLgFcBS4ARiUf0C4MrkdiOsxWX1JEnS5l4EPfdD\n",
       "5yDMXAkcPHLXmrvgp2thOLl9+UZgbFG9LbzrUBj6JXAQsAdULmpF4BJAH5t/KCEW1tsl5/OT22NN\n",
       "paf6Zgi7p3+eJDVcGXtty9hm5UMHdC+Drw3DcIBrA8waAjat0jELem+C56+CPZ+EWcuB51U9/2B4\n",
       "9xp4bBg+uQ7mnDjm9feFeT+Hra+DaUe0okFquLbPX7sBNwDnAjcD3wC6gSeqHlMZc3uTqRTVv4Kw\n",
       "T/rnSVLDtX2CboIytln5MD9OQKze2fANK4G3VT1mOjAAvJnYKZi4eC6ctx6eSp73UIC+IeBlyQNe\n",
       "BT1D8M0APwqwYAimfaA1zVIDtX3+2gNYD+yZ3D4d+BSbF9GPj/PcqRTVV0LYL/3zJKnh2j5BN0EZ\n",
       "26x8mAnT18JdSWE8GGC7QeI8rwmEPWDd7+FbG0YX5ANP8kxB3nMWnFZ13xUB5t7Z7Aap4VLlr2nN\n",
       "imICy5LjxuT2xcCJwCPEYR+PANsDK2o8f1HV+eLkmMha4gQESWq1geSQ1H7WQuVfYK8vwpuG4YYK\n",
       "rLkY+PX4Dw+dwEeBY2H9MfD+r8E2fXAAcB9w0zTiAgxA2ABPB56Z/LgOqGxobnNUVtcQJyRCLJJP\n",
       "S47jk2snAKeO87yp9FRfHBdgl6TMlbHXtoxtVr7sChwBvJGaG7yEBRCuGrMz4uuh60lYsApmroGt\n",
       "Plj1hFfArEH4/DCcG2CbIai8q5mNUFPkIn/tSuypvhX4AXGc0jzgChq/pN4FEN4z1UAlqYFykaDH\n",
       "cQ6wHPhd1bVFxG8dlyTHgTWem9c2S4lw6AQ7I3YTx1FvPc4Td4e+78K8S4G3Nj1MNUOh89dUiupv\n",
       "QHBygKR2kNcE/TpgIaOL6lOAYyfx3Ly2WaUXeiCcleyMuIVx1iqoXKxT3UprcEy1JNXjF4y/IlON\n",
       "r8qlvAt7EFco6wQWQuWGjANSDpShqHbzF0lqjqOIw/jOZvwhe1LObNoZkf8DTobKkVBZnXVUyocs\n",
       "Vv9oNXuqJanxvgZ8Mjn/FPB54O9rPHZR1flitrxqk5SBsAC4gNjhuAdU/phxQGq9AUq0YtNUxlSf\n",
       "COEzjQ9FklLL8/jifkaPqZ7sfXlus0pjwsmIKq+2X6e61dYwsv25JKkxtgceTs4PoXZRLbWx0EPc\n",
       "hG4AeKtjp1WPshTVDv+QpKm7ENgX2AZ4kLjyxwCwG7En537gH7MKTpqasAfwbeBa4mREx06rVKYy\n",
       "/ONwCOc3PhRJSq2MQyHK2Ga1tU2TEcMKCO/IOhq1NZfUG2MImJV1EJIkqSl2gL7rYMYa6L0f2Gec\n",
       "x2wTH7dmAXGjuYOIkxG/18pApXYylZ7qAyBc1vhQJCm1MvbalrHNap0K9C6FkzfAYwG+H6BrNXHM\n",
       "P0AH9H4TZj4N73kaHlsPj33SyYiapIbnrx7i4ucALyRutTm90T9kkqZSVO8N4frGhyJJqZWxwCxj\n",
       "m9U6z4JZa2E4QEiON6wkTp4FKv8ArxmCp0N8zH+sgz57pzVZDc9fNxOHT+wAPABcBHyr0T9kkqZS\n",
       "VL8Uwu2ND0WSUitjgVnGNqt1umD6OliWFNTrAvSv5pm1hg+9FJ6oKrhvDjDngQzjVb40PH8tSf49\n",
       "CjguOb+10T9kkqZSVO8I4cHGhyJJqZWxwCxjm9VSM0+C+YPw4fWwcBBm/wy+Pi1ORhwchFPXwcak\n",
       "qP7MBphzZdYRKzeaUlS/GrgeeGlyLav1SKdSVPdBeLLxoUhSamUsMMvYZrXe/sBJwLvhkedAuArC\n",
       "1XDD86H3Bnjeatj9SeheDjwv41iVHw3PX/sCPwaOT27vDHyp0T9kkqZSVHdC2Aih0vhwJCmVMhaY\n",
       "ZWyzMjPuzojTgNcDBwCzs4tNOdTU/NVBth/IKTYuDCa7JklSlspYYJaxzWq50APhLAj3Qtgr62hU\n",
       "GA1fp/pCYiHdDdwG3MnI2Oq8WI1/nUqSVEBhD+KiCp3EnRF/nXFAUk2bJiW+G/g8cTm9HI2pBgh3\n",
       "QXhRY0ORpNSy6rU9Gpib0c+2p1pN4s6IarpU+WvaJB8zHXgbcAawPu0PaQOrgN6sg5CkjGwH3Ejs\n",
       "zTsH+Bn5y+NSlbAAuID4jfseUPljxgFJk3I08BDwE+KHtx/4RUaxTLWn+ucQ9mtsKJKUWpaFbAdw\n",
       "IPAd4F7g08SJ581m8a4GG3cyotQMTc9fFXK1oyJAuATCIY0NRZJSy7rA3A34InAX8DXikqmfa/LP\n",
       "zLrNKoaD4Tm/hYtWwmN/hvVORlQrNDx/zQG+ANyUHJ8H+hr9QyZpqkX1+RDe29hQJCm1rArMY4j5\n",
       "+zLgHYx0jHQA9zX5Z1tUq16vgYE1sDrAIwFeOQhdJ2YdlEqh4fnrB8AniIul7wwsSq5lYapF9RkQ\n",
       "/qWxoUhSalkVmJ8Anlvjvpc0+WdbVKsOoRPOvh4Gq7Yavy7A3Huzjkyl0PD8Nd6W5DnaphwgfCaO\n",
       "vZKkTJWxwCxjm9UQYUHcGfGuZfCZjSNF9f8GmHt71tGpFBq++sca4HWMTE58LfBUyqCythpX/5Ak\n",
       "KSfCocBXgS/Ba/4Bhn4Dq3vgWR3wiadg5clZRyhNxW7Ab4E/JMctwK4ZxTLVnup/iUNAJClTZey1\n",
       "LWObNTWdcMjz4elzk50RX1V13y7Q9SXoPQd4Y1YBqnSalr/6GJmg+KFm/ZAtmGpR/V4IFzQ2FElK\n",
       "rYwFZhnbrPQWwr6Pwz3DcP4GeP77sw5IokX568FW/JBxTLWoPiQuqydJmSpjgVnGNiuVr0+DT6yG\n",
       "NcmY6dsC9AwBL8g6MpVeqvzV0awo2swqYHbWQUiSpGphAbx3MezfDTOTay8F9tkAvCK7uKT0LKol\n",
       "SVIGwqHATdDxM9h/LdycXH8CuHkacR6XVAiDxFUzxjs2ZhTTVId/vAjCXY0NRZJSK+NQiDK2WRMK\n",
       "PRDOGj0ZseOvoXsI9l0JWw9B7+nZxigBBc9fUy2qnw3h4caGIkmp5TVBnwMsB35XdW0ecDlwN3Gn\n",
       "xjk1npvXNqspwish3A3hmxDGLnXbD7wVWNjysKTxFTp/TbWo7oUw1NhQJCm1vCbo1xELneqi+jTg\n",
       "uOT8eODUGs/Na5vVUKEDvvc/sGIY3r0eer/DyCBqqV0VOn9NtajugLAxbncqSZnJc4LuZ3RRvRTY\n",
       "Ljmfn9weT57brPS2Bl5D/LwkwgL402/hho1wb4DHArz5KZj9tYxilCar0PmrjsaFJyHU+npSkloh\n",
       "zwm6n9FF9RNV55Uxt6vluc1KZz/oWg0vWQnda6D73+JkxLAczr0RTg8jW40vCdD3x6wDlrag4duU\n",
       "F8Vq4gogK7MORJIKJjDxL59FVeeLk0P5dDDMPRWYBWu/CWs+DQwDnTDzB/C/PfAG4hD8yz8Oax+B\n",
       "mX8FHzgYDnsFMCO+zO1A5dFsmiDVNJAcpVBPT/XtEF7euFAkKbU899r2s/nwj/nJ+fY4/KMM9oHZ\n",
       "Q/CjANcFeNkQdH08uW8b6Fo70hMdAly5Dp5/RHL/POh+EA4agiPXQNcgsHc2zZAmLTf5qxNYAlya\n",
       "3J7MTPJ6iurFEN4w9edLUt1yk6DH0c/mExWPT85PwImKJdD1ZfhMVdH86wBzHkju7IDux+HO5L5H\n",
       "A/QNAdWdWX3A+4GjgF1aHLw0FbnZUfEY4A5GAj6BWFS/ALgyud1IjwLbNvg1JakMLgR+BbwQeBA4\n",
       "klhEv4nYEfJGahfVKoyNT8GjVftUPAawNp6HZ8Mf7oeVG+E1q2HHp2HdyYz+Q+xJ4Czgy8C9LQpa\n",
       "KrwFwBXEgVebeqonM5O8np7qMyAcNfXnS1LdythrW8Y2F1U/dK2Ej2yAzweYMwT8NYS/jpMRw8nw\n",
       "ytnE7cXtxFIR5CJ/XURc83RfRorqycwkr6eoPgXCp6b+fEmqWy4SdIOVsc1FthPM/C/oPhNecBCE\n",
       "b0C4D4Ljo1VEbb/6x1uAFcTx1AM1HrOlmeRTsQLYtcGvKUlSmdwPa4+NOyPybeB6YCFUVmUcl5S5\n",
       "LIrqfYjbkB5M3E1pNnABcf2d+cAjxJnkK2o8f1HV+WImvzSTY6oltdoAJVqeSWUQOoCPJMfRUPlO\n",
       "xgFJSlQP/5jMTPJ6hn+8HsIvp/58SapbGYdClLHNedYB0z4Ic74LW/0HseMrERZAuBLCLyD0ZxWg\n",
       "1EK5yl/7Aj9OzucRJy82a0m9F0G4e+rPl6S65SpBN0gZ25xjPV+Dlw/BmQH+bi30LAW6qiYj/huE\n",
       "Mm0cp3IrdP6qp6ieB6HWNrqS1AqFTtA1lLHN7e5ZwEeg4xRgt6rrXTBtAzyRrDU9HODVq+D2nzoZ\n",
       "USVV6PxVT1HdAWE9hBmNC0eSUil0gq6hjG1uZ9vBrOVw2Fo4biP0DBHXGwfohenr4emqDV4e3Ah3\n",
       "XAVh9kQvKhVUofNXnY0LD0PYoTGhSFJqhU7QNZSxzW2s85PwgXUjRfMlAebcPnL/7Mvhb9bAAwGG\n",
       "ArxvFbB1ZuFK2Wr7JfWytIK4AshDWQciSVLrzZgLu0wfub0To3uhf/DPMH8x/HkuvP0muPFIkq0T\n",
       "JRVLvT3Vl0M4oDGhSFJqZey1LWOb29l+sPUQXBvgvgCvewp6T493Ve+MGDqzDVNqC4XOX/UW1d+G\n",
       "cFhjQpGk1AqdoGsoY5vbXOd7YPZD0P0Y9H4VftTnzojSuAqdv+otqk+HcGxjQpGk1AqdoGsoY5tz\n",
       "JLwSwl0QznMyorSZVPmro1lRtKkVxKWEJEkqsdAB4TjgJ8ApUDncrcal+pRxouLOWQchSVJ2wgLg\n",
       "fGA6sCdU/pBxQFIhlK2nejmwfdZBSJLUADOAnnRPCX8N3ARcBbzBgloqr3rHVL8MwtLGhCJJqZVx\n",
       "fHEZ29xsFZj1aehcHzdrmX0NMGfip4RuJyNKqRU6f9VbVM+CsDaOJZOklit0gq6hjG1utnfA8wbh\n",
       "kQDrAxy+Fvouqv3wZyYjnu9kRCmVQuevBjQu/AnCjvW/jiSlVugEXUMZ29xkXWfA56q2Er8jwOyH\n",
       "N3/cpsmIYQWEv219nFLuufrHFvweeF7WQUiSNDVr74er1478vr82QMey0Y8JOwCXAX9JnIx4YWtj\n",
       "lNTuGtFTfR6E99X/OpKUWhl7bcvY5mabBb23wCtWwQGroOtJYNeRu8Mhyc6I/wahbKt8SY1U6PzV\n",
       "iKL64xD+o/7XkaTUCp2gayhjm1thK+AtwDuB+fFS6IZwZjIZ8dUZxiYVRaHzVyOK6sMg+DWYpCwU\n",
       "OkHXUMY2ZyDsHle3cjKi1ECFzl+NKKpfDeHX9b+OJKVW6ARdQxnb3EKhA8JHnYwoNUWq/FXGsVb3\n",
       "4URFSWqUB4BVwEZgPbBXptGUStgBOI84FGQvqDxQdedziOOslwFLWh+bpHbXiJ7qCoTVEPrqfy1J\n",
       "SqWIvbb3A/MmuL+IbW4D4W0QHknmCY3tIHsLzBqC162ErYeg98uZhCjlX6HzV4MaF26BsEdjXkuS\n",
       "Jq2ICfp+YOsJ7i9imzMUuiF8fYLJiB2w1SBcn6xhvTLAdoPAPq2OVCoA16mehCXAK7MOQpIKIABX\n",
       "AL8BPpBxLAUXdgduArqAhVC5bpwH9QHT4VVVN/ccBvpbEqJUYmUcUw1wA7A38PWsA5GknHsN8DDw\n",
       "LOByYCnwizGPWVR1vjg5FMdCHwB0A1cT38dxhA7gw8BHgWO2sJHLSpj2OPz3fDgMuAu4ehpwawPj\n",
       "lopqIDlKoVHDPxZCuKMxryVJk1b0oRCnEIu/akVv81QlG7jsugoOXAVdq4CFmz8s7ADhCgi/hNA/\n",
       "ydfeFWZKQ1IlAAAPm0lEQVQthzlDMGMtzHDDM2lqCp2/GlVUT4Mw6GRFSS1WtAQ9C+hNzruBa4k9\n",
       "r9WK1uYGqXwY/mINDCdjn88Zhjk3j37MhJMRt2QasCPxv5GkqSl0/mpg48I1EPZv3OtJ0hYVLUHv\n",
       "BNySHLcBJ47zmKK1uUFmfhE+mxTUIcCdAXofifdtcTKipNYodP5qZFH9OQgfa9zrSdIWFTpB11DG\n",
       "Nk/GodA/CA8HWB/giLXQd1EyPNGdEaX2UOj81cii+lAI/9u415OkLSp0gq6hjG2ejArM+iRMWx+P\n",
       "OdfAYydX7YxYAd4DvefCtI8zMsxGUusUOn81sqjug/Ck46oltVChE3QNZWzzJttA309g1krouwsY\n",
       "byjHdLhsl80nI3b/J7xgEL4U4G/WQu/twMzWhS6JguevBjcuXALh8Ma+piTVVOgEXUMZ25yYfT38\n",
       "0zr4Y4CLAnStJk4erDLuZMQZ0LkeHk3GWw8HWLgKeGuLGyCVXaHzV6OL6r+F8H+NfU1JqqnQCbqG\n",
       "MrYZoBumbYANVZMR37IK+Lt49zOTEX8/zmTE7jgkZF3Vc9+0Cnhni9sglV2h81eji+qeZAjINo19\n",
       "XUkaV6ETdA1lbDPANJiW9FKHABsDvGw18JbJTUacfSW8aw0sCfCVYZj1BLBtKxsgqdj5qwmNC+dC\n",
       "+ETjX1eSNlPoBF1DGduc6Doeth+CfxuGgSGYcwM8fVzVZMSJ9ELf+TDnDzDnl8BLWxGxpFFS5a9K\n",
       "s6JokkDDYw7PBW4GXgaVGlvESlJDNCGHtb0ytrnagVDZB/YagmvfBJ0zgcOg8kDWgUnaokLnryb1\n",
       "eIT/hHB2c15bkp5Rxl7bMrZ5jLp2RpSUnULnr2YV1XMg3AXhH5vz+pIEFDxB11DGNicmnIwoqf0V\n",
       "On81sXFhl6Qn4X0QCtvVLylThU7QNZShzbOB7Rj1NXFYCOFOd0aUcq3t89eOwFXA7cBtwNHJ9XnA\n",
       "5cDdwGXAnHGe2+TGhZdCuB3CRRBe0tyfJamE2j5BN0GR21yBni/A9HXQvRZ6b4Ftt4XwkUlORpTU\n",
       "3to+f80HdkvOe4C7gBcDpwHHJdePB04d57ktaFyYBeFjSa/1r5IxcIdAeBGE6c3/+ZIKrO0TdBMU\n",
       "uc3vgucPwp+TJfM+tg5uenT0zoiScix3q39cAnwlOfYFlhML78XAi8Y8toWzMMNWSTz7AS9JjucA\n",
       "g8ATwMrkGATWbeFYD2xo4rFx4vsrw41+dyRNSaFnkteQhzYfAn2fh+Fu4Huw+lhi3t6CrU6HTxwT\n",
       "+4Egptz/GoLj5kBlQ/PCldQiqfJX1omuH7gaeBnwR2Bucr0CPF51e5OMk3OYRhw7N5c4PGUuMAuY\n",
       "MYmjE5jWoqP6Z00nvm/NLOrHFvcbx5w361oDXqNS5F40tZ88FJiN1u5t3gdmXwHf74IFwAeeglvP\n",
       "gdVHVT3mrTDj9bBuGfB1YE1y/Sg46LPwP13QAfwwwJE3wZN7troRkpoiVf7KcmmfHuD7wDHA6jH3\n",
       "BWp3uS+qOl+cHC1S2UAs9h9v3c9shNBB8wv56cRiflNBP/Z8GqP/uJjocVu6r1HXOuNOZ80o1pv6\n",
       "h8F4x3CDr6V5bPCPk5oGkkNta8bb4NiZsH9y+8xZ8Kq3A0lRPfNk2OYE+Kdu+OUa+OURsHovYB0s\n",
       "uyH+r3npBjjjKbh2Izx1ZCbNkJS5rIrq6cSC+gLi8A8YGfbxCLA9sKLGcxc1O7jiqQwzMhRFzwgV\n",
       "YvdSswr4el5jxhYeV3101HGt3ud3ABUIY4vtegv1Zjy/ma9Z69/1wE+B1VC5HTgFtZkNq+CPG4i/\n",
       "l4BlQMdQcmcnbPg4XD8ddgBCF+z+PLjjYHh6F+A4WPd+6P8TsaPoOuCx1rdBUjvIoqiuAGcDdwCn\n",
       "V13/MXA48Nnk30s2f6rUSJXASHGkKQsVGl+ot+KPh+nAVg14zY4xj+kY53lLiXlNbWf4TPjOUTA8\n",
       "F547Db6wBlZ9OLlzRvymb7vkZgXYtQI/+hTwJLAXbPVA62OWpOi1xF6cW4AlyXEgcUm9K8h0ST1J\n",
       "aqoy5rA8tHlbqJwE008FxmzSMvsaOPJpuDfA1QGWD8ND/+nOiFIp5CF/TVmhGyep8MqYw/Le5jmw\n",
       "44/g/DWwbB185X1ZBySpZfKevyZU6MZJKrwy5rCctzkshLAUwgXujCiVTs7z18QK3ThJhVfGHJbT\n",
       "NocOd0aUSi+n+WtyCt04SYVXxhyWwzaHZ0O43J0RpdLLYf6avEI3TlLhlTGH5azN4W0QlkM4xcmI\n",
       "UunlLH+lU+jGSSq8IuawA4lLBt7DyH7d1XLS5tAN4esQfg9hn6yjkdQWcpK/pqbQjZNUeEXLYZ3A\n",
       "vUA/cd3vW4AXj3lMTtoc9ksmI/ZlHYmktpGT/DU1hW6cpMIrWg57NXHHyE1OSI5qRWuzpPJIlb86\n",
       "mhWFJKnwdgAerLq9LLkmSaXjJAxJ0lRNthdnUdX54uSQpHYzkByl4NeIkvKsaDlsb0YP/ziRzScr\n",
       "Fq3Nksqj0Pmr0I2TVHhFy2HTgPuIExVnkOuJipK0mULnr0I3TlLhFTGHHQTcRVwF5MRx7i9imyWV\n",
       "Q6HzV6EbJ6nwypjDythmScXg6h+SJElSK1lUS5IkSXWyqJYkSZLqZFEtSZIk1cmiWpIkSaqTRbUk\n",
       "SZJUJ4tqSZIkqU4W1ZIkSVKdLKolSZKkOllUS5IkSXWyqJYkSZLqZFEtSZIk1cmiWpIkSaqTRbUk\n",
       "SZJUJ4tqSZIkqU4W1ZIkSVKdLKolSZKkOllUS5IkSXWyqJYkSZLqZFEtSZIk1cmiWpIkSaqTRbUk\n",
       "SZJUp3Yrqg8ElgL3AMdnHIskqbZFwDJgSXIcmGk0kqRndAL3Av3AdOAW4MVjHhNaHFMRDGQdQM4M\n",
       "ZB1AzgxkHUDOFCmHnQIcO4nH5anNA1kHkNJA1gGkMJB1ACkNZB1ACgNZB5DSQNYBpJAqf7VTT/Ve\n",
       "xKL6AWA98B3gr7IMqCAGsg4gZwayDiBnBrIOQJmqZB1Agw1kHUBKA1kHkMJA1gGkNJB1ACkMZB1A\n",
       "SgNZB9As7VRU7wA8WHV7WXJNktSejgJuBc4G5mQciyRlqp2K6jx9RShJZXA58LtxjrcCXwN2AnYD\n",
       "HgY+n1GMktQW2umru72JE182TXY5ERgGPlv1mHuBnVsbliQ1zH3ALlkH0QT9wKXAy8e5z7wtKa9y\n",
       "m7OnEYPvB2Yw/kRFSVJ72L7q/F+Bb2cViCRpcwcBdxF7Nk7MOBZJUm3nA78ljqm+BNgu23AkSZIk\n",
       "SZJaxI1htuwBYs/REuDXybV5xMlGdwOXUe4Z+ucAy4kTrTaZ6P05kfh5Wwoc0KIY28l479ciRm/4\n",
       "cVDVfWV/v3YErgJuB24Djk6ul/0z9jngTmKP9g+AvmzDqSkvv2Nqfc7aWScxX1yadSCTMAe4mPiZ\n",
       "vYM436udnUj8LPyOOARrq2zDGSXt79ysjRdvXvJXKpPZGEZwP/EDW+004Ljk/Hjg1JZG1F5eByxk\n",
       "9P8wtd6flxA/Z9OJn7t7aa/VclphvPer1oYfvl8wn7gSBkAPcSjbi/Ez9iZG2nUq7ZmD8vQ7ptbn\n",
       "rJ0dC3wL+HHWgUzCecD7kvNptHcR1Q/8npFC+rvA4ZlFs7k0v3PbwXjx5iF/pfZq4KdVt09IDo12\n",
       "P7D1mGtLGRnrOD+5XWb9jP4fptb7cyKje6t+Svv3WDRDP5sX1R8e53G+X5u7BNgfP2PVDgH+O+sg\n",
       "xpHn3zGXAPtlHcQEFgBXAG+g/Xuq+4hFal7MI/5RNZf4B8ClxJzTTvqZ3O/cdtHP6HirbTF/5aVX\n",
       "xI1hJicQk9dvgA8k17Yjfp1B8q+TiUar9f48m/g528TP3IjxNvzw/Rqtn9jjcQN+xqq9D/i/rIMY\n",
       "R15/x/Qz8jlrV18APkpcIrfd7QQ8CpwL3Ax8A5iVaUQTe5y4PvwfgT8BK4k1QDvLc02yxfyVl6La\n",
       "jWEm5zXEBHsQ8P+IX2VUC/heTmRL74/vXboNP8r6fvUA3weOAVaPua+on7Fam8T8ZdVjPgasoz2X\n",
       "3svj+95DHPt7DDCYcSy1vAVYQRxP3U77YtQyDdgd+Gry7xDt/Y3FzsCHiH9cPZv4mXh3lgGllKea\n",
       "ZFL5a1prYqnbQ8TJGZvsyOgeHkUPJ/8+CvwQ2Iv4l+B84BHiurIrsgmtbdV6f8Z+5hYk18qu+vNz\n",
       "FiNf5/p+RdOJBfUFxK/loRyfsTdt4f4jgINp32EKefsds+lz9t+MfM7a0T7E3TcPBmYCs4lLMb43\n",
       "y6AmsCw5bkxuX0x7F9V7AL8CHktu/4D4nn8rs4i2LI81yRG0d/5KzY1htmwW0JucdwPXElcTOI2R\n",
       "cZsnUJBB9nXoZ/NJE+O9P5smkc0g9szeRz56Whqtn9HvV60NP3y/YnvPJ37dXa3sn7EDiasTbJN1\n",
       "IBPI0++YWp+zdrcv7T+mGuAa4AXJ+SJG7+rcbnYlrgDTRfxcnEf8lrqd9DO537ntop/R8eYhf02J\n",
       "G8NMbCfiL4JbiP+TbXqP5hHHWLXj8jWtdiFx3Nk64vjJI5n4/TmJ+HlbCry5pZG2h7Hv1/uYeMOP\n",
       "sr9fryWOG72FkSUHD8TP2D3AHxh5T76abTg15eV3TK3PWbvbl3ys/rErsac6L0uoHcfIknrnEb/F\n",
       "aBdpf+dmbbzfeXnJX5IkSZIkSZIkSZIkSZIkSZIkSZIkSZIkSZIkSZIkSYo2MrKW5BLi+p2N0s/o\n",
       "BeElSZKekZdtyqXJeApYmHUQkiSpfDqyDkBqgQeIW83+FrgB2Dm53g/8nLhz1hXAjsn17YAfMrJD\n",
       "5d7J9U7gTOKOlT8DZibXjybuaHUrcUcmSVLz7EnMt1sB3cSc/JJMI5KkgtnA6OEfb0+u38/ItsPv\n",
       "AS5Nzi9NbkPcPvWHyfl3iYUyxD88ZxML8PXAK6oe8+7k/CFGtoad3ZCWSJIm8ingc8BXgOMzjkWS\n",
       "Cmd1jev3E4tiiMXvn5PzR4m9z5uuP5qcr2CkSN6kH7i76vZxwMeS858AFxGL7O70YUuSUppO7K2+\n",
       "HqhkHIsEOPxD5RSqzmsl4/GuP111vpGROQl/AZwB7A7cyEihLklqjm2InRg9QFfGsUiARbXK451V\n",
       "//4qOf8V8K7k/N3ANcn5lcA/JeedTDykowI8B1gMnAD0YW+1JDXb14GTgW8T58xImXP1DxVJF3Es\n",
       "9SY/AU5KzucSvypcC/xtcu0o4Fzgo8QhH0cm148hTkj8e2KP9AeB5Yzu4Sa53QlcQCymK8AXgVWN\n",
       "apAkaTPvJX5z+B1i5+CvgAFi54YkqYnuB+ZlHYQkSSouh3+oDMb2MEuSJEmSJEmSJEmSJEmSJEmS\n",
       "JEmSJEmSJEmSJEmSJEmSJPj/cECJ9LS/S2QAAAAASUVORK5CYII=\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x108dc2890>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(12,4))\n",
    "fig.add_subplot(121)\n",
    "plt.plot(results[0:200,1])\n",
    "plt.ylim([0,100])\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.ylabel(\"Loss\")\n",
    "\n",
    "fig.add_subplot(122)\n",
    "ablineValues = np.zeros_like(x_data)\n",
    "for i,x_val in enumerate(x_data):\n",
    "  ablineValues[i] = res[1]*x_val+res[2]\n",
    "plt.scatter(x_data,y_data)\n",
    "plt.plot(x_data, ablineValues)\n",
    "plt.xlabel('x')\n",
    "plt.ylabel('y')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What is the correct value for the fit?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 2.3581152 ,  0.93678743], dtype=float32),\n",
       " [19.037403, 2.4355152, 0.41587028])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.polyfit(x_data,y_data,1), res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Out procedure has not excatly found the optimum, but that's usually not a problem for neural networks, since we do not need to find the global minimum for a good model. Note that TensorFlow currently does not support advanced optimization methods like 'L-BFGS' yet see http://ktsiegel.com/6869.pdf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}