Tf_2.0_sequential_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2.1 -0.5 10.   3.2 22.  10. ]]\n"
     ]
    }
   ],
   "source": [
    "full_training_inputs = np.array([[-1.2, -37]\n",
    "                                 , [1.,1.]\n",
    "                                 , [2., 1.]\n",
    "                                 , [4, 1]\n",
    "                                 , [1.,2.]\n",
    "                                 , [4, 6]                                 \n",
    "                              ])\n",
    "#print(full_training_inputs)\n",
    "full_training_outputs = np.array([[2.1\n",
    "                                   , -0.5\n",
    "                                   , 10\n",
    "                                   , 3.2\n",
    "                                   , 22\n",
    "                                   , 10                                   \n",
    "                                ]])\n",
    "print(full_training_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNet_init(tf.keras.Model):\n",
    "  def __init__(self):\n",
    "    super(NNet_init, self).__init__()\n",
    "    self.weights_var = tf.Variable([[1.,  0.9]\n",
    "                                    ], name = \"weights_var\")\n",
    "    self.bias_var = tf.Variable([[1.]\n",
    "                                 ], name = \"bias_var\")\n",
    "    self.weights_out_var = tf.Variable([[1.\n",
    "                                         ]], name = \"weights_out_var\")\n",
    "    \n",
    "    \n",
    "    \n",
    "  def call(self, input_data):\n",
    "    input_data = tf.transpose(input_data)\n",
    "    layer_1_var = tf.linalg.matmul(self.weights_var, input_data) + self.bias_var\n",
    "    layer_1_var_activated = tf.nn.leaky_relu(layer_1_var)\n",
    "    layer_out_var = tf.linalg.matmul(self.weights_out_var, layer_1_var_activated)\n",
    "    \n",
    "    return layer_out_var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([[ -1.2 -37. ]], shape=(1, 2), dtype=float32)\n",
      "tf.Tensor([[2.1]], shape=(1, 1), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "training_inputs = tf.constant(full_training_inputs[0,:].reshape((1,2))\n",
    "                             , dtype = \"float32\")\n",
    "print(training_inputs)\n",
    "training_outputs = tf.constant(full_training_outputs[0,0].reshape((1,1))\n",
    "                              , dtype = \"float32\")\n",
    "print(training_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The loss function to be optimized\n",
    "def loss(model, inputs, targets):\n",
    "  error = model(inputs) - targets\n",
    "  return tf.reduce_mean(tf.square(error))\n",
    "\n",
    "def grad(model, inputs, targets):\n",
    "  with tf.GradientTape() as tape:\n",
    "    loss_value = loss(model, inputs, targets)\n",
    "  return tape.gradient(loss_value, [model.weights_var\n",
    "                                    , model.bias_var\n",
    "                                    , model.weights_out_var\n",
    "                                    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = NNet_init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial loss: 77.440\n"
     ]
    }
   ],
   "source": [
    "optimizer = (tf.keras.optimizers.Adam()\n",
    "            )\n",
    "\n",
    "print(\"Initial loss: {:.3f}\".format(loss(model, training_inputs, training_outputs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([[  4.2240005 130.24     ]], shape=(1, 2), dtype=float32)\n",
      "tf.Tensor([[-3.5200002]], shape=(1, 1), dtype=float32)\n",
      "tf.Tensor([[117.920006]], shape=(1, 1), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "grads = grad(model, training_inputs, training_outputs)\n",
    "for i in range(len(grads)):\n",
    "    print(grads[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "77.18445"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.apply_gradients(zip(grads, [model.weights_var, \n",
    "    model.bias_var\n",
    "                                    , model.weights_out_var\n",
    "                                     ]))\n",
    "best_loss = loss(model, training_inputs, training_outputs).numpy()\n",
    "best_iter = -1\n",
    "best_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at step 0, 000: 76.929604\n"
     ]
    }
   ],
   "source": [
    "j = 0\n",
    "while best_loss > 0.0 and j < 10:\n",
    "  for i in range(80000):\n",
    "    grads = grad(model, training_inputs, training_outputs)\n",
    "    optimizer.apply_gradients(zip(grads, [model.weights_var,\n",
    "      model.bias_var\n",
    "                                    , model.weights_out_var\n",
    "                                       ]))\n",
    "    current_loss = loss(model, training_inputs, training_outputs).numpy()\n",
    "    if current_loss < best_loss:\n",
    "      best_loss = current_loss\n",
    "      best_iter = i\n",
    "    if i % 8000 == 0:\n",
    "      print(\"Loss at step {}, {:03d}: {:5f}\".format(j, i, loss(model, training_inputs, training_outputs)))\n",
    "    if i - best_iter > 2000:\n",
    "      break\n",
    "  j += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Halted at step: 6671 with loss 0.000, W = [[ 0.00293999 -0.09705605]], B = [[1.9970503]], W_out = [[0.3760343]]\n"
     ]
    }
   ],
   "source": [
    "print(\"Halted at step: {} with loss {:.3f}, W = {}, B = {}, W_out = {}\".format(i,\n",
    "                                                                               current_loss\n",
    "                                                                               , model.weights_var.numpy()\n",
    "                                                                               , model.bias_var.numpy()\n",
    "                                                                               , model.weights_out_var.numpy())\n",
    "     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.00293999 -0.09705605]]\n"
     ]
    }
   ],
   "source": [
    "results_weights_var = model.weights_var.numpy()\n",
    "results_bias_var = model.bias_var.numpy()\n",
    "results_weights_out_var = model.weights_out_var.numpy()\n",
    "print(results_weights_var)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Initialization complete__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNet(tf.keras.Model):\n",
    "  def __init__(self):\n",
    "    super(NNet, self).__init__()\n",
    "    self.weights_const = tf.constant(results_weights_var)\n",
    "    self.biases_const = tf.constant(results_bias_var)\n",
    "    self.weights_out_const = tf.constant(results_weights_out_var)\n",
    "    \n",
    "    self.weights_var = tf.Variable(np.random.uniform(size = (1,2))\n",
    "                                   , name = \"weights_var\"\n",
    "                                   , dtype = \"float32\")\n",
    "    self.bias_var = tf.Variable(np.random.uniform(size = (1,1))\n",
    "                                , name = \"bias_var\"\n",
    "                               , dtype = \"float32\")\n",
    "    self.weights_out_var = tf.Variable(np.random.uniform(size = (1,1))\n",
    "                                       , name = \"weights_out_var\"\n",
    "                                      , dtype = \"float32\")\n",
    "    \n",
    "  def call(self, input_data):\n",
    "    input_data = tf.transpose(input_data)\n",
    "    layer_1_const = tf.linalg.matmul(self.weights_const, input_data, transpose_b = False) + self.biases_const\n",
    "    layer_1_const = tf.nn.leaky_relu(layer_1_const)\n",
    "    layer_out_const = tf.linalg.matmul(self.weights_out_const, layer_1_const, transpose_b = False)\n",
    "    \n",
    "    layer_1_var = tf.linalg.matmul(self.weights_var, input_data, transpose_b = False) + self.bias_var\n",
    "    layer_1_var_activated = tf.nn.leaky_relu(layer_1_var)\n",
    "    layer_out_var = tf.linalg.matmul(self.weights_out_var, layer_1_var_activated, transpose_b = False)\n",
    "    #layer_out_var = self.weights_out_var * layer_1_var_activated\n",
    "    \n",
    "    return layer_out_const + layer_out_var\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[ -1.2 -37. ]\n",
      " [  1.    1. ]], shape=(2, 2), dtype=float32)\n",
      "tf.Tensor([[ 2.1 -0.5]], shape=(1, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "training_inputs = tf.constant(full_training_inputs[:2,:]\n",
    "                             , dtype = \"float32\")\n",
    "print(training_inputs)\n",
    "training_outputs = tf.constant(full_training_outputs[:,:2]\n",
    "                              , dtype = \"float32\")\n",
    "print(training_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = NNet()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial loss: 4.159\n"
     ]
    }
   ],
   "source": [
    "optimizer = (tf.keras.optimizers.Adam()\n",
    "            )\n",
    "\n",
    "print(\"Initial loss: {:.3f}\".format(loss(model, training_inputs, training_outputs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([[ 2.2186797 10.214375 ]], shape=(1, 2), dtype=float32)\n",
      "tf.Tensor([[1.7273241]], shape=(1, 1), dtype=float32)\n",
      "tf.Tensor([[6.768092]], shape=(1, 1), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "grads = grad(model, training_inputs, training_outputs)\n",
    "for i in range(len(grads)):\n",
    "    print(grads[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.137771"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.apply_gradients(zip(grads, [model.weights_var, \n",
    "    model.bias_var\n",
    "                                    , model.weights_out_var\n",
    "                                     ]))\n",
    "best_loss = loss(model, training_inputs, training_outputs).numpy()\n",
    "best_iter = -1\n",
    "best_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at step 0, 000: 4.117011\n",
      "Loss at step 0, 8000: 0.000000\n"
     ]
    }
   ],
   "source": [
    "j = 0\n",
    "while best_loss > 0.000001 and j < 10:\n",
    "  for i in range(80000):\n",
    "    grads = grad(model, training_inputs, training_outputs)\n",
    "    optimizer.apply_gradients(zip(grads, [model.weights_var,\n",
    "      model.bias_var\n",
    "                                    , model.weights_out_var\n",
    "                                       ]))\n",
    "    current_loss = loss(model, training_inputs, training_outputs).numpy()\n",
    "    if current_loss < best_loss:\n",
    "      best_loss = current_loss\n",
    "      best_iter = i\n",
    "    if i % 8000 == 0:\n",
    "      print(\"Loss at step {}, {:03d}: {:5f}\".format(j, i, loss(model, training_inputs, training_outputs)))\n",
    "    if i - best_iter > 2000:\n",
    "      break\n",
    "  j += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Halted at step: 14332 with loss 0.000, W = [[-1.8064003   0.01491064]], B = [[-1.615987]], W_out = [[1.7836784]]\n"
     ]
    }
   ],
   "source": [
    "print(\"Halted at step: {} with loss {:.3f}, W = {}, B = {}, W_out = {}\".format(i,\n",
    "                                                                               current_loss\n",
    "                                                                               , model.weights_var.numpy()\n",
    "                                                                               , model.bias_var.numpy()\n",
    "                                                                               , model.weights_out_var.numpy())\n",
    "     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.00293999 -0.09705605]\n",
      " [-1.8064003   0.01491064]]\n"
     ]
    }
   ],
   "source": [
    "results_weights_var = np.concatenate((results_weights_var\n",
    "                                     , model.weights_var.numpy()\n",
    "                                     )\n",
    "                                    )\n",
    "\n",
    "results_bias_var = np.concatenate((results_bias_var\n",
    "                                  , model.bias_var.numpy()))\n",
    "\n",
    "results_weights_out_var = np.concatenate((results_weights_out_var\n",
    "                                          , model.weights_out_var.numpy()\n",
    "                                          )\n",
    "                                        , axis = 1)\n",
    "print(results_weights_var)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[4.055589 4.055589]]\n"
     ]
    }
   ],
   "source": [
    "tmp = np.concatenate(([[4.055589]], [[4.055589]])\n",
    "                     , axis = 1\n",
    "                    )\n",
    "#.reshape((1,2))\n",
    "print(tmp)\n",
    "#np.array([[1,2,3]]).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Attempt as a loop__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 rows of data\n",
      "with 1 neuron(s)\n",
      "Loss at step 0, 000: 34.047142\n",
      "Loss at step 0, 8000: 0.000000\n",
      "Loss at step 0, 16000: 0.000000\n",
      "Fitting 4 rows of data\n",
      "with 1 neuron(s)\n",
      "Loss at step 0, 000: 212.954254\n",
      "Loss at step 0, 8000: 2.864986\n",
      "Loss at step 1, 000: 1.756455\n",
      "Loss at step 2, 000: 1.756455\n",
      "Loss at step 3, 000: 1.756474\n",
      "with 2 neuron(s)\n",
      "Loss at step 0, 000: 210.846954\n",
      "Loss at step 0, 8000: 13.801014\n",
      "Loss at step 1, 000: 13.801014\n",
      "Loss at step 2, 000: 13.801010\n",
      "Loss at step 3, 000: 13.801012\n",
      "with 3 neuron(s)\n",
      "Loss at step 0, 000: 339.963654\n",
      "Loss at step 0, 8000: 3.595809\n",
      "Loss at step 0, 16000: 0.000001\n",
      "Fitting 5 rows of data\n",
      "with 1 neuron(s)\n",
      "Loss at step 0, 000: 120.838135\n",
      "Loss at step 0, 8000: 62.609173\n",
      "Loss at step 1, 000: 62.609180\n",
      "Loss at step 2, 000: 62.609169\n",
      "Loss at step 3, 000: 62.609180\n",
      "with 2 neuron(s)\n",
      "Loss at step 0, 000: 117.149513\n",
      "Loss at step 0, 8000: 0.000000\n",
      "Fitting 6 rows of data\n",
      "with 1 neuron(s)\n",
      "Loss at step 0, 000: 550.352966\n",
      "Loss at step 0, 8000: 291.629395\n",
      "Loss at step 0, 16000: 274.579102\n",
      "Loss at step 1, 000: 274.577698\n",
      "Loss at step 2, 000: 274.577667\n",
      "Loss at step 3, 000: 274.577698\n",
      "with 2 neuron(s)\n",
      "Loss at step 0, 000: 564.043457\n",
      "Loss at step 0, 8000: 280.724976\n",
      "Loss at step 1, 000: 274.577637\n",
      "Loss at step 2, 000: 274.577637\n",
      "Loss at step 3, 000: 274.577698\n",
      "with 3 neuron(s)\n",
      "Loss at step 0, 000: 638.245300\n",
      "Loss at step 0, 8000: 24.056620\n",
      "Loss at step 1, 000: 24.055830\n",
      "Loss at step 2, 000: 24.055822\n",
      "Loss at step 3, 000: 24.055832\n",
      "with 4 neuron(s)\n",
      "Loss at step 0, 000: 767.394531\n",
      "Loss at step 0, 8000: 24.086237\n",
      "Loss at step 1, 000: 24.050713\n",
      "Loss at step 2, 000: 24.050713\n",
      "Loss at step 3, 000: 24.050714\n",
      "with 5 neuron(s)\n",
      "Loss at step 0, 000: 715.554504\n",
      "Loss at step 0, 8000: 10.318854\n",
      "Loss at step 1, 000: 10.318851\n",
      "Loss at step 2, 000: 10.318860\n",
      "Loss at step 3, 000: 10.318855\n",
      "with 6 neuron(s)\n",
      "Loss at step 0, 000: 1063.015259\n",
      "Loss at step 0, 8000: 9.911113\n",
      "Loss at step 0, 16000: 0.000000\n"
     ]
    }
   ],
   "source": [
    "for rows_of_data_to_fit in range(3, 1 + full_training_outputs.shape[1]):\n",
    "  print(\"Fitting {} rows of data\".format(rows_of_data_to_fit))\n",
    "  number_neurons_to_fit_with = 1\n",
    "  best_loss = 1\n",
    "  \n",
    "  while number_neurons_to_fit_with <= 10 and best_loss > 0.001:\n",
    "    print(\"with {} neuron(s)\".format(number_neurons_to_fit_with))\n",
    "    class NNet(tf.keras.Model):\n",
    "      def __init__(self):\n",
    "        super(NNet, self).__init__()\n",
    "        self.weights_const = tf.constant(results_weights_var, dtype = \"float32\")\n",
    "        self.biases_const = tf.constant(results_bias_var, dtype = \"float32\")\n",
    "        self.weights_out_const = tf.constant(results_weights_out_var, dtype = \"float32\")\n",
    "    \n",
    "        self.weights_var = tf.Variable(np.random.uniform(size = (number_neurons_to_fit_with, 2))\n",
    "                                       , name = \"weights_var\"\n",
    "                                       , dtype = \"float32\")\n",
    "        self.bias_var = tf.Variable(np.random.uniform(size = (number_neurons_to_fit_with, 1))\n",
    "                                    , name = \"bias_var\"\n",
    "                                   , dtype = \"float32\")\n",
    "        self.weights_out_var = tf.Variable(np.random.uniform(size = (1, number_neurons_to_fit_with))\n",
    "                                           , name = \"weights_out_var\"\n",
    "                                          , dtype = \"float32\")    \n",
    "    \n",
    "      def call(self, input_data):\n",
    "        input_data = tf.transpose(input_data)\n",
    "        layer_1_const = tf.linalg.matmul(self.weights_const, input_data) + self.biases_const\n",
    "        layer_1_const = tf.nn.leaky_relu(layer_1_const)\n",
    "        layer_out_const = tf.linalg.matmul(self.weights_out_const, layer_1_const, transpose_b = False)\n",
    "    \n",
    "        layer_1_var = tf.linalg.matmul(self.weights_var, input_data) + self.bias_var\n",
    "        layer_1_var_activated = tf.nn.leaky_relu(layer_1_var)\n",
    "        layer_out_var = tf.linalg.matmul(self.weights_out_var, layer_1_var_activated)\n",
    "    \n",
    "        return layer_out_const + layer_out_var\n",
    "\n",
    "    training_inputs = tf.constant(full_training_inputs[:rows_of_data_to_fit,:]\n",
    "                                 , dtype = \"float32\")\n",
    "    training_outputs = tf.constant(full_training_outputs[:,:rows_of_data_to_fit]\n",
    "                                  , dtype = \"float32\")\n",
    "\n",
    "    model = NNet()\n",
    "    optimizer = tf.keras.optimizers.Adam()\n",
    "\n",
    "    grads = grad(model, training_inputs, training_outputs)\n",
    "\n",
    "    optimizer.apply_gradients(zip(grads, [model.weights_var, \n",
    "        model.bias_var\n",
    "                                        , model.weights_out_var\n",
    "                                         ]))\n",
    "    best_loss = loss(model, training_inputs, training_outputs).numpy()\n",
    "    best_iter = -1\n",
    "\n",
    "    j = 0\n",
    "    while best_loss > 0.000001 and j < 4:\n",
    "      for i in range(80000):\n",
    "        grads = grad(model, training_inputs, training_outputs)\n",
    "        optimizer.apply_gradients(zip(grads, [model.weights_var,\n",
    "          model.bias_var\n",
    "                                        , model.weights_out_var\n",
    "                                           ]))\n",
    "        current_loss = loss(model, training_inputs, training_outputs).numpy()\n",
    "        if current_loss < best_loss:\n",
    "          best_loss = current_loss\n",
    "          best_iter = i\n",
    "        if i % 8000 == 0:\n",
    "          print(\"Loss at step {}, {:03d}: {:5f}\".format(j, i, loss(model, training_inputs, training_outputs)))\n",
    "        if i - best_iter > 2000:\n",
    "          break\n",
    "      j += 1\n",
    "      best_iter = -1\n",
    "\n",
    "    number_neurons_to_fit_with += 1    \n",
    "    \n",
    "  results_weights_var = np.concatenate((results_weights_var\n",
    "                                       , model.weights_var.numpy()\n",
    "                                       )\n",
    "                                      )\n",
    "\n",
    "  results_bias_var = np.concatenate((results_bias_var\n",
    "                                    , model.bias_var.numpy()))\n",
    "\n",
    "  results_weights_out_var = np.concatenate((results_weights_out_var\n",
    "                                            , model.weights_out_var.numpy()\n",
    "                                            )\n",
    "                                          , axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2.9399912e-03 -9.7056054e-02]\n",
      " [-1.8064003e+00  1.4910637e-02]\n",
      " [ 3.2217562e+00 -1.8651757e-01]\n",
      " [ 2.9733770e+00 -7.3177719e-01]\n",
      " [ 2.1737335e+00 -1.1891392e+00]\n",
      " [ 7.9192072e-01  2.3103638e+00]\n",
      " [-3.7493932e+00  5.0147133e+00]\n",
      " [ 1.1052321e+00 -1.4782224e+00]\n",
      " [ 3.3738452e-01 -2.4047546e+00]\n",
      " [-7.3040712e-01  4.5971361e-01]\n",
      " [ 3.0730014e+00 -1.8441094e+00]\n",
      " [ 2.0833323e+00  3.0476074e+00]\n",
      " [-6.7727047e-01  5.3606176e-01]\n",
      " [-3.1413722e+00  1.8844931e+00]]\n",
      "[[ 1.9970503 ]\n",
      " [-1.615987  ]\n",
      " [-3.0352924 ]\n",
      " [-5.2151594 ]\n",
      " [-0.9846479 ]\n",
      " [-0.4650485 ]\n",
      " [-1.2654616 ]\n",
      " [ 0.37303057]\n",
      " [-1.5036277 ]\n",
      " [ 2.4619    ]\n",
      " [-1.2288816 ]\n",
      " [-1.8901116 ]\n",
      " [ 2.1730328 ]\n",
      " [ 1.2566622 ]]\n",
      "[[ 0.3760343  1.7836784  3.458818  -5.8438025  2.0788293 -1.3177881\n",
      "   5.264634   3.5719504 -2.6457014  1.6120778  3.5091817 -2.823831\n",
      "   1.8356752  3.6632023]]\n"
     ]
    }
   ],
   "source": [
    "print(results_weights_var)\n",
    "print(results_bias_var)\n",
    "print(results_weights_out_var)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__Final Prediction__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NNet(tf.keras.Model):\n",
    "  def __init__(self):\n",
    "    super(NNet, self).__init__()\n",
    "    self.weights_const = tf.constant(results_weights_var)\n",
    "    self.biases_const = tf.constant(results_bias_var)\n",
    "    self.weights_out_const = tf.constant(results_weights_out_var)\n",
    "    \n",
    "  def call(self, input_data):\n",
    "    input_data = tf.transpose(input_data)\n",
    "    layer_1_const = tf.linalg.matmul(self.weights_const, input_data, transpose_b = False) + self.biases_const\n",
    "    layer_1_const = tf.nn.leaky_relu(layer_1_const)\n",
    "    layer_out_const = tf.linalg.matmul(self.weights_out_const, layer_1_const, transpose_b = False)\n",
    "    \n",
    "    return layer_out_const"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = NNet()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer n_net_13 is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2.1984581e-08"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss(model, full_training_inputs, full_training_outputs).numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ -1.2 -37. ]\n",
      " [  1.    1. ]\n",
      " [  2.    1. ]\n",
      " [  4.    1. ]\n",
      " [  1.    2. ]\n",
      " [  4.    6. ]]\n",
      "[[ 2.1 -0.5 10.   3.2 22.  10. ]]\n"
     ]
    }
   ],
   "source": [
    "print(full_training_inputs)\n",
    "print(full_training_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "number_points_to_plot = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.         1.33333333 1.66666667 2.         2.33333333 2.66666667\n",
      " 3.         3.33333333 3.66666667 4.        ]\n"
     ]
    }
   ],
   "source": [
    "input_data_to_graph = np.concatenate((np.linspace(start = 1, stop = 4, num=number_points_to_plot, \n",
    "                                                  endpoint=True, dtype=None, axis=0).reshape((number_points_to_plot, 1)),\n",
    "                                      np.ones(number_points_to_plot).reshape((number_points_to_plot, 1))),\n",
    "                                     axis = 1)\n",
    "print(input_data_to_graph[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3xUVeL+8c9JIQEEqVKkRAUNVcTAAgGxIIINVlfERUCUDYquXVBTIIZI+bkuuioaMdRQRFTUL4gooiQ0A1JDRJQWaaGX0JKc3x9kXYUEApnkzp0879eLl8mdYe5zufJwMufOPcZai4iIuI+f0wFEROTiqMBFRFxKBS4i4lIqcBERl1KBi4i4VEBJ7qxatWo2JCSkJHcpIuJ6y5cv32OtrX7m9hIt8JCQEFJTU0tylyIirmeM2ZLfdr2FIiLiUipwERGXUoGLiLiUClxExKVU4CIiLqUCFxFxKRW4iIhLqcBFRFzqvAVujEk0xuw2xqz9w7Yqxph5xpif8/5buXhjijdKHjiKjICq5BpDRkBVkgeOcjqSSKlSmBH4eKDLGdteBL6x1jYEvsn7XkqR5IGjuG5MFHVy9uEH1MnZx3VjolTiIiXovAVurf0e2HfG5m7AhLyvJwDdPZxLvFxIwkiCyeYDWvErVQAozylCEkY6nEyk9LjY98BrWGt3AOT997KCnmiMiTDGpBpjUjMzMy9yd+JtaufsYyrX0Z8eNOF5hnMTJ/Gnds6Z/9aLSHEp9klMa22CtTbMWhtWvfpZN9MSl9riV40hdKY527mTNF7mdlryNLP8mjkdTaTUuNgC32WMqQWQ99/dnoskbjCmwwv8SlVeZQ4zmMznJHKIIO7J7UNExFfs23fM6YgiPu9iC/wzoG/e132BWZ6JI25w/Hg2U3+5lKY1LM39dpMLtPDfxQf9L+X558NITFxDo0bjmDJlPdZap+OK+KzCXEY4FVgMXGOMyTDGPAKMAG41xvwM3Jr3vZQS7767ioyMw7wx5X7q5uzFz1rqZO/l1vcH8f/+340sX96bK664lF69/o/OnT9i48b9TkcW8UmmJEdIYWFhVgs6uNuRIye56qqxNG1ajW++6VHg83JycnnvvVW89NJCTpzIISqqDYMGtaZMGf8STCviG4wxy621YWdu1ycx5YL85z8/snt3FvHx7c/5PH9/PwYOvI709Ifp1q0B0dEptGgxge+/31ZCSUV8nwpcCu3AgeOMGrWMO++8kjZtahfq99SqdQnTp9/F7Nn3cOxYNh07TueRR75k715NcooUlQpcCu1f/0rlwIETxMWde/Sdn65dr2Tdun4MHtyaiRPTCA1NZOLEdZrkFCkCFbgUSmZmFqNHL6dHj2to0aLAz22dU7lygYwYcQMrVvSmYcPK9O07h06dZrBhgz78I3IxVOBSKCNGLCUrK5vY2HZFfq1mzaqTnPwA7757K8uX76JZswnExi7ixIlsDyQVKT1U4HJev/12mHfeWUXv3o0JDa3qkdf08zMMGHAt6ekPc++9DRk6dBHXXjuRBQu2euT1RUoDFbicV3z8EnJychkypK3HX7tmzfJMmXInc+f+jVOncrjppg956KE57NmT5fF9ifgaFbic06ZNB3j//TX079+MK66oVGz76dw5hLVrH+Lll/9CUtJ6QkPHMX78Wk1yipyDClzOKTZ2MQEBfkRFeX70faayZQOJj+/AypV9CA2tQr9+X3LTTdNJT99b7PsWcSMVuBRo/fq9TJqUxuOPt6B27UtKbL9NmlTj++978v77nVm9eg/Nm09gyJAUjh/XJKfIH6nApUBDhqRQrlwAgwe3LvF9+/kZ+vdvTnp6P+6/P5RXXllM8+YT+OabLSWeRcRbqcAlXytX7mbGjA08/fT1VK9ezrEcl11WnkmTbmfevPuw1tKp0wz69JlNZqYmOUVU4JKv6OhkKlUK4rnnzrp/jiM6darPmjUPER3dhmnT0rnmmkQ++GANkyatIyQkAT+/1wgJSSApKc3pqCIlJsDpAOJ9lizZzhdf/Mqrr3agUqVgp+P8Ljg4gFdeac8DDzTi0Ufn0b//XPz8IDf39ONbthwiIuIrAHr1auxgUpGSoRG4nCUyMpnLLivHk09e53SUfDVqVJUFC+6natXg38v7v7KysomMTHYmmEgJ0whc/mT+/K3Mn7+Vf//7JsqXL+N0nAIZY9i373i+j23deqiE04g4QyNw+Z21lsjIhdSpU4FHH73W6TjnVa9exXy3ly0bwK5dR0s4jUjJU4HL72bP/pUlS3YQHd2G4GDv/+EsPr495cr9OWdAgOHkyVxCQxN5//3V5Obqk5ziu1TgAkBuriUqKoWrrqpEv35NnY5TKL16NSYhoTP161fEGKhfvyLjx3dl7dqHaNHiMiIivuKGG6axbt0ep6OKFAutiSkAzJjxEz16fM6kSbfz4IPuv4LDWsvEiet47rnvOHjwBC+80Iro6DaULRvodDSRC6Y1MaVAOTm5xMSk0LhxVR54INTpOB5hjKFv36akp/fjwQcbMXz4Upo2Hc/cuZucjibiMSpwYfLkNNLT9/HKK+H4+/vW/xLVqpVj3LiufPttDwID/enSZSZ///sX7NypSU5xP9/62yoX7OTJHIYOXUTLljW4556GTscpNjfeWI9Vq/oQG9uOmTN/JjQ0kffeW6VJTnE1FXgpl5i4hs2bDzFsWDjGGKfjFKugoABiYtqxenVfWra8jEcfnUf79lNZsybT6WgiF0UFXoodO3aKuLglhIdfTpcuVzgdp8Rcc00VvvmmBxMmdGXDhv20bDmJl176nqysU05HE7kgKvBSbMyYVWzffoT4+PY+P/o+kzGGPn2akJ7ej969GzNixDKaNh3Pl19qklPcQwVeSh0+fJLhw5fSqVN9Onas63Qcx1SrVo7ExC4sWHA/QUH+dO06k549P2fHjiNORxM5LxV4KfXGG8vZs+cY8fHtnY7iFTp2rMvKlX145ZVwPv10I40ajWPMmJWa5BSvVqQCN8Y8Y4xZZ4xZa4yZaozxnnuPSoH27z/Oa6+lcvfdV9G6dS2n43iNoKAAoqPbsnp1X66/vgYDB35NePgUVq/WJKd4p4sucGPM5cCTQJi1tingD/T0VDApPq+99gOHDp0gLk6j7/xcfXUVvv76PiZO7MrGjQdo2XIigwd/x9GjJ52OJvInRX0LJQAoa4wJAMoB24seSYrT7t1HeeONFdx/fyjNm1d3Oo7XMsbQu3cT0tMf5qGHmjJq1A80bTqe2bN/dTqayO8uusCttb8BrwFbgR3AQWvtV2c+zxgTYYxJNcakZmbqR1GnDR++jGPHsomNbed0FFeoWrUsY8fexnff3U9wcAB33PExPXp8xvbtmuQU5xXlLZTKQDfgCqA2UN4Y8+CZz7PWJlhrw6y1YdWra8TnpIyMw4wZs5K+fZtw9dVVnI7jKjfccHqSMy4unM8++4VGjRJ5550fycnJPf9vFikmRXkLpROwyVqbaa09BXwMaFjnxeLiFpOba4mJaet0FFcKCgogKqota9Y8ROvWtXj88W9o124Kr766RAsriyOKctf+rUAbY0w54BhwC6B7xXqpX345QGLiWgYMaE5IyKVOx3G1hg0r89VXf2PKlPUMHPg1y5b9bw1OLawsJako74EvBT4CVgBr8l4rwUO5xMNiYxcRGOhHZGQbp6P4BGMMvXo1pmLFoLMe08LKUlKKtG6WtXYIMMRDWaSYpKXtYfLkNJ5/vhW1al3idByf8ttvh/PdvmWLFlaW4qdPYpYCMTEpXHJJGQYNauV0FJ9T0MLKxsBbb63QJKcUKxW4j1uxYhczZ/7MM89cT7Vq5ZyO43PyW1g5ONifJk2q8c9/zqdt2ymsXLnboXTi61TgPi4qKpnKlYN59tmzltMTD8hvYeWxY29j9eq+TJlyB1u2HCIsbBLPPfctR47ok5ziWVrU2IelpPxG+/ZTGTGiA4MH/8XpOKXS/v3HefHF70lIWE3duhV4++1O3HXXVU7HEpfRosaljLWWqKhkatQoxxNPXOd0nFKrcuVg3nuvM8nJD1CxYhnuvvsT7rlnFhkZ+U9+ilwIFbiP+uabrSxYsI3IyDaUL1/G6TilXnj45axY0YfhwzswZ84mGjVK5M03NckpRaMC90HWWiIjF1K3bgUiIpo7HUfylCnjz4sv/oV16x4iPPxynnpqPm3aJLFixS6no4lLqcB90Oef/8KyZTuJiWlLUFCRLvWXYnDllZWYM+depk69k23bDtOq1WSefVaTnHLhVOA+JjfXEh2dQoMGlejbt4nTcaQAxhh69gwlPf1hIiKa8+9/L6dRo3HMmrXR6WjiIipwHzNjxk+sXp1JbGw4gYH+TseR86hUKZgxY24lJeUBKlUKonv3T/nrXz9l2zZ9klPOTwXuQ7Kzc4mJSaFp02r07BnqdBy5AO3aXc6KFb0ZMaIDc+dupnHjcYwevZzsbE1ySsFU4D5k0qR1bNiwn1deCcfPzzgdRy5QYKA/gwefnuRs3/5ynnnmW/7ylySWL9/pdDTxUipwH3HiRDaxsYsJC6tB9+4NnI4jRXDFFZWYPftepk+/k+3bj9C6dRJPPz2fw4c1ySl/pgL3EWPHrmHLlkMMG9YeYzT6djtjDD16hLJ+fT8effRa3nxzBY0aJfLppz87HU28iArcB2RlnWLYsCV06FCHzp1DnI4jHlSpUjBvv92JRYv+TtWqZfnrX2fRrdsnbN2qSU5RgfuEd95Zyc6dR4mP1+jbV7VpU5vU1AcZNeoG5s3bQuPG43j99VRNcpZyKnCXO3ToBCNGLOO220Lo0KGO03GkGAUG+vPCC61JS+tHx451eO65BbRqNZkffthBUlKa1uUshfQxPZcbPXo5e/ceIy4u3OkoUkJCQi7liy/uYebMDTz55Hz+8pck/P0N2dmn7yyqdTlLD91O1oWSktKIjExmy5ZDGAMtW9YgNbW307HEAQcPnqBu3Xc5fPjUWY/Vr1+RzZsjHEglnqbbyfqIpKQ0IiK++n3NRWth3bo9+pG5lLr00iCOHDm7vAFNdJYCKnCXiYxMJisr+0/bjh/P0SropVhB63JWqhSkSU4fpwJ3mYJGVRptlV75rcvp52fYv/8EYWGTWLp0h0PJpLipwF2moNFWQdvF9+W3LueECV346KO7ycw8Rtu2STzxxNccPHjC6ajiYZrEdJmkpDT69p1DTs7/zlu5cgEkJHTWFQdylkOHThAVlcxbb/1IzZrlefPNm7n33qv1eQGX0SSmj2jduhbWQoUKZX4fbam8pSAVKwbx5pu3sHRpL2rWLM99933OXXd9wubNB52OJh6g68BdZujQRQQF+bNhwyPUrFne6TjiEq1a1WLZsgf5z39WEB2dQpMm4xg6tB1PP3297hvvYhqBu8jatZlMnbqeJ59sqfKWCxYQ4Mczz4SRltaPW26pz6BB3xMWNpklS7Y7HU0uUpEK3BhTyRjzkTEm3Riz3hjT1lPB5GwxMYuoUKEMgwa1cjqKuFi9ehWZNas7H3/cjb17j9Gu3RQef1yTnG5U1BH4G8CX1tpQ4FpgfdEjSX5SU3fyySc/8+yzYVSpUtbpOOJyxhj++teGrF//ME8+2ZJ3311FaGgiM2b8REle2CBFc9EFboypCNwAfABgrT1prT3gqWDyZ1FRyVStWpZnnrne6SjiQypUKMPo0TezdGkvate+hB49PueOOz5m0yb9VXaDoozArwQygXHGmB+NMWONMWe9MWuMiTDGpBpjUjMzM4uwu9Jr4cIM5s7dzODBrahYMcjpOOKDwsJqsnRpL0aPvomFCzNo0mQ8I0cu5dSpHKejyTlc9HXgxpgwYAkQbq1daox5AzhkrY0u6PfoOvALZ62lY8fp/Pzzfn75pT/lygU6HUl83LZth3jyyfl8+ulGmjWrxnvvdaZt29pOxyrViuM68Awgw1q7NO/7j4CWRXg9yce8eVtYuDCDqKg2Km8pEXXrVuSTT7rzySfd2L//BOHhU3jssXkcOHDc6WhyhosucGvtTmCbMeaavE23ALolngdZa4mMXEj9+hXp37+Z03GklOnevSFpaf14+unrSUhYTWhoItOnp2uS04sU9SqUfwJJxpjVQAvg1aJHkv+aNWsjqam7iIlpS1CQPnMlJa9ChTK8/vpN/PDDg9SpU4GePb+ga9eZ/PqrJjm9ge6F4qVycnJp0WIiJ0/msG5dPwIC9JkrcVZOTi7vvLOSl19eSHa2ZciQtjz3XJg+yVkCdC8Ul5k+/SfWrt1DbGy4ylu8gr+/H//8Z0vWr3+Y22+/gpdeWkjLlpNYtOg3p6OVWmoGL3TqVA5DhqTQvHl1evS45vy/QaQE1alTgZkzuzFrVncOHjxBePhUBgz4ioSEVVpYuYTpjVUvNHFiGhs3HmDWrO74+em2n+Kd7r67ATffXI8hQ1L497+Xk5Dwv8e0sHLJ0HvgXubEiWyuvvoDatYsz5IlvXTfZnGF2rXHsGPH0bO2a2FlzyjoPXCNwL1MQsJqtm49zNixt6m8xTV27jy7vIHfF9+W4qH3wL3I0aMniY9fQseOdejUqb7TcUQKraAl/QID/UhOzijhNKWHCtyLvPXWj+zalUV8fAeNvsVV8ltYOSjIj4oVy9ChwzQiIr5i375jDqXzXSpwL3Hw4AlGjvyBrl2vIDz8cqfjiFyQ/BZW/uCDLmzZEsHzz4eRmLiG0NBEkpLS9ElOD9IkppcYOjSF2NjFLF/em5YtazgdR8SjVq7czYABX7Fs2U46darPmDGdaNCgstOxXEMf5PFie/ce4/XXl3PvvQ1V3uKTWrS4jEWL/s7bb9/CsmU7aNp0PMOGLebkSd2utihU4F5g5MhlHDlyktjYcKejiBQbf38/Bg68jvXrH6ZbtwZER6fQosUEFi7UJOfFUoE7bMeOI7z11o/06tWYJk2qOR1HpNjVrn0J06ffxf/93z1kZWVzww3T6N9/riY5L4IK3GHx8Us4dSqXoUPbOR1FpETdfvuVrFv3EIMGtWL8+LWEhiYyebImOS+ECtxBmzcfJCFhNQ8/3JSrrqrkdByREle+fBlGjuzIihV9uPLKSvTuPZtbb53Bhg37nI7mCipwB8XFLcbPzxAd3dbpKCKOat68OosW/Z0xYzqRmrqL5s0nEBe3mBMnsp2O5tVU4A7ZsGEfEyas47HHWlCnTgWn44g4zs/P8OijLVi/vh/duzcgJiaFa6+dyHffbXM6mtdSgTtkyJBFBAX58+KLrZ2OIuJVatW6hGnT7mLOnHs5eTKHG2+czsMPf8nevZrkPJMK3AGrV2cybVo6Tz11PTVqlHc6johX6tLlCtaufYgXX2zNpElphIYmMnHiOk1y/oEK3AHR0clcemkQL7zQyukoIl6tXLlAhg+/gRUretOwYWX69p3DLbd8qEnOPCrwErZ06Q4+++wXnn8+jMqVg52OI+IKzZpVJzn5Ad5991ZWrNhNs2YTiI1dVOonOVXgJSw6Oplq1cry1FPXOx1FxFX8/AwDBlxLevrD3HtvQ4YOXUTz5hNYsGCr09EcowIvQd99t41587bw4outqVChjNNxRFypZs3yTJlyJ19+eS/Z2bncdNOHPPTQHPbsyXI6WonT3QhLiLWWDh2msWnTQTZufISyZQOdjiTieseOnWLYsCWMGvUDl14axL33NuTLLzexbdth6tWrSHx8e59Yk1N3I3TYl19uIiXlN6Ki2qi8RTykbNlA4uM7sHJlH6pWDf59SUJr/7ewclJSmtMxi40KvARYa4mKSiEkpCKPPNLM6TgiPqdJk2ocP372hGZWVjaRkckOJCoZWtS4BHzyyc+sWLGL8eO7UKaMv9NxRHzStm2H893uywsrawRezHJycomOTiE0tAoPPuj+9+JEvFVBCysD9Okzm8xM35vkVIEXs6lT00lL20tsbDv8/fXHLVJc8ltYuWxZf7p1a8C0aemEhiaSmLjGpz7JWeRGMcb4G2N+NMZ84YlAvuTUqRyGDEnh2mur87e/XeN0HBGflt/Cyu+/fxufftqdlSv70LhxVR55ZC433jid9ev3Oh3XI4p8GaEx5lkgDKhorb3zXM8tbZcRJiSsYsCAeXz++V+5886rnI4jUqrl5lrGjVvLCy98x5EjJ3nxxda8/HIbgoO9fyqwWC4jNMbUAe4AxhbldXzR8ePZxMUtoU2bWtxxx5VOxxEp9fz8DI880oz09H7cf38ocXFLaNZsPF9/vcXpaBetqG+hjAYGAbkFPcEYE2GMSTXGpGZmZhZxd+7x3nuryMg4THx8B4wxTscRkTyXXVaeSZNuZ968+wC49dYZ9O49m927jzqc7MJddIEbY+4Edltrl5/redbaBGttmLU2rHr16he7O1c5evQkr766lJtvrsfNN9dzOo6I5KNTp/qsWfMQMTFtmT49ndDQcYwdu5rcXPdMchZlBB4O3G2M2QxMA242xkz2SCqXe/PNH9m9O4thw9o7HUVEziE4OIDY2HBWrepLs2bV+Mc/vqJjx2mkpe1xOlqhXHSBW2tfstbWsdaGAD2B+dbaBz2WzKUOHDjOqFHLuOOOK2nbtrbTcUSkEBo1qsqCBfeTmHgbaWl7adFiIlFRyRw7dsrpaOekC5M97F//SuXAgRMafYu4jDGGfv2akZ7+MH//eyPi45fQrNkE5s3b7HS0AnmkwK21C853CWFpkJmZxejRy7nvvqtp0eIyp+OIyEWoXr0c48d3Zf78Hvj7Gzp3/ohevf6PXbu8b5JTI3APGjlyGVlZ2bzySrjTUUSkiG66qR6rVvVlyJC2fPTRBkJDE3n/fe+a5FSBe8j27Ud4++2V9O7dmNDQqk7HEREPCA4OYOjQcFav7kuLFpcREfEVN9wwjbVrveOSaBW4hwwbtpjs7FxiYto6HUVEPOyaa6owf34Pxo/vQnr6Pq67bhIvv7yQrCxnJzlV4B6wadMB3n9/Df37N+PKKys5HUdEioExhr59m5Ke3o8HH2zE8OFLadZsPHPnbnIskwrcA2JjFxMQ4EdUVBuno4hIMatWrRzjxnXl2297EBjoT5cuM3nggS/YubPkJzlV4EW0fv1eJk1KY+DAa7n88gpOxxGREnLjjfVYtaoPsbHt+PjjnwkNTeS991aRm2tJHjiKjICq5BpDRkBVkgeOKpYMWtS4iO6//3Nmz/6VX3/9B9Wrl3M6jog4YMOGfTz22NfMn7+VpjUsY3e9yV/I+P3xowTy42PDaP/OoIt6fS1qXAxWrtzNhx/+xNNPX6/yFinFrr66Cl9/fR8TJ3Zl+65jtOcJXqIrWZxewLw8pwhJGOnx/arAiyA6OplKlYJ47rmz/mEUkVLGGEPv3k1IZyR9SWUEN/Ml/1vIpXbOPo/v0/vvZO5lkpLSiIxM/n2h1Pvuu5pKlYIdTiUi3uKEfzBjcz7in6TQnB2/b9/uX4U6Ht6XRuAXICkpjYiIr/60yvUXX/xCUlKag6lExJtsjhjMUQK5lh38dyWAowSyOWKwx/elAr8AkZHJZGVl/2nbsWM5REYmO5RIRLxN+3cG8eNjw8jwr0IukOFfpUgTmOeiq1AugJ/fa+T3x2UM5OY+X/KBRKRU0FUoHlCvXsUL2i4iUpxU4BcgLi6cM5e3LFcugPh43ftbREqeCvwCBAcHYC1UrRqMMVC/fkUSEjrTq1djp6OJSCmkywgLKScnl5iYFBo3rsrq1X3x99e/fSLiLBV4IU2enEZ6+j4++uhulbeIeAU1USGcPJlDbOxiWraswT33NHQ6jogIoBF4oSQmrmHTpoO8/fYtmDNnMUVEHKIR+HkcO3aKuLgltGtXmy5drnA6jojI7zQCP48xY1axffsRkpJu1+hbRLyKRuDncPjwSYYPX0qnTvW58cZ6TscREfkTFfg5vPHGcvbsOaYP6oiIV1KBF2D//uO89loqd999Fa1b13I6jojIWVTgBXjttR84ePAEcXEafYuId1KB52P37qO88cYK7r//Gpo3r+50HBGRfF10gRtj6hpjvjXGrDfGrDPGPOXJYE4aPnwZx45lExsb7nQUEZECFeUywmzgOWvtCmNMBWC5MWaetdbVy9NkZBxmzJiV9O3bhGuuqeJ0HBGRAl30CNxau8NauyLv68PAeuByTwVzSlzcYnJzLTExbZ2OIiJyTh55D9wYEwJcByzN57EIY0yqMSY1MzPTE7srNr/8coDExLVERDQnJORSp+OIiJxTkQvcGHMJMBN42lp76MzHrbUJ1towa21Y9erePSEYG7uIgAA/IiPbOB1FROS8ilTgxphATpd3krX2Y89EckZa2h4mT07jiSdaUKvWJU7HERE5r6JchWKAD4D11trXPRfJGTExKVxySRkGD27tdBQRkUIpygg8HOgN3GyMWZn363YP5SpRK1bsYubMn3nmmeupVq2c03FERArloi8jtNYmAz5xe76oqGQqVw7m2WfDnI4iIlJopf6TmCkpvzFnziYGD27FpZcGOR1HRKTQSnWBW2uJikqmRo1yPPHEdU7HERG5IKV6QYdvvtnKggXbeOONmylfvozTcURELkipHYFba4mMXEjduhUYMKC503FERC5YqR2Bf/75LyxbtpP33+9MUFCp/WMQERcrlSPw3FxLdHQKDRpUom/fJk7HERG5KKVy6Dljxk+sXp1JUtIdBAb6Ox1HROSilLoReHZ2LjExKTRtWo2ePUOdjiMictFK3Qh80qR1bNiwn48/7oafn098DklESqlSNQI/cSKb2NjFhIXVoHv3Bk7HEREpklI1Ah87dg1bthzivfdu5fS9uERE3KvUjMCzsk4xbNgSOnSoQ+fOIU7HEREpslIzAn/nnZXs3HmUDz+8S6NvEfEJpWIEfujQCUaMWMZtt4XQoUMdp+OIiHhEqSjw0aOXs3fvMeLiwp2OIiLiMT5f4Pv2HeNf/0qle/cGtGpVy+k4IiIe4/MFPmrUDxw+fFKjbxHxOT5d4Dt3HuXNN1fwwAONaNq0utNxREQ8yqcLfPjwpZw8mUNsbDuno4iIeJzPFvjWrYd4991V9OvXlAYNKjsdR0TE43y2wOPiFgMQHd3W4SQiIsXDJwv855/3M27cWgYMaE69ehWdjiMiUj0FKZkAAAaJSURBVCx8ssCHDl1EmTL+vPxyG6ejiIgUG58r8LVrM5k6dT1PPtmSmjXLOx1HRKTY+FyBx8QsokKFMgwa1MrpKCIixcqnCjw1dSeffPIzzz0XRpUqZZ2OIyJSrHyqwKOikqlatSxPP32901FERIqdzxT4woUZzJ27mcGDW1GxYpDTcUREil2RCtwY08UY85MxZqMx5kVPhbpQ1loiI5OpWbM8jz9+nVMxRERK1EUXuDHGH3gb6Ao0Bh4wxjT2VLAL8dVXm1m4MIOoqDaUKxfoRAQRkRJXlBF4a2CjtfZXa+1JYBrQzTOxCs9aS1RUMvXrV+Qf/2he0rsXEXFMUQr8cmDbH77PyNv2J8aYCGNMqjEmNTMzswi7y9+sWRtJTd3FkCFtKVPG3+OvLyLirYpS4PktLGnP2mBtgrU2zFobVr26Z2/pmpOTS3R0CldfXZnevZt49LVFRLxdURY1zgDq/uH7OsD2osW5MNOn/8TatXuYOvVOAgJ85oIaEZFCKUrr/QA0NMZcYYwpA/QEPvNMrPM7dSqHIUNSaN68Oj16XFNSuxUR8RoXPQK31mYbY54A5gL+QKK1dp3Hkp3HhAnr2LjxALNmdcfPL793c0REfFtR3kLBWjsbmO2hLIV24kQ2r7yymNata3LXXVeV9O5FRLyCq944Th44ioyAqrwXfCPbth3mgRrbMUajbxEpnVxT4MkDR3HdmCgq5xzmVW6hI7/Q//MokgeOcjqaiIgjXFPgIQkjKc8p3iKcXVQgni+5hFOEJIx0OpqIiCNcU+C1c/YBUIMj9GMZ4Wz+03YRkdLGNQW+3b8KAA+RSiIzztouIlLauKbAN0cM5ih/vlHVUQLZHDHYoUQiIs5yTYG3f2cQPz42jAz/KuQCGf5V+PGxYbR/Z5DT0UREHGGsPev2JcUmLCzMpqamltj+RER8gTFmubU27MztrhmBi4jIn6nARURcSgUuIuJSKnAREZdSgYuIuJQKXETEpVTgIiIupQIXEXGpEv0gjzEmE9jigZeqBuzxwOs4TcfhPXzhGEDH4W08dRz1rbVnrQpfogXuKcaY1Pw+leQ2Og7v4QvHADoOb1Pcx6G3UEREXEoFLiLiUm4t8ASnA3iIjsN7+MIxgI7D2xTrcbjyPXAREXHvCFxEpNRTgYuIuJTXFrgxJtEYs9sYs7aAx40x5k1jzEZjzGpjTMuSzlgYhTiOG40xB40xK/N+xZR0xvMxxtQ1xnxrjFlvjFlnjHkqn+d4/fko5HG44XwEG2OWGWNW5R1HbD7PccP5KMxxeP35ADDG+BtjfjTGfJHPY8V3Lqy1XvkLuAFoCawt4PHbgTmAAdoAS53OfJHHcSPwhdM5z3MMtYCWeV9XADYAjd12Pgp5HG44Hwa4JO/rQGAp0MaF56Mwx+H15yMv57PAlPyyFue58NoRuLX2e2DfOZ7SDZhoT1sCVDLG1CqZdIVXiOPwetbaHdbaFXlfHwbWA5ef8TSvPx+FPA6vl/dnfCTv28C8X2dejeCG81GY4/B6xpg6wB3A2AKeUmznwmsLvBAuB7b94fsMXPiXMU/bvB8j5xhjmjgd5lyMMSHAdZweLf2Rq87HOY4DXHA+8n5kXwnsBuZZa115PgpxHOD952M0MAjILeDxYjsXbi5wk8821/3rDazg9H0OrgX+A3zqcJ4CGWMuAWYCT1trD535cD6/xSvPx3mOwxXnw1qbY61tAdQBWhtjmp7xFFecj0Ich1efD2PMncBua+3ycz0tn20eORduLvAMoO4fvq8DbHcoy0Wz1h7674+R1trZQKAxpprDsc5ijAnkdOklWWs/zucprjgf5zsOt5yP/7LWHgAWAF3OeMgV5+O/CjoOF5yPcOBuY8xmYBpwszFm8hnPKbZz4eYC/wzokzfD2wY4aK3d4XSoC2WMqWmMMXlft+b0OdnrbKo/y8v3AbDeWvt6AU/z+vNRmONwyfmoboyplPd1WaATkH7G09xwPs57HN5+Pqy1L1lr61hrQ4CewHxr7YNnPK3YzkWAJ16kOBhjpnJ6BrqaMSYDGMLpSQ6ste8Cszk9u7sRyAL6OZP03ApxHH8DHjPGZAPHgJ42b+rai4QDvYE1ee9XArwM1ANXnY/CHIcbzkctYIIxxp/ThfahtfYLY8yj4KrzUZjjcMP5OEtJnQt9lF5ExKXc/BaKiEippgIXEXEpFbiIiEupwEVEXEoFLiLiUipwERGXUoGLiLjU/weG9x61NInxSAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(input_data_to_graph[:,0],\n",
    "        model(input_data_to_graph).numpy()[0],\n",
    "        color = \"darkblue\")\n",
    "plt.scatter(input_data_to_graph[:,0],\n",
    "        model(input_data_to_graph).numpy()[0],\n",
    "        color = \"darkblue\")\n",
    "plt.scatter(full_training_inputs[1:4,0],\n",
    "           full_training_outputs[0][1:4],\n",
    "           color = \"red\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tf2_2] *",
   "language": "python",
   "name": "conda-env-tf2_2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}