project01_MNIST_digit_recognizer/feedforward_MNIST_with_tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feed forward neural network example using tensorflow\n",
    "\n",
    "Tue 25 Apr 2017\n",
    "\n",
    "\n",
    "The steps below are **heavily** inspired by this tutorial:\n",
    "  - https://github.com/sherrym/tf-tutorial/blob/master/2_mnist.ipynb\n",
    "  - video: https://youtu.be/eyovmAtoUx0?t=6h25m20s\n",
    "\n",
    "Some updates were performed because tf 1.0.1 has a slightly different API.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 2.1 Import libraries.\n",
    "import math\n",
    "import os\n",
    "from six.moves import xrange\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib.learn.python.learn.datasets.mnist import read_data_sets\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from IPython.display import clear_output, Image, display, HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 2.2 Define some constants.\n",
    "# The MNIST dataset has 10 classes, representing the digits 0 through 9.\n",
    "NUM_CLASSES = 10\n",
    "\n",
    "# The MNIST images are always 28x28 pixels.\n",
    "IMAGE_SIZE = 28\n",
    "IMAGE_PIXELS = IMAGE_SIZE * IMAGE_SIZE\n",
    "\n",
    "# Batch size. Must be evenly dividable by dataset sizes.\n",
    "BATCH_SIZE = 200\n",
    "LEARNING_RATE = 0.4\n",
    "EVAL_BATCH_SIZE = 1\n",
    "\n",
    "# Number of units in hidden layers.\n",
    "HIDDEN1_UNITS = 128\n",
    "HIDDEN2_UNITS = 64\n",
    "\n",
    "# Maximum number of training steps.\n",
    "MAX_STEPS = 3000\n",
    "\n",
    "# Directory to put the training data.\n",
    "TRAIN_DIR=\"/tmp/mnist\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/mnist/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/mnist/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/mnist/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/mnist/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# 2.3 Get input data: get the sets of images and labels for training, validation, and\n",
    "# test on MNIST.\n",
    "data_sets = read_data_sets(TRAIN_DIR, validation_size=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 2.4 Build inference graph.\n",
    "def mnist_inference(images, hidden1_units, hidden2_units):\n",
    "    \"\"\"Build the MNIST model up to where it may be used for inference.\n",
    "    Args:\n",
    "        images: Images placeholder.\n",
    "        hidden1_units: Size of the first hidden layer.\n",
    "        hidden2_units: Size of the second hidden layer.\n",
    "    Returns:\n",
    "        logits: Output tensor with the computed logits.\n",
    "    \"\"\"\n",
    "    # Hidden 1\n",
    "    with tf.name_scope('hidden1'):\n",
    "        weights = tf.Variable(\n",
    "            tf.truncated_normal([IMAGE_PIXELS, hidden1_units],\n",
    "                                stddev=1.0 / math.sqrt(float(IMAGE_PIXELS))),\n",
    "            name='weights')\n",
    "        biases = tf.Variable(tf.zeros([hidden1_units]),\n",
    "                             name='biases')\n",
    "        hidden1 = tf.nn.relu(tf.matmul(images, weights) + biases)\n",
    "    # Hidden 2\n",
    "    with tf.name_scope('hidden2'):\n",
    "        weights = tf.Variable(\n",
    "            tf.truncated_normal([hidden1_units, hidden2_units],\n",
    "                                stddev=1.0 / math.sqrt(float(hidden1_units))),\n",
    "            name='weights')\n",
    "        biases = tf.Variable(tf.zeros([hidden2_units]),\n",
    "                             name='biases')\n",
    "        hidden2 = tf.nn.relu(tf.matmul(hidden1, weights) + biases)\n",
    "    # Linear\n",
    "    with tf.name_scope('softmax_linear'):\n",
    "        weights = tf.Variable(\n",
    "            tf.truncated_normal([hidden2_units, NUM_CLASSES],\n",
    "                                stddev=1.0 / math.sqrt(float(hidden2_units))),\n",
    "            name='weights')\n",
    "        biases = tf.Variable(tf.zeros([NUM_CLASSES]),\n",
    "                             name='biases')\n",
    "        logits = tf.matmul(hidden2, weights) + biases\n",
    "    #\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 2.5 Build training graph.\n",
    "def mnist_training(logits, labels, learning_rate):\n",
    "    \"\"\"Build the training graph.\n",
    "\n",
    "    Args:\n",
    "        logits: Logits tensor, float - [BATCH_SIZE, NUM_CLASSES].\n",
    "        labels: Labels tensor, int32 - [BATCH_SIZE], with values in the\n",
    "          range [0, NUM_CLASSES).\n",
    "        learning_rate: The learning rate to use for gradient descent.\n",
    "    Returns:\n",
    "        train_op: The Op for training.\n",
    "        loss: The Op for calculating loss.\n",
    "    \"\"\"\n",
    "    # Create an operation that calculates loss.\n",
    "    labels = tf.to_int64(labels)\n",
    "    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "        logits=logits,\n",
    "        labels=labels,\n",
    "        name='xentropy')\n",
    "    loss = tf.reduce_mean(cross_entropy, name='xentropy_mean')\n",
    "    # Create the gradient descent optimizer with the given learning rate.\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    # Create a variable to track the global step.\n",
    "    global_step = tf.Variable(0, name='global_step', trainable=False)\n",
    "    # Use the optimizer to apply the gradients that minimize the loss\n",
    "    # (and also increment the global step counter) as a single training step.\n",
    "    train_op = optimizer.minimize(loss, global_step=global_step)\n",
    "\n",
    "    # Uncomment the following line to see what we have constructed.\n",
    "    # tf.train.write_graph(tf.get_default_graph().as_graph_def(),\n",
    "    #                      \"/tmp\", \"train.pbtxt\", as_text=True)\n",
    "\n",
    "    return train_op, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 2.6 Build the complete graph for feeding inputs, training, and saving checkpoints.\n",
    "mnist_graph = tf.Graph()\n",
    "with mnist_graph.as_default():\n",
    "    # Generate placeholders for the images and labels.\n",
    "    images_placeholder = tf.placeholder(tf.float32)                                       \n",
    "    labels_placeholder = tf.placeholder(tf.int32)\n",
    "    tf.add_to_collection(\"images\", images_placeholder)  # Remember this Op.\n",
    "    tf.add_to_collection(\"labels\", labels_placeholder)  # Remember this Op.\n",
    "\n",
    "    # Build a Graph that computes predictions from the inference model.\n",
    "    logits = mnist_inference(images_placeholder,\n",
    "                             HIDDEN1_UNITS,\n",
    "                             HIDDEN2_UNITS)\n",
    "    tf.add_to_collection(\"logits\", logits)  # Remember this Op.\n",
    "\n",
    "    # Add to the Graph the Ops that calculate and apply gradients.\n",
    "    train_op, loss = mnist_training(logits, labels_placeholder, LEARNING_RATE)\n",
    "    \n",
    "    # setup logging of loss\n",
    "    tf.summary.scalar('loss', loss)\n",
    "    \n",
    "    # Add the variable initializer Op.\n",
    "    init = tf.global_variables_initializer()\n",
    "\n",
    "    # Create a saver for writing training checkpoints.\n",
    "    saver = tf.train.Saver()\n",
    "    \n",
    "    # Uncomment the following line to see what we have constructed.\n",
    "    tf.train.write_graph(tf.get_default_graph().as_graph_def(),\n",
    "                         \"/tmp\", \"complete.pbtxt\", as_text=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.45743773907577145&quot;).pbtxt = 'node {\\n  name: &quot;Placeholder&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Placeholder_1&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000\\\\200\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0357142873108387\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;hidden1/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;hidden1/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/truncated_normal/mul&quot;\\n  input: &quot;hidden1/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 128\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/weights&quot;\\n  input: &quot;hidden1/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/weights/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/weights&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 128\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/biases&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 128\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/biases/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/biases&quot;\\n  input: &quot;hidden1/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/biases/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/biases&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;Placeholder&quot;\\n  input: &quot;hidden1/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/MatMul&quot;\\n  input: &quot;hidden1/biases/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;hidden1/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\200\\\\000\\\\000\\\\000@\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0883883461356163\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;hidden2/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;hidden2/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/truncated_normal/mul&quot;\\n  input: &quot;hidden2/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 128\\n        }\\n        dim {\\n          size: 64\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/weights&quot;\\n  input: &quot;hidden2/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/weights/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/weights&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 64\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/biases&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 64\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/biases/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/biases&quot;\\n  input: &quot;hidden2/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/biases/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/biases&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;hidden1/Relu&quot;\\n  input: &quot;hidden2/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/MatMul&quot;\\n  input: &quot;hidden2/biases/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;hidden2/add&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/truncated_normal/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;@\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/truncated_normal/mean&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/truncated_normal/stddev&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.125\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/truncated_normal/TruncatedNormal&quot;\\n  op: &quot;TruncatedNormal&quot;\\n  input: &quot;softmax_linear/truncated_normal/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/truncated_normal/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;softmax_linear/truncated_normal/TruncatedNormal&quot;\\n  input: &quot;softmax_linear/truncated_normal/stddev&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/truncated_normal&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;softmax_linear/truncated_normal/mul&quot;\\n  input: &quot;softmax_linear/truncated_normal/mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/weights&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 64\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/weights/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;softmax_linear/weights&quot;\\n  input: &quot;softmax_linear/truncated_normal&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@softmax_linear/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/weights/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;softmax_linear/weights&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@softmax_linear/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/biases&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/biases/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;softmax_linear/biases&quot;\\n  input: &quot;softmax_linear/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@softmax_linear/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/biases/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;softmax_linear/biases&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@softmax_linear/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;hidden2/Relu&quot;\\n  input: &quot;softmax_linear/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;softmax_linear/add&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;softmax_linear/MatMul&quot;\\n  input: &quot;softmax_linear/biases/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;ToInt64&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;Placeholder_1&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;xentropy/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;ToInt64&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;xentropy/xentropy&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;softmax_linear/add&quot;\\n  input: &quot;ToInt64&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;xentropy_mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;xentropy/xentropy&quot;\\n  input: &quot;Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;global_step/initial_value&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;global_step&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;global_step/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;global_step&quot;\\n  input: &quot;global_step/initial_value&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@global_step&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;global_step/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;global_step&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@global_step&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;gradients/Shape&quot;\\n  input: &quot;gradients/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/Fill&quot;\\n  input: &quot;gradients/xentropy_mean_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;xentropy/xentropy&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;gradients/xentropy_mean_grad/Reshape&quot;\\n  input: &quot;gradients/xentropy_mean_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;xentropy/xentropy&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;gradients/xentropy_mean_grad/Shape_1&quot;\\n  input: &quot;gradients/xentropy_mean_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;gradients/xentropy_mean_grad/Shape_2&quot;\\n  input: &quot;gradients/xentropy_mean_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;gradients/xentropy_mean_grad/Prod_1&quot;\\n  input: &quot;gradients/xentropy_mean_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;gradients/xentropy_mean_grad/Prod&quot;\\n  input: &quot;gradients/xentropy_mean_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;gradients/xentropy_mean_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy_mean_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;gradients/xentropy_mean_grad/Tile&quot;\\n  input: &quot;gradients/xentropy_mean_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;xentropy/xentropy:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy/xentropy_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;xentropy/xentropy:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy/xentropy_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy/xentropy_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;gradients/xentropy_mean_grad/truediv&quot;\\n  input: &quot;gradients/xentropy/xentropy_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/xentropy/xentropy_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;gradients/xentropy/xentropy_grad/ExpandDims&quot;\\n  input: &quot;gradients/xentropy/xentropy_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;softmax_linear/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 10\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;gradients/softmax_linear/add_grad/Shape&quot;\\n  input: &quot;gradients/softmax_linear/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;gradients/xentropy/xentropy_grad/mul&quot;\\n  input: &quot;gradients/softmax_linear/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/softmax_linear/add_grad/Sum&quot;\\n  input: &quot;gradients/softmax_linear/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;gradients/xentropy/xentropy_grad/mul&quot;\\n  input: &quot;gradients/softmax_linear/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/softmax_linear/add_grad/Sum_1&quot;\\n  input: &quot;gradients/softmax_linear/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/softmax_linear/add_grad/Reshape&quot;\\n  input: &quot;^gradients/softmax_linear/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/softmax_linear/add_grad/Reshape&quot;\\n  input: &quot;^gradients/softmax_linear/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/softmax_linear/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/softmax_linear/add_grad/Reshape_1&quot;\\n  input: &quot;^gradients/softmax_linear/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/softmax_linear/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/softmax_linear/add_grad/tuple/control_dependency&quot;\\n  input: &quot;softmax_linear/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;hidden2/Relu&quot;\\n  input: &quot;gradients/softmax_linear/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/softmax_linear/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/softmax_linear/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/softmax_linear/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/softmax_linear/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/softmax_linear/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/softmax_linear/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/softmax_linear/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/softmax_linear/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/softmax_linear/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;gradients/softmax_linear/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;hidden2/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 64\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;gradients/hidden2/add_grad/Shape&quot;\\n  input: &quot;gradients/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;gradients/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;gradients/hidden2/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/hidden2/add_grad/Sum&quot;\\n  input: &quot;gradients/hidden2/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;gradients/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;gradients/hidden2/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/hidden2/add_grad/Sum_1&quot;\\n  input: &quot;gradients/hidden2/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^gradients/hidden2/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/hidden2/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/hidden2/add_grad/Reshape&quot;\\n  input: &quot;^gradients/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/hidden2/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/hidden2/add_grad/Reshape_1&quot;\\n  input: &quot;^gradients/hidden2/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/hidden2/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/hidden2/add_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;hidden1/Relu&quot;\\n  input: &quot;gradients/hidden2/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;gradients/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/add_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;hidden1/MatMul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/add_grad/Shape_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 128\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  op: &quot;BroadcastGradientArgs&quot;\\n  input: &quot;gradients/hidden1/add_grad/Shape&quot;\\n  input: &quot;gradients/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/add_grad/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;gradients/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;gradients/hidden1/add_grad/BroadcastGradientArgs&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/add_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/hidden1/add_grad/Sum&quot;\\n  input: &quot;gradients/hidden1/add_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/add_grad/Sum_1&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;gradients/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;gradients/hidden1/add_grad/BroadcastGradientArgs:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/add_grad/Reshape_1&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/hidden1/add_grad/Sum_1&quot;\\n  input: &quot;gradients/hidden1/add_grad/Shape_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/add_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^gradients/hidden1/add_grad/Reshape_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/hidden1/add_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/hidden1/add_grad/Reshape&quot;\\n  input: &quot;^gradients/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/hidden1/add_grad/Reshape&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/hidden1/add_grad/Reshape_1&quot;\\n  input: &quot;^gradients/hidden1/add_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/hidden1/add_grad/Reshape_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/hidden1/add_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/weights/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;Placeholder&quot;\\n  input: &quot;gradients/hidden1/add_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^gradients/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^gradients/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.4000000059604645\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden1/weights/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/weights&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;gradients/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden1/biases/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden1/biases&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;gradients/hidden1/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden2/weights/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/weights&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;gradients/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_hidden2/biases/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;hidden2/biases&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;gradients/hidden2/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_softmax_linear/weights/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;softmax_linear/weights&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;gradients/softmax_linear/MatMul_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@softmax_linear/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_softmax_linear/biases/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;softmax_linear/biases&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;gradients/softmax_linear/add_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@softmax_linear/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^GradientDescent/update_hidden1/weights/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden1/biases/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden2/weights/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_hidden2/biases/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_softmax_linear/weights/ApplyGradientDescent&quot;\\n  input: &quot;^GradientDescent/update_softmax_linear/biases/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;GradientDescent/value&quot;\\n  op: &quot;Const&quot;\\n  input: &quot;^GradientDescent/update&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@global_step&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent&quot;\\n  op: &quot;AssignAdd&quot;\\n  input: &quot;global_step&quot;\\n  input: &quot;GradientDescent/value&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@global_step&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/tags&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;loss&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss&quot;\\n  op: &quot;ScalarSummary&quot;\\n  input: &quot;loss/tags&quot;\\n  input: &quot;xentropy_mean&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden1/weights/Assign&quot;\\n  input: &quot;^hidden1/biases/Assign&quot;\\n  input: &quot;^hidden2/weights/Assign&quot;\\n  input: &quot;^hidden2/biases/Assign&quot;\\n  input: &quot;^softmax_linear/weights/Assign&quot;\\n  input: &quot;^softmax_linear/biases/Assign&quot;\\n  input: &quot;^global_step/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 7\\n          }\\n        }\\n        string_val: &quot;global_step&quot;\\n        string_val: &quot;hidden1/biases&quot;\\n        string_val: &quot;hidden1/weights&quot;\\n        string_val: &quot;hidden2/biases&quot;\\n        string_val: &quot;hidden2/weights&quot;\\n        string_val: &quot;softmax_linear/biases&quot;\\n        string_val: &quot;softmax_linear/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 7\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;global_step&quot;\\n  input: &quot;hidden1/biases&quot;\\n  input: &quot;hidden1/weights&quot;\\n  input: &quot;hidden2/biases&quot;\\n  input: &quot;hidden2/weights&quot;\\n  input: &quot;softmax_linear/biases&quot;\\n  input: &quot;softmax_linear/weights&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_INT32\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;global_step&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_INT32\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;global_step&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@global_step&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden1/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_1/tensor_names&quot;\\n  input: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/biases&quot;\\n  input: &quot;save/RestoreV2_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden1/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_2/tensor_names&quot;\\n  input: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/weights&quot;\\n  input: &quot;save/RestoreV2_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden2/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_3/tensor_names&quot;\\n  input: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/biases&quot;\\n  input: &quot;save/RestoreV2_3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden2/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_4/tensor_names&quot;\\n  input: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/weights&quot;\\n  input: &quot;save/RestoreV2_4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;softmax_linear/biases&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_5/tensor_names&quot;\\n  input: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;softmax_linear/biases&quot;\\n  input: &quot;save/RestoreV2_5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@softmax_linear/biases&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_6/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;softmax_linear/weights&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_6/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_6&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_6/tensor_names&quot;\\n  input: &quot;save/RestoreV2_6/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_6&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;softmax_linear/weights&quot;\\n  input: &quot;save/RestoreV2_6&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@softmax_linear/weights&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n  input: &quot;^save/Assign_6&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.45743773907577145&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Display in the browser (works only in Chrome)\n",
    "from tfutils import show_graph\n",
    "show_graph(mnist_graph.as_graph_def())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 0: loss = 2.29\n",
      "Step 500: loss = 0.10\n",
      "Step 1000: loss = 0.05\n",
      "Step 1500: loss = 0.04\n",
      "Step 2000: loss = 0.04\n",
      "Step 2500: loss = 0.04\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11c127978>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VPW9//HXZxJ2kEWCIggBRS2KLCLq1au0xQ1bl157\nRVtrbf1pq7b12t5erFatWrW31iquV3GvWreqKCCyCoogYd8h7IQtLAkJkHW+vz/mzDBJZiaTMCE5\nw/v5eOTBzJkzM9+TCe/5ns/5nu8x5xwiIpJeAo3dABERST2Fu4hIGlK4i4ikIYW7iEgaUriLiKQh\nhbuISBpSuIuIpCGFu4hIGlK4i4ikoczGeuPOnTu77Ozsxnp7ERFfmjt37k7nXFZt6zVauGdnZ5OT\nk9NYby8i4ktmtiGZ9VSWERFJQwp3EZE0pHAXEUlDCncRkTSkcBcRSUMKdxGRNKRwFxFJQ74L95Xb\nivjb5yvZVVza2E0REWmyfBfua/KLeWpKLvkKdxGRuHwX7i2bhZpcUh5s5JaIiDRd/gv3zAwASsor\nG7klIiJNl+/CvUUzhbuISG18F+4qy4iI1M6H4R7quZdWqOcuIhKP78K9eUaoyaXquYuIxOW/cM8M\nNbk8qHAXEYnHd+GeGTAAKipdI7dERKTp8l24Nwv33CvVcxcRicd/4R4Ih7t67iIi8fgv3DPCZRn1\n3EVE4vFduGd4NXeVZURE4vNduJsZzTMClAdVlhERicd34Q6QmWGUV6jnLiISjz/DPWBUqOcuIhKX\nL8O9eWaAMtXcRUTi8mW4ZwYCGi0jIpKAL8O9WabpDFURkQT8Ge4BlWVERBLxZbhnZqjnLiKSiC/D\nPWBG0CncRUTiqTXczex4M5tqZsvMbKmZ/SbGOmZmo8ws18wWmdmghmluSCjcG/IdRET8LTOJdSqA\n3zrn5plZO2CumU10zi2LWudSoI/3cxbwnPdvgwgEUM9dRCSBWnvuzrmtzrl53u0iYDnQrdpqVwCv\nu5BZQAcz65ry1noyVJYREUmoTjV3M8sGBgKzqz3UDdgUdX8zNb8AUsZUlhERSSjpcDeztsAHwB3O\nub31eTMzu9nMcswsJz8/vz4vAUDAIKh0FxGJK6lwN7NmhIL9Tefcv2KskgccH3W/u7esCufcC865\nwc65wVlZWfVpLxCa9ldlGRGR+JIZLWPAS8By59zjcVYbA/zEGzVzNlDonNuawnZWb5PCXUQkgWRG\ny5wLXA8sNrMF3rI/AD0AnHPPA+OA4UAusB+4MfVNPShUlmnIdxAR8bdaw9059yVgtazjgNtS1aja\nZARME4eJiCSgM1RFRNKQL8PdzNDUMiIi8fky3DMMnHruIiJx+TLcVZYREUnMl+FuZuh4qohIfL4M\n94yAyjIiIon4MtxVlhERScy34V6puWVEROLyZ7gHDHXcRUTi82e4my7WISKSiE/D3ahUuIuIxOXb\ncNfEYSIi8fk03DUUUkQkEZ+Gu8oyIiKJ+DPcA7qGqohIIv4Md5VlREQS8mm46yQmEZFEfBruqCwj\nIpKAP8M9oLllREQS8We4mxFU111EJC6fhrvKMiIiifgz3FWWERFJyJ/hrvncRUQS8mm4qywjIpKI\nL8M9Qz13EZGEfBnuZqGLdegsVRGR2HwZ7gEzQKUZEZF4fBnuGV6rVZoREYnNl+FukZ67wl1EJBZf\nhnukLKOrMYmIxOTLcA+XZXTBDhGR2HwZ7uFMf/CTZY3bEBGRJsqX4R4eJfNOzqbGbYiISBPly3AX\nEZHEag13M3vZzHaY2ZI4jw81s0IzW+D93Jv6ZlalUTIiIollJrHOq8DTwOsJ1pnhnPteSlokIiKH\nrNaeu3NuOrD7MLRFRERSJFU193PMbKGZjTezU1P0mnHpKkwiIoklU5apzTygp3Ou2MyGAx8BfWKt\naGY3AzcD9OjRo95vqGgXEUnskHvuzrm9zrli7/Y4oJmZdY6z7gvOucHOucFZWVmH8J71fqqIyBHh\nkMPdzI41b7IXMxviveauQ33dRJz67iIiCdValjGzt4GhQGcz2wzcBzQDcM49D1wN/NLMKoADwAjX\nwBOtq+cuIpJYreHunLu2lsefJjRU8rBRtouIJObPM1TVdRcRSciX4a6RkCIiifky3APW2C0QEWna\nfBnuWe1aNHYTRESaNF+G+7VDQidAnXdizOH0IiJHPF+Ge2ZGgNO7t6dZhuozIiKx+DLcIXSR7JXb\ninDOkT1yLA9+qqsyiYiE+TbcF24qYEthCW/O3gjAS1+ua+QWiYg0Hb4N97ClW/Y2dhNERJoc34e7\nqewuIlKD78NdJ6uKiNTk+3AXEZGaFO4iImnI9+GumruISE2+D3cREalJ4S4ikoZ8H+6qyoiI1OT7\ncNdISBGRmnwf7iIiUpPvw11lGRGRmnwf7iIiUpPCXUQkDSncRUTSkMJdRCQNKdxFRNKQ78Nd49xF\nRGryfbiLiEhNvg93XaxDRKQm34e7iIjU5NtwD+jUVBGRuHwc7kp3EZF4FO4iImnIv+EeabmOqIqI\nVOffcFfPXUQkrlrD3cxeNrMdZrYkzuNmZqPMLNfMFpnZoNQ3s6YML9w1FFJEpKZkeu6vApckePxS\noI/3czPw3KE3KwnquIuIxFVruDvnpgO7E6xyBfC6C5kFdDCzrqlqYDzhsszybUUN/VYiIr6Tipp7\nN2BT1P3N3rIGleENdF+4qaCh30pExHcO6wFVM7vZzHLMLCc/P/+QXksnMYmIxJeKcM8Djo+6391b\nVoNz7gXn3GDn3OCsrKxDelPTaBkRkbhSEe5jgJ94o2bOBgqdc1tT8LoJZSjcRUTiyqxtBTN7GxgK\ndDazzcB9QDMA59zzwDhgOJAL7AdubKjGRlNZRkQkvlrD3Tl3bS2PO+C2lLUoSSrLiIjE598zVH3b\nchGRhufbiNT0AyIi8SncRUTSkG/DXdkuIhKfb8P9qgENfhKsiIhv+Tbcb7nghMZugohIk+XbcNc4\ndxGR+Hwb7onGuc9Ync+u4tLD2BoRkabFv+EeZ3lFZZDrX/qGH7/0zWFtj4hIU+LfcI+T7kHvyky5\nOzTPu4gcuXwc7iq6i4jE49twj8cR6rrr2qoiciRLv3BXqIuIpF+4xzJr7S6+XrOrsZshInLY1Drl\nr9+Ee+7RHfgRL8wCYP2jlx3+BomINIK067k7VJcREUm/cA/33FV8F5EjWNqFe5iiXUSOZGkX7gp1\nEZF0DHence4iIr4O96euHVhjmTJdRMTn4R5rBgL12EVEfD7OvTJYNcn/97MVnNatfSO1RkSk6Uir\ncH922ppGaomISNPi67JM9XAXEZEQX4d7UAV2EZGYfB3uFeq5i4jE5Otwjy7LtG6e0YgtERFpWtIm\n3NWLFxE5KG3Cvawi2IgtERFpWtIm3EVE5CBfh3ttpZjb3pqnqX9F5Ijk63AP1hLuYxdtVe9eRI5I\nvg73UtXZRURiSirczewSM1tpZrlmNjLG4z81s3wzW+D93JT6ptZUUl5Z6zrqt4vIkajWcDezDOAZ\n4FKgL3CtmfWNseo7zrkB3s/oFLczpkv7HVvrOk9OWn0YWiIi0rQk03MfAuQ659Y658qAfwJXNGyz\nknNGz06sfXg4vxx6Qtx1np6aexhbJCLSNCQT7t2ATVH3N3vLqvsPM1tkZu+b2fEpaV0SAgGjeYav\nDx2IiKRcqlLxEyDbOXc6MBF4LdZKZnazmeWYWU5+fn6K3hoCsa7aEcP+soqUvaeISFOWTLjnAdE9\n8e7esgjn3C7nXKl3dzRwRqwXcs694Jwb7JwbnJWVVZ/2xpRsx/2Cv05L2XuKiDRlycTiHKCPmfUy\ns+bACGBM9Apm1jXq7uXA8tQ1sXaWZM89v6i09pVERNJArVdics5VmNntwAQgA3jZObfUzB4Acpxz\nY4Bfm9nlQAWwG/hpA7a5hmTLMiIiR4qkLrPnnBsHjKu27N6o23cBd6W2acmr6/FU5xxmxhuzNvDH\nj5aw8qFLaJGpKYNFJH0cccNMPpqfR6+7xrF8616emhwaA1+4vzzyePbIsdz02hwAFm4q4PkvdF1W\nEfEfX18gO+y9nM1Jr3vHOwsAmLdxDxmBUDmn+gRkk5bvoHB/OVc88xUAv7gg/jh6EZGmKC3Cfc/+\nsno9LxzulUHHmvxiWjU7WJrp/8DnKWmbiEhjSItwr0+93DnIjOq5f/dvX6S6WSIijSYtau71uX5q\n0LkqPXcRkXSSFuF+XIdWdX5OZdCRGQhtfkUw8dTBuuCHiPhNWoR7faL3gU+XEQiXZSoTv4I69iLi\nN2kR7v27t4/c/uvVpyf1nOiae1ll4p57UD13EfGZtAj3O4adxKM/6Efnti0Y9q1jkn5eeJTNV6t3\nJlyvMuhwzvHU5NVsLTxwSG0VETkc0iLcMwLGiCE9yLlnGB3bNE/6eZv3hIL6mWmJ53x3DlZtL+Zv\nE1dx25vzDqmtIiKHQ1qE+6EqKU9clikpr6TcK90cqGVdEZGmQOGehEEPTYzc1hRlIuIHaRnuz/5o\nUEpfL/p4qiagFBE/SMtwb4gpgMMjZhTuIuIHaRnuDaH65GLx7Nhbwuy1u+r9PqUVlQDc89HiQ3od\nETmyKdyTVF4ROpBqtVTdh4+awTUvzKrXe8zbuIeT7/mMqSt38I9ZG+v9OiIiaRruqT/pKNxzX5xX\nSH5RKc65KvPAh+0srjpDZXiMfLQdRSWR0TfRctbvBmDaih2ASkAiUn9pGe5ltUwnUB8Pjzt4Wdj/\neG4mr3+9gf4PfM6M1fmR5U9PWV3jeSf8YRx/+mRZ5H5pRSVD/jyZkR8sBkLDLPeVVlR5TviLJKOe\n6T5/4x5Kyivr9VwRSQ9pGe5HtUz9TMZLt+yN3N64ez/3jVkKwPUvfcN/v7eQd+ds4rHPV1V5TtAL\n6VdnrqeopJz1O/dR7n3xfDBvMx8vyOM7j03j1PsmcKDsYBiH57oJz1qZyMzcnewtObgHkV9UylXP\nzuS37y2s55aKSDpIy3C/4KSsw/p+783dzO8/WFRl2egZa6tcROSHz3/N0MemVVnnN/9cwJbCEgC+\nde9nzFobKstEeu7Vwr2sIsjKbUWR+4X7y7lu9Gxu/cfBs2bD5Z55G/bUa1vKK4NsKdAUCw2hoJ4X\nlRGpj7QMdzNjUI8ONZaf0bPjYWvDQ2OXc8Ffp0Xur/BCefnWvXGeAV+vCY2OCU9BvL+sknfmbIzU\n7P/0yVIufmI6WwsPcOHjX3DPx0uA0LVel+QVApCZ4U2GVlG/M2kf/HQZ//bolLQJov1lFTz++cp6\n/z5SZcLSbQx4YCJzvOMqIg0tLcMd4Gfn9apyf/2jl/HeLecc1jYUV6ulQ6gHH0+lF+Lb95ZElv3P\nB4vpddc4Xv1qHTnrQ73xgv3lrN5RzCcLtwBQVFrB9576ku17SyInXIV7/yXllTz06TKKoko3J909\nnscnVi0hhX3pTaJW/cCwXz0zNZdRU3J5Z87GRm1H+It7qfclLNLQ0jbcv3f6cax/9LIqywIBY/2j\nl9GyWdPc7HDvMlyeiXb/J8tYub2oxvJoRSXlNaYnfi9nE6O/XMeoyQcP9pZVBhk1eTXZI8dWqdcD\ntPKuatVQB2S3FZYwfvHWBnntWA6UhX6npY3ccw9f7SuQxHEUkVRomimXQq//bAhv3nRWlWX3f//U\nGuuNuf3cw9WkQ3bpkzNiLv/HrI0UeMMzi0rKufOdBSz2eor//GYTm3bvr/GcaSvzq9zPzAj9SYRr\n9zPX7OTjBXnsiNqbqIuZa3Yya+0uPpy/meVb93Ldi7P45ZvzYg4FTWfxjqOINJS0uEB2IufHOLja\nIkbP/fTuNWv0fvPqzPW8OnM9ELp61L/m50UeKyqt4MpnvmLuHy+s8pzoMfhFJeUs3FQAwH1jlnLn\nhSfx01fmANCtQyu+GvmdGu85fvFWfvnmPObcPYysdi1qPH7di7Or3G/ufXmUlFeyZ18Zndo0j3yh\nHG6Tl29nV3EZ/3nm8Q3+XsFDHN5am5LySlo2q/u1hCV9pX3PPZaGmHvGD3btK6txQtUnC7eydEsh\n2SPH0u/+zyPLF20ujAQ7QF7BAa59YRb7yyooqwiSX1TKloIDjP5yHQBr84urvO7HC/LIHjm2RhvC\nV73K3VHMkIcnc+Ld48keOZb1O/dF1pm3cQ+7iksTbsvM3J3c5x1QBnhz9gben7s54XOCQVflYug/\nfy2nxiin+tpZXMq2wvh7N+HjKQ3Rc39/7mZO+eNnrIv6HYqkfc89kYv6HsPny7Y3djMOq153jaty\nf9Ly7Uxantzv4Ou1u/hy9U5Gf7mOb9ZVPS7QLPNgP+HTRVuqnPQVy1XPzqxyf+hj0zixS1tydxz8\nkggYzLl7GEe3De0ROOcw74v5utGhPYI/XXEa+0oruPvDUNBffUZ3ACoqg/zuvYV8tCB00Lm4tIIR\nL87im3W7axyLqavHJ65i1OTVrHroUpp72z34oUkAcV+7MomyTHFpBW1bJP9fsqS8EjP4bMk2AFZt\nL6JX5zZJP1/S2xEd7i2bZTDrru9GRpK8euOZFJVU8Oj4FeRprHdMv3p7fsyDk+FyS2XQcftb8+v1\n2tHBDqHS0tXPf02H1s24rF9XHhq7nK/v+g5d27eKrFN97yD8BfDLN+cxMeqL+4lJBw8o/9sjk/n7\nNQPitsM5R+GBcjq0jn1Vr1e8vZVz/zKFti0ymfq7oZHH/jx2GXdf1rfGc2oL99wdxQx7/Asu738c\no64dGLdt0frdP4G2LTIZnN0psmzU5NWc3r09Q0/uUuvzD5RVUlpRGXc7U8U5x18nrOSErLb8+0md\n6dKuJa/NXE+7lpn8YFD3Bn3vsKKSctq1bHZY3qupOCLLMtHT9x7bviV9jmkHwNCTu/D9/scx+bcX\nsPRPF/PkiAE8cc0ALj714HVZf3jG4fljbKrijTr53lNf8tt3F/Lpoi0pfb91O/cxf2MBD40N7Qmc\n88gU+tw9Lu76ve4ax8PjllcJ9uq2FJbwl89WRO7f8kYOnyzcwr7SCr5cvZNnp61hwAMT45ZZMrxz\nCfKLSmuUQl6csS7mc8JlmXBJcPqqfF756uC6q7yRUGMWbqlytnI8y7fupbzSsafa/EaPT1xVpZyW\nyEVPfMGABybWvmIdLMkrZPBDk9i97+BQ2h1FpTw7bQ2/fW8hw73BAPeNWcqd78Y/i3r+xj1kjxzL\nW7M3Mm3lDjbuqjkYIFlLtxTS7/7PGbNwC6UVlWSPHMvoGWvr/Xp+cUT23Nu3Cn2DH3tUy5iPhw9M\nXTGgGwBXDuwW6SH+YugJvOfVdjMDltRUwJPuvIBhj39xyO1u6j6Yt5kP5iWue6dCeS1zB70wvfb/\nuNGhPGHpdiYs3c5ZvToxe91uOrQO/X385bMVXHPm8ezeV8bEZdu5Y1gfeh7dpsZB0UWbC2K+x96S\nct74egO/uOCEyAHVez9ewq/ePrhnc8M52QQCVuWCMAUHymjVvFX1l6si3oipRAr2l5ERsEgPdtPu\n1OydfrEqnwNlFZxy7FE898UadhaX8lXuTr7f/7ga6yZ7/kS4bPfM1FzyCg7QPCPAqj9fWq/2feQN\nLJi+Kp9zTzgagGenreGmf+9dr9err2DQkV9cyjFxcifVjsie+7dP7sIT1wzgzotOqvNznXOM8EZX\njLz0lBqP9zy6deT2ukeGs/bh4ZzYpW292/ofh2m39UhTvccLMNs7jhAeTvrh/DxGvDCLW9+cx4fz\n87jz3YWUlFeya1/VgLr86a+q3P9ofh6FB8q55fW5/HXCSmau2RnpBFR/395/GMe4xVvZWngwaMsr\nHPd8tJgHP11W5QAwhA4297tvQpVl4b2UW96YG1m2Nr+4yl7UvtIKBjwwkTMemsQL09dw4yvfJPjt\nVLWzuJQvVuXXWL54cyGPTVjJDS9/wy/+MY+hj01j7KLQOQy/ens+O4tDezaJDiFf/dzMmEN0w8Lb\nX1bHobOrtheRPXIsCzcVRPamMqNKYtUHFsQTDDo+X7ot6fXDCveXc95fpkTOHAcYNWU1Zz08+bCV\nfI/IcDczrhzYjRaZdR86FnTw8FX9mPq7oVW++b/f/zhWPHgJI87sAcBzPxqEmcU8aeX9X9Q8U/bM\n7NhTI/ztP/sz5bcX1LmdknpzN+zhlD9+Vut6d7yzgP5/+pyvvYutBB0Jy0S3vjkvUnYCeGLyKv4x\nayMvfbmOjxfkMdebJ2hfaQU/eHYmRTHOfI7Vhtvfms/q7UW8O2cTp3pfCGUVQR4et4KpUec3nPvo\nFH7+6hz63hvath+Pns1J94xn+94SXvlqHT8ePZsbXv6GyqDj+pdm81Pvi+HyZ77k6am5cdsw+KFJ\nfPuxaQx5eHKV5VNWHPxd5GzYw70fL6GkvJLC/eW8OH1tleMolVGh+vGCPHbvK2O690WzYFMBz8R4\n/zX5xfzXOwsAqnzBZQQsMhl4+N+pK3aQPXIshQdqftlDaBTWzW/M5b25m2t80SYyc81ONu85wFNT\nVlNWEeSm1+ZEjvskGlWVSkdkWeZQVAYdgYBFRiU0yzDKKx1PXjOAQMC45fzeXHzqMfTOqtpbf+yH\n/Rm3eCtXDDguMk9776w2rM3fxx+Gn8LN558Q+aN+8IpT+ePHSyPP7Z3Vlh6dWrMxQQ9Hmq4bXk6+\nlwzwr3kHz08I16X/fNVprM1Pfqjjos2hHuOFf59e67p5BQcivcnoYD2rWiiXVwaZ4U1P8dCny6hj\nZzbiZ6/mVLk/dWU+p/zxs8j/h2j5RQeHxP7mnwsYenIW01bmM/eeYVz5TGiP6TundKFLuxa0ap5B\n33ur7tVEHwPJDFhkdteC/eVsKyzhxldDxyeWb93L2b2Pjqx702s5nHvi0ez0huT+/v1F/GX8iirn\niWzfW8Kk5dv50Vk9I8v2lVbw+/cX8dWa0O9pyoodnHTP+CptquteQH0p3JN07FEt2ba3pMbp/R/f\ndh7TVu2I9NADAasR7BAaohcephc+2HTb0BO5cmA3wp37zm1bsLO4lOvPyea0bu3pElWb+/y/zq/S\na3zmukHc9tY8hn2rC5OWhy7u8dFt50b+4Kv73uld+f3Fp3DrW3NZkhd/8jJpmsJDPRvTuzmbIrfD\n5zekUjJfXuEzqqP3dJI9/jBm4RZe+3pD5P7Zjxz88vr7xFW8/vMh7C+tpF3LzMgQ4V9/t09knXA5\nbm1+MT2PbhP58jvtuPb0P74Dzjk+XrCFsVHTa8Q6PlSHHYBDYofrW6S6wYMHu5ycnNpXbCIuGzWD\npVv28snt59Gve/sGeY+thQdYsa2Ib8cZxjZ15Q5ufGUOD15xKtcO6cH/TljJLef3ZtveEnp1bkPr\n5pks3lxIj06tWbm9iLGLQn/Mo38ymGF9QyN+dhaXMnXFDv77/YMn76x48BIeHb8icnbriz8ZzP97\nveE+m7G/Po/fv7+oyhz5fbq0ZXW1oZAije34Tq1qPfDcslmAkvLQMYHwnnwif736dH44uP5nRZvZ\nXOfc4FrXSybczewS4EkgAxjtnHu02uMtgNeBM4BdwDXOufWJXtNv4X7rm3MZt3gbE//r/MjQycYQ\nDDrMiJzMU1/b95Zw1sOT+c4pXXj5p2cyesbaSG9ozO3ncuMrc9i1r4wbz83mla/WR543sEcH5m8s\n4LJ+XRl17UACBvM2FtCxdTO6HNWSeRv2UOkcN74yh79f05+yiiD/88FiHrziVLq2b8XQk7Mi0w3s\nLSnndO+s2PWPXhY5Oai+wntX0a47qwdvzW7cGSFFqlv3yPB6/x9OWbibWQawCrgQ2AzMAa51zi2L\nWudW4HTn3C/MbARwlXPumkSv67dwLy6tYPqqfIb369rYTUmZ5Vv30vPo1rRunkll0PHJwi0cc1RL\nzjnhaMoqgjgcy7cWceUzX3HnhSdxef/jyO7chvkb93Dqce0jZ2cm4pzjQHklrZvHrgB+vnQbndu1\nYFCPjgSDjnvHLKGi0jHsW8dwwclZ9Ll7POeflEWXdi24+fzeXPT36ZzduxOz1u7m4av68YcPQ5cr\n/M13+zBiyPEUl1SQX1xKeaXjqJaZDOzRkVGTV9OqWQZn9e7E/01fy+3fPpEnJ63ms6WhMzuH9zuW\ncYu3Rdo04PgOvHTDYFZuK4qcCVvdsG8dwx3D+nBat/Ys3FTA6d3b1zj7d3DPjuQkcdGU49q3pOBA\nOc0yAnEP7PlJwA5f6cGvnrluEJedXr8sSWW4nwPc75y72Lt/F4Bz7pGodSZ463xtZpnANiDLJXhx\nv4W7NB3OHZwj5oUZa7nx33pFpiquy2tMWr6Db3t7EmUVQcxCByJP7NI2ci4EhA6iZwSMTxdt4Vdv\nz+eBy0/l+nOya7zmkrxC9uwvo//xHVi5rYgzszuRX1RK84wAa3YWk9W2BSXllZE9v/9+byGd2jTn\nruHfirxGRWWQPveM58Sstgzs0YGOrZvz47N78vXaXfz+/UV8++QsfvXdPnRt35Ku7VtRURlk7c59\n7C8LTcS2t6ScZ6eu4bw+nRnYowO9Ordh4rLtPDFpNZf3P462LTN5a/ZGHvlBPy7sewxXPfsVT44Y\nSM9OrTnjoUnceeFJzFidzxzv2gG3XNCb//sidN7Ap786DwidsAahg7zRxwJOyGrDGz8/i46tm/O7\n9xZGas+jfzKYc044OjJi5+JTj6Fr+1YsySukvDLIws2FXH92T96YdbAeXh/XndWD3110Mi/OWMtz\n09ZUeSzWwdrGNLBHBz68tX4z0aYy3K8GLnHO3eTdvx44yzl3e9Q6S7x1Nnv313jr7Kz2WjcDNwP0\n6NHjjA0bDu3DFJGGEb62QPPMAOWVQTIDFikjLMkrZGthCRf2PSbRSwChETbNvDJccWkFQec4qmXV\nL86lWwprzMpaVFJOq2YZNWYM3VFUwrbCEvp2PYr5mwoY3LNj3PLG0i2F9OrchhaZGRihwQ4VlUHM\nLDINxJK8QrLatWDV9iK6d2zNU5NX84NB3enXvT179pVR4ZVBHxm3goevOo15G/dwdNsWHN2mOQUH\nyhnUoyP5RaU8On4FF/btQla7lrTIDDB3wx6G9+tKR++EuH1llbRpnkHhgXKenprLz87txfGdWsds\nd22aZLiSSwR/AAAFN0lEQVRHU89dRKTukg33ZE5iygOiD+1295bFXMcry7QndGBVREQaQTLhPgfo\nY2a9zKw5MAIYU22dMcAN3u2rgSmJ6u0iItKwaj2JyTlXYWa3AxMIDYV82Tm31MweAHKcc2OAl4A3\nzCwX2E3oC0BERBpJUmeoOufGAeOqLbs36nYJ8MPUNk1EROrriJw4TEQk3SncRUTSkMJdRCQNKdxF\nRNJQo80KaWb5QH1PUe0MxD1Byme0LU1TumxLumwHaFvCejrnsmpbqdHC/VCYWU4yZ2j5gbalaUqX\nbUmX7QBtS12pLCMikoYU7iIiaciv4f5CYzcghbQtTVO6bEu6bAdoW+rElzV3ERFJzK89dxERScB3\n4W5ml5jZSjPLNbORjd2eZJjZejNbbGYLzCzHW9bJzCaa2Wrv347ecjOzUd72LTKzQY3Y7pfNbIc3\nX394WZ3bbWY3eOuvNrMbYr1XI23L/WaW530uC8xseNRjd3nbstLMLo5a3uh/f2Z2vJlNNbNlZrbU\nzH7jLffVZ5NgO3z3uZhZSzP7xswWetvyJ295LzOb7bXrHW9mXcyshXc/13s8u7ZtrDPnnG9+CM1K\nuQboDTQHFgJ9G7tdSbR7PdC52rL/BUZ6t0cCf/FuDwfGAwacDcxuxHafDwwCltS33UAnYK33b0fv\ndscmsi33A7+LsW5f72+rBdDL+5vLaCp/f0BXYJB3ux2haxz39dtnk2A7fPe5eL/btt7tZsBs73f9\nLjDCW/488Evv9q3A897tEcA7ibaxPm3yW899CJDrnFvrnCsD/glc0chtqq8rgNe8268BV0Ytf92F\nzAI6mFmjXJXbOTed0BTO0era7ouBic653c65PcBE4JKGb31VcbYlniuAfzrnSp1z64BcQn97TeLv\nzzm31Tk3z7tdBCwHuuGzzybBdsTTZD8X73db7N1t5v044DvA+97y6p9J+LN6H/iumRnxt7HO/Bbu\n3YBNUfc3k/iPoalwwOdmNtdC15EFOMY5t9W7vQ0IX5CyqW9jXdvd1Lfndq9U8XK4jIGPtsXbnR9I\nqKfo28+m2naADz8XM8swswXADkJflGuAAudcRYx2RdrsPV4IHE0Kt8Vv4e5X5znnBgGXAreZ2fnR\nD7rQ/pjvhi35td1RngNOAAYAW4G/NW5z6sbM2gIfAHc45/ZGP+anzybGdvjyc3HOVTrnBhC6FOkQ\n4JTGbI/fwj2Z67k2Oc65PO/fHcCHhD747eFyi/fvDm/1pr6NdW13k90e59x27z9kEHiRg7u/TX5b\nzKwZoUB80zn3L2+x7z6bWNvh588FwDlXAEwFziFUAgtfFCm6XfGuO52ybfFbuCdzPdcmxczamFm7\n8G3gImAJVa87ewPwsXd7DPATb4TD2UBh1K52U1DXdk8ALjKzjt7u9UXeskZX7VjGVYQ+Fwhtywhv\nREMvoA/wDU3k78+rzb4ELHfOPR71kK8+m3jb4cfPxcyyzKyDd7sVcCGhYwhTCV1XGmp+JrGuOx1v\nG+vucB5RTsUPoSP/qwjVs+5u7PYk0d7ehI5+LwSWhttMqL42GVgNTAI6uYNH3Z/xtm8xMLgR2/42\nod3ickK1v5/Xp93AzwgdGMoFbmxC2/KG19ZF3n+qrlHr3+1ty0rg0qb09wecR6jksghY4P0M99tn\nk2A7fPe5AKcD8702LwHu9Zb3JhTOucB7QAtveUvvfq73eO/atrGuPzpDVUQkDfmtLCMiIklQuIuI\npCGFu4hIGlK4i4ikIYW7iEgaUriLiKQhhbuISBpSuIuIpKH/D0IW3IUqSXRSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11ac62fd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 2.7 Run training for MAX_STEPS and save checkpoint at the end.\n",
    "with tf.Session(graph=mnist_graph) as sess:\n",
    "    # Run the Op to initialize the variables.\n",
    "    sess.run(init)\n",
    "    \n",
    "\n",
    "    # manual data save\n",
    "    losses =[]\n",
    "\n",
    "    # Setup summary \n",
    "    summary = tf.summary.merge_all()\n",
    "    summary_writer = tf.summary.FileWriter('logs', sess.graph)\n",
    "    \n",
    "\n",
    "    # Start the training loop.\n",
    "    for step in xrange(MAX_STEPS):\n",
    "        # Read a batch of images and labels.\n",
    "        images_feed, labels_feed = data_sets.train.next_batch(BATCH_SIZE)\n",
    "\n",
    "        # Run one step of the model.  The return values are the activations\n",
    "        # from the `train_op` (which is discarded) and the `loss` Op.  To\n",
    "        # inspect the values of your Ops or variables, you may include them\n",
    "        # in the list passed to sess.run() and the value tensors will be\n",
    "        # returned in the tuple from the call.\n",
    "        _, loss_value = sess.run([train_op, loss],\n",
    "                                 feed_dict={images_placeholder: images_feed,\n",
    "                                            labels_placeholder: labels_feed})\n",
    "\n",
    "        # save loss\n",
    "        losses.append(loss_value)\n",
    "\n",
    "        # Write the summaries every 100 steps\n",
    "        if step % 100 == 0:\n",
    "            # Update the events file.\n",
    "            summary_str = sess.run(summary, feed_dict={images_placeholder: images_feed,\n",
    "                                                        labels_placeholder: labels_feed})\n",
    "            summary_writer.add_summary(summary_str, step)\n",
    "            summary_writer.flush()\n",
    "\n",
    "        # Print out loss value every 500 steps\n",
    "        if step % 500 == 0:\n",
    "            print('Step %d: loss = %.2f' % (step, loss_value))\n",
    "\n",
    "\n",
    "    # Write a checkpoint.\n",
    "    checkpoint_file = os.path.join(TRAIN_DIR, 'checkpoint')\n",
    "    saver.save(sess, checkpoint_file, global_step=step)\n",
    "    \n",
    "# plot losses over time\n",
    "plt.plot(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ground truth: 7\n",
      "Prediction: 7\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADPpJREFUeJzt3W+oXPWdx/HPx2yDmD/+K4aLya7dIAvFYCIXWdkgkd0U\nV0uiD5SISlZKbpEqW+iDjQpuHpbQP/RR8YaGJks37UpbzANdq0HQQKkmYhLNNY0bUpp4TRoSiCKa\n1Xz3wT12b82d34wzZ+bMzff9gsudOd9zznyZ5HPPmfmdmZ8jQgDyuaTpBgA0g/ADSRF+ICnCDyRF\n+IGkCD+QFOEHkiL8QFKEH0jqrwb5YLa5nBDos4hwJ+v1dOS3fbvtQ7bfsb2xl30BGCx3e22/7TmS\nfi9ptaRjkl6TdF9EHCxsw5Ef6LNBHPlvlvRORByJiHOSfi5pbQ/7AzBAvYT/Wkl/nHb/WLXsL9ge\ns73H9p4eHgtAzfr+hl9EjEsalzjtB4ZJL0f+45KWTLu/uFoGYBboJfyvSbre9ldsz5W0TtLOetoC\n0G9dn/ZHxCe2H5H0vKQ5krZGxFu1dQagr7oe6uvqwXjND/TdQC7yATB7EX4gKcIPJEX4gaQIP5AU\n4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+Q\nFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5BU11N0S5Lto5Lel/SppE8iYrSOpgD0X0/hr9wW\nEadq2A+AAeK0H0iq1/CHpN/Y3mt7rI6GAAxGr6f9KyPiuO1rJL1g++2IeHn6CtUfBf4wAEPGEVHP\njuxNkj6IiO8V1qnnwQC0FBHuZL2uT/ttz7O94LPbkr4m6c1u9wdgsHo57V8k6de2P9vPf0bEf9fS\nFYC+q+20v6MH47Qf6Lu+n/YDmN0IP5AU4QeSIvxAUoQfSIrwA0nV8am+FMbGWl+h/Oijjxa3PXHi\nRLH+4YcfFuvj4+PF+pEjR1rWDh48WNwWeXHkB5Ii/EBShB9IivADSRF+ICnCDyRF+IGk+Ehvh86c\nOdOydvnllw+wkwudO3euZe348eMD7GS4lK6veOKJJ4rbvvTSS3W3MzB8pBdAEeEHkiL8QFKEH0iK\n8ANJEX4gKcIPJMU4f4fuueeelrWbbrqpuO2BAweK9WXLlhXrt9xyS7G+YsWKlrUFCxYUtz179myx\nvnDhwmK9F+fPny/W233Pwfz587t+7B07dhTr999/f9f7bhrj/ACKCD+QFOEHkiL8QFKEH0iK8ANJ\nEX4gqbbf2297q6SvSzoZETdUy66S9AtJ10k6KuneiGj9gfeLwNNPP91VbRCuvvrqlrXbbrutuO2L\nL75YrK9evbqrnjrRbhx/7969xXppvgJJuvTSS1vWDh06VNw2g06O/D+VdPvnlm2UtCsirpe0q7oP\nYBZpG/6IeFnS6c8tXitpW3V7m6S7au4LQJ91+5p/UURMVrffk7Sopn4ADEjPc/VFRJSu2bc9Jqn1\nRHcAGtHtkf+E7RFJqn6fbLViRIxHxGhEjHb5WAD6oNvw75S0vrq9XtIz9bQDYFDaht/2Dkm/lfR3\nto/Z/oak70pabfuwpH+q7gOYRfg8P4bWhg0bivWnnnqqWJ+cnGxZu/HGG4vbnjp1qlgfZnyeH0AR\n4QeSIvxAUoQfSIrwA0kRfiAphvrQmJGRkWL98OHDxfq8efOK9bGx1leVb9mypbjtbMZQH4Aiwg8k\nRfiBpAg/kBThB5Ii/EBShB9Iquev8QK69eSTTxbrl112WbH+0UcfFev79u37wj1lwpEfSIrwA0kR\nfiApwg8kRfiBpAg/kBThB5JinB99deedd7astftq7nbWrVtXrL/66qs97f9ix5EfSIrwA0kRfiAp\nwg8kRfiBpAg/kBThB5JqO85ve6ukr0s6GRE3VMs2Sdog6U/Vao9HxLP9ahKz1913392ydskl5WPP\nxMREsf7ss/yX60UnR/6fSrp9huU/jIjl1Q//CsAs0zb8EfGypNMD6AXAAPXymv8R2/ttb7V9ZW0d\nARiIbsP/Y0lLJS2XNCnp+61WtD1me4/tPV0+FoA+6Cr8EXEiIj6NiPOStki6ubDueESMRsRot00C\nqF9X4bc9fXrVuyW9WU87AAalk6G+HZJWSfqy7WOS/l3SKtvLJYWko5K+2cceAfSBI2JwD2YP7sEw\nEO2+W//tt99uWRsZGWlZk6Q1a9YU688991yxnlVEuJP1uMIPSIrwA0kRfiApwg8kRfiBpAg/kBRf\n3Y2ebN68uVhfvHhxy9r+/fuL2zKU118c+YGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcb5UfTggw8W\n6w8//HCx/vHHH7esbdy4saueUA+O/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFF/dndw111xTrJe+\neluSrrjiimJ99+7dLWu33nprcVt0h6/uBlBE+IGkCD+QFOEHkiL8QFKEH0iK8ANJtR3nt71E0nZJ\niySFpPGI+JHtqyT9QtJ1ko5KujcizrTZF+P8AzZnzpxi/ciRI8X6kiVLivUzZ4r/5Fq5cmXL2sTE\nRHFbdKfOcf5PJH0nIr4q6e8lfcv2VyVtlLQrIq6XtKu6D2CWaBv+iJiMiNer2+9LmpB0raS1krZV\nq22TdFe/mgRQvy/0mt/2dZJWSPqdpEURMVmV3tPUywIAs0TH3+Fne76kX0r6dkSctf//ZUVERKvX\n87bHJI312iiAenV05Lf9JU0F/2cR8atq8QnbI1V9RNLJmbaNiPGIGI2I0ToaBlCPtuH31CH+J5Im\nIuIH00o7Ja2vbq+X9Ez97QHol06G+lZKekXSAUnnq8WPa+p1/39J+mtJf9DUUN/pNvtiqG/Ali1b\nVqzv27evp/0/9NBDxfq2bduKddSv06G+tq/5I2K3pFY7+8cv0hSA4cEVfkBShB9IivADSRF+ICnC\nDyRF+IGkmKL7IrB06dKWtVdeeaWnfW/evLlY3759e0/7R3M48gNJEX4gKcIPJEX4gaQIP5AU4QeS\nIvxAUozzXwQee+yxlrWFCxf2tO/nn3++WB/kFO+oF0d+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK\ncf5ZYM2aNcX6Aw88MKBOcDHhyA8kRfiBpAg/kBThB5Ii/EBShB9IivADSbUd57e9RNJ2SYskhaTx\niPiR7U2SNkj6U7Xq4xHxbL8azWzVqlXF+ty5c7ve95kzZ3qqY/bq5CKfTyR9JyJet71A0l7bL1S1\nH0bE9/rXHoB+aRv+iJiUNFndft/2hKRr+90YgP76Qq/5bV8naYWk31WLHrG93/ZW21e22GbM9h7b\ne3rqFECtOg6/7fmSfinp2xFxVtKPJS2VtFxTZwbfn2m7iBiPiNGIGK2hXwA16Sj8tr+kqeD/LCJ+\nJUkRcSIiPo2I85K2SLq5f20CqFvb8Nu2pJ9ImoiIH0xbPjJttbslvVl/ewD6pZN3+/9B0oOSDth+\no1r2uKT7bC/X1PDfUUnf7EuH6Mm7775brC9fvrxYP3XqVJ3tYIh08m7/bkmeocSYPjCLcYUfkBTh\nB5Ii/EBShB9IivADSRF+ICkPcopl28znDPRZRMw0NH8BjvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/\nkNSgp+g+JekP0+5/uVo2jIa1t2HtS6K3btXZ2990uuJAL/K54MHtPcP63X7D2tuw9iXRW7ea6o3T\nfiApwg8k1XT4xxt+/JJh7W1Y+5LorVuN9Nboa34AzWn6yA+gIY2E3/bttg/Zfsf2xiZ6aMX2UdsH\nbL/R9BRj1TRoJ22/OW3ZVbZfsH24+j3jNGkN9bbJ9vHquXvD9h0N9bbE9ku2D9p+y/a/Vssbfe4K\nfTXyvA38tN/2HEm/l7Ra0jFJr0m6LyIODrSRFmwflTQaEY2PCdu+VdIHkrZHxA3Vss2STkfEd6s/\nnFdGxL8NSW+bJH3Q9MzN1YQyI9NnlpZ0l6R/UYPPXaGve9XA89bEkf9mSe9ExJGIOCfp55LWNtDH\n0IuIlyWd/tzitZK2Vbe3aeo/z8C16G0oRMRkRLxe3X5f0mczSzf63BX6akQT4b9W0h+n3T+m4Zry\nOyT9xvZe22NNNzODRdW06ZL0nqRFTTYzg7YzNw/S52aWHprnrpsZr+vGG34XWhkRN0n6Z0nfqk5v\nh1JMvWYbpuGajmZuHpQZZpb+syafu25nvK5bE+E/LmnJtPuLq2VDISKOV79PSvq1hm/24ROfTZJa\n/T7ZcD9/NkwzN880s7SG4Lkbphmvmwj/a5Kut/0V23MlrZO0s4E+LmB7XvVGjGzPk/Q1Dd/swzsl\nra9ur5f0TIO9/IVhmbm51czSavi5G7oZryNi4D+S7tDUO/7/I+mJJnpo0dffStpX/bzVdG+Sdmjq\nNPB/NfXeyDckXS1pl6TDkl6UdNUQ9fYfkg5I2q+poI001NtKTZ3S75f0RvVzR9PPXaGvRp43rvAD\nkuINPyApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSf0fnI02wbzE1UIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11ab73c50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 2.8 Run evaluation based on the saved checkpoint.\n",
    "with tf.Session(graph=tf.Graph()) as sess:\n",
    "    saver = tf.train.import_meta_graph(\n",
    "        os.path.join(TRAIN_DIR, \"checkpoint-2999.meta\")\n",
    "    )\n",
    "    saver.restore(\n",
    "        sess,\n",
    "        os.path.join(TRAIN_DIR, \"checkpoint-2999\")\n",
    "    )\n",
    "\n",
    "    # Retrieve the Ops we 'remembered'.\n",
    "    logits = tf.get_collection(\"logits\")[0]\n",
    "    images_placeholder = tf.get_collection(\"images\")[0]\n",
    "    labels_placeholder = tf.get_collection(\"labels\")[0]\n",
    "    \n",
    "    # Add an Op that chooses the top k predictions.\n",
    "    eval_op = tf.nn.top_k(logits)\n",
    "    \n",
    "    # Run evaluation.\n",
    "    images_feed, labels_feed = data_sets.test.next_batch(EVAL_BATCH_SIZE)\n",
    "    imgplot = plt.imshow(np.reshape(images_feed, (28, 28)), cmap=\"Greys_r\")\n",
    "    prediction = sess.run(eval_op,\n",
    "                          feed_dict={images_placeholder: images_feed,\n",
    "                                     labels_placeholder: labels_feed})\n",
    "    print(\"Ground truth: %d\\nPrediction: %d\" % (labels_feed, prediction.indices[0][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Build graph to count correct answers.\n",
    "def mnist_evaluation(logits, labels):\n",
    "    \"\"\"Evaluate the quality of the logits at predicting the label.\n",
    "    Args:\n",
    "        logits: Logits tensor, float - [BATCH_SIZE, NUM_CLASSES].\n",
    "        labels: Labels tensor, int32 - [BATCH_SIZE], with values in the\n",
    "        range [0, NUM_CLASSES).\n",
    "    Returns:\n",
    "        A scalar int32 tensor with the number of examples (out of batch_size)\n",
    "        that were predicted correctly.\n",
    "    \"\"\"\n",
    "    # For a classifier model, we can use the in_top_k Op.\n",
    "    # It returns a bool tensor with shape [batch_size] that is true for\n",
    "    # the examples where the label is in the top k (here k=1)\n",
    "    # of all logits for that example.\n",
    "    correct = tf.nn.in_top_k(logits, labels, 1)\n",
    "\n",
    "    # Return the number of true entries.\n",
    "    return tf.reduce_sum(tf.cast(correct, tf.int32))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Data Eval:\n",
      " Num examples: 2000000 Num correct: 1956201 Precision @ 1: 97.81 %\n"
     ]
    }
   ],
   "source": [
    "# Evaluation on the whole evaluation set.\n",
    "\n",
    "with tf.Session(graph=tf.Graph()) as sess:\n",
    "    saver = tf.train.import_meta_graph(\n",
    "        os.path.join(TRAIN_DIR, \"checkpoint-2999.meta\")\n",
    "    )\n",
    "    saver.restore(\n",
    "        sess,\n",
    "        os.path.join(TRAIN_DIR, \"checkpoint-2999\")\n",
    "    )\n",
    "    print('Validation Data Eval:')\n",
    "    # And run one epoch of eval.\n",
    "    true_count = 0\n",
    "    # Counts the number of correct predictions.\n",
    "    steps_per_epoch = data_sets.test.num_examples\n",
    "    num_examples = steps_per_epoch*BATCH_SIZE\n",
    "    \n",
    "    # Retrieve the Ops we 'remembered'.\n",
    "    logits = tf.get_collection(\"logits\")[0]\n",
    "    images_placeholder = tf.get_collection(\"images\")[0]\n",
    "    labels_placeholder = tf.get_collection(\"labels\")[0]\n",
    "    eval_op = mnist_evaluation(logits, labels_placeholder)\n",
    "    for step in xrange(steps_per_epoch):\n",
    "        images_feed, labels_feed = data_sets.test.next_batch(BATCH_SIZE)\n",
    "        true_count += sess.run(eval_op,\n",
    "                               feed_dict={images_placeholder: images_feed,\n",
    "                                          labels_placeholder: labels_feed})\n",
    "    precision = true_count*100.0/num_examples\n",
    "    print(' Num examples: %d Num correct: %d Precision @ 1: %0.02f %%' % (num_examples, true_count, precision))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Observed results\n",
    "\n",
    "\n",
    "| BATCH_SIZE    | LEARNING_RATE | HIDDEN2_UNITS  | Precision |\n",
    "| ------------- | ------------- | -------------- | --------- |\n",
    "| 200           | 0.5           | 64             | 97.77     |\n",
    "| 200           | 0.4           | 64             | **97.99** |\n",
    "| 200           | 0.4           | 32             | 97.96     |\n",
    "| 200           | 0.3           | 64             | 97.71     |\n",
    "| 200           | 0.2           | 64             | 97.69     |\n",
    "| 200           | 0.1           | 64             | 97.03     |\n",
    "| 300           | 0.4           | 64             | 97.85     |\n",
    "| 200           | 0.1           | 32             | 96.89     |\n",
    "| 100           | 0.1           | 64             | 96.38     |\n",
    "| 200           | 0.01          | 64             | 91.28     |\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}