02_neural_network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  MNIST "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/Users/i550012/.pyenv/versions/3.7.17/lib/python3.7/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.6 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.7\n",
      "  return f(*args, **kwds)\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.examples.tutorials.mnist.input_data as input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_76580/744223613.py:1: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /Users/i550012/virtualenv/tensorflow1/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_76580/2474181960.py:4: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# placeholder\n",
    "n_pixels = 28*28 # the image has 28*28 pixels   \n",
    "with tf.name_scope('input'):\n",
    "    x = tf.placeholder(tf.float32, [None, n_pixels], name=\"x\")\n",
    "    y = tf.placeholder(tf.float32, [None, 10], name=\"y\") # the label is one-hot vector of 10 elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('input_reshape'):\n",
    "    x_image = tf.reshape(x, [-1, 28, 28, 1], name='x_image') \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper functions\n",
    "\n",
    "def weight_variable(shape, name=None):\n",
    "    initial = tf.truncated_normal(shape, stddev=0.1, name=name)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "def bias_variable(shape, name=None):\n",
    "    initial = tf.constant(0.1, shape=shape, name=name)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "def conv(x,W, name=None):   \n",
    "    return tf.nn.conv2d(x,W, strides=[1,1,1,1], padding='SAME', name=name)\n",
    "\n",
    "def max_pool(x, name=None):\n",
    "    return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME', name=name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_76580/3186703957.py:15: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# define layers\n",
    "with tf.name_scope('conv1'):\n",
    "    with tf.name_scope('weights'):\n",
    "        W_conv1 = weight_variable([5,5,1,32], name='Weight')\n",
    "    with tf.name_scope('biases'):\n",
    "        b_conv1 = bias_variable([32], name='Bias')\n",
    "    conv1 = conv(x_image, W_conv1, name='Conv1') + b_conv1\n",
    "    h_conv1 = tf.nn.relu(conv1, name='Relu')\n",
    "    h_pool1 = max_pool(h_conv1, name='Pool')\n",
    "    \n",
    "with tf.name_scope('conv2'):\n",
    "    with tf.name_scope('weights'):\n",
    "        W_conv2 = weight_variable([5,5,32,64], name='Weight')\n",
    "    with tf.name_scope('biases'):\n",
    "        b_conv2 = bias_variable([64], name='Bias')\n",
    "    conv2 = conv(h_pool1, W_conv2, name='Conv') + b_conv2\n",
    "    h_conv2 = tf.nn.relu(conv2, name='Relu')\n",
    "    h_pool2 = max_pool(h_conv2, name='Pool')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# connect layers\n",
    "with tf.name_scope('fc'):\n",
    "    W_c1 = weight_variable([7*7*64, 1024], name='Weight')\n",
    "    b_c1 = bias_variable([1024], name='Bias')\n",
    "    h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])\n",
    "    h_c1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_c1) + b_c1, name='relu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_76580/413170830.py:4: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "# dropout\n",
    "with tf.name_scope('dropout'):\n",
    "    keep_prob = tf.placeholder(tf.float32, name='keep_prob')\n",
    "    h_c1_drop = tf.nn.dropout(h_c1, keep_prob, name='Dropout')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# readout layer\n",
    "with tf.name_scope('readout'):\n",
    "    W_c2 = weight_variable([1024, 10], name='Weight')\n",
    "    b_c2 = bias_variable([10], name='Bias')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_76580/4147314702.py:4: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n",
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_76580/4147314702.py:6: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_76580/4147314702.py:10: The name tf.summary.merge_all is deprecated. Please use tf.compat.v1.summary.merge_all instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# model\n",
    "y_conv = tf.matmul(h_c1_drop, W_c2) + b_c2\n",
    "with tf.name_scope('cross_entropy'):\n",
    "    cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_conv))\n",
    "with tf.name_scope('train'):\n",
    "    train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "with tf.name_scope('accuracy'):\n",
    "    accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(y_conv,1), tf.argmax(y,1)),tf.float32))\n",
    "\n",
    "summarize = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_76580/3443439190.py:1: The name tf.InteractiveSession is deprecated. Please use tf.compat.v1.InteractiveSession instead.\n",
      "\n",
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_76580/3443439190.py:2: The name tf.global_variables_initializer is deprecated. Please use tf.compat.v1.global_variables_initializer instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-03-22 09:27:19.302800: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n"
     ]
    }
   ],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "sess.run(tf.global_variables_initializer())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/nl/yn0d4tg1107g2jk38nj1hc740000gn/T/ipykernel_76580/3145455974.py:2: The name tf.summary.FileWriter is deprecated. Please use tf.compat.v1.summary.FileWriter instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "log_path = 'logs/'\n",
    "tb = tf.summary.FileWriter(log_path, sess.graph)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0, training accuracy 0.10000000149011612\n",
      "step 100, training accuracy 0.8600000143051147\n",
      "step 200, training accuracy 0.8799999952316284\n",
      "step 300, training accuracy 0.8399999737739563\n",
      "step 400, training accuracy 0.8999999761581421\n",
      "step 500, training accuracy 0.9399999976158142\n",
      "step 600, training accuracy 0.9399999976158142\n",
      "step 700, training accuracy 0.9200000166893005\n",
      "step 800, training accuracy 0.9399999976158142\n",
      "step 900, training accuracy 0.9800000190734863\n",
      "step 1000, training accuracy 0.9200000166893005\n",
      "step 1100, training accuracy 0.9800000190734863\n",
      "step 1200, training accuracy 0.9599999785423279\n",
      "step 1300, training accuracy 0.9800000190734863\n",
      "step 1400, training accuracy 0.9399999976158142\n",
      "accuracy: 0.970%\n"
     ]
    }
   ],
   "source": [
    "# train\n",
    "for i in range(1500):\n",
    "    batch = mnist.train.next_batch(50)\n",
    "    train_step.run(feed_dict={x:batch[0], y:batch[1], keep_prob:0.5})\n",
    "    if i%100 == 0:\n",
    "        train_accuracy = accuracy.eval(feed_dict={x:batch[0], y:batch[1], keep_prob:1.0})\n",
    "        print(\"step {}, training accuracy {}\".format(i, train_accuracy))\n",
    "\n",
    "print(\"accuracy: {0:.3f}%\".format(accuracy.eval(feed_dict={x:mnist.test.images, y:mnist.test.labels, keep_prob:1.0})))\n",
    "sess.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}