DL_book_Nikolenko/DL_chapter_4_minibatch_normalization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DL_chapter_4_minibatch_normalization.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/LenaVolzhina/playing-with-neural-networks/blob/master/DL_chapter_4_minibatch_normalization.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MTIZHiOq1CBE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from datetime import datetime"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h0vE0Z4QwSaz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "\n",
        "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rI1j8WvdwqJt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def fullyconnected_layer(tensor, input_size, out_size):\n",
        "  W = tf.Variable(tf.truncated_normal([input_size, out_size], stddev=0.1))\n",
        "  b = tf.Variable(tf.truncated_normal([out_size], stddev=0.1))\n",
        "  return tf.nn.tanh(tf.matmul(tensor, W) + b)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wp59EAhRwuYX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def batchnorm_layer(tensor, size):\n",
        "  batch_mean, batch_var = tf.nn.moments(tensor, [0])\n",
        "  beta = tf.Variable(tf.zeros([size]))\n",
        "  scale = tf.Variable(tf.ones([size]))\n",
        "  return tf.nn.batch_normalization(\n",
        "    tensor, batch_mean, batch_var, beta, scale, 0.001\n",
        "  )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4fSW-eQEzVpk",
        "colab_type": "text"
      },
      "source": [
        "Заглушки для данных"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HS96mPBCzSGL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = tf.placeholder(tf.float32, [None, 784])\n",
        "\n",
        "# заглушка для истинной разметки\n",
        "y = tf.placeholder(tf.float32, [None, 10])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "25g-ugYBzYP4",
        "colab_type": "text"
      },
      "source": [
        "Описываю модели"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j5nEAuGZwwtZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "h1 = fullyconnected_layer(x, 784, 100)\n",
        "\n",
        "# с нормализацией\n",
        "h1_bn = batchnorm_layer(h1, 100)     # нормализация между двумя промежуточными слоями\n",
        "h2_bn = fullyconnected_layer(h1_bn, 100, 100)\n",
        "y_logit_bn = fullyconnected_layer(h2_bn, 100, 10)\n",
        "\n",
        "# без нормализации\n",
        "h2 = fullyconnected_layer(h1, 100, 100)\n",
        "y_logit = fullyconnected_layer(h2, 100, 10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zvZK3hks0aPX",
        "colab_type": "text"
      },
      "source": [
        "Описываю, как замерять промежуточное качество"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JMMAImu_yyM_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "accuracy_history = {'bn': {'train': [], 'test': []}, 'no_bn': {'train': [], 'test': []}}\n",
        "\n",
        "correct_prediction = tf.equal(tf.argmax(y_logit, 1), tf.argmax(y, 1))\n",
        "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
        "\n",
        "correct_prediction_bn = tf.equal(tf.argmax(y_logit_bn, 1), tf.argmax(y, 1))\n",
        "accuracy_bn = tf.reduce_mean(tf.cast(correct_prediction_bn, tf.float32))\n",
        "\n",
        "def calc_accuracy(session, ds, accuracy_fun):  \n",
        "  accuracy_calc = session.run(accuracy_fun, feed_dict={\n",
        "      x: ds.images, y: ds.labels\n",
        "  })\n",
        "  return accuracy_calc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZhWyRN74zzpr",
        "colab_type": "text"
      },
      "source": [
        "Обучаю без нормализации"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-zWGlY0yxPpq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 867
        },
        "outputId": "b1a50ad4-1761-4fa2-fee9-b5c87d682376"
      },
      "source": [
        "loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=y_logit, labels=y)\n",
        "train_op = tf.train.GradientDescentOptimizer(0.00001).minimize(loss)    # было 0.01\n",
        "\n",
        "# обучение\n",
        "init = tf.global_variables_initializer()\n",
        "\n",
        "sess = tf.Session()\n",
        "sess.run(init)\n",
        "\n",
        "for i in range(1000):\n",
        "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
        "  sess.run(train_op, feed_dict={x: batch_xs, y: batch_ys})\n",
        "\n",
        "  if (i + 1) % 20 == 0:\n",
        "    acc_train = calc_accuracy(sess, mnist.train, accuracy)\n",
        "    acc_test = calc_accuracy(sess, mnist.test, accuracy)\n",
        "    accuracy_history['no_bn']['train'].append(acc_train)\n",
        "    accuracy_history['no_bn']['test'].append(acc_test)\n",
        "  \n",
        "    print(f\"[{datetime.now()}] Step {i + 1}, train accuracy {acc_train:.4f}, test accuracy {acc_test:.4f}\")"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2019-11-27 12:56:27.717031] Step 20, train accuracy 0.0831, test accuracy 0.0845\n",
            "[2019-11-27 12:56:28.064679] Step 40, train accuracy 0.0970, test accuracy 0.0981\n",
            "[2019-11-27 12:56:28.404204] Step 60, train accuracy 0.1077, test accuracy 0.1096\n",
            "[2019-11-27 12:56:28.751914] Step 80, train accuracy 0.1159, test accuracy 0.1161\n",
            "[2019-11-27 12:56:29.095892] Step 100, train accuracy 0.1213, test accuracy 0.1233\n",
            "[2019-11-27 12:56:29.441695] Step 120, train accuracy 0.1265, test accuracy 0.1288\n",
            "[2019-11-27 12:56:29.783856] Step 140, train accuracy 0.1314, test accuracy 0.1324\n",
            "[2019-11-27 12:56:30.194865] Step 160, train accuracy 0.1350, test accuracy 0.1354\n",
            "[2019-11-27 12:56:30.535902] Step 180, train accuracy 0.1403, test accuracy 0.1403\n",
            "[2019-11-27 12:56:30.878216] Step 200, train accuracy 0.1449, test accuracy 0.1451\n",
            "[2019-11-27 12:56:31.222822] Step 220, train accuracy 0.1495, test accuracy 0.1501\n",
            "[2019-11-27 12:56:31.570305] Step 240, train accuracy 0.1543, test accuracy 0.1542\n",
            "[2019-11-27 12:56:31.912367] Step 260, train accuracy 0.1587, test accuracy 0.1592\n",
            "[2019-11-27 12:56:32.260670] Step 280, train accuracy 0.1623, test accuracy 0.1633\n",
            "[2019-11-27 12:56:32.607958] Step 300, train accuracy 0.1675, test accuracy 0.1674\n",
            "[2019-11-27 12:56:32.946403] Step 320, train accuracy 0.1716, test accuracy 0.1728\n",
            "[2019-11-27 12:56:33.290702] Step 340, train accuracy 0.1758, test accuracy 0.1765\n",
            "[2019-11-27 12:56:33.632306] Step 360, train accuracy 0.1805, test accuracy 0.1812\n",
            "[2019-11-27 12:56:33.971056] Step 380, train accuracy 0.1842, test accuracy 0.1851\n",
            "[2019-11-27 12:56:34.310083] Step 400, train accuracy 0.1876, test accuracy 0.1900\n",
            "[2019-11-27 12:56:34.651592] Step 420, train accuracy 0.1918, test accuracy 0.1939\n",
            "[2019-11-27 12:56:35.027049] Step 440, train accuracy 0.1955, test accuracy 0.1977\n",
            "[2019-11-27 12:56:35.363901] Step 460, train accuracy 0.1989, test accuracy 0.2024\n",
            "[2019-11-27 12:56:35.705237] Step 480, train accuracy 0.2019, test accuracy 0.2056\n",
            "[2019-11-27 12:56:36.050746] Step 500, train accuracy 0.2046, test accuracy 0.2078\n",
            "[2019-11-27 12:56:36.393266] Step 520, train accuracy 0.2076, test accuracy 0.2098\n",
            "[2019-11-27 12:56:36.735087] Step 540, train accuracy 0.2101, test accuracy 0.2118\n",
            "[2019-11-27 12:56:37.073411] Step 560, train accuracy 0.2124, test accuracy 0.2142\n",
            "[2019-11-27 12:56:37.413932] Step 580, train accuracy 0.2144, test accuracy 0.2155\n",
            "[2019-11-27 12:56:37.752791] Step 600, train accuracy 0.2169, test accuracy 0.2180\n",
            "[2019-11-27 12:56:38.098375] Step 620, train accuracy 0.2183, test accuracy 0.2220\n",
            "[2019-11-27 12:56:38.442937] Step 640, train accuracy 0.2202, test accuracy 0.2249\n",
            "[2019-11-27 12:56:38.782439] Step 660, train accuracy 0.2217, test accuracy 0.2272\n",
            "[2019-11-27 12:56:39.122948] Step 680, train accuracy 0.2235, test accuracy 0.2293\n",
            "[2019-11-27 12:56:39.460657] Step 700, train accuracy 0.2238, test accuracy 0.2317\n",
            "[2019-11-27 12:56:39.880284] Step 720, train accuracy 0.2249, test accuracy 0.2336\n",
            "[2019-11-27 12:56:40.221035] Step 740, train accuracy 0.2261, test accuracy 0.2348\n",
            "[2019-11-27 12:56:40.566841] Step 760, train accuracy 0.2273, test accuracy 0.2359\n",
            "[2019-11-27 12:56:40.919539] Step 780, train accuracy 0.2281, test accuracy 0.2361\n",
            "[2019-11-27 12:56:41.259806] Step 800, train accuracy 0.2288, test accuracy 0.2353\n",
            "[2019-11-27 12:56:41.615152] Step 820, train accuracy 0.2293, test accuracy 0.2357\n",
            "[2019-11-27 12:56:41.954682] Step 840, train accuracy 0.2299, test accuracy 0.2355\n",
            "[2019-11-27 12:56:42.291884] Step 860, train accuracy 0.2300, test accuracy 0.2351\n",
            "[2019-11-27 12:56:42.634988] Step 880, train accuracy 0.2300, test accuracy 0.2350\n",
            "[2019-11-27 12:56:42.976122] Step 900, train accuracy 0.2301, test accuracy 0.2346\n",
            "[2019-11-27 12:56:43.319332] Step 920, train accuracy 0.2303, test accuracy 0.2340\n",
            "[2019-11-27 12:56:43.672233] Step 940, train accuracy 0.2299, test accuracy 0.2334\n",
            "[2019-11-27 12:56:44.021498] Step 960, train accuracy 0.2303, test accuracy 0.2337\n",
            "[2019-11-27 12:56:44.371764] Step 980, train accuracy 0.2298, test accuracy 0.2326\n",
            "[2019-11-27 12:56:44.728284] Step 1000, train accuracy 0.2298, test accuracy 0.2334\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "otDNRpol1Jib",
        "colab_type": "text"
      },
      "source": [
        "Обучаю с нормализацией"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sa7PXMa2xRCg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 867
        },
        "outputId": "9d89f048-fc95-43df-d6b5-1ab5d26a01fb"
      },
      "source": [
        "loss_bn = tf.nn.sigmoid_cross_entropy_with_logits(logits=y_logit_bn, labels=y)\n",
        "train_op_bn = tf.train.GradientDescentOptimizer(0.01).minimize(loss_bn)\n",
        "\n",
        "# обучение\n",
        "init = tf.global_variables_initializer()\n",
        "\n",
        "sess = tf.Session()\n",
        "sess.run(init)\n",
        "\n",
        "for i in range(1000):\n",
        "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
        "  sess.run(train_op_bn, feed_dict={x: batch_xs, y: batch_ys})\n",
        "\n",
        "  if (i + 1) % 20 == 0:\n",
        "    acc_train = calc_accuracy(sess, mnist.train, accuracy_bn)\n",
        "    acc_test = calc_accuracy(sess, mnist.test, accuracy_bn)\n",
        "    accuracy_history['bn']['train'].append(acc_train)\n",
        "    accuracy_history['bn']['test'].append(acc_test)\n",
        "  \n",
        "    print(f\"[{datetime.now()}] Step {i + 1}, train accuracy {acc_train:.4f}, test accuracy {acc_test:.4f}\")"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2019-11-27 12:31:20.374065] Step 20, train accuracy 0.8430, test accuracy 0.8531\n",
            "[2019-11-27 12:31:20.723469] Step 40, train accuracy 0.8715, test accuracy 0.8827\n",
            "[2019-11-27 12:31:21.104601] Step 60, train accuracy 0.8878, test accuracy 0.8963\n",
            "[2019-11-27 12:31:21.490234] Step 80, train accuracy 0.8944, test accuracy 0.8996\n",
            "[2019-11-27 12:31:21.878171] Step 100, train accuracy 0.9061, test accuracy 0.9103\n",
            "[2019-11-27 12:31:22.360951] Step 120, train accuracy 0.9043, test accuracy 0.9102\n",
            "[2019-11-27 12:31:22.749724] Step 140, train accuracy 0.9170, test accuracy 0.9197\n",
            "[2019-11-27 12:31:23.127503] Step 160, train accuracy 0.9225, test accuracy 0.9249\n",
            "[2019-11-27 12:31:23.508479] Step 180, train accuracy 0.9245, test accuracy 0.9285\n",
            "[2019-11-27 12:31:23.903821] Step 200, train accuracy 0.9316, test accuracy 0.9343\n",
            "[2019-11-27 12:31:24.295109] Step 220, train accuracy 0.9329, test accuracy 0.9326\n",
            "[2019-11-27 12:31:24.689619] Step 240, train accuracy 0.9358, test accuracy 0.9367\n",
            "[2019-11-27 12:31:25.061724] Step 260, train accuracy 0.9353, test accuracy 0.9346\n",
            "[2019-11-27 12:31:25.441511] Step 280, train accuracy 0.9403, test accuracy 0.9410\n",
            "[2019-11-27 12:31:25.823191] Step 300, train accuracy 0.9407, test accuracy 0.9406\n",
            "[2019-11-27 12:31:26.212291] Step 320, train accuracy 0.9444, test accuracy 0.9436\n",
            "[2019-11-27 12:31:26.590934] Step 340, train accuracy 0.9450, test accuracy 0.9448\n",
            "[2019-11-27 12:31:26.989013] Step 360, train accuracy 0.9455, test accuracy 0.9443\n",
            "[2019-11-27 12:31:27.374845] Step 380, train accuracy 0.9460, test accuracy 0.9420\n",
            "[2019-11-27 12:31:27.779120] Step 400, train accuracy 0.9468, test accuracy 0.9455\n",
            "[2019-11-27 12:31:28.164185] Step 420, train accuracy 0.9493, test accuracy 0.9484\n",
            "[2019-11-27 12:31:28.553272] Step 440, train accuracy 0.9493, test accuracy 0.9496\n",
            "[2019-11-27 12:31:28.942475] Step 460, train accuracy 0.9521, test accuracy 0.9476\n",
            "[2019-11-27 12:31:29.323527] Step 480, train accuracy 0.9532, test accuracy 0.9478\n",
            "[2019-11-27 12:31:29.703743] Step 500, train accuracy 0.9538, test accuracy 0.9516\n",
            "[2019-11-27 12:31:30.086094] Step 520, train accuracy 0.9538, test accuracy 0.9494\n",
            "[2019-11-27 12:31:30.474819] Step 540, train accuracy 0.9568, test accuracy 0.9520\n",
            "[2019-11-27 12:31:30.856853] Step 560, train accuracy 0.9553, test accuracy 0.9499\n",
            "[2019-11-27 12:31:31.252408] Step 580, train accuracy 0.9554, test accuracy 0.9514\n",
            "[2019-11-27 12:31:31.638157] Step 600, train accuracy 0.9584, test accuracy 0.9569\n",
            "[2019-11-27 12:31:32.019019] Step 620, train accuracy 0.9571, test accuracy 0.9550\n",
            "[2019-11-27 12:31:32.406813] Step 640, train accuracy 0.9589, test accuracy 0.9570\n",
            "[2019-11-27 12:31:32.866465] Step 660, train accuracy 0.9585, test accuracy 0.9552\n",
            "[2019-11-27 12:31:33.251936] Step 680, train accuracy 0.9603, test accuracy 0.9565\n",
            "[2019-11-27 12:31:33.628884] Step 700, train accuracy 0.9602, test accuracy 0.9568\n",
            "[2019-11-27 12:31:34.004067] Step 720, train accuracy 0.9606, test accuracy 0.9580\n",
            "[2019-11-27 12:31:34.384766] Step 740, train accuracy 0.9622, test accuracy 0.9575\n",
            "[2019-11-27 12:31:34.765413] Step 760, train accuracy 0.9610, test accuracy 0.9568\n",
            "[2019-11-27 12:31:35.136628] Step 780, train accuracy 0.9615, test accuracy 0.9566\n",
            "[2019-11-27 12:31:35.539241] Step 800, train accuracy 0.9638, test accuracy 0.9606\n",
            "[2019-11-27 12:31:35.917998] Step 820, train accuracy 0.9643, test accuracy 0.9602\n",
            "[2019-11-27 12:31:36.299872] Step 840, train accuracy 0.9644, test accuracy 0.9601\n",
            "[2019-11-27 12:31:36.688502] Step 860, train accuracy 0.9656, test accuracy 0.9616\n",
            "[2019-11-27 12:31:37.069259] Step 880, train accuracy 0.9659, test accuracy 0.9616\n",
            "[2019-11-27 12:31:37.457493] Step 900, train accuracy 0.9633, test accuracy 0.9567\n",
            "[2019-11-27 12:31:37.845896] Step 920, train accuracy 0.9653, test accuracy 0.9598\n",
            "[2019-11-27 12:31:38.226143] Step 940, train accuracy 0.9659, test accuracy 0.9603\n",
            "[2019-11-27 12:31:38.608243] Step 960, train accuracy 0.9658, test accuracy 0.9610\n",
            "[2019-11-27 12:31:38.985748] Step 980, train accuracy 0.9636, test accuracy 0.9586\n",
            "[2019-11-27 12:31:39.366298] Step 1000, train accuracy 0.9662, test accuracy 0.9610\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-wUfJ3Czgnap",
        "colab_type": "text"
      },
      "source": [
        "![](https://i.imgur.com/j1a8yFu.jpg)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cp5So4f2y-ng",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SSYYy_18zIbO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}