MNIST_VGG.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MNIST_VGG.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "WZcAr0fi04Mf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 63
        },
        "outputId": "90426c6b-8a91-43d3-ea71-3f46b7421a13"
      },
      "source": [
        "import tensorflow"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zoupn01I1mIM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6b5dcbb4-a0cf-443b-8d1e-4a295cf42e9a"
      },
      "source": [
        "print(tensorflow.__version__)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.15.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "py08wp4M1ogV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 962
        },
        "outputId": "4f532979-1394-49fb-bd89-193dbe9020e3"
      },
      "source": [
        "!pip install tensorflow-gpu==2.0"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorflow-gpu==2.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/25/44/47f0722aea081697143fbcf5d2aa60d1aee4aaacb5869aee2b568974777b/tensorflow_gpu-2.0.0-cp36-cp36m-manylinux2010_x86_64.whl (380.8MB)\n",
            "\u001b[K     |████████████████████████████████| 380.8MB 47kB/s \n",
            "\u001b[?25hRequirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (1.1.0)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (0.8.1)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (0.9.0)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (3.1.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (0.34.2)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (3.10.0)\n",
            "Collecting tensorboard<2.1.0,>=2.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/76/54/99b9d5d52d5cb732f099baaaf7740403e83fe6b0cedde940fabd2b13d75a/tensorboard-2.0.2-py3-none-any.whl (3.8MB)\n",
            "\u001b[K     |████████████████████████████████| 3.8MB 45.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (1.0.8)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (1.1.0)\n",
            "Collecting tensorflow-estimator<2.1.0,>=2.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fc/08/8b927337b7019c374719145d1dceba21a8bb909b93b1ad6f8fb7d22c1ca1/tensorflow_estimator-2.0.1-py2.py3-none-any.whl (449kB)\n",
            "\u001b[K     |████████████████████████████████| 450kB 58.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (0.1.8)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (1.12.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (1.27.1)\n",
            "Requirement already satisfied: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (0.2.2)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (1.11.2)\n",
            "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0) (1.17.5)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu==2.0) (45.1.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (1.7.2)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (1.0.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (0.4.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (3.2.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (2.21.0)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow-gpu==2.0) (2.8.0)\n",
            "Requirement already satisfied: cachetools<3.2,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (3.1.1)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (0.2.8)\n",
            "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (4.0)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (1.3.0)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (2.8)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (2019.11.28)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (3.0.4)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (0.4.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.1.0,>=2.0.0->tensorflow-gpu==2.0) (3.1.0)\n",
            "\u001b[31mERROR: tensorflow 1.15.0 has requirement tensorboard<1.16.0,>=1.15.0, but you'll have tensorboard 2.0.2 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: tensorflow 1.15.0 has requirement tensorflow-estimator==1.15.1, but you'll have tensorflow-estimator 2.0.1 which is incompatible.\u001b[0m\n",
            "Installing collected packages: tensorboard, tensorflow-estimator, tensorflow-gpu\n",
            "  Found existing installation: tensorboard 1.15.0\n",
            "    Uninstalling tensorboard-1.15.0:\n",
            "      Successfully uninstalled tensorboard-1.15.0\n",
            "  Found existing installation: tensorflow-estimator 1.15.1\n",
            "    Uninstalling tensorflow-estimator-1.15.1:\n",
            "      Successfully uninstalled tensorflow-estimator-1.15.1\n",
            "Successfully installed tensorboard-2.0.2 tensorflow-estimator-2.0.1 tensorflow-gpu-2.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h_IK6HNy2NUE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HCvL09tq2QtL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "bd31fd30-2faa-4823-b752-969d6b715423"
      },
      "source": [
        "print(tensorflow.__version__)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "su9ngwXY2W7B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 注释：安装tensorflow2.0，import tensorflow.keras as keras，可以使用tensorflow内集成的keras模块，神经网络编写接口友善"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l-hFvcLh2iQI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow.keras as keras"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p_vIzGsQ2lY4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "00aa4f7f-94e7-4068-f97a-42c82c84a53d"
      },
      "source": [
        "from keras.datasets import mnist"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kswTCc952nPd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#引入Keras自带手写数字数据集，且自动按照七三开比例划分训练数据集与测试数据集，初次运行，下载时间在5分钟左右"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UNtr7PVk2sn0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "17e5d6ab-15fe-451a-bae3-16fb016b24fc"
      },
      "source": [
        "from keras.datasets import mnist\n",
        "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
        "print(\"X_train original shape\", X_train.shape)\n",
        "print(\"y_train original shape\", y_train.shape)\n",
        "print(\"X_test original shape\", X_test.shape)\n",
        "print(\"y_test original shape\", y_test.shape)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 1s 0us/step\n",
            "X_train original shape (60000, 28, 28)\n",
            "y_train original shape (60000,)\n",
            "X_test original shape (10000, 28, 28)\n",
            "y_test original shape (10000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UAO5WB222wVu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#引入python-matlab接口，实现画图功能，看看图像和标签的模样"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2e968OiQ232a",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "outputId": "3eedd8fd-820a-466e-b3aa-1eeafa13c47a"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "plt.imshow(X_train[0], cmap='gray')\n",
        "plt.title('Class '+ str(y_train[0]))"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Class 5')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEICAYAAACZA4KlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAQaElEQVR4nO3df6xUdX7G8fdTlGxEFFm7SFyVxRqM\nGmUbxK6ldY1FxWgVNWbZtaHRiE0gcaMlNbTpYhusWcV2ie4GNurK1rraqBXJbsEVFLu2xCuiItbV\nNZiFXEFF5Ic/gU//mIO9q3e+c5k5M2e43+eV3NyZ85kz53OPPJ4z58d8FRGY2eD3e1U3YGad4bCb\nZcJhN8uEw26WCYfdLBMOu1kmHPaMSJor6V+r7sOq4bAPMpK+LalH0k5JvZJ+IWlSRb1skPRh0ctO\nScur6MNqHPZBRNL1wL8ANwOjgGOBHwIXV9jWRRFxaPFzboV9ZM9hHyQkHQ78AzAzIh6OiF0R8WlE\nPBYRs+vM8++S3pL0vqRVkk7uU7tA0npJOyRtkvTXxfQjJS2VtE3SVklPS/K/owOA/yMNHt8AvgQ8\nsh/z/AI4AfgKsAa4r0/tLuDaiBgOnAKsKKbfAGwEfp/a3sMcIHXN9X2S3pa0XNJp+9GblcxhHzy+\nDLwTEbsHOkNE3B0ROyLiY2AucFqxhwDwKXCSpMMi4r2IWNNn+mjguGLP4emof4PFd4AxwHHASmCZ\npBH7/ZdZKRz2weNd4EhJBw3kxZKGSLpF0m8kbQc2FKUji9+XARcAb0p6StI3ium3Aq8DyyW9IenG\nesuIiF9FxIcR8UFE/BOwDfiT/f/TrAwO++Dx38DHwCUDfP23qR24+zPgcGpbYAABRMSzEXExtV38\n/wAeLKbviIgbImIs8OfA9ZLOGeAyY9/7W+c57INERLwP/D1wp6RLJB0i6WBJUyR9v59ZhlP7n8O7\nwCHUjuADIGmopO9IOjwiPgW2A3uL2oWS/kCSgPeBPftqfUk6VtIfF+/1JUmzqe01/Krcv9wGymEf\nRCJiPnA98HfA28BvgVnUtsyftxh4E9gErAf+53P1vwA2FLv4f0Xt8zfUDuj9EthJbW/ihxGxsp/3\nHw78CHivWMb5wJSIeLfZv89aI395hVkevGU3y4TDbpYJh90sEw67WSYGdAFGWST5aKBZm0VEv9cy\ntLRll3S+pFclvZ66ksrMqtf0qTdJQ4BfA5Op3RjxLDAtItYn5vGW3azN2rFlnwi8HhFvRMQnwM+o\n9r5pM0toJexHU7tCa5+NxbTfIWlG8c0pPS0sy8xa1PYDdBGxCFgE3o03q1IrW/ZNwDF9nn+1mGZm\nXaiVsD8LnCDpa5KGAt8ClpTTlpmVrend+IjYLWkWsAwYAtwdES+X1pmZlaqjd735M7tZ+7Xlohoz\nO3A47GaZcNjNMuGwm2XCYTfLhMNulgmH3SwTDrtZJhx2s0w47GaZcNjNMuGwm2XCYTfLhMNulgmH\n3SwTDrtZJhx2s0w47GaZcNjNMuGwm2XCYTfLhMNulgmH3SwTDrtZJhx2s0w47GaZcNjNMuGwm2XC\nYTfLRNNDNtuBYciQIcn64Ycf3tblz5o1q27tkEMOSc47bty4ZH3mzJnJ+m233Va3Nm3atOS8H330\nUbJ+yy23JOs33XRTsl6FlsIuaQOwA9gD7I6ICWU0ZWblK2PLfnZEvFPC+5hZG/kzu1kmWg17AMsl\nPSdpRn8vkDRDUo+knhaXZWYtaHU3flJEbJL0FeBxSf8bEav6viAiFgGLACRFi8szsya1tGWPiE3F\n7y3AI8DEMpoys/I1HXZJwyQN3/cYOBdYV1ZjZlauVnbjRwGPSNr3Pv8WEf9ZSleDzLHHHpusDx06\nNFk/88wzk/VJkybVrY0YMSI572WXXZasV2njxo3J+oIFC5L1qVOn1q3t2LEjOe8LL7yQrD/11FPJ\nejdqOuwR8QZwWom9mFkb+dSbWSYcdrNMOOxmmXDYzTLhsJtlQhGdu6htsF5BN378+GR9xYoVyXq7\nbzPtVnv37k3Wr7rqqmR9586dTS+7t7c3WX/vvfeS9VdffbXpZbdbRKi/6d6ym2XCYTfLhMNulgmH\n3SwTDrtZJhx2s0w47GaZ8Hn2EowcOTJZX716dbI+duzYMtspVaPet23blqyfffbZdWuffPJJct5c\nrz9olc+zm2XOYTfLhMNulgmH3SwTDrtZJhx2s0w47GaZ8JDNJdi6dWuyPnv27GT9wgsvTNaff/75\nZL3RVyqnrF27NlmfPHlysr5r165k/eSTT65bu+6665LzWrm8ZTfLhMNulgmH3SwTDrtZJhx2s0w4\n7GaZcNjNMuH72bvAYYcdlqw3Gl544cKFdWtXX311ct4rr7wyWb///vuTdes+Td/PLuluSVskresz\nbaSkxyW9Vvw+osxmzax8A9mN/wlw/uem3Qg8EREnAE8Uz82sizUMe0SsAj5/PejFwL3F43uBS0ru\ny8xK1uy18aMiYt9gWW8Bo+q9UNIMYEaTyzGzkrR8I0xEROrAW0QsAhaBD9CZVanZU2+bJY0GKH5v\nKa8lM2uHZsO+BJhePJ4OPFpOO2bWLg134yXdD3wTOFLSRuB7wC3Ag5KuBt4Ermhnk4Pd9u3bW5r/\n/fffb3rea665Jll/4IEHkvVGY6xb92gY9oiYVqd0Tsm9mFkb+XJZs0w47GaZcNjNMuGwm2XCYTfL\nhG9xHQSGDRtWt/bYY48l5z3rrLOS9SlTpiTry5cvT9at8zxks1nmHHazTDjsZplw2M0y4bCbZcJh\nN8uEw26WCZ9nH+SOP/74ZH3NmjXJ+rZt25L1lStXJus9PT11a3feeWdy3k7+2xxMfJ7dLHMOu1km\nHHazTDjsZplw2M0y4bCbZcJhN8uEz7NnburUqcn6Pffck6wPHz686WXPmTMnWV+8eHGy3tvbm6zn\nyufZzTLnsJtlwmE3y4TDbpYJh90sEw67WSYcdrNM+Dy7JZ1yyinJ+u23356sn3NO84P9Lly4MFmf\nN29esr5p06aml30ga/o8u6S7JW2RtK7PtLmSNklaW/xcUGazZla+gezG/wQ4v5/p/xwR44ufn5fb\nlpmVrWHYI2IVsLUDvZhZG7VygG6WpBeL3fwj6r1I0gxJPZLqfxmZmbVds2H/EXA8MB7oBebXe2FE\nLIqICRExocllmVkJmgp7RGyOiD0RsRf4MTCx3LbMrGxNhV3S6D5PpwLr6r3WzLpDw/Psku4Hvgkc\nCWwGvlc8Hw8EsAG4NiIa3lzs8+yDz4gRI5L1iy66qG6t0b3yUr+niz+zYsWKZH3y5MnJ+mBV7zz7\nQQOYcVo/k+9quSMz6yhfLmuWCYfdLBMOu1kmHHazTDjsZpnwLa5WmY8//jhZP+ig9Mmi3bt3J+vn\nnXde3dqTTz6ZnPdA5q+SNsucw26WCYfdLBMOu1kmHHazTDjsZplw2M0y0fCuN8vbqaeemqxffvnl\nyfrpp59et9boPHoj69evT9ZXrVrV0vsPNt6ym2XCYTfLhMNulgmH3SwTDrtZJhx2s0w47GaZ8Hn2\nQW7cuHHJ+qxZs5L1Sy+9NFk/6qij9rungdqzZ0+y3tub/vbyvXv3ltnOAc9bdrNMOOxmmXDYzTLh\nsJtlwmE3y4TDbpYJh90sEw3Ps0s6BlgMjKI2RPOiiPiBpJHAA8AYasM2XxER77Wv1Xw1Opc9bVp/\nA+3WNDqPPmbMmGZaKkVPT0+yPm/evGR9yZIlZbYz6A1ky74buCEiTgL+CJgp6STgRuCJiDgBeKJ4\nbmZdqmHYI6I3ItYUj3cArwBHAxcD9xYvuxe4pF1Nmlnr9uszu6QxwNeB1cCoiNh3veJb1HbzzaxL\nDfjaeEmHAg8B342I7dL/DycVEVFvHDdJM4AZrTZqZq0Z0JZd0sHUgn5fRDxcTN4saXRRHw1s6W/e\niFgUERMiYkIZDZtZcxqGXbVN+F3AKxFxe5/SEmB68Xg68Gj57ZlZWRoO2SxpEvA08BKw757BOdQ+\ntz8IHAu8Se3U29YG75XlkM2jRqUPZ5x00knJ+h133JGsn3jiifvdU1lWr16drN966611a48+mt4+\n+BbV5tQbsrnhZ/aI+C+g35mBc1ppysw6x1fQmWXCYTfLhMNulgmH3SwTDrtZJhx2s0z4q6QHaOTI\nkXVrCxcuTM47fvz4ZH3s2LFN9VSGZ555JlmfP39+sr5s2bJk/cMPP9zvnqw9vGU3y4TDbpYJh90s\nEw67WSYcdrNMOOxmmXDYzTKRzXn2M844I1mfPXt2sj5x4sS6taOPPrqpnsrywQcf1K0tWLAgOe/N\nN9+crO/ataupnqz7eMtulgmH3SwTDrtZJhx2s0w47GaZcNjNMuGwm2Uim/PsU6dObaneivXr1yfr\nS5cuTdZ3796drKfuOd+2bVtyXsuHt+xmmXDYzTLhsJtlwmE3y4TDbpYJh90sEw67WSYGMj77McBi\nYBQQwKKI+IGkucA1wNvFS+dExM8bvFeW47ObdVK98dkHEvbRwOiIWCNpOPAccAlwBbAzIm4baBMO\nu1n71Qt7wyvoIqIX6C0e75D0ClDtV7OY2X7br8/sksYAXwdWF5NmSXpR0t2SjqgzzwxJPZJ6WurU\nzFrScDf+sxdKhwJPAfMi4mFJo4B3qH2O/0dqu/pXNXgP78abtVnTn9kBJB0MLAWWRcTt/dTHAEsj\n4pQG7+Owm7VZvbA33I2XJOAu4JW+QS8O3O0zFVjXapNm1j4DORo/CXgaeAnYW0yeA0wDxlPbjd8A\nXFsczEu9l7fsZm3W0m58WRx2s/ZrejfezAYHh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TD\nbpYJh90sEw67WSYcdrNMOOxmmXDYzTLR6SGb3wHe7PP8yGJaN+rW3rq1L3BvzSqzt+PqFTp6P/sX\nFi71RMSEyhpI6NbeurUvcG/N6lRv3o03y4TDbpaJqsO+qOLlp3Rrb93aF7i3ZnWkt0o/s5tZ51S9\nZTezDnHYzTJRSdglnS/pVUmvS7qxih7qkbRB0kuS1lY9Pl0xht4WSev6TBsp6XFJrxW/+x1jr6Le\n5kraVKy7tZIuqKi3YyStlLRe0suSriumV7ruEn11ZL11/DO7pCHAr4HJwEbgWWBaRKzvaCN1SNoA\nTIiIyi/AkPSnwE5g8b6htSR9H9gaEbcU/6M8IiL+pkt6m8t+DuPdpt7qDTP+l1S47soc/rwZVWzZ\nJwKvR8QbEfEJ8DPg4gr66HoRsQrY+rnJFwP3Fo/vpfaPpePq9NYVIqI3ItYUj3cA+4YZr3TdJfrq\niCrCfjTw2z7PN9Jd470HsFzSc5JmVN1MP0b1GWbrLWBUlc30o+Ew3p30uWHGu2bdNTP8eat8gO6L\nJkXEHwJTgJnF7mpXitpnsG46d/oj4HhqYwD2AvOrbKYYZvwh4LsRsb1vrcp1109fHVlvVYR9E3BM\nn+dfLaZ1hYjYVPzeAjxC7WNHN9m8bwTd4veWivv5TERsjog9EbEX+DEVrrtimPGHgPsi4uFicuXr\nrr++OrXeqgj7s8AJkr4maSjwLWBJBX18gaRhxYETJA0DzqX7hqJeAkwvHk8HHq2wl9/RLcN41xtm\nnIrXXeXDn0dEx3+AC6gdkf8N8LdV9FCnr7HAC8XPy1X3BtxPbbfuU2rHNq4Gvgw8AbwG/BIY2UW9\n/ZTa0N4vUgvW6Ip6m0RtF/1FYG3xc0HV6y7RV0fWmy+XNcuED9CZZcJhN8uEw26WCYfdLBMOu1km\nHHazTDjsZpn4P/utVN5YYpVyAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l0AgGoTB288h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#图像reshape到28*28*1灰度图（宽度、高度、像素通道）"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VqI-vNcu3Dg2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)\n",
        "X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NwtuL54P3Haf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#图像像素是0-255（2^8)，在实际处理中，为了便于梯度相关的优化算法，需要对图像像素做归一化处理"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZRpvsz7W3LeC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "3dee3f02-a639-47e7-a99a-058df796391a"
      },
      "source": [
        "X_train = X_train.astype('float32')\n",
        "X_test = X_test.astype('float32')\n",
        "\n",
        "X_train/=255\n",
        "X_test/=255\n",
        "\n",
        "X_train.shape"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(60000, 28, 28, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UCTJLPVw3PBo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#0-9的数字一共十类，对标签进行独热编码"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vsZlvVtW3VsN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#引入numpy库，python矩阵运算相关的library"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M1pKQeHO3bzW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RL2K4vyk3c_g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.utils import np_utils"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_4jLMgqd3iSB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1677d4a1-1f47-4852-ea1c-b95639a9f405"
      },
      "source": [
        "number_of_classes = 10\n",
        "\n",
        "Y_train = np_utils.to_categorical(y_train, number_of_classes)\n",
        "Y_test = np_utils.to_categorical(y_test, number_of_classes)\n",
        "\n",
        "y_train[0], Y_train[0]"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5, array([0., 0., 0., 0., 0., 1., 0., 0., 0., 0.], dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VaQZe0zj3m5i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten\n",
        "\n",
        "from tensorflow.keras.layers import BatchNormalization\n",
        "from tensorflow.keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D\n",
        "from tensorflow.keras.layers import LeakyReLU"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wCWAemN13oZ1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# CNN三步走\n",
        "# 1. 卷积层\n",
        "# 2. 非线性激活函数\n",
        "# 3. 池化层\n",
        "# 重复堆叠操作\n",
        "#这构成了VGG的基本结构，本模型在VGG的基本框架上加入Dropout和BatcchNormalization，Dropout可以起到正则化作用，BN则进行数据归一化，使得模型训练梯度稳定\n",
        "#BN可以跟Dropout结合用，但在使用BN的情况下，Dropout的参数设置成0.05-0.1\n",
        "\n",
        "# 4.全连接层\n",
        "#为了正则化，可以加入Dropout\n",
        "#为了防止分布漂移，可以加入BathcNormalization\n",
        "#这里先是使用32个感受野为3*3的卷积滤波器，再加入Relu激活函数，BatcchNormalization为正则化的操作，再卷积+激活函数+最大池化，在64个滤波器的卷积核操作上重复"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PPqQaxmB5F_a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras import backend"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jFkV0-4O4aZ0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = Sequential()\n",
        "\n",
        "model.add(Conv2D(32, (3, 3), input_shape=(28,28,1)))\n",
        "model.add(Activation('relu'))\n",
        "model.add(BatchNormalization())\n",
        "model.add(Conv2D(32, (3, 3)))\n",
        "model.add(Activation('relu'))\n",
        "model.add(MaxPooling2D(pool_size=(2,2)))\n",
        "\n",
        "model.add(BatchNormalization())\n",
        "model.add(Conv2D(64,(3, 3)))\n",
        "model.add(Activation('relu'))\n",
        "model.add(BatchNormalization())\n",
        "model.add(Conv2D(64, (3, 3)))\n",
        "model.add(Activation('relu'))\n",
        "model.add(MaxPooling2D(pool_size=(2,2)))\n",
        "\n",
        "model.add(Flatten())\n",
        "# Fully connected layer\n",
        "\n",
        "model.add(BatchNormalization())\n",
        "model.add(Dense(512))\n",
        "model.add(Activation('relu'))\n",
        "model.add(BatchNormalization())\n",
        "model.add(Dropout(0.05))\n",
        "model.add(Dense(10))\n",
        "\n",
        "# model.add(Convolution2D(10,3,3, border_mode='same'))\n",
        "# model.add(GlobalAveragePooling2D())\n",
        "model.add(Activation('softmax'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4LQlbZzP7GPD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#model.summary()看结构"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iReO9l8Q7KOA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 867
        },
        "outputId": "ea6f8810-6a46-4e3d-88a7-c09871cb0536"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d (Conv2D)              (None, 26, 26, 32)        320       \n",
            "_________________________________________________________________\n",
            "activation (Activation)      (None, 26, 26, 32)        0         \n",
            "_________________________________________________________________\n",
            "batch_normalization (BatchNo (None, 26, 26, 32)        128       \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 24, 24, 32)        9248      \n",
            "_________________________________________________________________\n",
            "activation_1 (Activation)    (None, 24, 24, 32)        0         \n",
            "_________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D) (None, 12, 12, 32)        0         \n",
            "_________________________________________________________________\n",
            "batch_normalization_1 (Batch (None, 12, 12, 32)        128       \n",
            "_________________________________________________________________\n",
            "conv2d_2 (Conv2D)            (None, 10, 10, 64)        18496     \n",
            "_________________________________________________________________\n",
            "activation_2 (Activation)    (None, 10, 10, 64)        0         \n",
            "_________________________________________________________________\n",
            "batch_normalization_2 (Batch (None, 10, 10, 64)        256       \n",
            "_________________________________________________________________\n",
            "conv2d_3 (Conv2D)            (None, 8, 8, 64)          36928     \n",
            "_________________________________________________________________\n",
            "activation_3 (Activation)    (None, 8, 8, 64)          0         \n",
            "_________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2 (None, 4, 4, 64)          0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "batch_normalization_3 (Batch (None, 1024)              4096      \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 512)               524800    \n",
            "_________________________________________________________________\n",
            "activation_4 (Activation)    (None, 512)               0         \n",
            "_________________________________________________________________\n",
            "batch_normalization_4 (Batch (None, 512)               2048      \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 512)               0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 10)                5130      \n",
            "_________________________________________________________________\n",
            "activation_5 (Activation)    (None, 10)                0         \n",
            "=================================================================\n",
            "Total params: 601,578\n",
            "Trainable params: 598,250\n",
            "Non-trainable params: 3,328\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qzG-6eRZ7NRY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.optimizers import Adam"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fT81dEjA7Raz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Adam 作为优化器，更多其介绍，参考Adam论文https://arxiv.org/abs/1412.6980"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "evT0doiU7Wo7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aqSs-e1A7X3h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#ImageDataGenerator引入Keras图像处理接口，参考https://keras.io/zh/preprocessing/image/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YbaRK5TD4i_T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.preprocessing.image import ImageDataGenerator"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SPzeYDlv7vkW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#训练数据增强应对潜在的过拟合风险，测试数据不增强"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OudAUdsO70kf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "gen = ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,\n",
        "                         height_shift_range=0.08, zoom_range=0.08)\n",
        "\n",
        "test_gen = ImageDataGenerator()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u6GFklSK7-MY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_generator=gen.flow(X_train, Y_train, batch_size=64)\n",
        "test_generator = test_gen.flow(X_test, Y_test, batch_size=64)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mCNXINlU7_yZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#训练5个epoch，看效果"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b3kpKoQT8GKM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "183452e0-96cb-41f5-9eaf-65af6780eac0"
      },
      "source": [
        "model.fit_generator(train_generator, steps_per_epoch=60000//64, epochs=5, \n",
        "                    validation_data=test_generator, validation_steps=10000//64)"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "937/937 [==============================] - 78s 83ms/step - loss: 0.1326 - accuracy: 0.9592 - val_loss: 0.0445 - val_accuracy: 0.9866\n",
            "Epoch 2/5\n",
            "937/937 [==============================] - 71s 75ms/step - loss: 0.0559 - accuracy: 0.9830 - val_loss: 0.0302 - val_accuracy: 0.9898\n",
            "Epoch 3/5\n",
            "937/937 [==============================] - 69s 74ms/step - loss: 0.0446 - accuracy: 0.9856 - val_loss: 0.0263 - val_accuracy: 0.9907\n",
            "Epoch 4/5\n",
            "937/937 [==============================] - 69s 74ms/step - loss: 0.0392 - accuracy: 0.9876 - val_loss: 0.0191 - val_accuracy: 0.9936\n",
            "Epoch 5/5\n",
            "937/937 [==============================] - 69s 74ms/step - loss: 0.0388 - accuracy: 0.9878 - val_loss: 0.0364 - val_accuracy: 0.9880\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7fe01974d400>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "az71GVzG8MV5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#在测试集上看效果"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YyxRDZAw8Slj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "c463ed61-c676-41fe-c368-68cc61e935a4"
      },
      "source": [
        "score = model.evaluate(X_test, Y_test)\n",
        "print()\n",
        "print('Test accuracy: ', score[1])"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r10000/1 [================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 1s 122us/sample - loss: 0.0199 - accuracy: 0.9880\n",
            "\n",
            "Test accuracy:  0.988\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JmNJZg0m8Uax",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#如果同时安装2.0以上tensorflow和keras，需要显式指定tesorflow,keras，不然一些同名的函数内部实现和归属的域不一样，可能会发生命名冲突"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}