7-2_Deep_Neural_Network.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iNFOeMfl3tIu"
      },
      "source": [
        "# 심층 신경망"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zKfwb8gS3tI2"
      },
      "source": [
        "<table align=\"left\">\n",
        "  <td>\n",
        "    <a target=\"_blank\" href=\"https://colab.research.google.com/github/rickiepark/hg-mldl/blob/master/7-2.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />구글 코랩에서 실행하기</a>\n",
        "  </td>\n",
        "</table>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dPE5XsFhcfVD"
      },
      "source": [
        "## 2개의 층"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4sNOMcdaFVKa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6611c882-d354-4559-8ee6-c3968373f0dd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz\n",
            "32768/29515 [=================================] - 0s 0us/step\n",
            "40960/29515 [=========================================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz\n",
            "26427392/26421880 [==============================] - 0s 0us/step\n",
            "26435584/26421880 [==============================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz\n",
            "16384/5148 [===============================================================================================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz\n",
            "4423680/4422102 [==============================] - 0s 0us/step\n",
            "4431872/4422102 [==============================] - 0s 0us/step\n"
          ]
        }
      ],
      "source": [
        "from tensorflow import keras\n",
        "(train_input, train_target), (test_input, test_target) = keras.datasets.fashion_mnist.load_data()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aJJiRMa6FkWx"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "train_scaled = train_input / 255.0\n",
        "train_scaled = train_scaled.reshape(-1, 28*28)\n",
        "train_scaled, val_scaled, train_target, val_target = train_test_split(train_scaled, train_target, test_size=0.2, random_state=42)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MCZWQiEwF6MD"
      },
      "outputs": [],
      "source": [
        "dense1 = keras.layers.Dense(100, activation='sigmoid', input_shape=(784,))\n",
        "dense2 = keras.layers.Dense(10, activation='softmax')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Agy5gCVhcrm-"
      },
      "source": [
        "## 심층 신경망 만들기"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xmWL7kOoGB4P"
      },
      "outputs": [],
      "source": [
        "model = keras.Sequential([dense1, dense2])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "em0xDl8qa12J",
        "outputId": "da2a3534-d77e-4a71-8a79-b21f56db005c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " dense (Dense)               (None, 100)               78500     \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 10)                1010      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 79,510\n",
            "Trainable params: 79,510\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# 케라스 모델의 정보를 요약해주는 메서드\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qAi41rBTdk7k"
      },
      "source": [
        "## 층을 추가하는 다른 방법"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0XeV6V4ha3I8"
      },
      "outputs": [],
      "source": [
        "model = keras.Sequential([\n",
        "                          keras.layers.Dense(100, activation='sigmoid', input_shape=(784,),name='hidden'),\n",
        "                          keras.layers.Dense(10, activation='softmax', name='output')\n",
        "                        ], name = '패션 MNIST 모델')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bwXDLSOWbm3L",
        "outputId": "8501aaae-518d-4eae-e3fa-2c2b196fe1cc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"패션 MNIST 모델\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " hidden (Dense)              (None, 100)               78500     \n",
            "                                                                 \n",
            " output (Dense)              (None, 10)                1010      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 79,510\n",
            "Trainable params: 79,510\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yZSAxgZCbax7"
      },
      "outputs": [],
      "source": [
        "model = keras.Sequential()\n",
        "model.add(keras.layers.Dense(100, activation='sigmoid', input_shape=(784,)))\n",
        "model.add(keras.layers.Dense(10, activation='softmax'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bW2coaNQboe5",
        "outputId": "93bd11ed-7e89-4492-f681-9b3f5c313ea4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " dense_2 (Dense)             (None, 100)               78500     \n",
            "                                                                 \n",
            " dense_3 (Dense)             (None, 10)                1010      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 79,510\n",
            "Trainable params: 79,510\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kkYrPJembpYk",
        "outputId": "807d0eef-72be-47f3-bc29-4307611633da"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "1500/1500 [==============================] - 6s 2ms/step - loss: 0.5588 - accuracy: 0.8096\n",
            "Epoch 2/5\n",
            "1500/1500 [==============================] - 4s 2ms/step - loss: 0.4096 - accuracy: 0.8529\n",
            "Epoch 3/5\n",
            "1500/1500 [==============================] - 4s 2ms/step - loss: 0.3762 - accuracy: 0.8638\n",
            "Epoch 4/5\n",
            "1500/1500 [==============================] - 4s 2ms/step - loss: 0.3543 - accuracy: 0.8717\n",
            "Epoch 5/5\n",
            "1500/1500 [==============================] - 4s 2ms/step - loss: 0.3368 - accuracy: 0.8788\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f89e02b2e50>"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "model.compile(loss='sparse_categorical_crossentropy', metrics='accuracy')\n",
        "model.fit(train_scaled, train_target, epochs=5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_dfXJDhwcyAK"
      },
      "source": [
        "## ReLU 활성화 함수"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dG7yF8g6b062"
      },
      "outputs": [],
      "source": [
        "model = keras.Sequential()\n",
        "model.add(keras.layers.Flatten(input_shape=(28,28)))\n",
        "model.add(keras.layers.Dense(100, activation='relu'))\n",
        "model.add(keras.layers.Dense(10, activation='softmax'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zHogWhu6g90a",
        "outputId": "10d97b68-abd9-4c9f-f3b0-4dc4db7f1c64"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " flatten (Flatten)           (None, 784)               0         \n",
            "                                                                 \n",
            " dense_4 (Dense)             (None, 100)               78500     \n",
            "                                                                 \n",
            " dense_5 (Dense)             (None, 10)                1010      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 79,510\n",
            "Trainable params: 79,510\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JfPe_ruQdhqA"
      },
      "outputs": [],
      "source": [
        "(train_input, train_target), (test_input, test_target) = keras.datasets.fashion_mnist.load_data()\n",
        "train_scaled = train_input / 255.0\n",
        "train_scaled, val_scaled, train_target, val_target = train_test_split(train_scaled, train_target, test_size = 0.2, random_state=42)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9PGejuuhdvvk",
        "outputId": "119c881b-98d2-430d-bcfd-b3bf481e9e7e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "1500/1500 [==============================] - 4s 2ms/step - loss: 0.5321 - accuracy: 0.8141\n",
            "Epoch 2/5\n",
            "1500/1500 [==============================] - 3s 2ms/step - loss: 0.3975 - accuracy: 0.8570\n",
            "Epoch 3/5\n",
            "1500/1500 [==============================] - 4s 2ms/step - loss: 0.3580 - accuracy: 0.8719\n",
            "Epoch 4/5\n",
            "1500/1500 [==============================] - 4s 2ms/step - loss: 0.3364 - accuracy: 0.8794\n",
            "Epoch 5/5\n",
            "1500/1500 [==============================] - 4s 2ms/step - loss: 0.3225 - accuracy: 0.8857\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f896e309690>"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ],
      "source": [
        "model.compile(loss='sparse_categorical_crossentropy', metrics='accuracy')\n",
        "model.fit(train_scaled, train_target, epochs=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lVYLpnjeep4y",
        "outputId": "019c2542-b5bf-4357-9a7d-3edcc0d3c9b7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "375/375 [==============================] - 1s 2ms/step - loss: 0.3494 - accuracy: 0.8808\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.34936821460723877, 0.8808333277702332]"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ],
      "source": [
        "model.evaluate(val_scaled, val_target)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3YtLsmGAoavz"
      },
      "source": [
        "## 옵티마이저"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### SGD"
      ],
      "metadata": {
        "id": "WA2pG9QkLixN"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NxVj04Haocwa"
      },
      "outputs": [],
      "source": [
        "model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics='accuracy')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1426O4G8Hpfi"
      },
      "outputs": [],
      "source": [
        "sgd = keras.optimizers.SGD()\n",
        "model.compile(optimizer=sgd, loss='sparse_categorical_crossentropy', metrics='accuracy')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "- 학습률(`learning_rate`) 기본값은 0.01"
      ],
      "metadata": {
        "id": "Fl8EdykbLp2k"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Sh-HDiULlp18"
      },
      "outputs": [],
      "source": [
        "sgd = keras.optimizers.SGD(learning_rate=0.1)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Nesterov Momentum"
      ],
      "metadata": {
        "id": "NGV4tlL0L0-w"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uF1XolBXsl3a"
      },
      "outputs": [],
      "source": [
        "sgd = keras.optimizers.SGD(momentum=0.9, nesterov=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Adagrad (적응적 학습률)"
      ],
      "metadata": {
        "id": "QJz4LyLIL5Td"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Hy2MENbL170j"
      },
      "outputs": [],
      "source": [
        "adagrad = keras.optimizers.Adagrad()\n",
        "model.compile(optimizer=adagrad, loss='sparse_categorical_crossentropy', metrics='accuracy')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### RMSProp"
      ],
      "metadata": {
        "id": "BV_etCfFL_iR"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KkpbSMXWtakr"
      },
      "outputs": [],
      "source": [
        "rmsprop = keras.optimizers.RMSprop()\n",
        "model.compile(optimizer=rmsprop, loss='sparse_categorical_crossentropy', metrics='accuracy')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Gdu0hQIAz4JW"
      },
      "outputs": [],
      "source": [
        "model = keras.Sequential()\n",
        "model.add(keras.layers.Flatten(input_shape=(28,28)))\n",
        "model.add(keras.layers.Dense(100, activation='relu'))\n",
        "model.add(keras.layers.Dense(10, activation='softmax'))"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Adam = Momentum + RMSProp"
      ],
      "metadata": {
        "id": "3qGtiIEUMFXh"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2tcxIfILoi1t",
        "outputId": "2a928a14-8c6b-41ab-bd23-7fcd383e87b7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "1500/1500 [==============================] - 4s 2ms/step - loss: 0.5292 - accuracy: 0.8171\n",
            "Epoch 2/5\n",
            "1500/1500 [==============================] - 3s 2ms/step - loss: 0.3940 - accuracy: 0.8589\n",
            "Epoch 3/5\n",
            "1500/1500 [==============================] - 3s 2ms/step - loss: 0.3535 - accuracy: 0.8729\n",
            "Epoch 4/5\n",
            "1500/1500 [==============================] - 3s 2ms/step - loss: 0.3252 - accuracy: 0.8831\n",
            "Epoch 5/5\n",
            "1500/1500 [==============================] - 3s 2ms/step - loss: 0.3059 - accuracy: 0.8881\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f896e112b50>"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ],
      "source": [
        "model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics='accuracy')\n",
        "model.fit(train_scaled, train_target, epochs=5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8gxAWehsv9Gi",
        "outputId": "660c3693-da10-4ca0-d592-a3942e31e51b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "375/375 [==============================] - 1s 2ms/step - loss: 0.3516 - accuracy: 0.8755\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.35156118869781494, 0.8755000233650208]"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ],
      "source": [
        "model.evaluate(val_scaled, val_target)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "7-2 _Deep_Neural_Network",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}