variational_autoencoder.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "variational_autoencoder.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "VRfRA3GT-TKo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#!pip install tensorflow-gpu==2.0.0beta"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZGdua1xaErC7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.layers import Lambda, Input, Dense\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.datasets import cifar10\n",
        "from tensorflow.keras.losses import mse, binary_crossentropy\n",
        "from tensorflow.keras.utils import plot_model\n",
        "from tensorflow.keras import backend as K\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import argparse\n",
        "import os"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IXoflh7aCL3_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "8db38b8a-1721-497a-eb10-90a36dc56568"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "994fi-mLKss4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "os.chdir(r'/content/gdrive/My Drive/Colab Notebooks/Autoencoders/')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M6PR7E0fmVeO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# z = z_mean + sqrt(var) * epsilon\n",
        "\n",
        "def sampling(args):\n",
        "\n",
        "    z_mean, z_log_var = args\n",
        "    batch = K.shape(z_mean)[0]\n",
        "    dim = K.int_shape(z_mean)[1]\n",
        "    # by default, random_normal has mean = 0 and std = 1.0\n",
        "    epsilon = K.random_normal(shape=(batch, dim))\n",
        "    return z_mean + K.exp(0.5 * z_log_var) * epsilon"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oCLRxys6mSKN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 578
        },
        "outputId": "586314b1-e485-453a-fe62-af8084b8f181"
      },
      "source": [
        "\n",
        "# CIFAR10 dataset\n",
        "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
        "\n",
        "image_size = x_train.shape[1]\n",
        "original_dim = image_size * image_size\n",
        "x_train = np.reshape(x_train, [-1, original_dim])\n",
        "x_test = np.reshape(x_test, [-1, original_dim])\n",
        "x_train = x_train.astype('float32') / 255\n",
        "x_test = x_test.astype('float32') / 255\n",
        "\n",
        "# network parameters\n",
        "input_shape = (original_dim, )\n",
        "intermediate_dim = 512\n",
        "batch_size = 128\n",
        "latent_dim = 2\n",
        "epochs = 50\n",
        "\n",
        "# VAE model = encoder + decoder\n",
        "# build encoder model\n",
        "inputs = Input(shape=input_shape, name='encoder_input')\n",
        "x = Dense(intermediate_dim, activation='relu')(inputs)\n",
        "z_mean = Dense(latent_dim, name='z_mean')(x)\n",
        "z_log_var = Dense(latent_dim, name='z_log_var')(x)\n",
        "\n",
        "# use reparameterization trick to push the sampling out as input\n",
        "# note that \"output_shape\" isn't necessary with the TensorFlow backend\n",
        "z = Lambda(sampling, output_shape=(latent_dim,), name='z')([z_mean, z_log_var])\n",
        "\n",
        "# instantiate encoder model\n",
        "encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder')\n",
        "encoder.summary()\n",
        "plot_model(encoder, to_file='vae_mlp_encoder.png', show_shapes=True)\n",
        "\n",
        "# build decoder model\n",
        "latent_inputs = Input(shape=(latent_dim,), name='z_sampling')\n",
        "x = Dense(intermediate_dim, activation='relu')(latent_inputs)\n",
        "outputs = Dense(original_dim, activation='sigmoid')(x)\n",
        "\n",
        "# instantiate decoder model\n",
        "decoder = Model(latent_inputs, outputs, name='decoder')\n",
        "decoder.summary()\n",
        "plot_model(decoder, to_file='vae_mlp_decoder.png', show_shapes=True)\n",
        "\n",
        "# instantiate VAE model\n",
        "outputs = decoder(encoder(inputs)[2])\n",
        "vae = Model(inputs, outputs, name='vae_mlp')"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"encoder\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "encoder_input (InputLayer)      [(None, 1024)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 512)          524800      encoder_input[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "z_mean (Dense)                  (None, 2)            1026        dense_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "z_log_var (Dense)               (None, 2)            1026        dense_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "z (Lambda)                      (None, 2)            0           z_mean[0][0]                     \n",
            "                                                                 z_log_var[0][0]                  \n",
            "==================================================================================================\n",
            "Total params: 526,852\n",
            "Trainable params: 526,852\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "Model: \"decoder\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "z_sampling (InputLayer)      [(None, 2)]               0         \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 512)               1536      \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 1024)              525312    \n",
            "=================================================================\n",
            "Total params: 526,848\n",
            "Trainable params: 526,848\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5dWRWKg4mLCX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "7d2160dc-a126-4409-cd29-e04f7ca056db"
      },
      "source": [
        "models = (encoder, decoder)\n",
        "data = (x_test, y_test)\n",
        "\n",
        "\n",
        "reconstruction_loss = binary_crossentropy(inputs,\n",
        "                                              outputs)\n",
        "\n",
        "reconstruction_loss *= original_dim\n",
        "kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)\n",
        "kl_loss = K.sum(kl_loss, axis=-1)\n",
        "kl_loss *= -0.5\n",
        "vae_loss = K.mean(reconstruction_loss + kl_loss)\n",
        "vae.add_loss(vae_loss)\n",
        "vae.compile(optimizer='adam')\n",
        "vae.summary()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0711 19:35:42.456447 139832624809856 training_utils.py:1237] Output decoder missing from loss dictionary. We assume this was done on purpose. The fit and evaluate APIs will not be expecting any data to be passed to decoder.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"vae_mlp\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "encoder_input (InputLayer)      [(None, 1024)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "encoder (Model)                 [(None, 2), (None, 2 526852      encoder_input[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "decoder (Model)                 (None, 1024)         526848      encoder[1][2]                    \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_clip_by_value_1/Min [(None, 1024)]       0           decoder[1][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_clip_by_value_1 (Te [(None, 1024)]       0           tf_op_layer_clip_by_value_1/Minim\n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_add_5 (TensorFlowOp [(None, 1024)]       0           tf_op_layer_clip_by_value_1[0][0]\n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_Log_2 (TensorFlowOp [(None, 1024)]       0           tf_op_layer_add_5[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_mul_4 (TensorFlowOp [(None, 1024)]       0           encoder_input[0][0]              \n",
            "                                                                 tf_op_layer_Log_2[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_sub_4 (TensorFlowOp [(None, 1024)]       0           encoder_input[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_sub_5 (TensorFlowOp [(None, 1024)]       0           tf_op_layer_clip_by_value_1[0][0]\n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_add_6 (TensorFlowOp [(None, 1024)]       0           tf_op_layer_sub_5[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_Log_3 (TensorFlowOp [(None, 1024)]       0           tf_op_layer_add_6[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_mul_5 (TensorFlowOp [(None, 1024)]       0           tf_op_layer_sub_4[0][0]          \n",
            "                                                                 tf_op_layer_Log_3[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_add_7 (TensorFlowOp [(None, 1024)]       0           tf_op_layer_mul_4[0][0]          \n",
            "                                                                 tf_op_layer_mul_5[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_Neg_1 (TensorFlowOp [(None, 1024)]       0           tf_op_layer_add_7[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_Mean_2 (TensorFlowO [(None,)]            0           tf_op_layer_Neg_1[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_mul_6 (TensorFlowOp [(None,)]            0           tf_op_layer_Mean_2[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_add_8 (TensorFlowOp [(None, 2)]          0           z_log_var[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_Square_1 (TensorFlo [(None, 2)]          0           z_mean[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_sub_6 (TensorFlowOp [(None, 2)]          0           tf_op_layer_add_8[0][0]          \n",
            "                                                                 tf_op_layer_Square_1[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_Exp_1 (TensorFlowOp [(None, 2)]          0           z_log_var[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_sub_7 (TensorFlowOp [(None, 2)]          0           tf_op_layer_sub_6[0][0]          \n",
            "                                                                 tf_op_layer_Exp_1[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_Sum_1 (TensorFlowOp [(None,)]            0           tf_op_layer_sub_7[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_mul_7 (TensorFlowOp [(None,)]            0           tf_op_layer_Sum_1[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_add_9 (TensorFlowOp [(None,)]            0           tf_op_layer_mul_6[0][0]          \n",
            "                                                                 tf_op_layer_mul_7[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "tf_op_layer_Mean_3 (TensorFlowO [()]                 0           tf_op_layer_add_9[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "add_loss_1 (AddLoss)            ()                   0           tf_op_layer_Mean_3[0][0]         \n",
            "==================================================================================================\n",
            "Total params: 1,053,700\n",
            "Trainable params: 1,053,700\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jKQCB9X4CXUY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "efca8f84-c25a-4c6f-a684-c23ab43885ee"
      },
      "source": [
        "vae.fit(x_train,\n",
        "        epochs=epochs,\n",
        "        batch_size=batch_size,\n",
        "        validation_data=(x_test, None))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 150000 samples, validate on 30000 samples\n",
            "Epoch 1/50\n",
            "150000/150000 [==============================] - 44s 294us/sample - loss: 650.6047 - val_loss: 645.3723\n",
            "Epoch 2/50\n",
            "150000/150000 [==============================] - 43s 286us/sample - loss: 644.1156 - val_loss: 643.5928\n",
            "Epoch 3/50\n",
            "150000/150000 [==============================] - 43s 287us/sample - loss: 642.9342 - val_loss: 643.2314\n",
            "Epoch 4/50\n",
            "150000/150000 [==============================] - 43s 288us/sample - loss: 642.2630 - val_loss: 642.6723\n",
            "Epoch 5/50\n",
            "150000/150000 [==============================] - 43s 289us/sample - loss: 641.7485 - val_loss: 642.9198\n",
            "Epoch 6/50\n",
            "150000/150000 [==============================] - 43s 290us/sample - loss: 641.4007 - val_loss: 641.4827\n",
            "Epoch 7/50\n",
            "150000/150000 [==============================] - 44s 290us/sample - loss: 641.0230 - val_loss: 641.7365\n",
            "Epoch 8/50\n",
            "150000/150000 [==============================] - 43s 289us/sample - loss: 640.6578 - val_loss: 641.1716\n",
            "Epoch 9/50\n",
            "150000/150000 [==============================] - 43s 288us/sample - loss: 640.4371 - val_loss: 640.7723\n",
            "Epoch 10/50\n",
            "150000/150000 [==============================] - 43s 285us/sample - loss: 640.0390 - val_loss: 640.5263\n",
            "Epoch 11/50\n",
            "150000/150000 [==============================] - 43s 287us/sample - loss: 639.7692 - val_loss: 640.1701\n",
            "Epoch 12/50\n",
            "150000/150000 [==============================] - 44s 291us/sample - loss: 639.6051 - val_loss: 640.5234\n",
            "Epoch 13/50\n",
            "150000/150000 [==============================] - 43s 290us/sample - loss: 639.3985 - val_loss: 639.9053\n",
            "Epoch 14/50\n",
            "150000/150000 [==============================] - 44s 293us/sample - loss: 639.1731 - val_loss: 639.6924\n",
            "Epoch 15/50\n",
            "150000/150000 [==============================] - 44s 294us/sample - loss: 638.9539 - val_loss: 639.8201\n",
            "Epoch 16/50\n",
            "150000/150000 [==============================] - 44s 292us/sample - loss: 639.0357 - val_loss: 639.4114\n",
            "Epoch 17/50\n",
            "150000/150000 [==============================] - 44s 291us/sample - loss: 638.8193 - val_loss: 640.2233\n",
            "Epoch 18/50\n",
            "150000/150000 [==============================] - 44s 291us/sample - loss: 638.6752 - val_loss: 639.0177\n",
            "Epoch 19/50\n",
            "150000/150000 [==============================] - 44s 291us/sample - loss: 638.5062 - val_loss: 639.2208\n",
            "Epoch 20/50\n",
            "150000/150000 [==============================] - 44s 290us/sample - loss: 638.5578 - val_loss: 639.8157\n",
            "Epoch 21/50\n",
            "150000/150000 [==============================] - 44s 293us/sample - loss: 638.4992 - val_loss: 639.0639\n",
            "Epoch 22/50\n",
            "150000/150000 [==============================] - 44s 294us/sample - loss: 638.4634 - val_loss: 638.9695\n",
            "Epoch 23/50\n",
            "150000/150000 [==============================] - 44s 292us/sample - loss: 638.3369 - val_loss: 639.0571\n",
            "Epoch 24/50\n",
            "150000/150000 [==============================] - 43s 285us/sample - loss: 638.2256 - val_loss: 639.3618\n",
            "Epoch 25/50\n",
            "150000/150000 [==============================] - 43s 290us/sample - loss: 638.2321 - val_loss: 638.6097\n",
            "Epoch 26/50\n",
            "150000/150000 [==============================] - 43s 290us/sample - loss: 638.2129 - val_loss: 638.7158\n",
            "Epoch 27/50\n",
            "150000/150000 [==============================] - 43s 288us/sample - loss: 638.2633 - val_loss: 640.3296\n",
            "Epoch 28/50\n",
            "150000/150000 [==============================] - 43s 285us/sample - loss: 638.1203 - val_loss: 638.6249\n",
            "Epoch 29/50\n",
            "150000/150000 [==============================] - 42s 281us/sample - loss: 638.0966 - val_loss: 638.8831\n",
            "Epoch 30/50\n",
            "150000/150000 [==============================] - 42s 282us/sample - loss: 638.0415 - val_loss: 640.2288\n",
            "Epoch 31/50\n",
            "150000/150000 [==============================] - 42s 282us/sample - loss: 637.9215 - val_loss: 639.0043\n",
            "Epoch 32/50\n",
            "150000/150000 [==============================] - 43s 284us/sample - loss: 637.9975 - val_loss: 638.3583\n",
            "Epoch 33/50\n",
            "150000/150000 [==============================] - 43s 284us/sample - loss: 637.7661 - val_loss: 639.9518\n",
            "Epoch 34/50\n",
            "150000/150000 [==============================] - 43s 290us/sample - loss: 637.9612 - val_loss: 638.2697\n",
            "Epoch 35/50\n",
            "150000/150000 [==============================] - 43s 284us/sample - loss: 637.9690 - val_loss: 638.3385\n",
            "Epoch 36/50\n",
            "150000/150000 [==============================] - 43s 286us/sample - loss: 637.8676 - val_loss: 638.9359\n",
            "Epoch 37/50\n",
            "150000/150000 [==============================] - 43s 286us/sample - loss: 637.8760 - val_loss: 639.3343\n",
            "Epoch 38/50\n",
            "150000/150000 [==============================] - 43s 287us/sample - loss: 637.8416 - val_loss: 638.2967\n",
            "Epoch 39/50\n",
            "150000/150000 [==============================] - 43s 284us/sample - loss: 637.6687 - val_loss: 638.0785\n",
            "Epoch 40/50\n",
            "150000/150000 [==============================] - 43s 286us/sample - loss: 637.5911 - val_loss: 638.6504\n",
            "Epoch 41/50\n",
            "150000/150000 [==============================] - 43s 288us/sample - loss: 637.7078 - val_loss: 638.8680\n",
            "Epoch 42/50\n",
            "150000/150000 [==============================] - 43s 288us/sample - loss: 637.7035 - val_loss: 638.8953\n",
            "Epoch 43/50\n",
            "150000/150000 [==============================] - 52s 344us/sample - loss: 637.6224 - val_loss: 638.3545\n",
            "Epoch 44/50\n",
            "150000/150000 [==============================] - 43s 287us/sample - loss: 637.5452 - val_loss: 638.2358\n",
            "Epoch 45/50\n",
            "150000/150000 [==============================] - 42s 283us/sample - loss: 637.5158 - val_loss: 638.0288\n",
            "Epoch 46/50\n",
            "150000/150000 [==============================] - 43s 284us/sample - loss: 637.5201 - val_loss: 639.2184\n",
            "Epoch 47/50\n",
            "150000/150000 [==============================] - 43s 285us/sample - loss: 637.6458 - val_loss: 638.7875\n",
            "Epoch 48/50\n",
            "150000/150000 [==============================] - 42s 282us/sample - loss: 637.4041 - val_loss: 637.8510\n",
            "Epoch 49/50\n",
            "150000/150000 [==============================] - 42s 282us/sample - loss: 637.5038 - val_loss: 638.4479\n",
            "Epoch 50/50\n",
            "150000/150000 [==============================] - 42s 281us/sample - loss: 637.5745 - val_loss: 638.5400\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f2ceb1960f0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H3sI1ZY7Zd8v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}