26-fine-tune.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras.models import Sequential, Model, load_model\n",
    "from keras.layers import Dense, Activation, BatchNormalization, Dropout, Input\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "import keras\n",
    "from datetime import datetime\n",
    "from sklearn import preprocessing\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import plot_model\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.utils import shuffle\n",
    "import pickle\n",
    "import glob\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load processed dataset\n",
    "The data we load here are already shuffled and normalized."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train: (966, 26)\n",
      "y_train: (966, 2)\n",
      "x_test: (242, 26)\n",
      "y_test: (242, 2)\n"
     ]
    }
   ],
   "source": [
    "data_directory = 'dataset/3. normalized/'\n",
    "x_train = pickle.load(open(data_directory + \"x_train.p\", \"rb\"))\n",
    "y_train = pickle.load(open(data_directory + \"y_train.p\", \"rb\"))\n",
    "x_test = pickle.load(open(data_directory + \"x_test.p\", \"rb\"))\n",
    "y_test = pickle.load(open(data_directory + \"y_test.p\", \"rb\"))\n",
    "print('x_train: ' + str(x_train.shape))\n",
    "print('y_train: ' + str(y_train.shape))\n",
    "print('x_test: ' + str(x_test.shape))\n",
    "print('y_test: ' + str(y_test.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tunable parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 5000\n",
    "batch_size = 20\n",
    "learning_rate = 0.0001\n",
    "decay = 0.00001"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load auto encoders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 26)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                270       \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 8)                 88        \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 8)                 0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 26)                234       \n",
      "=================================================================\n",
      "Total params: 592\n",
      "Trainable params: 592\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "autoencoder_path = 'results/25-stacked-autoencoder-01/model/2018-03-07-04-14-10/weights.hdf5'\n",
    "weights = load_model(autoencoder_path)\n",
    "weights.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set up constants for weight transfer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "autoencoder_layer_1_units = 10\n",
    "autoencoder_layer_2_units = 8\n",
    "autoencoder_layer_1_name = 'dense_4'\n",
    "autoencoder_layer_2_name = 'dense_5'\n",
    "autoencoder_layer_3_name = 'dense_6'\n",
    "autoencoder_layer_1_dropout = 0.3/2\n",
    "autoencoder_layer_2_dropout = 0.3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model for classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "root_directory = 'results/26-fine-tune-01/'\n",
    "model_directory = root_directory + \"model/\"\n",
    "tensorboard_directory = root_directory + \"tensorboard/\"\n",
    "\n",
    "def create_model(layers, dropout):\n",
    "    global date\n",
    "    global autoencoder_path\n",
    "    x = Input(shape=(x_train.shape[1],))\n",
    "    \n",
    "    y = Dense(units = autoencoder_layer_1_units, activation='relu')(x)\n",
    "    y = Dropout(autoencoder_layer_1_dropout)(y)\n",
    "    \n",
    "    y = Dense(units = autoencoder_layer_2_units, activation='relu')(y)\n",
    "    y = Dropout(autoencoder_layer_2_dropout)(y)\n",
    "    \n",
    "    y = Dense(units = x_train.shape[1], activation='relu')(y)\n",
    "\n",
    "    y = Dense(units = layers['layer_1'], activation='relu')(y)\n",
    "    y = Dropout(dropout * 0.5)(y)\n",
    "    \n",
    "    if (layers['layer_2'] > 0):\n",
    "        y = Dense(units = layers['layer_2'], activation='relu')(y)\n",
    "        y = Dropout(dropout * 0.75)(y)\n",
    "    \n",
    "    if (layers['layer_3'] > 0):\n",
    "        y = Dense(units = layers['layer_3'], activation='relu')(y)\n",
    "        y = Dropout(dropout)(y)\n",
    "    \n",
    "    if (layers['layer_4'] > 0):\n",
    "        y = Dense(units = layers['layer_4'], activation='relu')(y)\n",
    "        y = Dropout(dropout)(y)\n",
    "    \n",
    "    y = Dense(units = y_train.shape[1], activation='softmax')(y)\n",
    "    model = Model(x, y)\n",
    "    \n",
    "    # Create directory\n",
    "    directory = model_directory + date + '/'\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "    \n",
    "    # Write model hyper-parameters\n",
    "    df = pd.DataFrame({\n",
    "        'Layer 4': layers['layer_1'],\n",
    "        'Layer 5': layers['layer_2'],\n",
    "        'Layer 6': layers['layer_3'],\n",
    "        'Layer 8': layers['layer_4'],\n",
    "        'Dropout 4': dropout * 0.5,\n",
    "        'Dropout 5': dropout * 0.75,\n",
    "        'Dropout 6': dropout,\n",
    "        'Dropout 8': dropout,\n",
    "        'Learning Rate': learning_rate,\n",
    "        'Decay': decay,\n",
    "        'Batch size': batch_size,\n",
    "        'Autoencoder': autoencoder_path,\n",
    "        'Epochs': epochs,\n",
    "        'Fine tune': True\n",
    "    }, index=[0])\n",
    "    df.to_csv(model_directory + date + '/params.csv')\n",
    "    \n",
    "    # Write model summary\n",
    "    file2 = open(directory + \"summary.txt\", \"a\")\n",
    "    model.summary(print_fn=lambda line: file2.write(line + '\\n'))\n",
    "    file2.close()\n",
    "\n",
    "    # Write model diagram\n",
    "    plot_model(model, to_file=directory + 'model.png', show_shapes=True, show_layer_names=False)\n",
    "    \n",
    "    # Compile the model\n",
    "    model.compile(optimizer=keras.optimizers.Adam(lr=learning_rate, decay=decay), loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])\n",
    "    \n",
    "    # Set layer weights to pre trained autoencoder weights\n",
    "    model.layers[1].set_weights(weights.get_layer(autoencoder_layer_1_name).get_weights())\n",
    "    model.layers[3].set_weights(weights.get_layer(autoencoder_layer_2_name).get_weights())\n",
    "    model.layers[5].set_weights(weights.get_layer(autoencoder_layer_3_name).get_weights())\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class KerasClassifierTensorBoard(KerasClassifier):\n",
    "    def fit(self, x, y, **kwargs):\n",
    "        global date\n",
    "        date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "        tensorboard_callback = keras.callbacks.TensorBoard(log_dir=tensorboard_directory + date + '/')\n",
    "        csv_logger = keras.callbacks.CSVLogger(model_directory + date + '/epochs.csv')\n",
    "        model_checkpoint = keras.callbacks.ModelCheckpoint(model_directory + date + '/weights.hdf5', save_best_only=True)\n",
    "        callbacks = [tensorboard_callback, csv_logger, model_checkpoint]\n",
    "        return super(KerasClassifierTensorBoard, self).fit(x, y, callbacks=callbacks, verbose=0, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 1 folds for each of 24 candidates, totalling 24 fits\n",
      "[CV] dropout=0.3, layers={'layer_2': 16, 'layer_3': 8, 'layer_1': 32, 'layer_4': 0} \n",
      "880/966 [==========================>...] - ETA: 0s[CV]  dropout=0.3, layers={'layer_2': 16, 'layer_3': 8, 'layer_1': 32, 'layer_4': 0}, total=26.4min\n",
      "[CV] dropout=0.3, layers={'layer_2': 16, 'layer_3': 8, 'layer_1': 24, 'layer_4': 0} \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed: 26.4min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "900/966 [==========================>...] - ETA: 0s[CV]  dropout=0.3, layers={'layer_2': 16, 'layer_3': 8, 'layer_1': 24, 'layer_4': 0}, total=27.6min\n",
      "[CV] dropout=0.3, layers={'layer_2': 16, 'layer_3': 0, 'layer_1': 24, 'layer_4': 0} \n",
      "600/966 [=================>............] - ETA: 0s[CV]  dropout=0.3, layers={'layer_2': 16, 'layer_3': 0, 'layer_1': 24, 'layer_4': 0}, total=23.9min\n",
      "[CV] dropout=0.3, layers={'layer_2': 32, 'layer_3': 16, 'layer_1': 64, 'layer_4': 0} \n",
      "800/966 [=======================>......] - ETA: 0s[CV]  dropout=0.3, layers={'layer_2': 32, 'layer_3': 16, 'layer_1': 64, 'layer_4': 0}, total=27.6min\n",
      "[CV] dropout=0.3, layers={'layer_2': 32, 'layer_3': 16, 'layer_1': 64, 'layer_4': 8} \n",
      "840/966 [=========================>....] - ETA: 0s[CV]  dropout=0.3, layers={'layer_2': 32, 'layer_3': 16, 'layer_1': 64, 'layer_4': 8}, total=27.1min\n",
      "[CV] dropout=0.3, layers={'layer_2': 16, 'layer_3': 0, 'layer_1': 64, 'layer_4': 0} \n",
      "780/966 [=======================>......] - ETA: 0s[CV]  dropout=0.3, layers={'layer_2': 16, 'layer_3': 0, 'layer_1': 64, 'layer_4': 0}, total=23.2min\n",
      "[CV] dropout=0.3, layers={'layer_2': 8, 'layer_3': 4, 'layer_1': 16, 'layer_4': 0} \n",
      "860/966 [=========================>....] - ETA: 0s[CV]  dropout=0.3, layers={'layer_2': 8, 'layer_3': 4, 'layer_1': 16, 'layer_4': 0}, total=25.1min\n",
      "[CV] dropout=0.3, layers={'layer_2': 4, 'layer_3': 0, 'layer_1': 8, 'layer_4': 0} \n",
      "940/966 [============================>.] - ETA: 0s[CV]  dropout=0.3, layers={'layer_2': 4, 'layer_3': 0, 'layer_1': 8, 'layer_4': 0}, total=23.1min\n",
      "[CV] dropout=0.3, layers={'layer_2': 0, 'layer_3': 0, 'layer_1': 16, 'layer_4': 0} \n",
      "860/966 [=========================>....] - ETA: 0s[CV]  dropout=0.3, layers={'layer_2': 0, 'layer_3': 0, 'layer_1': 16, 'layer_4': 0}, total=22.0min\n",
      "[CV] dropout=0.3, layers={'layer_2': 12, 'layer_3': 16, 'layer_1': 8, 'layer_4': 0} \n",
      "840/966 [=========================>....] - ETA: 0s[CV]  dropout=0.3, layers={'layer_2': 12, 'layer_3': 16, 'layer_1': 8, 'layer_4': 0}, total=26.4min\n",
      "[CV] dropout=0.3, layers={'layer_2': 12, 'layer_3': 16, 'layer_1': 8, 'layer_4': 32} \n",
      "840/966 [=========================>....] - ETA: 0s[CV]  dropout=0.3, layers={'layer_2': 12, 'layer_3': 16, 'layer_1': 8, 'layer_4': 32}, total=27.9min\n",
      "[CV] dropout=0.3, layers={'layer_2': 8, 'layer_3': 16, 'layer_1': 16, 'layer_4': 0} \n",
      "820/966 [========================>.....] - ETA: 0s[CV]  dropout=0.3, layers={'layer_2': 8, 'layer_3': 16, 'layer_1': 16, 'layer_4': 0}, total=25.1min\n",
      "[CV] dropout=0.5, layers={'layer_2': 16, 'layer_3': 8, 'layer_1': 32, 'layer_4': 0} \n",
      "840/966 [=========================>....] - ETA: 0s[CV]  dropout=0.5, layers={'layer_2': 16, 'layer_3': 8, 'layer_1': 32, 'layer_4': 0}, total=25.9min\n",
      "[CV] dropout=0.5, layers={'layer_2': 16, 'layer_3': 8, 'layer_1': 24, 'layer_4': 0} \n",
      "840/966 [=========================>....] - ETA: 0s[CV]  dropout=0.5, layers={'layer_2': 16, 'layer_3': 8, 'layer_1': 24, 'layer_4': 0}, total=25.6min\n",
      "[CV] dropout=0.5, layers={'layer_2': 16, 'layer_3': 0, 'layer_1': 24, 'layer_4': 0} \n",
      "920/966 [===========================>..] - ETA: 0s[CV]  dropout=0.5, layers={'layer_2': 16, 'layer_3': 0, 'layer_1': 24, 'layer_4': 0}, total=23.9min\n",
      "[CV] dropout=0.5, layers={'layer_2': 32, 'layer_3': 16, 'layer_1': 64, 'layer_4': 0} \n",
      "800/966 [=======================>......] - ETA: 0s[CV]  dropout=0.5, layers={'layer_2': 32, 'layer_3': 16, 'layer_1': 64, 'layer_4': 0}, total=27.4min\n",
      "[CV] dropout=0.5, layers={'layer_2': 32, 'layer_3': 16, 'layer_1': 64, 'layer_4': 8} \n",
      "780/966 [=======================>......] - ETA: 0s[CV]  dropout=0.5, layers={'layer_2': 32, 'layer_3': 16, 'layer_1': 64, 'layer_4': 8}, total=29.4min\n",
      "[CV] dropout=0.5, layers={'layer_2': 16, 'layer_3': 0, 'layer_1': 64, 'layer_4': 0} \n",
      "860/966 [=========================>....] - ETA: 0s[CV]  dropout=0.5, layers={'layer_2': 16, 'layer_3': 0, 'layer_1': 64, 'layer_4': 0}, total=25.2min\n",
      "[CV] dropout=0.5, layers={'layer_2': 8, 'layer_3': 4, 'layer_1': 16, 'layer_4': 0} \n",
      "780/966 [=======================>......] - ETA: 0s[CV]  dropout=0.5, layers={'layer_2': 8, 'layer_3': 4, 'layer_1': 16, 'layer_4': 0}, total=26.6min\n",
      "[CV] dropout=0.5, layers={'layer_2': 4, 'layer_3': 0, 'layer_1': 8, 'layer_4': 0} \n",
      "966/966 [==============================] - 0s     \n",
      "960/966 [============================>.] - ETA: 0s[CV]  dropout=0.5, layers={'layer_2': 4, 'layer_3': 0, 'layer_1': 8, 'layer_4': 0}, total=25.4min\n",
      "[CV] dropout=0.5, layers={'layer_2': 0, 'layer_3': 0, 'layer_1': 16, 'layer_4': 0} \n",
      "820/966 [========================>.....] - ETA: 0s[CV]  dropout=0.5, layers={'layer_2': 0, 'layer_3': 0, 'layer_1': 16, 'layer_4': 0}, total=23.3min\n",
      "[CV] dropout=0.5, layers={'layer_2': 12, 'layer_3': 16, 'layer_1': 8, 'layer_4': 0} \n",
      "740/966 [=====================>........] - ETA: 0s[CV]  dropout=0.5, layers={'layer_2': 12, 'layer_3': 16, 'layer_1': 8, 'layer_4': 0}, total=27.3min\n",
      "[CV] dropout=0.5, layers={'layer_2': 12, 'layer_3': 16, 'layer_1': 8, 'layer_4': 32} \n",
      "780/966 [=======================>......] - ETA: 0s[CV]  dropout=0.5, layers={'layer_2': 12, 'layer_3': 16, 'layer_1': 8, 'layer_4': 32}, total=29.4min\n",
      "[CV] dropout=0.5, layers={'layer_2': 8, 'layer_3': 16, 'layer_1': 16, 'layer_4': 0} \n",
      "740/966 [=====================>........] - ETA: 0s[CV]  dropout=0.5, layers={'layer_2': 8, 'layer_3': 16, 'layer_1': 16, 'layer_4': 0}, total=29.2min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed: 624.0min finished\n"
     ]
    }
   ],
   "source": [
    "model = KerasClassifierTensorBoard(build_fn=create_model, epochs=epochs, batch_size=batch_size)\n",
    "\n",
    "layers = [\n",
    "    {'layer_1': 32, 'layer_2': 16, 'layer_3': 8, 'layer_4': 0},\n",
    "    {'layer_1': 24, 'layer_2': 16, 'layer_3': 8, 'layer_4': 0},\n",
    "    {'layer_1': 24, 'layer_2': 16, 'layer_3': 0, 'layer_4': 0},\n",
    "    {'layer_1': 64, 'layer_2': 32, 'layer_3': 16, 'layer_4': 0},\n",
    "    {'layer_1': 64, 'layer_2': 32, 'layer_3': 16, 'layer_4': 8},\n",
    "    {'layer_1': 64, 'layer_2': 16, 'layer_3': 0, 'layer_4': 0},\n",
    "    {'layer_1': 16, 'layer_2': 8, 'layer_3': 4, 'layer_4': 0},\n",
    "    {'layer_1': 8, 'layer_2': 4, 'layer_3': 0, 'layer_4': 0},\n",
    "    {'layer_1': 16, 'layer_2': 0, 'layer_3': 0, 'layer_4': 0},\n",
    "    {'layer_1': 8, 'layer_2': 12, 'layer_3': 16, 'layer_4': 0},\n",
    "    {'layer_1': 8, 'layer_2': 12, 'layer_3': 16, 'layer_4': 32},\n",
    "    {'layer_1': 16, 'layer_2': 8, 'layer_3': 16, 'layer_4': 0}\n",
    "]\n",
    "param_grid = dict(\n",
    "    layers=layers,\n",
    "    dropout=[0.3, 0.5]\n",
    ")\n",
    "    \n",
    "grid = GridSearchCV(estimator = model, param_grid = param_grid, error_score=0, verbose=2, cv=[(slice(None), slice(None))], n_jobs=1, fit_params=dict(validation_data=(x_test, y_test)))\n",
    "grid_result = grid.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}