Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import pickle\n",
    "import re\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib import rnn\n",
    "import tensorflow.contrib.slim as slim\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "time_window = 10\n",
    "max_len_review = 288\n",
    "char_mtx_row = 1000\n",
    "_len_alphabet = 70\n",
    "_meta_dim = 1038\n",
    "item_dim = 70000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "char_mtx = tf.placeholder(dtype=tf.float32, shape=[None, max_len_review, char_mtx_row, _len_alphabet])\n",
    "img_mtx = tf.placeholder(dtype=tf.float32, shape=[None, 32,32,3]) # input image shape: 32x32\n",
    "meta = tf.placeholder(dtype=tf.float32, shape=[None, _meta_dim])\n",
    "y = tf.placeholder(dtype=tf.float32, shape=[None, item_dim])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def review_CNN(char_mtx, filter_sizes = [2, 3, 7, 11], filter_nums = [100, 100, 100, 100], stride=[1,1,1,1],\n",
    "               mlp_units=[200,50], reuse=False, is_training=True):\n",
    "    ## CNN for review aspect, sentiment extraction\n",
    "    with tf.variable_scope('review-charCNN', reuse=reuse):\n",
    "        input_ = tf.reshape(char_mtx, shape=[-1, char_mtx_row, _len_alphabet])\n",
    "        input_ = tf.expand_dims(input_, axis=3)\n",
    "        with tf.name_scope('conv-filter-1'):\n",
    "            filter_shape1 = [filter_sizes[0], _len_alphabet, 1, filter_nums[0]]\n",
    "            W1 = tf.Variable(tf.truncated_normal(shape=filter_shape1), name='filter-1')\n",
    "            b1 = tf.Variable(tf.random_uniform(shape=[filter_nums[0]]), name='bias-1')\n",
    "            conv1 = tf.nn.conv2d(input_, W1, strides=stride, padding='VALID', name='conv-1')\n",
    "            out1 = tf.nn.relu(tf.nn.bias_add(conv1, b1))\n",
    "            max1 = tf.reduce_max(out1, axis=1) # batch_size, 1, 100\n",
    "        with tf.name_scope('conv-filter-2'):\n",
    "            filter_shape2= [filter_sizes[1], _len_alphabet, 1, filter_nums[1]]\n",
    "            W2 = tf.Variable(tf.truncated_normal(shape=filter_shape2), name='filter-2')\n",
    "            b2 = tf.Variable(tf.random_uniform(shape=[filter_nums[1]]), name='bias-2')\n",
    "            conv2 = tf.nn.conv2d(input_, W2, strides=stride, padding='VALID', name='conv-2')\n",
    "            out2 = tf.nn.relu(tf.nn.bias_add(conv2, b2))\n",
    "            max2 = tf.reduce_max(out2, axis=1) # batch_size, 1, 100\n",
    "        with tf.name_scope('conv-filter-3'):\n",
    "            filter_shape3 = [filter_sizes[2], _len_alphabet, 1, filter_nums[2]]\n",
    "            W3 = tf.Variable(tf.truncated_normal(shape=filter_shape3), name='filter-3')\n",
    "            b3 = tf.Variable(tf.random_uniform(shape=[filter_nums[2]]), name='bias-3')\n",
    "            conv3 = tf.nn.conv2d(input_, W3, strides=stride, padding='VALID', name='conv-3')\n",
    "            out3 = tf.nn.relu(tf.nn.bias_add(conv3, b3))\n",
    "            max3 = tf.reduce_max(out3, axis=1) # batch_size, 1, 100\n",
    "        with tf.name_scope('conv-filter-4'):\n",
    "            filter_shape4= [filter_sizes[3], _len_alphabet, 1, filter_nums[3]]\n",
    "            W4 = tf.Variable(tf.truncated_normal(shape=filter_shape4), name='filter-4')\n",
    "            b4 = tf.Variable(tf.random_uniform(shape=[filter_nums[3]]), name='bias-4')\n",
    "            conv4 = tf.nn.conv2d(input_, W4, strides=stride, padding='VALID', name='conv-4')\n",
    "            out4 = tf.nn.relu(tf.nn.bias_add(conv4, b4))\n",
    "            max4 = tf.reduce_max(out4, axis=1) # batch_size, 1, 100\n",
    "        ## concat\n",
    "        max_concat = tf.squeeze(tf.concat([max1, max2, max3, max4], axis=2), axis=1) # batch_size, 400\n",
    "    ## MLP for feature reduction\n",
    "    with tf.variable_scope('review-MLP', reuse=reuse):\n",
    "        fc1 = slim.fully_connected(max_concat, mlp_units[0])\n",
    "        fc2 = slim.fully_connected(fc1, mlp_units[1])\n",
    "    result = tf.reshape(fc2, shape=[-1, max_len_review, mlp_units[1]])\n",
    "    return tf.reduce_sum(result, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def image_CNN(img_mtx, mlp_units=[200,50], reuse=False, is_training=True):\n",
    "    with tf.variable_scope('image-CNN', reuse=reuse):\n",
    "        with slim.arg_scope([slim.conv2d], padding='SAME', activation_fn=None, \n",
    "                    stride = 2, weights_initializer=tf.contrib.layers.xavier_initializer()):\n",
    "            with slim.arg_scope([slim.batch_norm], decay=0.95, center=True, scale=True,\n",
    "                            updates_collections = None, activation_fn=tf.nn.relu,\n",
    "                            is_training =is_training):\n",
    "                conv1 = slim.conv2d(img_mtx, 6, [3,3], scope='conv-1')\n",
    "                bn1 = slim.batch_norm(conv1, scope='bn-1')\n",
    "                conv2 = slim.conv2d(bn1, 12, [3,3], scope='conv-2')\n",
    "                bn2 = slim.batch_norm(conv2, scope='bn-2')\n",
    "        with tf.variable_scope('MLP', reuse=reuse):\n",
    "            fc1 = slim.fully_connected(slim.flatten(bn2), mlp_units[0])\n",
    "            fc2 = slim.fully_connected(fc1, mlp_units[1])\n",
    "    return fc2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def userLSTM(lstm_input, hidden=128, reuse=False):\n",
    "    cell = rnn.BasicLSTMCell(hidden, reuse=reuse)\n",
    "    input_ = tf.unstack(lstm_input, axis=1)    \n",
    "    outputs, _ = rnn.static_rnn(cell, input_, dtype=tf.float32, scope='juungLSTM')\n",
    "    return outputs[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fc_pred(lstm_result, reuse=False):\n",
    "    fc1 = slim.fully_connected(lstm_result, 1024, scope='fc-1')\n",
    "    fc2 = slim.fully_connected(fc1, item_dim, scope='fc-2')\n",
    "    return fc2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def juung(char_mtx, img_mtx, meta, hidden=128, reuse=False):\n",
    "    review_result = review_CNN(char_mtx, reuse=False)\n",
    "    img_result = image_CNN(img_mtx)\n",
    "    concat_result = tf.concat([review_result, img_result, meta], axis=1)\n",
    "    lstm_input = tf.reshape(concat_result, shape=[-1, time_window, 1138]) # reshape 확인하기\n",
    "    lstm_result = userLSTM(lstm_input, hidden=hidden)\n",
    "    pred = fc_pred(lstm_result)\n",
    "    return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred = juung(char_mtx, img_mtx, meta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_idx = tf.cast(tf.argmax(y, axis=1), dtype=tf.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "top_k = tf.reduce_mean(tf.cast(tf.nn.in_top_k(pred, y_idx, k=10, name='top-k'), dtype=tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}