BasicBidirectionalRNN-MeanPooling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import urllib\n",
    "import tarfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "url = 'https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz'\n",
    "\n",
    "def getunzipped(theurl):\n",
    "    name = './dbpedia_csv.tar.gz'\n",
    "    try:\n",
    "        name, hdrs = urllib.request.urlretrieve(theurl, name)\n",
    "    except IOError as e:\n",
    "        print(\"Can't retrieve %r to %r: %s\" % (theurl, thedir, e))\n",
    "        return\n",
    "    try:\n",
    "        z = tarfile.open(name, \"r:gz\")\n",
    "        z.extractall()\n",
    "        z.close()\n",
    "    except tarfile.error as e:\n",
    "        print(\"Bad zipfile (from %r): %s\" % (theurl, e))\n",
    "        return\n",
    "\n",
    "    print(\"Data Downloaded and unzipped!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "getunzipped(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>E. D. Abbott Ltd</td>\n",
       "      <td>Abbott of Farnham E D Abbott Limited was a Br...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Schwan-Stabilo</td>\n",
       "      <td>Schwan-STABILO is a German maker of pens for ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Q-workshop</td>\n",
       "      <td>Q-workshop is a Polish company located in Poz...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Marvell Software Solutions Israel</td>\n",
       "      <td>Marvell Software Solutions Israel known as RA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Bergan Mercy Medical Center</td>\n",
       "      <td>Bergan Mercy Medical Center is a hospital loc...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   class                              title  \\\n",
       "0      1                   E. D. Abbott Ltd   \n",
       "1      1                     Schwan-Stabilo   \n",
       "2      1                         Q-workshop   \n",
       "3      1  Marvell Software Solutions Israel   \n",
       "4      1        Bergan Mercy Medical Center   \n",
       "\n",
       "                                                text  \n",
       "0   Abbott of Farnham E D Abbott Limited was a Br...  \n",
       "1   Schwan-STABILO is a German maker of pens for ...  \n",
       "2   Q-workshop is a Polish company located in Poz...  \n",
       "3   Marvell Software Solutions Israel known as RA...  \n",
       "4   Bergan Mercy Medical Center is a hospital loc...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('dbpedia_csv/train.csv',header=None,names=['class','title','text'])\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>TY KU</td>\n",
       "      <td>TY KU /taɪkuː/ is an American alcoholic bever...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Odd Lot Entertainment</td>\n",
       "      <td>OddLot Entertainment founded in 2001 by longt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Henkel</td>\n",
       "      <td>Henkel AG &amp; Company KGaA operates worldwide w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>GOAT Store</td>\n",
       "      <td>The GOAT Store (Games Of All Type Store) LLC ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>RagWing Aircraft Designs</td>\n",
       "      <td>RagWing Aircraft Designs (also called the Rag...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   class                     title  \\\n",
       "0      1                     TY KU   \n",
       "1      1     Odd Lot Entertainment   \n",
       "2      1                    Henkel   \n",
       "3      1                GOAT Store   \n",
       "4      1  RagWing Aircraft Designs   \n",
       "\n",
       "                                                text  \n",
       "0   TY KU /taɪkuː/ is an American alcoholic bever...  \n",
       "1   OddLot Entertainment founded in 2001 by longt...  \n",
       "2   Henkel AG & Company KGaA operates worldwide w...  \n",
       "3   The GOAT Store (Games Of All Type Store) LLC ...  \n",
       "4   RagWing Aircraft Designs (also called the Rag...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv('dbpedia_csv/test.csv',header=None,names=['class','title','text'])\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train = train['text'].get_values()\n",
    "X_test = test['text'].get_values()\n",
    "y_train = train['class'].get_values()\n",
    "y_test= test['class'].get_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "max_doc_length = 50\n",
    "\n",
    "train_lengths = train['text'].apply(lambda x: min(max_doc_length, len(x.split(' '))))\n",
    "test_lengths = test['text'].apply(lambda x: min(max_doc_length, len(x.split(' '))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_lengths = train_lengths.values\n",
    "test_lengths = test_lengths.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VocabularyProcessor \n",
    "Has an sklearn type interface - instantiate, call .fit() and .transform()\n",
    "\n",
    "Creates a mapping between words and ID's.\n",
    "\n",
    "Fit creates the mapping word-> ID, and transform replaces the words with relevant IDS.\n",
    "\n",
    "We can then use tf.nn.embedding_lookup(...) to look up the vector representations that correspond to each ID."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "preprocessor = tf.contrib.learn.preprocessing.VocabularyProcessor(max_doc_length,min_frequency=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train_transformed = np.array(list(preprocessor.fit_transform(X_train)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see, we have replaced each word with a numerical ID corresponding to a word in our vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The word 'the' corresponds to ID: 1\n",
      "ID 25 corresponds to: known\n"
     ]
    }
   ],
   "source": [
    "print(\"The word 'the' corresponds to ID:\",preprocessor.vocabulary_.get('the')) #Get the ID for a word\n",
    "print(\"ID 25 corresponds to:\",preprocessor.vocabulary_.reverse(25)) #Get the word that corresponds to an ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_length = len(preprocessor.vocabulary_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_test_transformed = np.array(list(preprocessor.transform(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[     0,  33095,      0,      4,     15,     34,  13638,  12026,\n",
       "            58,     32,   2092,      2,  34220,      6,    154,  19432,\n",
       "             8,      0,     58,      7,    108,      2,    254,      6,\n",
       "             4,    672,      2,     46,    111,    133,     46,    111,\n",
       "          1789,     69,      2,     46,    111,      0,  33095,   1012,\n",
       "         12873,     40,    218,      2,    342,    137,      5,   2554,\n",
       "          2603,     17],\n",
       "       [  7874,  13438,   1130,    108,      2,    346,      9,   5924,\n",
       "          3640,  26563,  53628,      6,   9446,   4389, 115134,      8,\n",
       "         10402, 102537,      4,      5,     22,    454,      6,  11213,\n",
       "            58,     69,      2,  14655,    133,    144,   7874,  13438,\n",
       "           138,      1,     22,    579,      3,   9822,   1198,  38006,\n",
       "         13657,     82,  27928,   2842,     56,     22,    579,      3,\n",
       "           155,     82]], dtype=int64)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_transformed[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Abbott of Farnham E D Abbott Limited was a British coachbuilding business based in Farnham Surrey trading under that name from 1929 A major part of their output was under sub-contract to motor vehicle manufacturers Their business closed in 1972 <UNK> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK>'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(preprocessor.reverse(X_train_transformed[0,None].tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now our data is prepared we can start modelling..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The basics have been implemented, but you need to implement the RNN functionality, and define the loss function.\n",
    "\n",
    "As a starting point, you several comments have been provided in the relevant name scopes."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This architecture is a simple extension of the normal bidirectional RNN."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "embedding_size=100\n",
    "rnn_cell_size = 100\n",
    "n_classes=15\n",
    "learning_rate = 0.005\n",
    "\n",
    "with graph.as_default():#, tf.device('/cpu:0'):\n",
    "    with tf.name_scope('placeholders'):\n",
    "        tf_X = tf.placeholder(shape=[None, max_doc_length], dtype=tf.int32) #This is a matrix of word IDs\n",
    "        tf_y = tf.placeholder(shape=[None], dtype=tf.int64) #This is a list of class labels e.g. 1,5,3 etc.\n",
    "        tf_lengths = tf.placeholder(shape=[None], dtype=tf.int32) #This is the lengths \n",
    "                                                                    #of each document in sample\n",
    "        \n",
    "    with tf.name_scope('variables'):\n",
    "        embedding_matrix = tf.Variable(tf.random_uniform([vocab_length,embedding_size],-1.0,1.0)) \n",
    "        \n",
    "        W = tf.Variable(tf.truncated_normal(shape=[rnn_cell_size*2, n_classes ], dtype=tf.float32))\n",
    "        b = tf.Variable(tf.random_uniform([n_classes], -1.0,1.0))\n",
    "        tf_y_one_hot = tf.one_hot(tf_y,n_classes, dtype=tf.int64)\n",
    "    \n",
    "    with tf.name_scope('embbeding_lookup'):\n",
    "        #You need to look up the word ID and get replace with the associated word vector.\n",
    "        #Verify the size is (?, max_doc_length, embedding_size)\n",
    "        tf_X_embedded = tf.nn.embedding_lookup(embedding_matrix, tf_X) \n",
    "        \n",
    "    with tf.name_scope('run_rnn'):\n",
    "        #You need to define a cell type - use BasicRNNCell\n",
    "        #Also need to feed data into the RNN and get the final_state - use bidirectional_dynamic_rnn\n",
    "        #Concatenate the outputs on the last dimension and run a reduce_mean on the sequence dimension (axis 1)\n",
    "        pass\n",
    "        \n",
    "        \n",
    "    with tf.name_scope('output_layer'):\n",
    "        #Create a variable logits which is final_state * W + b\n",
    "        #Pass this through a softmax to create prediction probabilities.\n",
    "        pass\n",
    "        \n",
    "    with tf.name_scope('loss'):\n",
    "        #Add a loss and an optimizer\n",
    "        pass\n",
    "        \n",
    "    with tf.name_scope('validation'):\n",
    "        correct_prediction = tf.equal(tf_y, predictions)\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        \n",
    "    with tf.name_scope('init'):\n",
    "        init_op = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|                                                                                          | 0/251 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 0.0604450702667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.156328\n",
      "---------------------------------------- step 0 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|████████████████▏                                                                | 50/251 [00:33<01:25,  2.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 1.09996471167\n",
      "accuracy: 0.820078\n",
      "---------------------------------------- step 50 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███████████████████████████████▊                                                | 100/251 [01:08<01:05,  2.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 0.458211859167\n",
      "accuracy: 0.88875\n",
      "---------------------------------------- step 100 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|███████████████████████████████████████████████▊                                | 150/251 [01:41<00:45,  2.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 0.332322835922\n",
      "accuracy: 0.915\n",
      "---------------------------------------- step 150 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████████████████████████████████████████████████████████████▋                | 200/251 [02:16<00:22,  2.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 0.272184424698\n",
      "accuracy: 0.929102\n",
      "---------------------------------------- step 200 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████▋| 250/251 [02:49<00:00,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average loss: 0.227578248978\n",
      "accuracy: 0.942695\n",
      "---------------------------------------- step 250 ----------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 251/251 [03:01<00:00,  4.09s/it]\n"
     ]
    }
   ],
   "source": [
    "batch_size = 128\n",
    "print_step = 50\n",
    "from tqdm import tqdm\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    number_of_steps = 251\n",
    "    train_indicies = np.arange(X_train_transformed.shape[0])\n",
    "    test_indicies = np.arange(X_test_transformed.shape[0])\n",
    "    \n",
    "    session.run(init_op)\n",
    "    average_loss = 0\n",
    "    \n",
    "    for step in tqdm(range(number_of_steps)):\n",
    "        ind =  np.random.choice(np.arange(X_train_transformed.shape[0]), batch_size)\n",
    "        feed_dict = {tf_X:X_train_transformed[ind], \n",
    "                     tf_y:y_train[ind], \n",
    "                     tf_lengths: train_lengths[ind] }\n",
    "        l,_ = session.run([loss,optimizer ], feed_dict=feed_dict)\n",
    "        average_loss+=l\n",
    "        \n",
    "        if step%print_step == 0:\n",
    "            print (\"average loss:\",average_loss/print_step)\n",
    "            ind2 =  np.random.choice(np.arange(X_test_transformed.shape[0]), batch_size*200)\n",
    "            test_dict = {tf_X:X_test_transformed[ind2], \n",
    "                     tf_y:y_test[ind2], \n",
    "                     tf_lengths: test_lengths[ind2] }\n",
    "            print(\"accuracy:\",accuracy.eval(feed_dict=test_dict))\n",
    "            #print(predictions.eval(feed_dict=test_dict))\n",
    "            #print(y_test[ind2])\n",
    "            print('-'*40, \"step \"+str(step), '-'*40)\n",
    "            \n",
    "            average_loss=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:p3]",
   "language": "python",
   "name": "conda-env-p3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}