FarewellEmailWriter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Farewell Email Writer in Gluon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import os\n",
    "\n",
    "import glob\n",
    "import time\n",
    "import math\n",
    "import itertools\n",
    "import random\n",
    "import nltk\n",
    "\n",
    "import mxnet as mx\n",
    "from mxnet import gluon, autograd\n",
    "from mxnet.gluon.utils import download\n",
    "\n",
    "import gluonnlp as nlp\n",
    "from mxnet.gluon import nn, Block\n",
    "\n",
    "PATH = 'C:\\\\Users\\\\pochetti\\\\WorkDocs\\\\Desktop\\\\Fra\\\\Francesco\\\\Farewell'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class LMDecoder(object):\n",
    "    def __init__(self, model): self._model = model\n",
    "    def __call__(self, inputs, states):\n",
    "        outputs, states = self._model(mx.nd.expand_dims(inputs, axis=0), states)\n",
    "        return outputs[0], states\n",
    "    def state_info(self, *arg, **kwargs): return self._model.state_info(*arg, **kwargs)\n",
    "\n",
    "def detach(hidden):\n",
    "    if isinstance(hidden, (tuple, list)):\n",
    "        hidden = [detach(i) for i in hidden]\n",
    "    else:\n",
    "        hidden = hidden.detach()\n",
    "    return hidden\n",
    "\n",
    "def train(model, train_data, epochs, lr):\n",
    "\n",
    "    start_train_time = time.time()\n",
    "    parameters = model.collect_params().values()\n",
    "    for epoch in range(epochs):\n",
    "        total_L = 0.0\n",
    "        start_epoch_time = time.time()\n",
    "        start_log_interval_time = time.time()\n",
    "        hiddens = [model.begin_state(batch_size//len(context), func=mx.nd.zeros, ctx=ctx)\n",
    "                   for ctx in context]\n",
    "        for i, (data, target) in enumerate(train_data):\n",
    "            data_list = gluon.utils.split_and_load(data, context,\n",
    "                                                   batch_axis=1, even_split=True)\n",
    "            target_list = gluon.utils.split_and_load(target, context,\n",
    "                                                     batch_axis=1, even_split=True)\n",
    "            hiddens = detach(hiddens)\n",
    "            L = 0\n",
    "            Ls = []\n",
    "            with autograd.record():\n",
    "                for j, (X, y, h) in enumerate(zip(data_list, target_list, hiddens)):\n",
    "                    output, h = model(X, h)\n",
    "                    batch_L = loss(output.reshape(-3, -1), y.reshape(-1,))\n",
    "                    L = L + batch_L.as_in_context(context[0]) / X.size\n",
    "                    Ls.append(batch_L / X.size)\n",
    "                    hiddens[j] = h\n",
    "            L.backward()\n",
    "            grads = [p.grad(x.context) for p in parameters for x in data_list]\n",
    "            gluon.utils.clip_global_norm(grads, grad_clip)\n",
    "\n",
    "            trainer.step(1)\n",
    "\n",
    "            total_L += sum([mx.nd.sum(l).asscalar() for l in Ls])\n",
    "\n",
    "        mx.nd.waitall()\n",
    "\n",
    "        print('[Epoch %d] loss %.2f '%(epoch, total_L/(i+1)))\n",
    "\n",
    "        lr = lr*0.8\n",
    "        trainer.set_learning_rate(lr)\n",
    "\n",
    "    print('Total training throughput %.2f samples/s'%(\n",
    "                            (batch_size * len(train_data) * epochs) /\n",
    "                            (time.time() - start_train_time)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Excerpt of the corpus I have used to fine tune the pre-trained AWD-LSTM model (~40 emails)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hi guys, i would like to thank you all for this amazing experience and for your help. it has been a pleasure to work with all of you. i wish you all the best for your career as well as for your personal life. let's keep in touch. <eos> hi all,   well, it’s my last day at amazon.  i’ve seen many of these “farewell” notes in my 8.5 year tenure at amazon, so am keenly aware that saying things like “i have truly enjoyed working with all of you” and “you are some of the smartest people i will likely ever work with” are at best overused, and at worst, down right trite.  but i don’t care – i’m going to say them anyway, because they are true.  though i’m moving on to a new challenge, in my years at amazon and across teams i have met some of the kindest, most fun, and smartest people i will likely ever have the pleasure of working with.  you have challenged me, amazed me, encouraged me, and shown me what hard work from smart people can accomplish on a daily basis.   i wish you all the best, as \n"
     ]
    }
   ],
   "source": [
    "f = open(f'{PATH}\\\\farewell_corpus.txt', encoding=\"utf8\")\n",
    "lines = f.readlines()\n",
    "print(lines[0][:1000])\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining key parameters for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "context = [mx.cpu()]\n",
    "batch_size = 20\n",
    "lr = 0.1\n",
    "epochs = 15\n",
    "bptt = 35\n",
    "grad_clip = 0.25"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading the emails dataset and preparing it for Gluon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Vocab(size=1031, unk=\"<unk>\", reserved=\"['<eos>']\")"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "moses_tokenizer = nlp.data.SacreMosesTokenizer()\n",
    "\n",
    "farewell_train = nlp.data.CorpusDataset(\n",
    "    f'{PATH}\\\\farewell_corpus.txt',\n",
    "    sample_splitter=nltk.tokenize.sent_tokenize,\n",
    "    tokenizer=moses_tokenizer,\n",
    "    flatten=True,\n",
    "    eos='<eos>')\n",
    "\n",
    "vocab = nlp.Vocab(nlp.data.Counter(farewell_train), padding_token=None, bos_token=None)\n",
    "\n",
    "bptt_batchify = nlp.data.batchify.CorpusBPTTBatchify(vocab, bptt, batch_size, last_batch='discard')\n",
    "\n",
    "farewell_train_data = bptt_batchify(farewell_train)\n",
    "\n",
    "vocab"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading the pre-trained AWD-LSTM language model from the GluonNLP zoo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AWDRNN(\n",
      "  (embedding): HybridSequential(\n",
      "    (0): Embedding(33278 -> 400, float32)\n",
      "    (1): Dropout(p = 0.65, axes=(0,))\n",
      "  )\n",
      "  (encoder): Sequential(\n",
      "    (0): LSTM(400 -> 1150, TNC)\n",
      "    (1): LSTM(1150 -> 1150, TNC)\n",
      "    (2): LSTM(1150 -> 400, TNC)\n",
      "  )\n",
      "  (decoder): HybridSequential(\n",
      "    (0): Dense(400 -> 33278, linear)\n",
      "  )\n",
      ")\n",
      "Vocab(size=33278, unk=\"<unk>\", reserved=\"['<eos>']\")\n"
     ]
    }
   ],
   "source": [
    "dataset_name = 'wikitext-2'\n",
    "awd_model_name = 'awd_lstm_lm_1150'\n",
    "awd_model, voc = nlp.model.get_model(\n",
    "    awd_model_name,\n",
    "    vocab=vocab,\n",
    "    dataset_name=dataset_name,\n",
    "    pretrained=True)\n",
    "print(awd_model)\n",
    "print(voc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Editing the last Dense layer to reflect the new (smaller) vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AWDRNN(\n",
       "  (embedding): HybridSequential(\n",
       "    (0): Embedding(33278 -> 400, float32)\n",
       "    (1): Dropout(p = 0.65, axes=(0,))\n",
       "  )\n",
       "  (encoder): Sequential(\n",
       "    (0): LSTM(400 -> 1150, TNC)\n",
       "    (1): LSTM(1150 -> 1150, TNC)\n",
       "    (2): LSTM(1150 -> 400, TNC)\n",
       "  )\n",
       "  (decoder): HybridSequential(\n",
       "    (0): Dense(None -> 1031, linear)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_decoder = mx.gluon.nn.HybridSequential()\n",
    "new_decoder.add(mx.gluon.nn.Dense(units=1031, flatten=False))\n",
    "new_decoder.initialize()\n",
    "awd_model.decoder = new_decoder\n",
    "\n",
    "awd_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Printing model params to figure out which ones to optimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "awdrnn0_ (\n",
       "  WeightDropParameter awdrnn0_hybridsequential0_embedding0_weight (shape=(33278, 400), dtype=float32, rate=0.1, mode=training)\n",
       "  Parameter awdrnn0_sequential0_lstm0_l0_i2h_weight (shape=(4600, 400), dtype=<class 'numpy.float32'>)\n",
       "  WeightDropParameter awdrnn0_sequential0_lstm0_l0_h2h_weight (shape=(4600, 1150), dtype=<class 'numpy.float32'>, rate=0.5, mode=training)\n",
       "  Parameter awdrnn0_sequential0_lstm0_l0_i2h_bias (shape=(4600,), dtype=<class 'numpy.float32'>)\n",
       "  Parameter awdrnn0_sequential0_lstm0_l0_h2h_bias (shape=(4600,), dtype=<class 'numpy.float32'>)\n",
       "  Parameter awdrnn0_sequential0_lstm1_l0_i2h_weight (shape=(4600, 1150), dtype=<class 'numpy.float32'>)\n",
       "  WeightDropParameter awdrnn0_sequential0_lstm1_l0_h2h_weight (shape=(4600, 1150), dtype=<class 'numpy.float32'>, rate=0.5, mode=training)\n",
       "  Parameter awdrnn0_sequential0_lstm1_l0_i2h_bias (shape=(4600,), dtype=<class 'numpy.float32'>)\n",
       "  Parameter awdrnn0_sequential0_lstm1_l0_h2h_bias (shape=(4600,), dtype=<class 'numpy.float32'>)\n",
       "  Parameter awdrnn0_sequential0_lstm2_l0_i2h_weight (shape=(1600, 1150), dtype=<class 'numpy.float32'>)\n",
       "  WeightDropParameter awdrnn0_sequential0_lstm2_l0_h2h_weight (shape=(1600, 400), dtype=<class 'numpy.float32'>, rate=0.5, mode=training)\n",
       "  Parameter awdrnn0_sequential0_lstm2_l0_i2h_bias (shape=(1600,), dtype=<class 'numpy.float32'>)\n",
       "  Parameter awdrnn0_sequential0_lstm2_l0_h2h_bias (shape=(1600,), dtype=<class 'numpy.float32'>)\n",
       "  Parameter dense0_weight (shape=(1031, 0), dtype=float32)\n",
       "  Parameter dense0_bias (shape=(1031,), dtype=float32)\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "awd_model.collect_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining a Gluon trainer and passing the last dense layer (i.e. DO NOT OPTIMIZE the rest of the network)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trainer = gluon.Trainer([awd_model.collect_params()['dense0_bias'],\n",
    "                        awd_model.collect_params()['dense0_weight']], 'adam', {\n",
    "    'learning_rate': lr,\n",
    "    'wd': 0\n",
    "})\n",
    "loss = gluon.loss.SoftmaxCrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fine tuning the AWD-LSTM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch 0] loss 6.26 \n",
      "[Epoch 1] loss 5.00 \n",
      "[Epoch 2] loss 4.03 \n",
      "[Epoch 3] loss 3.49 \n",
      "[Epoch 4] loss 3.18 \n",
      "[Epoch 5] loss 2.98 \n",
      "[Epoch 6] loss 2.93 \n",
      "[Epoch 7] loss 2.85 \n",
      "[Epoch 8] loss 2.87 \n",
      "[Epoch 9] loss 2.83 \n",
      "[Epoch 10] loss 2.67 \n",
      "[Epoch 11] loss 2.63 \n",
      "[Epoch 12] loss 2.67 \n",
      "[Epoch 13] loss 2.54 \n",
      "[Epoch 14] loss 2.54 \n",
      "Total training throughput 5.06 samples/s\n"
     ]
    }
   ],
   "source": [
    "train(awd_model, farewell_train_data, epochs, lr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining the sequence sampler to perform Beam Search and generate samples of sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = awd_model\n",
    "\n",
    "decoder = LMDecoder(model)\n",
    "\n",
    "sampler = nlp.model.SequenceSampler(beam_size=15,\n",
    "                                    decoder=decoder,\n",
    "                                    eos_id=vocab['<eos>'],\n",
    "                                    max_length=500,\n",
    "                                    temperature=0.97)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating a few email samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation Result:\n",
      "['hi , please . <eos>', -8.115891]\n",
      "['hi , since marketplaces yes trying enjoyed unchanged be of you know <eos>', -42.86605]\n",
      "['hi , move so i would much doing ’ ve like going to day so been a chance for a great personal horizons , and ’ on facebook last day you all for is and my but . <eos>', -128.69484]\n",
      "['hi , because from wish to stay an opportunity for these wonderful 1,280 that together , career 11 last day and innovate and in luxembourg t of you for , is up on a monday morning morning looking our paths cross again . <eos>', -127.031044]\n",
      "['hi , first today allowed day day at you working with lux and bbqs me around accomplish with end i start a &lt; cup games my experience here <eos>', -103.4374]\n"
     ]
    }
   ],
   "source": [
    "ctx=context[0]\n",
    "\n",
    "bos = 'hi , '.split()\n",
    "bos_ids = [vocab[ele] for ele in bos]\n",
    "begin_states = model.begin_state(batch_size=1, ctx=ctx)\n",
    "if len(bos_ids) > 1:\n",
    "    _, begin_states = model(mx.nd.expand_dims(mx.nd.array(bos_ids[:-1]), axis=1), begin_states)\n",
    "inputs = mx.nd.full(shape=(1,), ctx=ctx, val=bos_ids[-1])\n",
    "samples, scores, valid_lengths = sampler(inputs, begin_states)\n",
    "samples = samples[0].asnumpy()\n",
    "scores = scores[0].asnumpy()\n",
    "valid_lengths = valid_lengths[0].asnumpy()\n",
    "sentence = bos[:-1] + [vocab.idx_to_token[ele] for ele in samples[0][:valid_lengths[0]]]\n",
    "print('Generation Result:')\n",
    "for i in range(5):\n",
    "    sentence = bos[:-1] + [vocab.idx_to_token[ele] for ele in samples[i][:valid_lengths[i]]]\n",
    "    print([' '.join(sentence), scores[i]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}