notebooks/3_Score_consistency_test_set_baseline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## In this notebook, you will learn how to score translations with CADec\n",
    "\n",
    "You can use this code to score consistency test sets with your trained model.\n",
    "\n",
    "(If you don't know how to load a model or how to operate with vocabularies yet, look in the notebook __1_Load_model_and_translate_baseline__.)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.insert(0, 'path_to_good_translation_wrong_in_context') # insert your local path to the repo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load vocabularies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "DATA_PATH = # insert your path\n",
    "VOC_PATH =  # insert your path\n",
    "\n",
    "inp_voc = pickle.load(open(VOC_PATH + 'src.voc', 'rb'))\n",
    "out_voc = pickle.load(open(VOC_PATH + 'dst.voc', 'rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_VISIBLE_DEVICES=1\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_VISIBLE_DEVICES=0\n",
    "\n",
    "import tensorflow as tf\n",
    "import lib\n",
    "import lib.task.seq2seq.models.transformer as tr\n",
    "\n",
    "tf.reset_default_graph()\n",
    "gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.99, allow_growth=True)\n",
    "sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options))\n",
    "\n",
    "hp = {\n",
    "     \"num_layers\": 6,\n",
    "     \"num_heads\": 8,\n",
    "     \"ff_size\": 2048,\n",
    "     \"ffn_type\": \"conv_relu\",\n",
    "     \"hid_size\": 512,\n",
    "     \"emb_size\": 512,\n",
    "     \"res_steps\": \"nlda\", \n",
    "    \n",
    "     \"rescale_emb\": True,\n",
    "     \"inp_emb_bias\": True,\n",
    "     \"normalize_out\": True,\n",
    "     \"share_emb\": False,\n",
    "     \"replace\": 0,\n",
    "    \n",
    "     \"relu_dropout\": 0.1,\n",
    "     \"res_dropout\": 0.1,\n",
    "     \"attn_dropout\": 0.1,\n",
    "     \"label_smoothing\": 0.1,\n",
    "    \n",
    "     \"translator\": \"ingraph\",\n",
    "     \"beam_size\": 4,\n",
    "     \"beam_spread\": 3,\n",
    "     \"len_alpha\": 0.6,\n",
    "     \"attn_beta\": 0,\n",
    "}\n",
    "\n",
    "model = tr.Model('mod', inp_voc, out_voc, inference_mode='fast', **hp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_ckpt = # insert path to the final checkpoint\n",
    "var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)\n",
    "lib.train.saveload.load(path_to_ckpt, var_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Score <a name=\"translate\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load consistency test set (for example, we'll take deixis)\n",
    "path_to_testset = # path to your data\n",
    "test_src = open(path_to_testset + 'deixis_dev.src').readlines()\n",
    "test_dst = open(path_to_testset + 'deixis_dev.dst').readlines()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are groups of consecutive sentences and treir translations in our test set (sentences are separated with the `_eos` token):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"just leave them outside the door . _eos the rooms need to be cleaned , once a week in minimum . _eos - that 's a policy ... _eos - didn 't i clear your policy ?\\n\",\n",
       " \"just leave them outside the door . _eos the rooms need to be cleaned , once a week in minimum . _eos - that 's a policy ... _eos - didn 't i clear your policy ?\\n\",\n",
       " \"just leave them outside the door . _eos the rooms need to be cleaned , once a week in minimum . _eos - that 's a policy ... _eos - didn 't i clear your policy ?\\n\",\n",
       " \"just leave them outside the door . _eos the rooms need to be cleaned , once a week in minimum . _eos - that 's a policy ... _eos - didn 't i clear your policy ?\\n\"]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_src[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['просто оставьте их за дверью . _eos номера должны быть очищ `ены , раз в неделю минимум . _eos - это политика ... . _eos - разве я не ваша политика ?\\n',\n",
       " 'просто оставьте их за дверью . _eos номера должны быть очищ `ены , раз в неделю минимум . _eos - это политика ... . _eos - разве я не твоя политика ?\\n',\n",
       " 'просто оставь их за дверью . _eos номера должны быть очищ `ены , раз в неделю минимум . _eos - это политика ... . _eos - разве я не твоя политика ?\\n',\n",
       " 'просто оставь их за дверью . _eos номера должны быть очищ `ены , раз в неделю минимум . _eos - это политика ... . _eos - разве я не ваша политика ?\\n']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dst[:4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To score sentences, we have to get loss values from the problem."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lib.task.seq2seq.problems.default import DefaultProblem\n",
    "from lib.task.seq2seq.data import make_batch_placeholder\n",
    "problem = DefaultProblem({'mod': model})\n",
    "batch_ph = make_batch_placeholder(model.make_feed_dict(model._get_batch_sample()))\n",
    "loss_values = problem.loss_values(batch=batch_ph, is_train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since baseline is able to process only one sentence, we have to take only last sentence to score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def num_sents(text):\n",
    "    return len(text.split(' _eos '))\n",
    "\n",
    "def make_baseline_batch_data(src_lines, dst_lines):\n",
    "    \"\"\"\n",
    "    src_lines contain groups of N sentences, last of which is to be translated (' _eos '-separated)\n",
    "    dst_lines contain translations of sentences in src_lines (' _eos '-separated)\n",
    "    \n",
    "    returns: list of pairs (src, dst) which one can give to a model\n",
    "    \"\"\"\n",
    "    assert len(src_lines) == len(dst_lines), \"Different number of text fragments\"\n",
    "    batch_src = []\n",
    "    batch_dst = []\n",
    "    for src, dst in zip(src_lines, dst_lines):\n",
    "        assert num_sents(src) == num_sents(dst)\n",
    "        batch_src.append(src.split(' _eos ')[-1])\n",
    "        batch_dst.append(dst.split(' _eos ')[-1])\n",
    "    return list(zip(batch_src, batch_dst))\n",
    "\n",
    "def score_batch(src_lines, dst_lines):\n",
    "    feed_dict = model.make_feed_dict(make_baseline_batch_data(src_lines, dst_lines))\n",
    "    feed = {batch_ph[k]: feed_dict[k] for k in batch_ph}\n",
    "    scores = sess.run(loss_values, feed)\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10.607619, 12.479008, 12.479008, 10.60762 ], dtype=float32)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_batch(test_src[:4], test_dst[:4])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To score a test set, just do this for a sequence of batches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}