neural-machine-translation/26.gru-seq2seq-contrib-greedy-luong.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dataset-bpe.json') as fopen:\n",
    "    data = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = data['train_X']\n",
    "train_Y = data['train_Y']\n",
    "test_X = data['test_X']\n",
    "test_Y = data['test_Y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "EOS = 2\n",
    "GO = 1\n",
    "vocab_size = 32000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_Y = [i + [2] for i in train_Y]\n",
    "test_Y = [i + [2] for i in test_Y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensor2tensor.utils import beam_search\n",
    "\n",
    "def pad_second_dim(x, desired_size):\n",
    "    padding = tf.tile([[[0.0]]], tf.stack([tf.shape(x)[0], desired_size - tf.shape(x)[1], tf.shape(x)[2]], 0))\n",
    "    return tf.concat([x, padding], 1)\n",
    "\n",
    "class Translator:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, learning_rate):\n",
    "        \n",
    "        def cells(reuse=False):\n",
    "            return tf.nn.rnn_cell.GRUCell(size_layer, reuse=reuse)\n",
    "        \n",
    "        def attention(encoder_out, seq_len, reuse=False):\n",
    "            attention_mechanism = tf.contrib.seq2seq.LuongAttention(num_units = size_layer, \n",
    "                                                                    memory = encoder_out,\n",
    "                                                                    memory_sequence_length = seq_len)\n",
    "            return tf.contrib.seq2seq.AttentionWrapper(\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell([cells(reuse) for _ in range(num_layers)]), \n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = size_layer)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        \n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype = tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype = tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        \n",
    "        embeddings = tf.Variable(tf.random_uniform([vocab_size, embedded_size], -1, 1))\n",
    "        \n",
    "        encoder_out, encoder_state = tf.nn.dynamic_rnn(\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell([cells() for _ in range(num_layers)]), \n",
    "            inputs = tf.nn.embedding_lookup(embeddings, self.X),\n",
    "            sequence_length = self.X_seq_len,\n",
    "            dtype = tf.float32)\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        dense = tf.layers.Dense(vocab_size)\n",
    "        decoder_cells = attention(encoder_out, self.X_seq_len)\n",
    "        \n",
    "        states = decoder_cells.zero_state(batch_size, tf.float32).clone(cell_state=encoder_state)\n",
    "        \n",
    "        training_helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "                inputs = tf.nn.embedding_lookup(embeddings, decoder_input),\n",
    "                sequence_length = self.Y_seq_len,\n",
    "                time_major = False)\n",
    "        training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                cell = decoder_cells,\n",
    "                helper = training_helper,\n",
    "                initial_state = states,\n",
    "                output_layer = dense)\n",
    "        training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = training_decoder,\n",
    "                impute_finished = True,\n",
    "                maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "        self.training_logits = training_decoder_output.rnn_output\n",
    "        \n",
    "        predicting_helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(\n",
    "                embedding = embeddings,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS)\n",
    "        predicting_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                cell = decoder_cells,\n",
    "                helper = predicting_helper,\n",
    "                initial_state = states,\n",
    "                output_layer = dense)\n",
    "        predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = True,\n",
    "                maximum_iterations = 2 * tf.reduce_max(self.X_seq_len))\n",
    "        self.fast_result = predicting_decoder_output.sample_id\n",
    "        \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 512\n",
    "num_layers = 2\n",
    "embedded_size = 256\n",
    "learning_rate = 1e-3\n",
    "batch_size = 128\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n",
      "WARNING:tensorflow:From <ipython-input-8-e88c0049c12e>:11: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-8-e88c0049c12e>:32: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-8-e88c0049c12e>:35: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/rnn_cell_impl.py:559: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.add_weight` method instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/rnn_cell_impl.py:565: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/rnn_cell_impl.py:575: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/rnn.py:244: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Translator(size_layer, num_layers, embedded_size, learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[18354, 13600, 13600, 13600,   417, 16219, 16219, 16219,  1859,\n",
       "          1859, 22031, 23211, 22031,  9058,  9058, 30228, 30228,  9058,\n",
       "         28363, 23269, 23269, 23269,  5272,  5272,  5272, 25428, 21558,\n",
       "         21558, 21558, 29628, 26549, 27415, 27415, 27415, 28114,   483,\n",
       "         28114,  6313,  1044,  1044, 26406, 26406, 26406, 26406, 26406,\n",
       "         29222, 29222, 11350,  8810, 18488, 18488, 27101, 27101, 27101,\n",
       "         26135, 26135, 26058, 26058, 11819, 11819, 25234,  3246,  3246,\n",
       "          3246,  3246,  3246,  3246,  3246,  3246, 17864, 17864, 28098],\n",
       "        [17357, 17357, 26720, 26720,  5869, 19413, 19413, 19413, 19413,\n",
       "         19413, 20363, 20363, 12041, 12041, 12041, 12577, 12480, 10066,\n",
       "         10066, 10066, 13367, 13367, 20976, 20976, 17677,     0,     0,\n",
       "          2342,  2342, 14368, 14368, 14368,  5885,  5885,  5885, 24093,\n",
       "         24093,  7841, 20845, 20845, 20845, 20845, 20845, 29149,  4008,\n",
       "          4008,  9356, 31351, 31351,  9605,  9605,  9605,  9605,  9605,\n",
       "         11919, 11919, 11919, 11147, 11147, 14682, 14682, 15056, 14682,\n",
       "         14682,  2500,  2500,  2500,  2500,  2500,  2500,  2500,  2500],\n",
       "        [27355,  7923, 24749, 23714, 22542, 22542, 22542, 23714, 31549,\n",
       "         31549, 18849, 18849, 18849,  7569, 25943, 25943, 18849, 18849,\n",
       "         18849,  9707,  9707,  6211, 30083, 30083, 25823, 25823,  2194,\n",
       "         31166, 31166, 31166, 31166, 31166, 31166,   389,   389,   389,\n",
       "           389, 27909, 30856, 30856, 30856, 25961,  7584, 18412, 18412,\n",
       "          5784,  5784,  5784,  5784, 31100, 31100, 31100, 13682, 13682,\n",
       "         13682, 11347, 11347, 11347, 15767, 15767, 15767, 15767, 15767,\n",
       "         14487, 14487, 31726, 31726, 31726, 31726, 31481, 31481, 31481],\n",
       "        [ 1103, 20351, 20351,  1080,  8737, 21044, 23855, 23319,  6154,\n",
       "          6154,  6154,  6154, 19943, 19943, 19943, 30277,  2099,  2099,\n",
       "          2099,  2099, 31513, 18343, 18343, 18343, 18343, 10423, 10423,\n",
       "         10423, 26923, 26923, 26923, 26923,  2027,  3405,  3405,  3405,\n",
       "          4794,  4794,  1008,  1008,  1008,  1008,  5064, 20534, 29626,\n",
       "          7646,  7646,  7646, 16014, 16014,  9113, 28412, 28412, 28412,\n",
       "         28412,  3025,  3025,  9113, 28619, 28619,  3822,  3822,  6014,\n",
       "         28619,  6014,  6014, 17794, 11274, 11274, 30679, 30679, 30679],\n",
       "        [ 7521, 10529, 25824, 25824,  5332,  3069,  3069, 24733, 24733,\n",
       "         24733, 24733, 24733, 11689, 15593, 15593, 15593, 18130, 18130,\n",
       "          9532,  9532,   145,   145,   145,   145, 25494, 17207, 17207,\n",
       "          1703, 14073, 14073, 14073, 14073, 16355, 16355, 16355,  6099,\n",
       "          4387,  4387,  6099,  6099, 17598, 17598, 17598, 31189, 31189,\n",
       "          6541,  4387, 29400, 22214, 22214,  4387, 13901, 22214,  8632,\n",
       "           622,   622,   622, 11282, 11282, 15028, 19879, 30249, 30249,\n",
       "         30249, 30249, 30249, 30249, 30249, 30249, 16381,  3334,  3334],\n",
       "        [10013, 10013,  1901,  8588,  8588,  8588,  8588,  8588,  8588,\n",
       "          8588,  8588, 28382, 24272, 24272, 24272,  9432,  9432,  9432,\n",
       "         19742,  4340, 19042, 19042, 27848, 27588, 27588, 27848, 20624,\n",
       "         20624, 20624, 20624,  6605,  6605,  6605,  6605,  6605, 30705,\n",
       "         18215, 18215, 12454, 12454, 12454, 12454, 12454, 10049, 10049,\n",
       "         22313, 22313, 22313, 23218, 23218, 15510, 15510, 15510, 24146,\n",
       "         24146,  1739,  1739,  1739,  1739, 11645, 31941, 31941, 11645,\n",
       "         11645, 31941, 31941,  1149,  8867,  1149,   329, 15094, 15094],\n",
       "        [19985,  3017,  7412,  7412,  7412,  7412,  8582,  5077, 22409,\n",
       "         12552, 12552,  7775, 24167, 24167, 24167,  5833,  5833, 11465,\n",
       "         11465, 11465, 11465, 11465, 21544, 21544, 21544, 21544, 19222,\n",
       "         19222, 19222, 19222, 19222, 28927, 28927, 10437, 10437, 15837,\n",
       "         15837, 21544, 21544, 26441, 26441,  7754,  7754,  7754, 15899,\n",
       "         15899, 15899, 13096, 13096, 13096, 13096, 13096, 13096, 13096,\n",
       "         13096,  9716,  9716, 24091, 24091, 24091, 24091, 24091, 24091,\n",
       "         24091, 24091, 24091, 26110, 15048, 15048, 19724, 19724, 27131],\n",
       "        [ 1945, 30950,   790,  5929, 19974, 19974, 12724, 12724, 12724,\n",
       "         19400, 19400, 19400, 31886,  8447,  8447,  8447, 16083, 19218,\n",
       "         18833, 30237, 30237, 10209, 31885,  8814,  8814,  8814,  8814,\n",
       "          4746,  4746, 13466,  7568,  7568,  7568,  7568,  7568,  9525,\n",
       "         15695, 15695, 15695, 20126, 20126, 20126, 30214, 15695, 29650,\n",
       "         29650, 15695, 15151, 15151, 29044, 24874, 24874, 29650, 19721,\n",
       "         14559, 24874, 14559, 14559, 18815, 18815, 23854, 23854, 23854,\n",
       "         23854,  5198,  5198, 12385, 24223, 24223, 31726, 31726, 31726],\n",
       "        [31332,  2869, 22133, 12945, 12945, 27891, 27891, 27891, 16106,\n",
       "         16106, 14835, 11270, 11270, 11270, 22149, 22149, 22149, 26137,\n",
       "         22149, 15542,  1815, 22149, 22149, 22149, 22149, 19699, 19699,\n",
       "         22149, 22149, 22149, 24813, 24813, 17844, 29595, 29595, 13768,\n",
       "         13768, 13768, 13768, 13768, 13768, 13768, 27156, 29003, 29003,\n",
       "         29003, 29003, 19401, 19401, 19401, 19401, 19401, 24235, 24235,\n",
       "         24235, 24235,  5108,  5108,  6172,  6172,  6172, 25695, 25695,\n",
       "         25695, 25695, 25695,  8646,  8646,  8646,  8646, 14172, 14172],\n",
       "        [20513, 20513, 20513, 17418, 24530, 24530, 24530, 24530, 24530,\n",
       "         24530,   589,   589,   589,  9715,  9715,   589, 15599, 19544,\n",
       "         19544, 26378,  9711,  9711,  9711,   363,   363,   363, 13269,\n",
       "         13269, 13269,  3359,  3359,  4684,  4684,  4684,  4684, 10318,\n",
       "         16508, 16508, 22001, 22677, 29310, 29310, 29310, 29310, 29957,\n",
       "         29957, 29957, 13847,  2918,  4648, 15389, 24477, 18083, 18083,\n",
       "         18083, 18083,  7398,  7398,  7398,  7398,  7398,  7398,  7398,\n",
       "         10063, 10063, 10063, 21786, 24231, 21786, 13535, 25279,  6954]],\n",
       "       dtype=int32), 10.373433, 0.0]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_x = pad_sequences(train_X[:10], padding='post')\n",
    "batch_y = pad_sequences(train_Y[:10], padding='post')\n",
    "\n",
    "sess.run([model.fast_result, model.cost, model.accuracy], \n",
    "         feed_dict = {model.X: batch_x, model.Y: batch_y})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:36<00:00,  2.45it/s, accuracy=0.289, cost=4.42]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.95it/s, accuracy=0.328, cost=3.9] \n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, training avg loss 5.261971, training avg acc 0.218410\n",
      "epoch 1, testing avg loss 4.266010, testing avg acc 0.299207\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:36<00:00,  2.46it/s, accuracy=0.369, cost=3.6] \n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.05it/s, accuracy=0.382, cost=3.51]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2, training avg loss 3.918923, training avg acc 0.332834\n",
      "epoch 2, testing avg loss 3.796151, testing avg acc 0.351709\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:36<00:00,  2.46it/s, accuracy=0.409, cost=3.25]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.05it/s, accuracy=0.398, cost=3.35]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 3, training avg loss 3.496567, training avg acc 0.376602\n",
      "epoch 3, testing avg loss 3.664481, testing avg acc 0.367121\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:36<00:00,  2.46it/s, accuracy=0.362, cost=3.46]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.02it/s, accuracy=0.376, cost=3.45]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 4, training avg loss 3.420632, training avg acc 0.382155\n",
      "epoch 4, testing avg loss 3.894568, testing avg acc 0.341869\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:36<00:00,  2.46it/s, accuracy=0.123, cost=7.09] \n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.04it/s, accuracy=0.118, cost=6.46]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 5, training avg loss 6.110177, training avg acc 0.233588\n",
      "epoch 5, testing avg loss 7.106655, testing avg acc 0.118639\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:35<00:00,  2.46it/s, accuracy=0.164, cost=5.72]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.02it/s, accuracy=0.188, cost=5.34]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 6, training avg loss 6.271831, training avg acc 0.150770\n",
      "epoch 6, testing avg loss 5.849673, testing avg acc 0.169044\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:36<00:00,  2.46it/s, accuracy=0.199, cost=5.1] \n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.05it/s, accuracy=0.183, cost=4.83]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 7, training avg loss 5.407912, training avg acc 0.186804\n",
      "epoch 7, testing avg loss 5.318305, testing avg acc 0.195084\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:36<00:00,  2.46it/s, accuracy=0.227, cost=4.64]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.00it/s, accuracy=0.204, cost=4.5] \n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 8, training avg loss 4.911012, training avg acc 0.213430\n",
      "epoch 8, testing avg loss 4.979586, testing avg acc 0.216734\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:35<00:00,  2.46it/s, accuracy=0.267, cost=4.26]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.03it/s, accuracy=0.226, cost=4.21]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 9, training avg loss 4.531836, training avg acc 0.242016\n",
      "epoch 9, testing avg loss 4.643238, testing avg acc 0.247601\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:36<00:00,  2.46it/s, accuracy=0.318, cost=3.83]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.06it/s, accuracy=0.312, cost=3.77]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, training avg loss 4.129842, training avg acc 0.284768\n",
      "epoch 10, testing avg loss 4.294620, testing avg acc 0.286507\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:36<00:00,  2.46it/s, accuracy=0.375, cost=3.36]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.06it/s, accuracy=0.328, cost=3.67]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 11, training avg loss 3.749316, training avg acc 0.328900\n",
      "epoch 11, testing avg loss 4.037546, testing avg acc 0.315795\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:35<00:00,  2.46it/s, accuracy=0.419, cost=3.01]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.07it/s, accuracy=0.333, cost=3.55]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 12, training avg loss 3.437722, training avg acc 0.366906\n",
      "epoch 12, testing avg loss 3.859852, testing avg acc 0.338977\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:37<00:00,  2.45it/s, accuracy=0.458, cost=2.73]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.02it/s, accuracy=0.366, cost=3.42]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 13, training avg loss 3.189048, training avg acc 0.398683\n",
      "epoch 13, testing avg loss 3.732217, testing avg acc 0.356847\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:36<00:00,  2.46it/s, accuracy=0.488, cost=2.57]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.07it/s, accuracy=0.376, cost=3.24]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 14, training avg loss 2.993566, training avg acc 0.424840\n",
      "epoch 14, testing avg loss 3.652792, testing avg acc 0.369746\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:36<00:00,  2.46it/s, accuracy=0.512, cost=2.37]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.02it/s, accuracy=0.382, cost=3.14]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 15, training avg loss 2.851281, training avg acc 0.443805\n",
      "epoch 15, testing avg loss 3.633529, testing avg acc 0.378120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:35<00:00,  2.46it/s, accuracy=0.525, cost=2.25]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.05it/s, accuracy=0.414, cost=3.08]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 16, training avg loss 2.731226, training avg acc 0.459997\n",
      "epoch 16, testing avg loss 3.553645, testing avg acc 0.387049\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:34<00:00,  2.46it/s, accuracy=0.546, cost=2.15]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.01it/s, accuracy=0.387, cost=3.14]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 17, training avg loss 2.628165, training avg acc 0.473949\n",
      "epoch 17, testing avg loss 3.554105, testing avg acc 0.393430\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:35<00:00,  2.46it/s, accuracy=0.553, cost=2.1] \n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.05it/s, accuracy=0.398, cost=3.2] \n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 18, training avg loss 2.556624, training avg acc 0.483280\n",
      "epoch 18, testing avg loss 3.534196, testing avg acc 0.397046\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:35<00:00,  2.46it/s, accuracy=0.56, cost=2.06] \n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.03it/s, accuracy=0.43, cost=3.18] \n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 19, training avg loss 2.580282, training avg acc 0.478416\n",
      "epoch 19, testing avg loss 3.549078, testing avg acc 0.403124\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [10:35<00:00,  2.46it/s, accuracy=0.556, cost=2.03]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.04it/s, accuracy=0.441, cost=3.34]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 20, training avg loss 2.480351, training avg acc 0.493041\n",
      "epoch 20, testing avg loss 3.580384, testing avg acc 0.398440\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "\n",
    "for e in range(epoch):\n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'minibatch loop')\n",
    "    train_loss, train_acc, test_loss, test_acc = [], [], [], []\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = pad_sequences(train_X[i : index], padding='post')\n",
    "        batch_y = pad_sequences(train_Y[i : index], padding='post')\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y}\n",
    "        accuracy, loss, _ = sess.run([model.accuracy,model.cost,model.optimizer],\n",
    "                                    feed_dict = feed)\n",
    "        train_loss.append(loss)\n",
    "        train_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    \n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = pad_sequences(test_X[i : index], padding='post')\n",
    "        batch_y = pad_sequences(test_Y[i : index], padding='post')\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y,}\n",
    "        accuracy, loss = sess.run([model.accuracy,model.cost],\n",
    "                                    feed_dict = feed)\n",
    "\n",
    "        test_loss.append(loss)\n",
    "        test_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    print('epoch %d, training avg loss %f, training avg acc %f'%(e+1,\n",
    "                                                                 np.mean(train_loss),np.mean(train_acc)))\n",
    "    print('epoch %d, testing avg loss %f, testing avg acc %f'%(e+1,\n",
    "                                                              np.mean(test_loss),np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensor2tensor.utils import bleu_hook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 40/40 [00:19<00:00,  2.10it/s]\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "for i in tqdm.tqdm(range(0, len(test_X), batch_size)):\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = pad_sequences(test_X[i : index], padding='post')\n",
    "    feed = {model.X: batch_x}\n",
    "    p = sess.run(model.fast_result,feed_dict = feed)\n",
    "    result = []\n",
    "    for row in p:\n",
    "        result.append([i for i in row if i > 3])\n",
    "    results.extend(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "rights = []\n",
    "for r in test_Y:\n",
    "    rights.append([i for i in r if i > 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.101576895"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bleu_hook.compute_bleu(reference_corpus = rights,\n",
    "                       translation_corpus = results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}