neural-machine-translation/31.lstm-seq2seq-contrib-beam-bahdanau.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dataset-bpe.json') as fopen:\n",
    "    data = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = data['train_X']\n",
    "train_Y = data['train_Y']\n",
    "test_X = data['test_X']\n",
    "test_Y = data['test_Y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "EOS = 2\n",
    "GO = 1\n",
    "vocab_size = 32000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_Y = [i + [2] for i in train_Y]\n",
    "test_Y = [i + [2] for i in test_Y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensor2tensor.utils import beam_search\n",
    "\n",
    "def pad_second_dim(x, desired_size):\n",
    "    padding = tf.tile([[[0.0]]], tf.stack([tf.shape(x)[0], desired_size - tf.shape(x)[1], tf.shape(x)[2]], 0))\n",
    "    return tf.concat([x, padding], 1)\n",
    "\n",
    "class Translator:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, learning_rate,\n",
    "                beam_width = 5):\n",
    "        \n",
    "        def cells(reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size_layer,initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "        \n",
    "        def attention(encoder_out, seq_len, reuse=False):\n",
    "            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(num_units = size_layer, \n",
    "                                                                    memory = encoder_out,\n",
    "                                                                    memory_sequence_length = seq_len)\n",
    "            return tf.contrib.seq2seq.AttentionWrapper(\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell([cells(reuse) for _ in range(num_layers)]), \n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = size_layer)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        \n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype = tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype = tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        \n",
    "        embeddings = tf.Variable(tf.random_uniform([vocab_size, embedded_size], -1, 1))\n",
    "        \n",
    "        encoder_out, encoder_state = tf.nn.dynamic_rnn(\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell([cells() for _ in range(num_layers)]), \n",
    "            inputs = tf.nn.embedding_lookup(embeddings, self.X),\n",
    "            sequence_length = self.X_seq_len,\n",
    "            dtype = tf.float32)\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        dense = tf.layers.Dense(vocab_size)\n",
    "        \n",
    "        with tf.variable_scope('decode'):\n",
    "            decoder_cells = attention(encoder_out, self.X_seq_len)\n",
    "            states = decoder_cells.zero_state(batch_size, tf.float32).clone(cell_state=encoder_state)\n",
    "\n",
    "            training_helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "                    inputs = tf.nn.embedding_lookup(embeddings, decoder_input),\n",
    "                    sequence_length = self.Y_seq_len,\n",
    "                    time_major = False)\n",
    "            training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                    cell = decoder_cells,\n",
    "                    helper = training_helper,\n",
    "                    initial_state = states,\n",
    "                    output_layer = dense)\n",
    "            training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                    decoder = training_decoder,\n",
    "                    impute_finished = True,\n",
    "                    maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "            self.training_logits = training_decoder_output.rnn_output\n",
    "        \n",
    "        with tf.variable_scope('decode', reuse=True):\n",
    "            encoder_out_tiled = tf.contrib.seq2seq.tile_batch(encoder_out, beam_width)\n",
    "            encoder_state_tiled = tf.contrib.seq2seq.tile_batch(encoder_state, beam_width)\n",
    "            X_seq_len_tiled = tf.contrib.seq2seq.tile_batch(self.X_seq_len, beam_width)\n",
    "            decoder_cell = attention(encoder_out_tiled, X_seq_len_tiled, reuse=True)\n",
    "            states = decoder_cell.zero_state(batch_size * beam_width, tf.float32).clone(\n",
    "                    cell_state = encoder_state_tiled)\n",
    "            predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                cell = decoder_cell,\n",
    "                embedding = embeddings,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS,\n",
    "                initial_state = states,\n",
    "                beam_width = beam_width,\n",
    "                output_layer = dense,\n",
    "                length_penalty_weight = 0.0)\n",
    "            predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = False,\n",
    "                maximum_iterations = 2 * tf.reduce_max(self.X_seq_len))\n",
    "            self.fast_result = predicting_decoder_output.predicted_ids[:, :, 0]\n",
    "        \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 512\n",
    "num_layers = 2\n",
    "embedded_size = 256\n",
    "learning_rate = 1e-3\n",
    "batch_size = 128\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n",
      "WARNING:tensorflow:From <ipython-input-7-d6797cae2309>:12: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-7-d6797cae2309>:33: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-7-d6797cae2309>:36: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/rnn_cell_impl.py:958: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.add_weight` method instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/rnn_cell_impl.py:962: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/rnn.py:244: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/seq2seq/python/ops/beam_search_decoder.py:971: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Translator(size_layer, num_layers, embedded_size, learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[ 2997,  2997,  4652,  5103,  5103,  5103,  5103,  5103,  5103,\n",
       "          3027,  3027,  3027,  3027, 28179, 28179, 28179, 24611, 24611,\n",
       "         31060, 24611, 31060, 31060, 31060,  6973,  6973,  6973,  2311,\n",
       "         23258,  4617,  4617,  4617,  4617,  4617,  4617,  4617,  4617,\n",
       "          4617,  8479,  8479, 22923, 22923, 22923, 24483, 24483, 24483,\n",
       "         24483, 24483, 15174, 17543,  2812,  2812,  2812, 15174,  2812,\n",
       "          2812,  2812,  2812,  2812, 13765,  2812, 13765, 13765,  2084,\n",
       "         17770, 24367, 24367, 24367, 24367, 24367, 21129, 21129, 21129],\n",
       "        [ 1553,  1553,  1553,  1553, 16429, 16429, 16429, 16429, 16429,\n",
       "         12654, 24705, 24705, 24705, 24705, 24705, 24705,  9250,  9250,\n",
       "         25577,  9250,  9250,  9250,  9250, 15755, 15755, 15755, 15755,\n",
       "         11352, 11352, 11352, 11352, 11352, 11352,  3858,  3858,  3858,\n",
       "          3858,  3858,  3858, 29150, 31673, 31673, 31673, 31673, 31673,\n",
       "         29150,  7190, 23623, 23623, 23623, 23623, 25644, 25644, 22335,\n",
       "         22335, 22335, 27412, 27412, 27412, 27412, 27412, 27412, 24346,\n",
       "         24346, 12629, 12629, 12629, 12629,  2604,  2604,  2604, 17781],\n",
       "        [ 4140,  1484,  1484, 21634, 21634, 21634, 21880, 21880, 21880,\n",
       "          2786,  2786,  2228,  2228,  2228,  2228,  2058,  2058,  2058,\n",
       "          2058,  9165,  9165,  9165,  9165, 22339, 22339, 22339, 22339,\n",
       "         22339, 22339, 22339,  5595, 10937, 10937, 10937, 10715, 10715,\n",
       "         10715, 10715, 10715, 27656, 27656, 19640, 19640, 19640,  9923,\n",
       "         31576, 31576, 31576, 31576, 31576, 31576,  6295, 31576,  6295,\n",
       "          6295, 10556, 10556, 28289, 15285, 15285,  2737,  2737,  2737,\n",
       "          2737,  2737,  2737,  2737, 16878,  2737, 16878,  6221,  6221],\n",
       "        [22904, 17045, 17045, 17045, 17045, 17045,  3171, 18115, 18115,\n",
       "         25266, 25266, 25266, 25266, 25266,  7293,  7293,  4233,  4233,\n",
       "         10656, 10656, 10656, 10656, 10656, 23989, 26040, 26040, 12499,\n",
       "          3482,  3541,  3541,  3541, 22434, 22434, 22434,  8291,  8291,\n",
       "          8291, 26757, 26757, 26757,  4877,  4877,  4877,  4877, 10754,\n",
       "         10754, 10754, 10754, 10754, 10754, 10754, 15769, 15769, 27376,\n",
       "         27376, 27376, 23759,  4035,  4035,  4035, 13612, 13612, 31528,\n",
       "         31528, 31528, 16748, 16748, 16748, 16748, 28249, 28249, 28249],\n",
       "        [26703, 26703, 26357, 29984, 29984, 15102, 15102,  1079, 24911,\n",
       "         24911, 24911, 26612, 31914, 31914, 31914, 31914, 31914, 17197,\n",
       "         17197, 17197, 17197, 31914,  6772,  6772,  6772, 31018,  5871,\n",
       "          5871,  5871,  5871,  5871, 25278, 25278,  9995,  9995,  9995,\n",
       "          9995,  9995,  9995,  9995, 26190, 26749, 26749, 26749, 26190,\n",
       "         26190, 14261, 14261, 14261, 28729, 28729, 28729, 28729, 22649,\n",
       "         22649, 22649, 21017, 21017, 21017, 21017, 21017, 31582, 31582,\n",
       "         31582, 31582,  6117,  6117,  6117,  6117,  2280,  2280, 13174],\n",
       "        [ 7393, 12241, 12241, 17159, 17159, 17159, 17159, 28226, 28226,\n",
       "         28226, 28226, 28226, 28226, 11368, 18059, 18059, 18059, 18059,\n",
       "          5129,  5129,  8490,  8490, 16984, 16984, 16984, 26621,  3031,\n",
       "          3031,  3031, 20376, 20376, 20376, 20376,  9806,  9806, 17704,\n",
       "         30030, 30030, 30030, 30030, 24407, 24407, 16391, 16391,  7512,\n",
       "          7512,  1612, 20703, 20703, 27551, 27551, 27551, 27551, 27551,\n",
       "         27551, 27551, 27551, 27551, 27551, 27551, 27551, 24491, 24491,\n",
       "         24491, 24491, 25428, 25428, 25428,  1649, 27893, 27893, 27893],\n",
       "        [14886, 14886, 14886, 23258, 23258, 23258, 23258, 18532, 19509,\n",
       "         24500, 24500, 24500, 22117, 22117, 22117, 22117, 22117, 23012,\n",
       "         23012,  7585,  7585,  7585,  7585,  7585, 13328, 13328, 21484,\n",
       "         29890, 29890, 29890, 29890,  1654,  1654,  1654,   275,  7305,\n",
       "          7305,  7305,  7305,  7305,   490,   490,   490,  4471,  4471,\n",
       "          4471, 10010, 10010, 18907, 24886, 24886, 24886, 24886, 24886,\n",
       "         24886,  8956,  8956, 25379, 25379, 25379, 25379, 25379, 20499,\n",
       "          6210,  6210,  6210,  6210,  6210,  2762,  2762, 17508, 17508],\n",
       "        [31045, 31045, 22084, 22084, 22084, 22084, 22084,  2812,  2812,\n",
       "          2812,  2812,  2812, 28529,  2492,  2492,  2492,  2492,  2492,\n",
       "          9295,  9295,  9295,  9295,  9295,  9295, 28784, 28784, 28784,\n",
       "         28784, 28784, 28784, 28784,  2576, 18565, 18565, 23616, 28784,\n",
       "         28784, 28784, 18565, 23993, 23993, 23993, 23993, 22703, 22703,\n",
       "         22703, 26231, 26231, 29668, 26231, 29668, 10915, 10915, 10915,\n",
       "         10915, 10915, 14178, 14178, 14178, 14178,  4964,  4964,  4964,\n",
       "         17344, 17344, 28848, 28848, 15382, 10671, 10671, 15382, 15382],\n",
       "        [28734, 11778, 11778, 11778, 25668,   191,   191,   191,   191,\n",
       "         13759,  9201,  9201,  9201,  9201,  9201,  9201, 29232, 29232,\n",
       "         29232, 29232, 15658, 15658, 15658, 15658, 11093, 21547, 21547,\n",
       "         19215, 19215, 19215, 19215, 13108, 13108, 13108, 13108,  7668,\n",
       "          7668,  7668,  7668, 20033, 20033, 20033, 19669, 19669, 19669,\n",
       "         18538, 18538, 28481, 23415, 23415, 23415, 23415, 23415, 13046,\n",
       "         13046, 21391, 21391, 21391, 21391, 21391,  7989,  7989, 16703,\n",
       "         16703, 16703, 16703, 16703, 16703, 16703, 16703, 17625, 10000],\n",
       "        [14525, 21880, 21880, 17764, 17764, 17764, 17764,  1658,  1658,\n",
       "         23757, 23757, 23757, 29346, 29346, 18311, 18311, 18075, 18075,\n",
       "         18075, 18075,  5181,  5181,  5181,  5181,  5181,  7744, 26404,\n",
       "         26404, 26404, 26404, 26404, 26404, 18559, 18559, 26404, 18559,\n",
       "         18559, 18559,  8841, 14291, 28707, 28707, 28707, 28036, 28036,\n",
       "         28036, 28036, 28036,  4907, 31020, 31020, 31020, 31020, 12547,\n",
       "         12547, 27428, 29022, 11029, 11029, 29022, 26715, 26715, 26715,\n",
       "         26435, 26435, 26435, 26435,  9292,  9292,  9292,  3008,  3008]],\n",
       "       dtype=int32), 10.373898, 0.0]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_x = pad_sequences(train_X[:10], padding='post')\n",
    "batch_y = pad_sequences(train_Y[:10], padding='post')\n",
    "\n",
    "sess.run([model.fast_result, model.cost, model.accuracy], \n",
    "         feed_dict = {model.X: batch_x, model.Y: batch_y})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:03<00:00,  2.36it/s, accuracy=0.337, cost=4.09]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.91it/s, accuracy=0.392, cost=3.65]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, training avg loss 5.297149, training avg acc 0.218395\n",
      "epoch 1, testing avg loss 3.919522, testing avg acc 0.353800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:15<00:00,  2.31it/s, accuracy=0.477, cost=2.95]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.98it/s, accuracy=0.468, cost=2.89]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2, training avg loss 3.364603, training avg acc 0.420088\n",
      "epoch 2, testing avg loss 3.045408, testing avg acc 0.460986\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:16<00:00,  2.31it/s, accuracy=0.542, cost=2.42]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.97it/s, accuracy=0.527, cost=2.6] \n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 3, training avg loss 2.708504, training avg acc 0.498823\n",
      "epoch 3, testing avg loss 2.743939, testing avg acc 0.499034\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:17<00:00,  2.31it/s, accuracy=0.6, cost=2.04]  \n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.98it/s, accuracy=0.559, cost=2.42]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 4, training avg loss 2.354952, training avg acc 0.542819\n",
      "epoch 4, testing avg loss 2.632721, testing avg acc 0.513618\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:17<00:00,  2.31it/s, accuracy=0.633, cost=1.77]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.95it/s, accuracy=0.554, cost=2.47]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 5, training avg loss 2.106150, training avg acc 0.576165\n",
      "epoch 5, testing avg loss 2.588290, testing avg acc 0.522604\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:17<00:00,  2.31it/s, accuracy=0.682, cost=1.5] \n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.99it/s, accuracy=0.575, cost=2.55]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 6, training avg loss 1.909658, training avg acc 0.603780\n",
      "epoch 6, testing avg loss 2.639336, testing avg acc 0.521015\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:16<00:00,  2.31it/s, accuracy=0.71, cost=1.3]  \n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.99it/s, accuracy=0.597, cost=2.41]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 7, training avg loss 1.746557, training avg acc 0.627920\n",
      "epoch 7, testing avg loss 2.654864, testing avg acc 0.525218\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:17<00:00,  2.31it/s, accuracy=0.752, cost=1.09]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.95it/s, accuracy=0.581, cost=2.45]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 8, training avg loss 1.603603, training avg acc 0.649982\n",
      "epoch 8, testing avg loss 2.715373, testing avg acc 0.522406\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:17<00:00,  2.31it/s, accuracy=0.787, cost=0.945]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.99it/s, accuracy=0.597, cost=2.46]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 9, training avg loss 1.488779, training avg acc 0.667858\n",
      "epoch 9, testing avg loss 2.781770, testing avg acc 0.519081\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:17<00:00,  2.31it/s, accuracy=0.795, cost=0.852]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.96it/s, accuracy=0.602, cost=2.45]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, training avg loss 1.365468, training avg acc 0.688779\n",
      "epoch 10, testing avg loss 2.871268, testing avg acc 0.516922\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:17<00:00,  2.31it/s, accuracy=0.813, cost=0.76]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.98it/s, accuracy=0.575, cost=2.53]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 11, training avg loss 1.268506, training avg acc 0.705160\n",
      "epoch 11, testing avg loss 2.962152, testing avg acc 0.511506\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:17<00:00,  2.31it/s, accuracy=0.83, cost=0.681] \n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.98it/s, accuracy=0.591, cost=2.66]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 12, training avg loss 1.178764, training avg acc 0.720956\n",
      "epoch 12, testing avg loss 3.096416, testing avg acc 0.501996\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:17<00:00,  2.31it/s, accuracy=0.839, cost=0.608]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.97it/s, accuracy=0.586, cost=2.56]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 13, training avg loss 1.105056, training avg acc 0.733323\n",
      "epoch 13, testing avg loss 3.213587, testing avg acc 0.504646\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:17<00:00,  2.31it/s, accuracy=0.868, cost=0.547]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.98it/s, accuracy=0.554, cost=2.71]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 14, training avg loss 1.040652, training avg acc 0.744844\n",
      "epoch 14, testing avg loss 3.247687, testing avg acc 0.506079\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:16<00:00,  2.31it/s, accuracy=0.866, cost=0.521]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.98it/s, accuracy=0.591, cost=2.68]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 15, training avg loss 0.978841, training avg acc 0.756209\n",
      "epoch 15, testing avg loss 3.326649, testing avg acc 0.499364\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:17<00:00,  2.31it/s, accuracy=0.879, cost=0.457]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.97it/s, accuracy=0.559, cost=2.85]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 16, training avg loss 0.920878, training avg acc 0.767512\n",
      "epoch 16, testing avg loss 3.424382, testing avg acc 0.496099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:17<00:00,  2.31it/s, accuracy=0.891, cost=0.423]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:07<00:00,  5.01it/s, accuracy=0.602, cost=2.73]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 17, training avg loss 0.868228, training avg acc 0.777795\n",
      "epoch 17, testing avg loss 3.506542, testing avg acc 0.493383\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:16<00:00,  2.31it/s, accuracy=0.891, cost=0.416]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.96it/s, accuracy=0.554, cost=3.09]\n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 18, training avg loss 0.823242, training avg acc 0.786328\n",
      "epoch 18, testing avg loss 3.598927, testing avg acc 0.492991\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:16<00:00,  2.31it/s, accuracy=0.912, cost=0.353]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  4.99it/s, accuracy=0.548, cost=3.2] \n",
      "minibatch loop:   0%|          | 0/1563 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 19, training avg loss 0.780483, training avg acc 0.794880\n",
      "epoch 19, testing avg loss 3.697737, testing avg acc 0.495180\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 1563/1563 [11:17<00:00,  2.31it/s, accuracy=0.919, cost=0.323]\n",
      "minibatch loop: 100%|██████████| 40/40 [00:08<00:00,  5.00it/s, accuracy=0.538, cost=3.4] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 20, training avg loss 0.743944, training avg acc 0.801999\n",
      "epoch 20, testing avg loss 3.792156, testing avg acc 0.490799\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "\n",
    "for e in range(epoch):\n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'minibatch loop')\n",
    "    train_loss, train_acc, test_loss, test_acc = [], [], [], []\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = pad_sequences(train_X[i : index], padding='post')\n",
    "        batch_y = pad_sequences(train_Y[i : index], padding='post')\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y}\n",
    "        accuracy, loss, _ = sess.run([model.accuracy,model.cost,model.optimizer],\n",
    "                                    feed_dict = feed)\n",
    "        train_loss.append(loss)\n",
    "        train_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    \n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = pad_sequences(test_X[i : index], padding='post')\n",
    "        batch_y = pad_sequences(test_Y[i : index], padding='post')\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y,}\n",
    "        accuracy, loss = sess.run([model.accuracy,model.cost],\n",
    "                                    feed_dict = feed)\n",
    "\n",
    "        test_loss.append(loss)\n",
    "        test_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    print('epoch %d, training avg loss %f, training avg acc %f'%(e+1,\n",
    "                                                                 np.mean(train_loss),np.mean(train_acc)))\n",
    "    print('epoch %d, testing avg loss %f, testing avg acc %f'%(e+1,\n",
    "                                                              np.mean(test_loss),np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensor2tensor.utils import bleu_hook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 40/40 [00:26<00:00,  1.51it/s]\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "for i in tqdm.tqdm(range(0, len(test_X), batch_size)):\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = pad_sequences(test_X[i : index], padding='post')\n",
    "    feed = {model.X: batch_x}\n",
    "    p = sess.run(model.fast_result,feed_dict = feed)\n",
    "    result = []\n",
    "    for row in p:\n",
    "        result.append([i for i in row if i > 3])\n",
    "    results.extend(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "rights = []\n",
    "for r in test_Y:\n",
    "    rights.append([i for i in r if i > 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.17929372"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bleu_hook.compute_bleu(reference_corpus = rights,\n",
    "                       translation_corpus = results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}