20-32-256-cat_eng_seq2seq_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GvEmSE6vTdTS"
   },
   "source": [
    "<h1>Cat to English TRANSLATION</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 358
    },
    "colab_type": "code",
    "id": "O6YEZIHOYXqA",
    "outputId": "0a70a234-4b27-4e00-a055-08a258b9e9f7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow==1.15 in /usr/local/lib/python3.6/dist-packages (1.15.0)\n",
      "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.18.4)\n",
      "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.0.8)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.28.1)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.9.0)\n",
      "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.34.2)\n",
      "Requirement already satisfied: tensorboard<1.16.0,>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.15.0)\n",
      "Requirement already satisfied: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.2.2)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.1.0)\n",
      "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.8.1)\n",
      "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.2.0)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.12.1)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (3.10.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.1.0)\n",
      "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.12.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (3.2.1)\n",
      "Requirement already satisfied: tensorflow-estimator==1.15.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.15.1)\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow==1.15) (2.10.0)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (46.3.0)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (1.0.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (3.2.1)\n"
     ]
    }
   ],
   "source": [
    "from distutils.version import LooseVersion\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.layers.core import Dense\n",
    "\n",
    "\n",
    "# Check TensorFlow Version\n",
    "assert LooseVersion(tf.__version__) >= LooseVersion('1.1'), 'Please use TensorFlow version 1.1 or newer'\n",
    "!python3.6 -m pip install tensorflow==1.15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "Nu3dP2ypavXH",
    "outputId": "a245ea3f-997a-478b-8788-c4366af175ed"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.15.0\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['AttentionCellWrapper',\n",
       " 'BasicLSTMCell',\n",
       " 'BasicRNNCell',\n",
       " 'BidirectionalGridLSTMCell',\n",
       " 'CompiledWrapper',\n",
       " 'Conv1DLSTMCell',\n",
       " 'Conv2DLSTMCell',\n",
       " 'Conv3DLSTMCell',\n",
       " 'ConvLSTMCell',\n",
       " 'CoupledInputForgetGateLSTMCell',\n",
       " 'DeviceWrapper',\n",
       " 'DropoutWrapper',\n",
       " 'EmbeddingWrapper',\n",
       " 'FusedRNNCell',\n",
       " 'FusedRNNCellAdaptor',\n",
       " 'GLSTMCell',\n",
       " 'GRUBlockCell',\n",
       " 'GRUBlockCellV2',\n",
       " 'GRUCell',\n",
       " 'GridLSTMCell',\n",
       " 'HighwayWrapper',\n",
       " 'IndRNNCell',\n",
       " 'IndyGRUCell',\n",
       " 'IndyLSTMCell',\n",
       " 'InputProjectionWrapper',\n",
       " 'IntersectionRNNCell',\n",
       " 'LSTMBlockCell',\n",
       " 'LSTMBlockFusedCell',\n",
       " 'LSTMBlockWrapper',\n",
       " 'LSTMCell',\n",
       " 'LSTMStateTuple',\n",
       " 'LayerNormBasicLSTMCell',\n",
       " 'LayerRNNCell',\n",
       " 'MultiRNNCell',\n",
       " 'NASCell',\n",
       " 'OutputProjectionWrapper',\n",
       " 'PhasedLSTMCell',\n",
       " 'RNNCell',\n",
       " 'ResidualWrapper',\n",
       " 'SRUCell',\n",
       " 'TimeFreqLSTMCell',\n",
       " 'TimeReversedFusedRNN',\n",
       " 'UGRNNCell',\n",
       " '__builtins__',\n",
       " '__cached__',\n",
       " '__doc__',\n",
       " '__file__',\n",
       " '__loader__',\n",
       " '__name__',\n",
       " '__package__',\n",
       " '__path__',\n",
       " '__spec__',\n",
       " 'best_effort_input_batch_size',\n",
       " 'stack_bidirectional_dynamic_rnn',\n",
       " 'stack_bidirectional_rnn',\n",
       " 'static_bidirectional_rnn',\n",
       " 'static_rnn',\n",
       " 'static_state_saving_rnn',\n",
       " 'transpose_batch_time']"
      ]
     },
     "execution_count": 2,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "print(tf.__version__)\n",
    "dir(tf.contrib.rnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "3aee62b9-47ce-e416-5816-8df7126fe690",
    "colab": {},
    "colab_type": "code",
    "id": "FQ7ytEbpTdTV"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import gzip\n",
    "import codecs\n",
    "import re\n",
    "import time\n",
    "from tensorflow.python.ops.rnn_cell_impl import _zero_state_tensors\n",
    "from tensorflow.python.layers.core import Dense\n",
    "from tensorflow.contrib.rnn import GRUCell, DropoutWrapper\n",
    "from tensorflow.contrib.seq2seq import TrainingHelper, GreedyEmbeddingHelper,BasicDecoder, dynamic_decode, \\\n",
    " BahdanauAttention, AttentionWrapper,sequence_loss\n",
    "import helper\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "from sklearn.model_selection import train_test_split\n",
    "import unicodedata\n",
    "import os\n",
    "import io\n",
    "\n",
    "\n",
    "TOKEN_GO = '<GO>'\n",
    "TOKEN_EOS = '<EOS>'\n",
    "TOKEN_PAD = '<PAD>'\n",
    "TOKEN_UNK = '<UNK>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 82
    },
    "colab_type": "code",
    "id": "flWSMJRkztR4",
    "outputId": "e9bf7be6-06fc-4824-b6ac-8f23a2ffbaf9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords \n",
    "from nltk.tokenize import word_tokenize \n",
    "from numpy import linalg as LA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "O9dyYsyuVIAg",
    "outputId": "9f9bfc66-ead2-4fdf-86ec-88ee92608b68"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/gdrive/; to attempt to forcibly remount, call drive.mount(\"/content/gdrive/\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 310
    },
    "colab_type": "code",
    "id": "vLDd9ViBV8aX",
    "outputId": "ad5ab192-0d91-4498-c87d-f5004795666a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['02_example.ipynb',\n",
       " 'data-protfolio.ppt',\n",
       " 'amazon-fine-food-reviews.zip',\n",
       " 'attention.py',\n",
       " 'cat-eng.zip',\n",
       " 'news.txt.gz',\n",
       " 'summary.txt.gz',\n",
       " 'deu-eng.zip',\n",
       " 'cc.ca.300.vec.gz',\n",
       " 'Colab Notebooks',\n",
       " 'data_new.csv',\n",
       " 'cc.en.300.vec.gz',\n",
       " 'cc.en.300.bin.gz',\n",
       " 'cc.ca.300.bin.gz',\n",
       " 'best_so_far_model.ckpt.index',\n",
       " 'checkpoint',\n",
       " 'best_so_far_model.ckpt.data-00000-of-00001',\n",
       " 'best_so_far_model.ckpt.meta']"
      ]
     },
     "execution_count": 6,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "! rm /content/gdrive/My\\ Drive/events.out*\n",
    "os.listdir('/content/gdrive/My Drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TqEa4u_DmgPx"
   },
   "outputs": [],
   "source": [
    "if 'cat.txt' not in os.listdir('/content/'):\n",
    "    !unzip /content/gdrive/My\\ Drive/cat-eng.zip\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-187pLnCnDsP"
   },
   "outputs": [],
   "source": [
    "path_to_file='/content/cat.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rd0jw-eC3jEh"
   },
   "outputs": [],
   "source": [
    "# Converts the unicode file to ascii\n",
    "def unicode_to_ascii(s):\n",
    "  return ''.join(c for c in unicodedata.normalize('NFD', s)\n",
    "      if unicodedata.category(c) != 'Mn')\n",
    "\n",
    "\n",
    "def preprocess_sentence(w):\n",
    "  w = unicode_to_ascii(w.lower().strip())\n",
    "\n",
    "  # creating a space between a word and the punctuation following it\n",
    "  # eg: \"he is a boy.\" => \"he is a boy .\"\n",
    "  # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
    "  w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "  w = re.sub(r'[\" \"]+', \" \", w)\n",
    "  # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\")\n",
    "  w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n",
    "  w = w.rstrip().strip()\n",
    "\n",
    "  # adding a start and an end token to the sentence\n",
    "  # so that the model know when to start and stop predicting.\n",
    "  w = '<GO> ' + w + ' <EOS>'\n",
    "  return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "opI2GzOt479E",
    "outputId": "b5b52ddf-a143-4bff-b4fc-a4c835594272"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<GO> may i borrow this book ? <EOS>\n",
      "b'<GO> \\xc2\\xbf podra prendre prestat aquest llibre ? <EOS>'\n"
     ]
    }
   ],
   "source": [
    "en_sentence = u\"May I borrow this book?\"\n",
    "cat_sentence = u\"¿Podrà prendre prestat aquest llibre?\"\n",
    "print(preprocess_sentence(en_sentence))\n",
    "print(preprocess_sentence(cat_sentence).encode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WHNdBMEq-at5"
   },
   "outputs": [],
   "source": [
    "num_examples=100000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OHn4Dct23jEm"
   },
   "outputs": [],
   "source": [
    "# 1. Remove the accents\n",
    "# 2. Clean the sentences\n",
    "# 3. Return word pairs in the format: [ENGLISH, CATALAN]\n",
    "def create_dataset(path, num_examples):\n",
    "  lines = io.open(path, encoding='UTF-8').read().strip().split('\\n')\n",
    "  linep=[w  for l in lines for w in l.split('\\t') if w.split()[0] != 'CC-BY']\n",
    "  words = [preprocess_sentence(w) for l in linep[:num_examples] for w in l.split('\\t')]\n",
    "  word_pairs=list(np.array(words).reshape(len(words)//2,2))\n",
    "  return zip(*word_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "cTbSbBz55QtF",
    "outputId": "5f5c9cd9-6532-4c1f-c4c2-90cb576122a7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<GO> we re gonna make sure that no one is taking advantage of the american people for their own short term gain . <EOS>\n",
      "<GO> ens assegurarem que ningu s estiga aprofitant del poble america per al seu propi interes a curt termini . <EOS>\n"
     ]
    }
   ],
   "source": [
    "en, cat = create_dataset(path_to_file, None)\n",
    "print(en[-1])\n",
    "print(cat[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "I6PXGIAlnXhp"
   },
   "outputs": [],
   "source": [
    "data=[]\n",
    "for eng,cata in zip(en,cat):\n",
    "  data.append([eng,cata])\n",
    "news=pd.DataFrame(data,columns=['eng','cat'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 191
    },
    "colab_type": "code",
    "id": "5fY2Sfu7Gs2H",
    "outputId": "80534bcf-5608-422e-914c-81ccfe4f2d88"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text_len</th>\n",
       "      <th>Summary_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;GO&gt; wow ! &lt;EOS&gt;</td>\n",
       "      <td>&lt;GO&gt; carai ! &lt;EOS&gt;</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;GO&gt; really ? &lt;EOS&gt;</td>\n",
       "      <td>&lt;GO&gt; de veritat ? &lt;EOS&gt;</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;GO&gt; thanks . &lt;EOS&gt;</td>\n",
       "      <td>&lt;GO&gt; gracies ! &lt;EOS&gt;</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;GO&gt; goodbye ! &lt;EOS&gt;</td>\n",
       "      <td>&lt;GO&gt; adeu ! &lt;EOS&gt;</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;GO&gt; hurry up . &lt;EOS&gt;</td>\n",
       "      <td>&lt;GO&gt; afanya t . &lt;EOS&gt;</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Text                  Summary  Text_len  Summary_len\n",
       "0       <GO> wow ! <EOS>       <GO> carai ! <EOS>         4            4\n",
       "1    <GO> really ? <EOS>  <GO> de veritat ? <EOS>         4            5\n",
       "2    <GO> thanks . <EOS>     <GO> gracies ! <EOS>         4            4\n",
       "3   <GO> goodbye ! <EOS>        <GO> adeu ! <EOS>         4            4\n",
       "4  <GO> hurry up . <EOS>    <GO> afanya t . <EOS>         5            5"
      ]
     },
     "execution_count": 15,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news=news.rename(columns={'eng':'Text','cat':'Summary'})\n",
    "news['Text_len'] = news.Text.apply(lambda x: len(x.split()))\n",
    "news['Summary_len'] = news.Summary.apply(lambda x: len(x.split()))\n",
    "news.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 82
    },
    "colab_type": "code",
    "id": "Rxa-JA_QTdT8",
    "outputId": "a20375e7-08da-43c2-c9c8-0f5c55494c0a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<GO> wow ! <EOS>' '<GO> really ? <EOS>' '<GO> thanks . <EOS>'\n",
      " '<GO> goodbye ! <EOS>' '<GO> hurry up . <EOS>']\n",
      "['<GO> carai ! <EOS>' '<GO> de veritat ? <EOS>' '<GO> gracies ! <EOS>'\n",
      " '<GO> adeu ! <EOS>' '<GO> afanya t . <EOS>']\n"
     ]
    }
   ],
   "source": [
    "print(news['Text'].head(5).values)\n",
    "print(news['Summary'].head(5).values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "48f10124-4c1b-1f09-512d-352c068de1b4",
    "colab": {},
    "colab_type": "code",
    "id": "ZHaGwlh2TdUI"
   },
   "outputs": [],
   "source": [
    "news_summaries = []\n",
    "for summary in news.Summary:\n",
    "    news_summaries.append(summary)\n",
    "news_texts = []\n",
    "for text in news.Text:\n",
    "    news_texts.append(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "4ek1lAYfYrK3",
    "outputId": "6367be06-8732-4a49-d7bd-a0ef3b4b8860"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[653, 653]"
      ]
     },
     "execution_count": 18,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(news_summaries), len(news_texts)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "3f980247-3c32-240d-7d3d-7b0c3c6c13e5",
    "colab": {},
    "colab_type": "code",
    "id": "yHNDse2lTdUT"
   },
   "outputs": [],
   "source": [
    "def count_words(words_dict, text):\n",
    "    for sentence in text:\n",
    "        for word in sentence.split():\n",
    "            if word not in words_dict:\n",
    "                words_dict[word] = 1\n",
    "            else:\n",
    "                words_dict[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "_cell_guid": "3e9ce130-88f4-8779-5b5f-86f2a23ab347",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "_P7WgumjTdUe",
    "outputId": "d86f49ec-5d78-4ff7-b560-691feb9a1d9d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total words in eng Vocabulary: 991\n"
     ]
    }
   ],
   "source": [
    "word_counts_dictx={}\n",
    "count_words(word_counts_dictx, news_texts)           \n",
    "print(\"Total words in eng Vocabulary:\", len(word_counts_dictx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "XOa2ZgKt9chb",
    "outputId": "4e49b004-3b00-4359-bbab-bbb5f5cedd5c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total words in cat Vocabulary: 1166\n"
     ]
    }
   ],
   "source": [
    "word_counts_dict = {}\n",
    "count_words(word_counts_dict, news_summaries)            \n",
    "print(\"Total words in cat Vocabulary:\", len(word_counts_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QY8Sk4lWpYLC"
   },
   "outputs": [],
   "source": [
    "if 'cc.ca.300.vec' not in os.listdir('/content/'):\n",
    "  !cp /content/gdrive/My\\ Drive/cc.ca.300.vec.gz /content/\n",
    "  !gunzip /content/cc.ca.300.vec.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qk9Bi7wh5AV9"
   },
   "outputs": [],
   "source": [
    "if 'cc.en.300.vec' not in os.listdir('/content/'):\n",
    "  !cp /content/gdrive/My\\ Drive/cc.en.300.vec.gz /content/\n",
    "  !gunzip /content/cc.en.300.vec.gz\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BGdGuM_NTdUm"
   },
   "outputs": [],
   "source": [
    "def build_word_vector_matrix(vector_file):\n",
    "    embedding_index = {}\n",
    "    with codecs.open(vector_file, 'r', 'utf-8') as f:\n",
    "        for i, line in enumerate(f):\n",
    "            sr = line.split()\n",
    "            if(len(sr)<26):\n",
    "                continue\n",
    "            word = sr[0]\n",
    "            embedding = np.asarray(sr[1:], dtype='float32')\n",
    "            embedding_index[word] = embedding\n",
    "    return embedding_index\n",
    "\n",
    "                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4iIXmDdz9KUg"
   },
   "outputs": [],
   "source": [
    "# Replace the path here to point to the faster vectors file on your system\n",
    "embeddings_index = build_word_vector_matrix('/content/cc.ca.300.vec')\n",
    "embeddings_indexx = build_word_vector_matrix('/content/cc.en.300.vec')\n",
    "                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 131
    },
    "colab_type": "code",
    "id": "kQa9b2Dv3z9T",
    "outputId": "ca70f67d-1757-44e9-97d9-e96890267da6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.config',\n",
       " 'gdrive',\n",
       " '_about.txt',\n",
       " 'cc.en.300.vec',\n",
       " 'cc.ca.300.vec',\n",
       " 'cat.txt',\n",
       " 'sample_data']"
      ]
     },
     "execution_count": 26,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('/content')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tBR8J3iFK3lD"
   },
   "outputs": [],
   "source": [
    "count_threshold = 20\n",
    "#special_codes = [TOKEN_UNK,TOKEN_PAD,TOKEN_EOS,TOKEN_GO] \n",
    "special_codes =  [TOKEN_UNK,TOKEN_PAD] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "0be42a13-70b2-e9cc-7468-1247b01f109c",
    "colab": {},
    "colab_type": "code",
    "id": "VdDgF69WTdUw"
   },
   "outputs": [],
   "source": [
    "def map_word_int(count_treshold,word_counts_dict,embeddings_index):\n",
    "          word2int = {} \n",
    "          value = 0\n",
    "          for word, count in word_counts_dict.items():\n",
    "                     if count >= count_threshold or word in embeddings_index:\n",
    "                          word2int[word] = value\n",
    "                          value +=1  \n",
    "\n",
    "          for code in special_codes:\n",
    "              word2int[code]=len(word2int)\n",
    "          int2word = {}\n",
    "          for word, value in word2int.items():\n",
    "                int2word[value] = word\n",
    "          return word2int,int2word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2Ni-nQ8zKCX1"
   },
   "outputs": [],
   "source": [
    "word2intx,int2wordx=map_word_int(20,word_counts_dictx,embeddings_indexx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "9QVjfuj90xPe",
    "outputId": "1af3f40b-2829-4971-dad0-a997b10c467a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[993, 993]"
      ]
     },
     "execution_count": 30,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(word2intx), len(int2wordx)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4uZ_z9LbKgf7"
   },
   "outputs": [],
   "source": [
    "word2int,int2word=map_word_int(20,word_counts_dict,embeddings_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "r_66qdkCKemV",
    "outputId": "4cd6e9f9-f8a0-483b-c963-aaadb1b90960"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1151, 1151]"
      ]
     },
     "execution_count": 32,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(word2int),len(int2word)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "4401990d-4baf-3f30-becc-3a6149716b56",
    "colab": {},
    "colab_type": "code",
    "id": "f4KKWkrTTdU3"
   },
   "outputs": [],
   "source": [
    "embedding_dim = 300\n",
    "\n",
    "def word_embeddings_matrix(embedding_dim, word2int,embeddings_index):\n",
    "        nwords = len(word2int)\n",
    "        word_emb_matrix = np.zeros((nwords, embedding_dim), dtype=np.float32)\n",
    "        for word, i in word2int.items():\n",
    "                  if word in embeddings_index:\n",
    "                          word_emb_matrix[i] = embeddings_index[word]\n",
    "                  else:\n",
    "                          new_embedding = np.array(np.random.uniform(-1.0, 1.0, embedding_dim))\n",
    "                          word_emb_matrix[i] = new_embedding\n",
    "        return word_emb_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nXhivPnNIQBv"
   },
   "outputs": [],
   "source": [
    "word_emb_matrixx=word_embeddings_matrix(300,word2intx,embeddings_indexx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "nHpbH6FpIRNS",
    "outputId": "ba91275f-8a32-4807-c9e5-4a41b36d3291"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of word embeddings on eng:  993\n"
     ]
    }
   ],
   "source": [
    "print(\"Length of word embeddings on eng: \", len(word_emb_matrixx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WVPBpFB7H4J6"
   },
   "outputs": [],
   "source": [
    "word_emb_matrix=word_embeddings_matrix(300,word2int,embeddings_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "eKTkrgQIHXev",
    "outputId": "fdd40e3b-7fc4-421d-e18b-e8df0d989afa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of word embeddings on cat:  1151\n"
     ]
    }
   ],
   "source": [
    "print(\"Length of word embeddings on cat: \", len(word_emb_matrix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "25cfd0e3-ae3d-8728-1c82-1a61bb06aa0e",
    "colab": {},
    "colab_type": "code",
    "id": "x_n1GMkcTdU_"
   },
   "outputs": [],
   "source": [
    "def convert_sentence_to_ids(text,word2int, eos=False):\n",
    "    wordints = []\n",
    "    word_count = 0\n",
    "    for sentence in text:\n",
    "        sentence2ints = []\n",
    "        for word in sentence.split():\n",
    "            word_count += 1\n",
    "            if word in word2int:\n",
    "                sentence2ints.append(word2int[word])\n",
    "            else:\n",
    "                sentence2ints.append(word2int[TOKEN_UNK])\n",
    "        if eos:\n",
    "            sentence2ints.append(word2int[TOKEN_EOS])\n",
    "        wordints.append(sentence2ints)\n",
    "    return wordints, word_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "360cfdf4-ad4c-0316-56d3-70b6206e75e4",
    "colab": {},
    "colab_type": "code",
    "id": "umsLAGX2TdVJ"
   },
   "outputs": [],
   "source": [
    "id_summaries, word_count = convert_sentence_to_ids(news_summaries,word2int)\n",
    "id_texts, word_countx = convert_sentence_to_ids(news_texts,word2intx,eos=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "s7sOQR3SbRpa",
    "outputId": "e4b78da1-92d1-45b6-c8cc-25826c3c86eb"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[653, 6076], [653, 6258]]"
      ]
     },
     "execution_count": 40,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[[len(id_summaries),word_count],[len(id_texts),word_countx]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "2a0ae7cd-a845-23dc-3563-ad133e2f02b4",
    "colab": {},
    "colab_type": "code",
    "id": "mPyYg9N3TdVV"
   },
   "outputs": [],
   "source": [
    "def unknown_tokens(sentence,word2int):\n",
    "    unk_token_count = 0\n",
    "    for word in sentence:\n",
    "        if word == word2int[TOKEN_UNK]:\n",
    "            unk_token_count += 1\n",
    "    return unk_token_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "50d631a2-fb5a-bb0d-6155-9cd15e70835b",
    "colab": {},
    "colab_type": "code",
    "id": "93RbMaQCTdVc"
   },
   "outputs": [],
   "source": [
    "news_summaries_filtered = []\n",
    "news_texts_filtered = []\n",
    "max_text_length = int(news.Text_len.mean() + news.Text_len.std())\n",
    "max_summary_length = int(int(news.Summary_len.mean() + news.Summary_len.std()))\n",
    "min_length = 4\n",
    "unknown_token_text_limit = 10\n",
    "unknown_token_summary_limit = 4\n",
    "\n",
    "for count,text in enumerate(id_texts):\n",
    "    unknown_token_text = unknown_tokens(id_texts[count],word2intx)\n",
    "    unknown_token_summary = unknown_tokens(id_summaries[count],word2int)\n",
    "    text_len = len(id_texts[count])\n",
    "    summary_len = len(id_summaries[count])\n",
    "    if((unknown_token_text>unknown_token_text_limit) or (unknown_token_summary>unknown_token_summary_limit)):\n",
    "        continue\n",
    "    if(text_len<min_length or summary_len<min_length or text_len>max_text_length or summary_len>max_summary_length):\n",
    "        continue\n",
    "    news_summaries_filtered.append(id_summaries[count])\n",
    "    news_texts_filtered.append(id_texts[count])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "RsgyQyUEC0se",
    "outputId": "f1d90092-d58a-40a5-b991-d88c42f9c343"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[514, 514]"
      ]
     },
     "execution_count": 43,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(news_summaries_filtered),len(news_texts_filtered)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PVZaQ851H-SC"
   },
   "outputs": [],
   "source": [
    "def convert(lang,int2word, tensor):\n",
    "  for t in tensor:\n",
    "    if t!=0:\n",
    "      print (\"%d ----> %s\" % (t, int2word[t]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 212
    },
    "colab_type": "code",
    "id": "jo0RqWYHIpHO",
    "outputId": "830ef809-cabb-4f41-cbf3-dcd454be12bc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Language; index to word mapping\n",
      "11 ----> too\n",
      "12 ----> late\n",
      "7 ----> .\n",
      "3 ----> <EOS>\n",
      "3 ----> <EOS>\n",
      "\n",
      "Target Language; index to word mapping\n",
      "12 ----> massa\n",
      "13 ----> tard\n",
      "11 ----> .\n",
      "3 ----> <EOS>\n"
     ]
    }
   ],
   "source": [
    "print (\"Input Language; index to word mapping\")\n",
    "convert(news_texts,int2wordx, news_texts_filtered[5])\n",
    "print ()\n",
    "print (\"Target Language; index to word mapping\")\n",
    "convert(news_summaries, int2word, news_summaries_filtered[5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "34d28c5f-8016-6b36-664e-3d5ee3db745d",
    "colab": {},
    "colab_type": "code",
    "id": "xuL2rU6mTdVk"
   },
   "outputs": [],
   "source": [
    "def model_inputs():\n",
    "    inputs_data = tf.placeholder(tf.int32, [None, None], name='input_data')\n",
    "    targets = tf.placeholder(tf.int32, [None, None], name='targets')\n",
    "    learning_rate = tf.placeholder(tf.float32, name='learning_rate')\n",
    "    dropout_probs = tf.placeholder(tf.float32, name='dropout_probs')\n",
    "    summary_len = tf.placeholder(tf.int32, (None,), name='summary_len')\n",
    "    max_summary_len = tf.reduce_max(summary_len, name='max_summary_len')\n",
    "    text_len = tf.placeholder(tf.int32, (None,), name='text_len')\n",
    "    return inputs_data, targets, learning_rate, dropout_probs, summary_len, max_summary_len, text_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "9c9b6087-3c28-478d-d311-4213e1c59654",
    "colab": {},
    "colab_type": "code",
    "id": "ANIGHjdQTdVr"
   },
   "outputs": [],
   "source": [
    "def process_decoding_input(target_data, word2int, batch_size):\n",
    "    ending = tf.strided_slice(target_data, [0, 0], [batch_size, -1], [1, 1])\n",
    "    decoding_input = tf.concat([tf.fill([batch_size, 1], word2int[TOKEN_GO]), ending], 1)\n",
    "    return decoding_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "d675562b-a9e0-df71-6979-a052fb78dcbc",
    "colab": {},
    "colab_type": "code",
    "id": "4-nmhhyfTdVz"
   },
   "outputs": [],
   "source": [
    "def get_cell(csize,dprob):\n",
    "    rnc = GRUCell(csize)\n",
    "    rnc = DropoutWrapper(rnc, input_keep_prob = dprob)\n",
    "    return rnc\n",
    "\n",
    "def encoding_layer(csize, len_s, nl, rinp, dprob):\n",
    "    for l in range(nl):\n",
    "        with tf.variable_scope('encoding_l_{}'.format(l)):\n",
    "            rnn_frnt = get_cell(csize,dprob)\n",
    "            rnn_bkwd = get_cell(csize,dprob)\n",
    "            eop, est = tf.nn.bidirectional_dynamic_rnn(rnn_frnt, rnn_bkwd, \n",
    "                                                                    rinp,\n",
    "                                                                    len_s,\n",
    "                                                                    dtype=tf.float32)\n",
    "    eop = tf.concat(eop,nl)\n",
    "    return eop, est"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "524d0246-ddae-b485-5ea4-11ad476447f4",
    "colab": {},
    "colab_type": "code",
    "id": "OBSH-yfcTdV-"
   },
   "outputs": [],
   "source": [
    "def trng_dec_layer(dec_emb_inp, summ_len, cell_dec, st_init, lyr_op, \n",
    "                            v_size, max_summ_len):\n",
    "    helper = TrainingHelper(inputs=dec_emb_inp,sequence_length=summ_len, time_major=False)\n",
    "    dec = BasicDecoder(cell_dec,helper,st_init,lyr_op) \n",
    "    logits, _, _ = dynamic_decode(dec,output_time_major=False,impute_finished=True, \n",
    "                                  maximum_iterations=max_summ_len)\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "6044b206-7f27-5304-4896-06d388af0949",
    "colab": {},
    "colab_type": "code",
    "id": "ExxIWv2YTdWK"
   },
   "outputs": [],
   "source": [
    "def infr_dec_layer(embeddings, start_token, end_token, decoding_cell, initial_state, op_layer,\n",
    "                             max_summary_len, batch_size):\n",
    "    \n",
    "    start_tokens = tf.tile(tf.constant([start_token], dtype=tf.int32), [batch_size], name='start_tokens')\n",
    "    inf_helper = GreedyEmbeddingHelper(embeddings,start_tokens,end_token)\n",
    "    inf_decoder = BasicDecoder(decoding_cell,inf_helper,initial_state,op_layer)       \n",
    "    inf_logits, _, _ = dynamic_decode(inf_decoder,output_time_major=False,impute_finished=True,\n",
    "                                                            maximum_iterations=max_summary_len)\n",
    "    return inf_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "4b50746f-8f78-0253-9178-56c62e4ac1bf",
    "colab": {},
    "colab_type": "code",
    "id": "IcMgwE8oTdWd"
   },
   "outputs": [],
   "source": [
    "def decoding_layer(dec_emb_op, embs, enc_op, enc_st, v_size, txt_len, \n",
    "                   summ_len,mx_summ_len, rnsize, word2int, dprob, batch_size, nlyrs):\n",
    "    \n",
    "    for l in range(nlyrs):\n",
    "        with tf.variable_scope('dec_rnn_layer_{}'.format(l)):\n",
    "            gru = tf.contrib.rnn.GRUCell(rnn_len)\n",
    "            cell_dec = tf.contrib.rnn.DropoutWrapper(gru,input_keep_prob = dprob)\n",
    "            \n",
    "    out_l = Dense(v_size, kernel_initializer = tf.truncated_normal_initializer(mean = 0.0, stddev=0.1))\n",
    "    \n",
    "    attention = BahdanauAttention(rnsize, enc_op,txt_len,\n",
    "                                                  normalize=False,\n",
    "                                                  name='BahdanauAttention')\n",
    "    cell_dec =  AttentionWrapper(cell_dec,attention,rnn_len)\n",
    "    attn_zstate = cell_dec.zero_state(batch_size , tf.float32 )\n",
    "    attn_zstate = attn_zstate.clone(cell_state = enc_st[0])\n",
    "    with tf.variable_scope(\"decoding_layer\"):\n",
    "        tr_dec_op = trng_dec_layer(dec_emb_op, \n",
    "                                                  summ_len, \n",
    "                                                  cell_dec, \n",
    "                                                  attn_zstate,\n",
    "                                                  out_l,\n",
    "                                                  v_size, \n",
    "                                                  mx_summ_len)\n",
    "    with tf.variable_scope(\"decoding_layer\", reuse=True):\n",
    "        inf_dec_op = infr_dec_layer(embs,  \n",
    "                                                    word2int[TOKEN_GO], \n",
    "                                                    word2int[TOKEN_EOS],\n",
    "                                                    cell_dec, \n",
    "                                                    attn_zstate, \n",
    "                                                    out_l,\n",
    "                                                    mx_summ_len,\n",
    "                                                    batch_size)\n",
    "\n",
    "    return tr_dec_op, inf_dec_op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "19ddcf22-4f6a-d531-071a-021b42b643e3",
    "colab": {},
    "colab_type": "code",
    "id": "pYVCMj7aTdWl"
   },
   "outputs": [],
   "source": [
    "def seq2seq_model(data_inp, data_summ_tgt, dprob, len_txt, len_summ, max_len_summ, \n",
    "                  v_size, rnsize, nlyrs, word2int, batch_size):\n",
    "    \n",
    "    inp_embx = word_emb_matrixx # modification \n",
    "    word_embs = tf.Variable(inp_embx, name=\"word_embs\") # modification\n",
    "    inp_enc_emb = tf.nn.embedding_lookup(word_embs, data_inp)\n",
    "    op_enc, st_enc = encoding_layer(rnsize, len_txt, nlyrs, inp_enc_emb, dprob)\n",
    "    \n",
    "    inp_dec = process_decoding_input(data_summ_tgt, word2int, batch_size)\n",
    "    inp_emb = word_emb_matrix # modification \n",
    "    inp_dec_emb = tf.nn.embedding_lookup(inp_emb, inp_dec)\n",
    "    \n",
    "    op_tr, op_inf  = decoding_layer(inp_dec_emb, \n",
    "                                                        inp_emb,\n",
    "                                                        op_enc,\n",
    "                                                        st_enc, \n",
    "                                                        v_size, \n",
    "                                                        len_txt, \n",
    "                                                        len_summ, \n",
    "                                                        max_len_summ,\n",
    "                                                        rnsize, \n",
    "                                                        word2int, \n",
    "                                                        dprob, \n",
    "                                                        batch_size,\n",
    "                                                        nlyrs)\n",
    "    \n",
    "    return op_tr, op_inf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "725e92bf-2309-1a78-c771-641a42b440c6",
    "colab": {},
    "colab_type": "code",
    "id": "t6c1ac3qTdWs"
   },
   "outputs": [],
   "source": [
    "def pad_sentences(sentences_batch,word2int):\n",
    "\n",
    "    max_sentence = max([len(sentence) for sentence in sentences_batch])\n",
    "    return [sentence + [word2int[TOKEN_PAD]] * (max_sentence - len(sentence)) for sentence in sentences_batch]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "47e4f70a-6377-68dd-c06c-eed674b2bb3f",
    "colab": {},
    "colab_type": "code",
    "id": "usG7exzOTdWy"
   },
   "outputs": [],
   "source": [
    "def get_batches(summaries, texts, batch_size):\n",
    "\n",
    "    for batch_idx in range(0, len(texts)//batch_size):\n",
    "\n",
    "        start_idx = batch_idx * batch_size\n",
    "\n",
    "        summaries_batch = summaries[start_idx:start_idx + batch_size]\n",
    "        texts_batch = texts[start_idx:start_idx + batch_size]\n",
    "\n",
    "        pad_summaries_batch = np.array(pad_sentences(summaries_batch,word2int))\n",
    "        pad_texts_batch = np.array(pad_sentences(texts_batch,word2intx))\n",
    "\n",
    "        pad_summaries_lens = []\n",
    "        for summary in pad_summaries_batch:\n",
    "            pad_summaries_lens.append(len(summary))\n",
    "        \n",
    "        pad_texts_lens = []\n",
    "        for text in pad_texts_batch:\n",
    "            pad_texts_lens.append(len(text))\n",
    "        \n",
    "        yield pad_summaries_batch, pad_texts_batch, pad_summaries_lens, pad_texts_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7SIkV3M9jYKl"
   },
   "outputs": [],
   "source": [
    "def train_on_batch(session, X, X_seq_len, Y, Y_seq_len, learning_rate, dropout_keep_probability):\n",
    "    feed_dict = {\n",
    "            data_inp: X,\n",
    "            tgts:Y,\n",
    "            lrt:learning_rate,\n",
    "            len_summ:Y_seq_len,\n",
    "            len_txt: X_seq_len,\n",
    "            dprobs: dropout_keep_probability}\n",
    "\n",
    "    _,pred,loss,summary = session.run([\n",
    "            train_op,\n",
    "            tr_cost,\n",
    "            inf_op,\n",
    "            merged_summary_op], feed_dict=feed_dict)\n",
    "    return _,pred, loss, summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6ejZsb2VqY7A"
   },
   "outputs": [],
   "source": [
    "def predict_for_batch(session, X, X_seq_len,Y, Y_seq_len,dropout_keep_probability):\n",
    "\n",
    "    feed_dict={data_inp:X, len_txt:X_seq_len,tgts:Y, len_summ: Y_seq_len,dprobs: dropout_keep_probability}\n",
    "\n",
    "    pred = session.run([inf_op], feed_dict=feed_dict)[0]\n",
    "\n",
    "    return pred\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xCltZfqbQpR1"
   },
   "outputs": [],
   "source": [
    "\n",
    "def predict_for_batch_with_loss(session, X, X_seq_len, Y, Y_seq_len,learning_rate,dropout_keep_probability):\n",
    "    \n",
    "    feed_dict={data_inp:X, len_txt:X_seq_len, tgts:Y, len_summ: Y_seq_len,lrt:learning_rate,dprobs: dropout_keep_probability}\n",
    "    \n",
    "    _,loss,pred = session.run([train_op,tr_cost,infer_predictions], feed_dict=feed_dict)\n",
    "    \n",
    "    return pred, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "77299c4b-a3cf-785b-981a-42a1bb3a2033",
    "colab": {},
    "colab_type": "code",
    "id": "HGM95d9MTdW5"
   },
   "outputs": [],
   "source": [
    "batch_size =32\n",
    "rnn_len = 256\n",
    "n_layers = 1\n",
    "lr = 0.0025\n",
    "dr_prob =0.75\n",
    "logs_path='/content/gdrive/My Drive/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "_cell_guid": "68781626-8bf4-0a23-4bb2-f24a5762fa1e",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 395
    },
    "colab_type": "code",
    "id": "AlrcY77-TdXF",
    "outputId": "585aa013-a75b-4145-8df5-c62c216944ba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-48-08a5f0b64492>:2: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-48-08a5f0b64492>:14: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn.py:464: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:559: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.add_weight` method instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:565: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:575: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn.py:244: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Graph created.\n"
     ]
    }
   ],
   "source": [
    "train_graph = tf.Graph()\n",
    "with train_graph.as_default():\n",
    "    \n",
    "    data_inp, tgts, lrt, dprobs, len_summ, max_len_summ, len_txt = model_inputs()\n",
    "\n",
    "    tr_opp, inf_opp = seq2seq_model(tf.reverse(data_inp, [-1]),\n",
    "                                                      tgts, \n",
    "                                                      dprobs,   \n",
    "                                                      len_txt,\n",
    "                                                      len_summ,\n",
    "                                                      max_len_summ,\n",
    "                                                      len(word2int)+1,\n",
    "                                                      rnn_len, \n",
    "                                                      n_layers, \n",
    "                                                      word2int,\n",
    "                                                      batch_size)\n",
    "    \n",
    "    tr_op = tf.identity(tr_opp.rnn_output, 'tr_op')\n",
    "    inf_op = tf.identity(inf_opp.sample_id, name='predictions')\n",
    "    \n",
    "    seq_masks = tf.sequence_mask(len_summ, max_len_summ, dtype=tf.float32, name='masks')\n",
    "\n",
    "    with tf.name_scope(\"optimizer\"):\n",
    "        tr_cost = sequence_loss(tr_op,tgts,seq_masks)\n",
    "        optzr = tf.train.AdamOptimizer(lrt)\n",
    "        grds = optzr.compute_gradients(tr_cost)\n",
    "        capped_grds = [(tf.clip_by_value(grd, -5., 5.), var) for grd, var in grds \n",
    "                        if grd is not None]\n",
    "        train_op = optzr.apply_gradients(capped_grds)\n",
    "    # Get predictions for evaluation.\n",
    "    train_predictions = tr_opp.sample_id\n",
    "    infer_predictions = inf_opp.sample_id\n",
    "    tf.summary.scalar(\"cost\", tr_cost)\n",
    "print(\"Graph created.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "989d2498-df4e-31e8-3133-56df524b8a28",
    "colab": {},
    "colab_type": "code",
    "id": "Tg98B19nTdXS"
   },
   "outputs": [],
   "source": [
    "def text_to_seq(text):\n",
    "    return [word2int.get(word, word2int[TOKEN_UNK]) for word in text.split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "s4afDgp2zidm"
   },
   "outputs": [],
   "source": [
    "# Program to measure similarity between  \n",
    "# two sentences using cosine similarity. \n",
    "def mean_absolute_error_texts(v1,v2):\n",
    "    meanerrorl2=[]\n",
    "    for X,Y in zip(v1,v2):\n",
    "        # tokenization \n",
    "        X_list = word_tokenize(X)  \n",
    "        Y_list = word_tokenize(Y) \n",
    "  \n",
    "        # sw contains the list of stopwords \n",
    "        sw = stopwords.words('english')  \n",
    "        l1 =[];l2 =[] \n",
    "  \n",
    "        # remove stop words from string \n",
    "        X_set = {w for w in X if not w in sw}  \n",
    "        Y_set = {w for w in Y if not w in sw} \n",
    "  \n",
    "        # form a set containing keywords of both strings  \n",
    "        rvector = X_set.union(Y_set)  \n",
    "        for w in rvector: \n",
    "            if w in X_set:\n",
    "               l1.append(1) # create a vector \n",
    "            else:\n",
    "               l1.append(0) \n",
    "            if w in Y_set:\n",
    "               l2.append(1) \n",
    "            else:\n",
    "               l2.append(0) \n",
    "        c = 0\n",
    "        # cosine formula  \n",
    "        for i in range(len(rvector)): \n",
    "                      c+= l1[i]*l2[i] \n",
    "        den=float((sum(l1)*sum(l2))**0.5)\n",
    "        try: \n",
    "           cosine = c /den\n",
    "        except:\n",
    "           cosine=0.0 \n",
    "        meanerrorl2.append(cosine) \n",
    "        #return LA.norm(meanerrorl2,2)*100.0,\n",
    "    return np.mean(meanerrorl2)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xALwgTyFOtLL"
   },
   "outputs": [],
   "source": [
    "epochs =50\n",
    "min_learning_rate = 0.0006\n",
    "display_step = 50\n",
    "early_stop_cnt = 0 \n",
    "early_stop_cnt_max = 3 \n",
    "per_epoch = 3 \n",
    "\n",
    "\n",
    "update_loss = 0 \n",
    "batch_loss = 0\n",
    "summary_update_loss = [] \n",
    "\n",
    "news_summaries_train = news_summaries_filtered[0:410]\n",
    "news_texts_train = news_texts_filtered[0:410]\n",
    "\n",
    "news_summaries_test =news_summaries_filtered[410:]\n",
    "news_texts_test=news_texts_filtered[410:]\n",
    "\n",
    "goo = word2int[TOKEN_GO]\n",
    "pad = word2int[TOKEN_PAD] \n",
    "eos = word2int[TOKEN_EOS]  \n",
    "unk = word2int[TOKEN_UNK]\n",
    "tokens_id=[eos,pad,unk,goo]\n",
    "\n",
    "update_check = (len(news_texts_train)//batch_size//per_epoch)-1\n",
    "checkpoint = logs_path + 'best_so_far_model.ckpt' \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "_cell_guid": "6368ba0d-4083-e182-ca38-5356b307e09f",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "9hbE8nF6TdXM",
    "outputId": "7d735d79-7a42-44a7-d870-1ea4c171e238"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start training... \n",
      "\n",
      "Average loss: 7.971\n",
      "Saving model\n",
      "Average loss: 4.007\n",
      "Saving model\n",
      "Average loss: 4.04\n",
      "No Improvement.\n",
      "history: [1, 4.04, 4.130271]\n",
      " \n",
      "Test: epoch 1 loss: 4.130271\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 132, 11]\n",
      "  Response Words: no no que .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 82, 11]\n",
      "  Response Words: no no no .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 132, 11]\n",
      "  Response Words: no no que .\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 4.341\n",
      "No Improvement.\n",
      "Average loss: 3.034\n",
      "Saving model\n",
      "Average loss: 3.323\n",
      "No Improvement.\n",
      "history: [2, 3.323, 3.692981]\n",
      " \n",
      "Test: epoch 2 loss: 3.692981\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 132, 11]\n",
      "  Response Words: no no que .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 132, 11]\n",
      "  Response Words: no no que .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 4]\n",
      "  Response Words: no no de\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 3.845\n",
      "No Improvement.\n",
      "Average loss: 2.836\n",
      "Saving model\n",
      "Average loss: 3.094\n",
      "No Improvement.\n",
      "history: [3, 3.094, 3.3838131]\n",
      " \n",
      "Test: epoch 3 loss: 3.3838131\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 132, 11]\n",
      "  Response Words: no no que .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 82, 132, 11]\n",
      "  Response Words: no no no que .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 40, 40, 132, 11]\n",
      "  Response Words: no es es que .\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 3.674\n",
      "No Improvement.\n",
      "Average loss: 2.698\n",
      "Saving model\n",
      "Average loss: 2.934\n",
      "No Improvement.\n",
      "history: [4, 2.934, 3.1686697]\n",
      " \n",
      "Test: epoch 4 loss: 3.1686697\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 82, 132, 132, 11]\n",
      "  Response Words: no no no que que .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 82, 132, 132, 11]\n",
      "  Response Words: no no no que que .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 40, 132, 6]\n",
      "  Response Words: no es que ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 3.453\n",
      "No Improvement.\n",
      "Average loss: 2.542\n",
      "Saving model\n",
      "Average loss: 2.824\n",
      "No Improvement.\n",
      "history: [5, 2.824, 2.9848843]\n",
      " \n",
      "Test: epoch 5 loss: 2.9848843\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 56, 58, 58, 58, 142, 11]\n",
      "  Response Words: no m ho ho ho fer .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 58, 58, 142, 11]\n",
      "  Response Words: no no ho ho fer .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [59, 59, 6]\n",
      "  Response Words: tens tens ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 3.158\n",
      "No Improvement.\n",
      "Average loss: 2.356\n",
      "Saving model\n",
      "Average loss: 2.63\n",
      "No Improvement.\n",
      "history: [6, 2.63, 2.7599735]\n",
      " \n",
      "Test: epoch 6 loss: 2.7599735\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [56, 56, 119, 132, 132, 11]\n",
      "  Response Words: m m anar que que .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 58, 58]\n",
      "  Response Words: no no ho ho\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [59, 30, 318, 6]\n",
      "  Response Words: tens un mobil ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 2.84\n",
      "No Improvement.\n",
      "Average loss: 2.144\n",
      "Saving model\n",
      "Average loss: 2.477\n",
      "No Improvement.\n",
      "history: [7, 2.477, 2.4775298]\n",
      " \n",
      "Test: epoch 7 loss: 2.4775298\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [56, 279, 119, 27, 16, 11]\n",
      "  Response Words: m agradaria anar a l .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 402, 11]\n",
      "  Response Words: no no agrada .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [132, 132, 27, 27, 27, 6]\n",
      "  Response Words: que que a a a ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 2.641\n",
      "No Improvement.\n",
      "Average loss: 1.996\n",
      "Saving model\n",
      "Average loss: 2.347\n",
      "No Improvement.\n",
      "history: [8, 2.347, 2.1967132]\n",
      " \n",
      "Test: epoch 8 loss: 2.1967132\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [277, 56, 98, 16, 11]\n",
      "  Response Words: voldria m la l .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 379, 105, 11]\n",
      "  Response Words: no se menjar .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [59, 183, 76, 142, 27, 6]\n",
      "  Response Words: tens et ha fer a ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 2.348\n",
      "No Improvement.\n",
      "Average loss: 1.702\n",
      "Saving model\n",
      "Average loss: 2.006\n",
      "No Improvement.\n",
      "history: [9, 2.006, 1.9512917]\n",
      " \n",
      "Test: epoch 9 loss: 1.9512917\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [56, 56, 199, 98, 94, 11]\n",
      "  Response Words: m m me la teva .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 379, 402, 11]\n",
      "  Response Words: no se agrada .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [98, 270, 270, 270, 6]\n",
      "  Response Words: la mare mare mare ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 1.91\n",
      "No Improvement.\n",
      "Average loss: 1.438\n",
      "Saving model\n",
      "Average loss: 1.747\n",
      "No Improvement.\n",
      "history: [10, 1.747, 1.6325569]\n",
      " \n",
      "Test: epoch 10 loss: 1.6325569\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [56, 199, 199, 94, 94, 94, 444, 11]\n",
      "  Response Words: m me me teva teva teva germana .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 82, 82, 79, 11]\n",
      "  Response Words: no no no seu .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [132, 355, 318, 6]\n",
      "  Response Words: que faries mobil ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 1.595\n",
      "No Improvement.\n",
      "Average loss: 1.244\n",
      "Saving model\n",
      "Average loss: 1.533\n",
      "No Improvement.\n",
      "history: [11, 1.533, 1.3033313]\n",
      " \n",
      "Test: epoch 11 loss: 1.3033313\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [56, 279, 199, 199, 98, 94, 11]\n",
      "  Response Words: m agradaria me me la teva .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [22, 379, 738, 11]\n",
      "  Response Words: jo se entenc .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [149, 27, 678, 27, 142, 6]\n",
      "  Response Words: per a disposat a fer ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 1.408\n",
      "No Improvement.\n",
      "Average loss: 1.11\n",
      "Saving model\n",
      "Average loss: 1.403\n",
      "No Improvement.\n",
      "history: [12, 1.403, 1.2196302]\n",
      " \n",
      "Test: epoch 12 loss: 1.2196302\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [277, 730, 731, 94, 94, 444, 444, 444, 444, 444, 444]\n",
      "  Response Words: voldria trobar coneixer teva teva germana germana germana germana germana germana\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [22, 737, 738, 79, 11]\n",
      "  Response Words: jo tampoc entenc seu .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [739, 481, 678, 27, 425, 6]\n",
      "  Response Words: algu nevara disposat a allo ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 1.209\n",
      "No Improvement.\n",
      "Average loss: 0.966\n",
      "Saving model\n",
      "Average loss: 1.14\n",
      "No Improvement.\n",
      "history: [13, 1.14, 1.486956]\n",
      " \n",
      "Test: epoch 13 loss: 1.486956\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [56, 279, 199, 98, 94, 11]\n",
      "  Response Words: m agradaria me la teva .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [115, 737, 738, 11]\n",
      "  Response Words: ja tampoc entenc .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [715, 263, 678, 11]\n",
      "  Response Words: vols ve disposat .\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 1.177\n",
      "No Improvement.\n",
      "Average loss: 0.798\n",
      "Saving model\n",
      "Average loss: 1.094\n",
      "No Improvement.\n",
      "history: [14, 1.094, 1.0701553]\n",
      " \n",
      "Test: epoch 14 loss: 1.0701553\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [277, 730, 199, 123, 196, 196, 196, 196, 196, 196, 196]\n",
      "  Response Words: voldria trobar me els gran gran gran gran gran gran gran\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [22, 23, 738, 11]\n",
      "  Response Words: jo tambe entenc .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [739, 132, 678, 142, 142, 142, 425, 6]\n",
      "  Response Words: algu que disposat fer fer fer allo ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 1.011\n",
      "No Improvement.\n",
      "Average loss: 0.758\n",
      "Saving model\n",
      "Average loss: 0.963\n",
      "No Improvement.\n",
      "history: [15, 0.963, 0.90179545]\n",
      " \n",
      "Test: epoch 15 loss: 0.90179545\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [56, 730, 98, 98, 94, 11]\n",
      "  Response Words: m trobar la la teva .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [22, 737, 738, 289, 11]\n",
      "  Response Words: jo tampoc entenc res .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [739, 183, 678, 27, 425, 425, 6]\n",
      "  Response Words: algu et disposat a allo allo ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 0.869\n",
      "No Improvement.\n",
      "Average loss: 0.595\n",
      "Saving model\n",
      "Average loss: 0.816\n",
      "No Improvement.\n",
      "history: [16, 0.816, 0.67093116]\n",
      " \n",
      "Test: epoch 16 loss: 0.67093116\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [277, 277, 731, 44, 444, 444, 444, 11]\n",
      "  Response Words: voldria voldria coneixer amb germana germana germana .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [22, 737, 738, 289, 11]\n",
      "  Response Words: jo tampoc entenc res .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [739, 481, 678, 27, 142, 425, 425, 11]\n",
      "  Response Words: algu nevara disposat a fer allo allo .\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 0.612\n",
      "No Improvement.\n",
      "Average loss: 0.491\n",
      "Saving model\n",
      "Average loss: 0.654\n",
      "No Improvement.\n",
      "history: [17, 0.654, 0.58363515]\n",
      " \n",
      "Test: epoch 17 loss: 0.58363515\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [56, 279, 199, 98, 94, 196, 196, 11]\n",
      "  Response Words: m agradaria me la teva gran gran .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [22, 737, 738, 289, 11]\n",
      "  Response Words: jo tampoc entenc res .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [739, 27, 678, 27, 142, 6]\n",
      "  Response Words: algu a disposat a fer ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 0.522\n",
      "No Improvement.\n",
      "Average loss: 0.364\n",
      "Saving model\n",
      "Average loss: 0.511\n",
      "No Improvement.\n",
      "history: [18, 0.511, 0.5226889]\n",
      " \n",
      "Test: epoch 18 loss: 0.5226889\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [56, 279, 731, 44, 94, 94, 444, 11]\n",
      "  Response Words: m agradaria coneixer amb teva teva germana .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [22, 737, 738, 289, 11]\n",
      "  Response Words: jo tampoc entenc res .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [739, 101, 678, 27, 142, 425, 425, 6]\n",
      "  Response Words: algu esta disposat a fer allo allo ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 0.419\n",
      "No Improvement.\n",
      "Average loss: 0.345\n",
      "Saving model\n",
      "Average loss: 0.395\n",
      "No Improvement.\n",
      "history: [19, 0.395, 0.4705306]\n",
      " \n",
      "Test: epoch 19 loss: 0.4705306\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [277, 199, 98, 98, 444, 196, 11]\n",
      "  Response Words: voldria me la la germana gran .\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [22, 737, 738, 11]\n",
      "  Response Words: jo tampoc entenc .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [739, 101, 678, 142, 142, 6]\n",
      "  Response Words: algu esta disposat fer fer ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Average loss: 0.392\n",
      "No Improvement.\n",
      "Average loss: 0.368\n",
      "No Improvement.\n",
      "history: [20, 0.368, 0.58033085]\n",
      " \n",
      "Test: epoch 20 loss: 0.58033085\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 241, 242, 138, 651, 81, 652, 399, 7, 3, 3]\n",
      "  Input Words: i d like to meet your older sister .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [56, 279, 731, 199, 44, 196, 196, 196, 196, 196, 196]\n",
      "  Response Words: m agradaria coneixer me amb gran gran gran gran gran gran\n",
      " Ground Truth: m agradaria coneixer la teva germana gran .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 24, 11, 24, 69, 70, 658, 466, 7, 3, 3]\n",
      "  Input Words: i , too , didn t understand anything .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [22, 737, 738, 289, 289, 289, 11]\n",
      "  Response Words: jo tampoc entenc res res res .\n",
      " Ground Truth: jo tampoc entenc res .\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 67, 659, 84, 598, 138, 143, 215, 5, 3, 3, 992]\n",
      "  Input Words: is anybody here willing to do that ? <PAD>\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [739, 101, 678, 27, 142, 425, 6]\n",
      "  Response Words: algu esta disposat a fer allo ?\n",
      " Ground Truth: algu esta disposat a fer allo ?\n",
      "\n",
      "Stopping Training.\n"
     ]
    }
   ],
   "source": [
    "invalid_number_prediction_counts = []\n",
    "all_model_predictions = []\n",
    "all_ground_truth = []\n",
    "history=[]\n",
    "print('Start training... \\n')\n",
    "\n",
    "with tf.Session(graph=train_graph) as sess:\n",
    "    tf_summary_writer = tf.summary.FileWriter(logs_path, graph=train_graph)\n",
    "    merged_summary_op = tf.summary.merge_all()\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for epoch_i in range(epochs):\n",
    "        update_loss = 0\n",
    "        batch_loss = 0\n",
    "        train_loss =0\n",
    "        for batch_i, (summaries_batch, texts_batch, summaries_len, texts_len) in enumerate(\n",
    "                get_batches(news_summaries_train, news_texts_train,batch_size)):\n",
    "            before = time.time()\n",
    "            _,loss,predictions,summary=train_on_batch(sess,texts_batch, texts_len, summaries_batch, summaries_len, lr, dr_prob)\n",
    "          \n",
    "            batch_loss += loss\n",
    "            update_loss += loss\n",
    "            after = time.time()\n",
    "            batch_time = after - before\n",
    "            tf_summary_writer.add_summary(summary, epoch_i * batch_size + batch_i)\n",
    "            \n",
    "            if batch_i % display_step == 0 and batch_i > 0:\n",
    "                print('** Epoch {:>3}/{} Batch {:>4}/{} - Batch Loss: {:>6.3f}, seconds: {:>4.2f}'\n",
    "                      .format(epoch_i+1,\n",
    "                              epochs, \n",
    "                              batch_i, \n",
    "                              len(news_texts_filtered) // batch_size, \n",
    "                              batch_loss / display_step, \n",
    "                              batch_time*display_step))\n",
    "                batch_loss = 0\n",
    "            \n",
    "            if batch_i % update_check == 0 and batch_i > 0:\n",
    "                print(\"Average loss:\", round(update_loss/update_check,3))\n",
    "                train_loss=round(update_loss/update_check,3)\n",
    "                summary_update_loss.append(update_loss)\n",
    "                \n",
    "                if update_loss <= min(summary_update_loss):\n",
    "                    print('Saving model') \n",
    "                    early_stop_cnt = 0\n",
    "                    saver = tf.train.Saver() \n",
    "                    saver.save(sess, checkpoint)\n",
    "\n",
    "                else:\n",
    "                    print(\"No Improvement.\")\n",
    "                    early_stop_cnt += 1\n",
    "                    if early_stop_cnt == early_stop_cnt_max:\n",
    "                        break\n",
    "                update_loss = 0\n",
    "\n",
    "        summaries_test_batch, texts_test_batch, summaries_test_len, texts_test_len =\\\n",
    "         next(get_batches(news_summaries_test, news_texts_test,batch_size))\n",
    "    \n",
    "        predictions,test_loss = predict_for_batch_with_loss(sess,texts_test_batch, texts_test_len,summaries_test_batch,summaries_test_len,lr,1.0)\n",
    "        history.append([epoch_i+1,train_loss,test_loss])\n",
    "        print('history:',history[-1])\n",
    "        print(\" \")  \n",
    "        print('Test: epoch', epoch_i+1, 'loss:', test_loss,)\n",
    "        print(\" \")\n",
    "        for text, summary, result_logits  in list(zip(texts_test_batch, summaries_test_batch, predictions))[-3:]: \n",
    "                     print('\\nText')\n",
    "                     print('  Word Ids:    {}'.format([i for i in text]))\n",
    "                     print('  Input Words: {}'.format(\" \".join( int2wordx[i] for i in text if i not in tokens_id)))\n",
    "\n",
    "                     print('\\nSummary')\n",
    "                     print('  Word Ids:       {}'.format([i for i in result_logits if i not in tokens_id]))\n",
    "                     print('  Response Words: {}'.format(\" \".join(int2word[i] for i in result_logits if i not in tokens_id )))\n",
    "                     print(' Ground Truth: {}'.format(\" \".join(int2word[i] for i in summary if i not in tokens_id)))\n",
    "                     print('')\n",
    "        model_predictions = []\n",
    "        ground_truth = []\n",
    "        invalid_number_prediction_count = 0\n",
    "\n",
    "        ind=0\n",
    "        for summaries_test_batch, texts_test_batch, summaries_test_len, texts_test_len\\\n",
    "         in get_batches(news_summaries_test,news_texts_test,batch_size=batch_size):\n",
    "                 \n",
    "  \n",
    "                 #summaries_test_len=[np.random.randint(5,8)]\n",
    "                 pred=predict_for_batch(sess, texts_test_batch, texts_test_len,summaries_test_batch,summaries_test_len,1.0)   \n",
    "                 \n",
    "                 for y,p in list(zip(summaries_test_batch,pred)): \n",
    "                     Y=''.join(int2word[i] for i in y if i not in tokens_id)\n",
    "                     O=''.join(int2word[i] for i in p if i not in tokens_id)\n",
    "                     \n",
    "                     if mean_absolute_error_texts([Y],[O])< 5 or len(O)< len(Y): \n",
    "                             invalid_number_prediction_count += 1\n",
    "                     \n",
    "                     lmin=min(len(Y),len(O))+1\n",
    "                     OO=O[:lmin]\n",
    "                     YY=Y[:lmin] \n",
    "                     model_predictions.append(O)\n",
    "                     ground_truth.append(Y)\n",
    "        all_model_predictions.append(model_predictions)\n",
    "        all_ground_truth.append(ground_truth)\n",
    "        invalid_number_prediction_counts.append(invalid_number_prediction_count)\n",
    "         \n",
    "        if early_stop_cnt == early_stop_cnt_max:\n",
    "            print(\"Stopping Training.\")\n",
    "            break        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 265
    },
    "colab_type": "code",
    "id": "wErDv9CAuhdy",
    "outputId": "e9192515-2585-4dc1-d0d3-21cb82b64b27"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3hU1dbH8e9KJ0AIJPQAoVepoYMCIlUpShOpForlWq7Yrlevvnpt14aKiooUFZEqUqQovZrQmxBCIKGG0JKQhJT9/nEGCTGNlJlksj7PM09m5uxzZmUcf5zs2WdvMcaglFKq6HNxdAFKKaXyhwa6Uko5CQ10pZRyEhroSinlJDTQlVLKSbg56oX9/f1NYGCgo15eKaWKpJCQkPPGmPIZbXNYoAcGBhIcHOyol1dKqSJJRI5ntk27XJRSyklooCullJPQQFdKKSfhsD50pZTKjaSkJCIjI0lISHB0KQXKy8uLgIAA3N3dc7yPBrpSqkiJjIykdOnSBAYGIiKOLqdAGGOIjo4mMjKSmjVr5ng/7XJRShUpCQkJ+Pn5OW2YA4gIfn5+t/xXiAa6UqrIceYwvy43v2PRC/Qrp2D5C5CS5OhKlFKqUCl6gX4yBLZ9DuvecXQlSqli6NKlS0yZMuWW9+vTpw+XLl0qgIpuKHqB3vAeaP4AbHgfIrY7uhqlVDGTWaAnJydnud+yZcvw9fUtqLKAohjoAL3eBp8AWDAOEmMdXY1Sqhh54YUXOHr0KM2bN6d169Z07tyZfv360ahRIwAGDBhAq1ataNy4MVOnTv1rv8DAQM6fP094eDgNGzbkkUceoXHjxvTo0YP4+Ph8qa1oDlv08oF7v4Rv+8CKl6DfZEdXpJRygNd+2c+BU1fy9ZiNqvjw6j2NM93+9ttvs2/fPnbt2sXatWvp27cv+/bt+2t44bRp0yhXrhzx8fG0bt2a++67Dz8/v5uOceTIEWbPns1XX33FkCFDmD9/PiNGjMhz7Tk+QxcRVxHZKSJLMtjmKSJzRCRURLaJSGCeK8vEhiNR9PpoPZfKB0HHJ2HHDDi0rKBeTimlstSmTZubxopPnjyZZs2a0a5dOyIiIjhy5Mjf9qlZsybNmzcHoFWrVoSHh+dLLbdyhv4kcBDwyWDbQ8BFY0wdERkGvAMMzYf6/savpCeHzsTwU3AE47q+BKG/weInIKA1lMpwRkmllJPK6kzaXkqWLPnX/bVr17J69Wq2bNmCt7c3Xbp0yXAsuaen51/3XV1d863LJUdn6CISAPQFvs6kSX9ghu3+POBOKaCBoo2q+NAmsBwztxwnxcUD7p0KiTFWqBtTEC+plFJ/KV26NDExMRluu3z5MmXLlsXb25tDhw6xdetWu9aW0y6Xj4DngNRMtlcFIgCMMcnAZcAvfSMRGSciwSISHBUVlYtyLaM7BBJ5MZ41h85BxUbQ/VU4vBx2zMz1MZVSKif8/Pzo2LEjTZo0YdKkSTdt69WrF8nJyTRs2JAXXniBdu3a2bU2Mdmc1YrI3UAfY8yjItIFeNYYc3e6NvuAXsaYSNvjo0BbY8z5zI4bFBRkcrvARVJKKp3fWUPdiqWY9VBbSE2FWf0hMgQmbAC/2rk6rlKq8Dt48CANGzZ0dBl2kdHvKiIhxpigjNrn5Ay9I9BPRMKBH4FuIvJdujYngWq2F3MDygDRt1Z6zrm7ujCiXXU2HDlP6LlYcHGBAZ+DqxssnAApWY8HVUopZ5RtoBtjXjTGBBhjAoFhwO/GmPTjaxYDo233B9naFGiH9rA21fFwdWHmlnDriTIB0PcDiNwOGz8syJdWSqlCKdcXFonI6yLSz/bwG8BPREKBZ4AX8qO4rPiX8uTuppWZHxJJTIJtXpfbBkGT+2Dd23ByR0GXoJRShcotBboxZu31/nNjzCvGmMW2+wnGmMHGmDrGmDbGmLCCKDa90R0CibuWwryQyBtP9n0fSlawriK9dtUeZSilVKFQNC/9t2lWzZfm1XyZteU4qam2Hp4SZWHg5xB9BFa/6tgClVLKjop0oAOM6RBI2Pk4NoSmGVBTqwu0exS2T4XQ1Y4qTSml7KrIB3qf2yrjX8qTGZvDb95w5ytQvgEsegyuXnBIbUop55Pb6XMBPvroI65eLbiu4CIf6B5uLgxvW501f57jeHTcjQ3uJayrSK9Gw5Kn9CpSpVS+0EAvYA+0rY6rCDO3HL95Q+Vm0PUlOPAz7JnjmOKUUk4l7fS5kyZN4r333qN169Y0bdqUV1+1vreLi4ujb9++NGvWjCZNmjBnzhwmT57MqVOn6Nq1K127di2Q2orm9LnpVPTxoleTSvwUHMEzd9WjpGeaX6vjk3BkJSybBDU6gG91xxWqlMpfy1+AM3vz95iVboPeb2e6Oe30uStXrmTevHls374dYwz9+vVj/fr1REVFUaVKFZYuXQpYc7yUKVOGDz74gDVr1uDv75+/Nds4xRk6WF+OxiQks2jXyZs3uLjCwC/ApMLCiZCa4pgClVJOZ+XKlaxcuZIWLVrQsmVLDh06xJEjR7jttttYtWoVzz//PBs2bKBMmTJ2qccpztABWtUoS+MqPszYHM7wNtVvXjG7bCD0fgd+fgy2fAYd/+GwOpVS+SiLM2l7MMbw4osvMn78+L9t27FjB8uWLePll1/mzjvv5JVXXinwepzmDF1EGN0hkMNnY9kSlsE0Ms0fgAZ3w+//B2f22b9ApZRTSDt9bs+ePZk2bRqxsdZSmCdPnuTcuXOcOnUKb29vRowYwaRJk9ixY8ff9i0IThPoAP2aVaGst/vfhzACiMA9H4OXLyx4RK8iVUrlStrpc1etWsXw4cNp3749t912G4MGDSImJoa9e/fSpk0bmjdvzmuvvcbLL78MwLhx4+jVq1eBfSma7fS5BSUv0+dm5Z1fD/HluqOsf64rAWW9/97gyCr4YQgEdobhc6zhjUqpIkOnz83b9LlFygNtrVEs3209kXGDundB/ylwbD38OByS/r48lFJKFUVOF+gBZb25q1FFfvzjBAlJmYxoaX4/9P8Ujq6BOQ9AcqJ9i1RKqQLgdIEO1iyMl64msXj3qcwbtRhh9amHroY5IzXUlSpCHNVVbE+5+R2dMtDb1/KjfsXSzNgcnvWb0mo03P0hHFkBc8dA8jW71aiUyh0vLy+io6OdOtSNMURHR+Pl5XVL+znNOPS0RIRRHWrwr4X7CDl+kaDAcpk3DnrQutho2bMwbywMng6u7narVSl1awICAoiMjCQvC80XBV5eXgQEBNzSPk4Z6AADW1Tl7eWHmL45POtAB2jziHUl6fLnYP5DcN83GupKFVLu7u7UrFnT0WUUStl2uYiIl4hsF5HdIrJfRF7LoM0YEYkSkV2228MFU27OeXu4MSSoGr/uO8PZKzkYydJ2PPT8rzWR14JxutC0UqrIyUkfeiLQzRjTDGgO9BKRdhm0m2OMaW67fZ2vVebSqPY1SDGG77cez74xQPvH4K7/g/0LYNEEnfdFKVWkZBvoxhJre+huuxWJbyNq+JWka/0K/LD9BInJOQznjv+A7v+BvXNh0aMa6kqpIiNHo1xExFVEdgHngFXGmG0ZNLtPRPaIyDwRqZbJccaJSLCIBNvrC43RHQI5H3uN5XvP5HynTk9Dt5dhz4+w+AlITS24ApVSKp/kKNCNMSnGmOZAANBGRJqka/ILEGiMaQqsAmZkcpypxpggY0xQ+fLl81J3jnWu408t/5JMz2h+l6zcPgm6vAS7vodf/qGhrpQq9G5pHLox5hKwBuiV7vloY8z1K3O+BlrlT3l55+IijGpfg10Rl9gdcenWdu7yPNz+HOycBUuf1lBXShVqORnlUl5EfG33SwB3AYfStamc5mE/4GB+FplX97UKoKSHa8azMGan60vQ6RkImW6NVXfiixmUUkVbTs7QKwNrRGQP8AdWH/oSEXldRPrZ2vzDNqRxN/APYEzBlJs7pb3cua9VAEv2nOZ87C1e4i8Cd75iLWUX/I01Vl1DXSlVCGV7YZExZg/QIoPnX0lz/0XgxfwtLX+Nah/IzC3H+XH7CR7vVvfWdhaB7q9ZI162fAriCr3esp5XSqlCwinncslInQql6FzXn++2niApJRd94SLQ4w1oOxG2fQ7fD4KYWxg5o5RSBazYBDrA6PaBnLmSwMr9Z3N3ABHrzLzv+xC+CT7vAIeW5m+RSimVS8Uq0Ls2qEC1ciVy9+XodSLQ+mEYvw7KBFiLZCx+AhJjs99XKaUKULEKdFcXYVS7QLaHX+CT346QmpqHLzfL14eHVlsXIe2YBV92hsiQ/CtWKaVuUbEKdICR7WvQv3kV3l91mNHfbr/1US9puXlY0wSMWQIpSfDNXbDuXZ3YSynlEMUu0L3cXfloaHPeuvc2th27QN/JG9gWFp23gwZ2ggkbocm9sOZNmN4HLobnS71KKZVTxS7QwVoA4/421Vn0aEdKerhx/1db+WxNaN66YEr4wn1fw71fw7mD8Hkn2PWDjllXStlNsQz06xpV8WHxE524u2kV3lvxJ6O/3U50XrpgAJoOhomboHJTWDTRWtru6oV8qVcppbJSrAMdoJSnGx8Pa85/B1pdMH3yowvGtzqM/gXufBUOLYHPO0LY2nypVymlMlPsAx2sLpjhba0uGO/86oJxcYXOz8DDv4FHSZjZH1b8C5Lz+BeAUkplQgM9jUZVfPjliU70tXXBjJ3+R967YKo0h/HrrbHrWz6Fr7rB2QP5U7BSSqWhgZ5OKU83Jtu6YLaERdN38kb+CM9jH7iHt3V16fCfIPYsTO0Ca9+GpBysdaqUUjmkgZ6B610wCx/tgJe7C8OmbmXK2jx2wQDU6wkTN0ODvrD2LZjSFg6vzJ+ilVLFngZ6FhpXKcMvT3Sid5NKvPur1QVzIe5a3g5aqgIM/hZG/QyuHvDDYJg9HC7mcCFrpZTKhAZ6Nkp7ufPJ/S14Y0ATtoRF0+fjDXnvggGo1QUmbLKuNA1bA5+1hfXv6ZemSqlc00DPARFhRLsaLJhodcEM/XILz83bzclL8Xk7sJuHNRfM439AvR7w+xswpT2E/pY/hSulihUxDrqSMSgoyAQHBzvktfMiJiGJD1cd4butVhfJA+2q81jXOviX8sz7wUNXw7Ln4MJRaNjPmqq3TEDej6uUchoiEmKMCcpwW3aBLiJewHrAE2uFo3nGmFfTtfEEZmItDh0NDDXGhGd13KIa6NedvBTP5NVHmBsSgZe7Kw91qskjt9fCx8s9bwdOToTNn8D6/1lT9d7xHLR7zDqbV0oVe3kNdAFKGmNiRcQd2Ag8aYzZmqbNo0BTY8wEERkGDDTGDM3quEU90K87GhXLB6sOs3TPacqUcGdil9qMbh9ICQ/XvB344nFY8ZJ1palfXej7P6vfXSlVrGUV6Nn2oRvL9dUb3G239P8K9Adm2O7PA+60/UPg9GqXL8Vnw1uy5IlOtKjuy9vLD3HHe2uYtfU415JzsdTddWVrwLDvYfhcSE22rjSdOxaunMq/4pVSTiVHX4qKiKuI7ALOAauMMdvSNakKRAAYY5KBy4BffhZa2DWpWobpY9vw0/j21PDz5t+L9tH9g3Us3BlJSl7Gr9frAY9uhS4vwZ/L4NPWsGmyzrmulPqbHAW6MSbFGNMcCADaiEiT3LyYiIwTkWARCY6KisrNIQq9NjXL8dP49nw7tjWlvdx4es5uen+8nhX7z5DrL6DdvaDL81awB3aCVf+GGffA5ZP5W7xSqki75VEuIvIKcNUY8780z60A/mOM2SIibsAZoLzJ4uDO0oeeldRUw/J9Z3h/1Z+ERcXRrJovz/WsT8c6/nk78O45sORpcPOEAZ9D/V75U7BSqtDLUx+6iJQXEV/b/RLAXcChdM0WA6Nt9wcBv2cV5sWFi4vQt2llVj51O+/e15SoKwk88PU27p+6laV7TpOQlJK7Azcbak34VaYqzB4Kv74IyXm8glUpVeTlZJRLU6wvPF2x/gH4yRjzuoi8DgQbYxbbhjbOAloAF4BhxpiwrI5bHM7Q00tMTuGHbSf4cl0YZ64kUNrTjT63VWZAi6q0rVkOF5db/B45KcHqftk+FSo3t6YUKFerYIpXShUKeRq2WFCKY6Bfl5Jq2BoWzcKdJ1m+9zRx11KoUsaL/i2qMrBFVepVLH1rBzz4C/z8GKSmwj0fwW2DCqZwpZTDaaAXYvHXUlh18CwLd0Sy/sh5UlINjav4MLBFVfo1q0IFH6+cHejSCZj/MERsg5ajoNc71rS9SimnooFeRJyPTWTJ7lMs3HmS3ZGXcRHoWMefgS2q0rNxJUp6umV9gJQkWPNf2PghlG9gdcFUaGif4pVSdqGBXgQdjYrl550nWbjrJBEX4inh7krPxhUZ0KIqner44+aaxffZR3+HBeMgMRZ6vw0tR1vTCCilijwN9CLMGEPI8Yss2HmSpXtOczk+ifKlPZnUsz6DWwWQ6QW5MWdh4ThrcerG98I9H4OXj11rV0rlPw10J5GYnMLaP6OYuj6MkOMX6VDbj/8OvI1A/5IZ75CaChs/sLphfKvBoG+hakv7Fq2Uyld5GoeuCg9PN1d6Nq7E3PHteWNAE/ZGXqbnR+uZsjaUpJQM5o1xcYHbn4Wxy6ypAr7pAVs+A71EQCmnpIFeBLm4WAturHrmDrrUL8+7v/5Jv083sTviUsY7VG8HEzZA3R7WDI4/DIX4TNoqpYosDfQirFIZL74cGcQXI1oRHZvIwCmbeP2XA8QlZjBxl3c5a/bG3u9aX5rOuBtiz9m/aKVUgdFAdwK9mlRi9T/vYHjb6kzbdIweH65nzaEMwloE2o6H4T9C9FGY1lMXp1bKiWigOwkfL3feGHAb8ya0p4SHK2On/8ETs3dyPjaDRafrdIeRi+BqtBXq59JPzaOUKoo00J1MUGA5lv6jE091r8uv+05z5/vr+Ck44u9T91ZvC2OXg0mFb3tBZIhjClZK5RsNdCfk6ebKU93rsfzJztStUIrn5u3hga+3EX4+7uaGFRvDgyvAq4w1v/rRNY4pWCmVLzTQnVidCqX5KbshjuVqWqFeNhB+GAIHfnZYvUqpvNFAd3IZDXG855ONrDscdaMbpnQlGLvUmoJ37hgImZHlMZVShZMGejGRdohjTEIyo6dt577PN7P+erCXKAujFkGtrvDLP2DjR44uWSl1i/TS/2LoWnIqc0Mi+Oz3UE5dTqBldV+e6l6PznX9kZQkWDge9i+Ajk9C99d0Yi+lCpGsLv3PZj5W5Yw83Fx4oG0NBrUKYG5wJJ+tCWXUtO20qlGWp7rXpdO9XyElfGHTxxB/Ee7+CFxcHV22UiobGujFmKebKyPa1WBwUAA/BUcyZU0oI7/ZTlCNsjx150t0LFEO2fA/SLgM935lLUqtlCq0crJIdDURWSMiB0Rkv4g8mUGbLiJyWUR22W6vFEy5qiB4urkysl0N1k7qwv/1b0zkxXhGTNvOkCN3cqzlS9bIlx+GWvOrK6UKrZx8KZoM/NMY0whoBzwmIo0yaLfBGNPcdns9X6tUduHp5srI9oGse64Lr/dvTMSFeLpubsJnPs9gjq3DzOwPVy84ukylVCayDXRjzGljzA7b/RjgIFC1oAtTjuPp5sqo9oGsndSF1/o1ZmZCR8YnPknyyd1c/bIHXDnl6BKVUhm4pWGLIhIItAC2ZbC5vYjsFpHlItI4k/3HiUiwiARHRUXdcrHKvrzcXRndIZB1k7rSoe9o/uH6L1IvRXDu4zvYvH0bKak6r7pShUmOhy2KSClgHfCmMWZBum0+QKoxJlZE+gAfG2PqZnU8HbZY9CQkpbBq1XI6bZ9IsoEXSvyH9h27MjioGmVKuDu6PKWKhTwvQSci7sASYIUx5oMctA8Hgowx5zNro4FedCWdPUTy9AGYhMuMTfgne9yaMLBlVcZ0CKRexdKOLk8pp5anJejEWoX4G+BgZmEuIpVs7RCRNrbjRue+ZFWYuVdsQIkJq/H2C2C297s8X/Mo80Ii6fHheoZ/tZUV+89od4xSDpDtGbqIdAI2AHuB67M6vQRUBzDGfCEijwMTsUbExAPPGGM2Z3VcPUN3AnHR8P0gOL2b2F4fMjO+I99tOc6pywlU9S3ByPY1GNa6Gr7eHo6uVCmnkecul4Kgge4kEmNgzggIWws93iS57aOsPniW6ZvD2Rp2AU83Fwa2qMroDoE0rOzj6GqVKvI00FXBSk6EBY9YFyB1egbufAVEOHTmCjM2H2fhzkgSklJpU7McYzoE0qNRRdxcdV44pXJDA10VvNQUWPpPCPkWWo6Guz/8a/6XS1ev8VNwBDO3HCfyYjxVynjx+YhWNKvm6+CilSp68vSlqFI54uJqhXjnZ2HHDGte9WRrPVNfbw/G3V6bdZO68tWoIFxdhbHT/+BY+hWUlFJ5ooGu8o8I3Plv6PlfOLgYvh9s9bHbuLoIdzWqyMwH2wIwato2omIyWMRaKZUrGugq/7V/DAZ8AeEbrbVK424ewVrTvyTTxrTmfMw1xk7fTmxisoMKVcq5aKCrgtH8fhj2PZw7CNN6wqWImzdX82XKAy05eDqGid+FcC05NZMDKaVySgNdFZz6vWHkQog9a4V61J83be7aoAJv3XsbG46c54X5e3DUF/RKOQsNdFWwanSAMUshJQmm9YKTITdtHhJUjX/eVY8FO0/yzq9/ZnIQpVROaKCrgle5KTy0AjxLw/R74OiamzY/3q0OD7StzhfrjjJ90zEHFalU0aeBruyjXC14aCWUDYQfhsCu2X9tEhFe79+EHo0q8tqSAyzdc9pxdSpVhGmgK/spXQnGLoVqbWHRBFj+gtUVgzWkcfL9LWhZvSxPz9nF1jCd202pW6WBruyrRFkYuQjaPQrbPodZA/8a1ujl7so3o4OoVq4Ej8wM5tCZKw4uVqmiRQNd2Z+rG/R6yxqrHrEdpnaB03sA66rSGQ+2wdvDlTHT/uDUpXjH1qpUEaKBrhyn+f3w4K9gUuCbHrB3HgABZb2ZPrYNcYnJjJ62nctXkxxcqFJFgwa6cqyqLWHcWqjSHOY/BKtegdQUGlb24ctRrTgefZVHZgaTkJTi6EqVKvQ00JXjlaoAoxZD64dh08fWohlXL9Chtj/vD2nG9vALPPXjLl0FSalsaKCrwsHNA/q+D/dMhmMb4KtucPYA9zSrwr/vbsSv+8/w2i/79WpSpbKQkzVFq4nIGhE5ICL7ReTJDNqIiEwWkVAR2SMiLQumXOX0Wo2GscsgKR6+7g4HFvNQp5qMu70WM7ccZ8rao46uUKlCKydn6MnAP40xjYB2wGMi0ihdm95AXdttHPB5vlapipdqbax+9QoN4aeR8PsbvNCzHv2bV+G9FX8yLyTS0RUqVSi5ZdfAGHMaOG27HyMiB4GqwIE0zfoDM4319/BWEfEVkcq2fZW6dT6VrTP1pc/A+vdwObOX9/p9wfnYRJ6fvwd3V6F/86qOrlKpQuWW+tBFJBBoAWxLt6kqkHZ+1Ejbc+n3HyciwSISHBUVdWuVquLHzRP6fQp9/gehq/H49i6m9vahVY2yPPnjLj5afVj71JVKI8eBLiKlgPnAU8aYXF3CZ4yZaowJMsYElS9fPjeHUMWNCLR5xBoFE3+RkjN68H3nC9zXMoCPVh/h6Tm7dEijUjY5CnQRcccK8++NMQsyaHISqJbmcYDtOaXyR2BHGL8O/Grj/tNw/ldrJ5N61mfRrlOM+Hob0bG6lJ1SORnlIsA3wEFjzAeZNFsMjLKNdmkHXNb+c5XvygRYV5bWuQtZ8hSPlQvh0+Et2HvyMgOmbCL0XEz2x1DKieXkDL0jMBLoJiK7bLc+IjJBRCbY2iwDwoBQ4Cvg0YIpVxV77iVg6CwI7ASLJnK3WzA/jmtH/LVUBk7ZzMYj5x1doVIOI476UikoKMgEBwc75LWVE0iMtWZqPLUT7p9NpH9HHpoeTGhULP/XvwnD21Z3dIVKFQgRCTHGBGW0Ta8UVUWTZyl4YC5UbARzRhBwKYR5E9vTqY4/Ly3cy5tLD+hUAarY0UBXRVcJXxixEMrWhB+GUjpqJ9+MDmJ0+xp8teEY42eFEJeY7OgqlbIbDXRVtJX0g1GLoHRF+G4Qbmf38Fr/Jvznnkb8fugsg7/YwunLOqe6Kh400FXRV7qSNU7dy8fqVz93kDEda/LN6NYcj45jwGeb2HfysqOrVKrAaaAr5+BbDUYvBlcPmNkfoo/StUEF5k3sgKsIg7/Ywsr9ZxxdpVIFSgNdOY9ytWDUz5CaDDP6waUTNKzsw6LHO1KvYinGfxfCV+vDdLoA5bQ00JVzqdAARi6EazFWqF85TYXSXvw4rj29m1TizWUHeWnhXq4lpzq6UqXynQa6cj6Vm8ED8yEuyup+iTtPCQ9XPr2/JY91rc3s7RHc+/kmQs/FOrpSpfKVBrpyTtVaw/A5cOk4zBoA8ZdwcREm9WzAFyNacvJiPHd/soFZW8K1C0Y5DQ105bwCO8HQ7+HcIWud0kRrrpdeTSqz4qnbaVPTj3//vJ+x0//gXEyCg4tVKu800JVzq9sdBk+Hkztg9v3W0nZABR8vZoxtzWv9GrPlaDS9PtrACh0Fo4o4DXTl/BreDfdOhfCNMGcEJFtT7YoIozsEsuSJTlTy8WL8rBBemL9Hry5VRZYGuioebhsE/SZD6GqY9yCkJP21qW7F0ix6rCMTu9RmTnAEfSZvYMeJiw4sVqnc0UBXxUfLUdD7XTi0BH5+HFJvDF30cHPh+V4N+PGRdiSnGAZ/sYUPVx0mKUWHN6qiQwNdFS9tx0PXf8GeH2HFS5BuhEvbWn4sf6oz/ZtV4ePfjjD4iy0cOx/noGKVujUa6Kr4uX0StJ0I2z6H9e/9bbOPlzsfDG3Op8NbcOx8HH0+3sDs7Sd0eKMq9DTQVfEjAj3/C83uhzVvwvavMmx2d9Mq/PpUZ1rW8OXFBXt5ZGYI53XtUlWI5WRN0Wkick5E9mWyvYuIXE6zPN0r+V+mUvnMxQX6fQr1+8CySbB3XobNKpcpwawH2/Jy34asPxJFr4/W8/uhs3YuVqmcyckZ+nSgVzZtNhhjmttur+e9LKXswNUNBn0LNTrCwvFweGWGzVxchIc712Lx4x3xL9oqUnUAABYVSURBVOXJg9ODeeDrrazcf0ZXRVKFSraBboxZD1ywQy1K2Z+7F9w/Gyo2hp9GwfEtmTZtUMmHnx/vyPO9GhAWFce4WSHc8d4apq4/yuWrSZnup5S95GiRaBEJBJYYY5pksK0LMB+IBE4Bzxpj9mdynHHAOIDq1au3On78eG7rVip/xUbBt72sn2OXQqXbsmyenJLKygNnmb45nO3HLuDl7sLAFlUZ3SGQBpV8Cq7OhCuweTI0H25NF6yKnawWic6PQPcBUo0xsSLSB/jYGFM3u2MGBQWZ4ODgbF9bKbu5FAHTeloXHT34K/jVztFuB05dYeaWcBbtOklCUirtapVjTIdAujesiJtrPo47uHIKvh8MZ/dBhcbwyG/gXiL/jq+KhKwCPc+fNmPMFWNMrO3+MsBdRPzzelyl7M63mjWXemqyNUPjlVM52q1RFR/evq8pW1+8kxd7NyDiQjwTvtvBHe+t5fO1R7kYdy3vtZ3dD193h4vhcPtzcG4/rHw578dVTiXPgS4ilUREbPfb2I4ZndfjKuUQ5evDiPlw9QLMutf6mUO+3h6Mv6M265/rypcjW1HDz5t3fj1Eu7d+47l5u9l/KpfrmoathWm9wKTC2OXQ7V/Q/nH442s4+EvujqmcUrZdLiIyG+gC+ANngVcBdwBjzBci8jgwEUgG4oFnjDGbs3th7XJRhVrYOmvK3UpNrWXtPEvl6jCHz8YwY3M4C3acJD4phdaBZRndIZDeTSrj6iLZH2DXbFj8OPjXgwfmQpkA6/nkazCtB1wIgwmbrL8uVLGQ5z70gqCBrgq9g0vgp5FQ8w5rsQw3z1wf6vLVJOaGRDBzy3FOXLhK94YVmXx/c7w93DLewRhY/z9Y8wbUvB2GzIISvje3iT4KX94OFZvAmKXWMEzl9Aq0D10pp9Xwbuj3CYStgQXjIDUl14cq4+3Ow51rsebZLrx6TyN+P3SWoV9u5dyVDBbWSEmCX/5hhXnTYdZyeunDHKwvbe/+CCK2wrp3cl2bch4a6EplpcUI6PEmHFgES57+22Ret8rVRRjbsSZfjw7iaFQsA6ds5s8zMTcaJMbA7GGwY6Y158zAL8DNI/MDNh0MzR+w5qQ5tj5PtamiTwNdqex0eBw6/xN2zIDfXsuXQ3ZrUJGfxrcnKSWVQZ9vZv3hKLhyGr7tDUfXwD0fQ7eXrXlnstP7XfCrY/0VEafjEYozDXSlcqLbv6HVWNj4IWyanC+HbFK1DIse60jVsiX474wFxE3pCtFhVn99qzE5P5BnKRg0Da5Gw8+P5vmvCFV0aaArlRMi0Pd9aHwvrPo3fNMT1r0LkcF56luv4luCBb2Tme/xGnHx8UxvMIXU2t1v/UCVm0KPN+Dwr7Dti1zXo4o2DXSlcsrFFQZ+CV1egpREWPNf+PpOeLcW/DQaQmZYV5veij1z8Z4zBG+/AKY3+pr//OHOEz/uJCEpF/9ItBlnzR656hU4tevW91dFng5bVCq34s5bF/0cXQNHf4OY09bz/vWgdjfrVqNjxmPYjYGNH8Bvr0ONTjDsO4yXL1PXh/HW8kO0qlGWqSNb4VfqFodKXr0An3e0pgQYvw48S+f511SFi45DV6qgGQNRh+Do79YtfBMkx4OLO1RvdyPgKzW1rvhc9iyEfAtNBsGAKTeNcV+29zRPz9lFpTJeTBvTmtrlb/GipvCNMOMeaDrUGiWjnIoGulL2lpQAJ7bcCPiztvVhvP2hdGU4uxc6PQ3dXrEW20hnx4mLPDIjmORUw9SRrWhby+/WXn/NW7DubauLqNmwfPiFVGGhga6Uo8WcsXXP/A6nd1uLVQc9mOUuJ6KvMnb6diIuxPPuoKYMaFE156+XkmydpZ/eDRM25HjmSFX4aaArVURdvprE+O+C2Rp2gWfuqscT3eogORmbDnA50upPL1sDHlqVp6kLVOGhl/4rVUSV8XZn5oNtubdFVT5YdZhn5+7hWnJqDncOsPrnT++G1flzQZQq3HQ2H6UKOQ83F94f0ozqft58tPoIpy7FM6FLbWr5l6SKb4msZ21s0Ncazrj1M6h1B9Trab/Cld1pl4tSRcjCnZE8P28v11Kss3QPNxdq+pWkpn9JapUvSa3ypajpX5La5Uvi622bAyYpwVocI+aUNdWuT2UH/gYqr7QPXSkncjHuGofPxhB2Po5j5+MIi4olLCqOExeukpx64//nciU9rKD3L0lL7ygG7xjBtYotcB3zM54eWUz4pQo1DXSlioGklFQiLly1hXwcYedjbT/jiIpJZLDrWt5zn8r7yUM4WHc8D3YMpH1tv5x/yaoKhawCXfvQlXIS7q4u1CpfilrlS3Fnw5u3XUlIIjyqAxG/nuTpk/P5v3A/RhzsSJ2KPozpUJMBLapkvtiGKjL0DF2p4iThirW0XsQ2LvnU573UEXx/vjZlSrgzrHU1RrSrQbVy3o6uUmUhT8MWRWSaiJwTkX2ZbBcRmSwioSKyR0Ra5rVgpVQB8fKBB1fAoGn4uiTwZuy/2VX7S4ZUu8LXG49xx3trGD8rmC1Ho3HUyZ7KvZwsEn07EAvMNMY0yWB7H+AJoA/QFvjYGNM2uxfWM3SlHCw5EbZ/Za12lHiFuEZDmeH5AF/tiufi1SQaVCrNmA6B9G9elRIero6uVtnk+UtREQkElmQS6F8Ca40xs22P/wS6GGNOZ3VMDXSlComrF2DD+7B9Kri4kdz2URaXHMxX26M4ePoKvt7uDGtdnZHta1DVt4Sjqy32CvpK0apA2kmgI23PZVTIOBEJFpHgqKiofHhppVSeeZeDnm/C439A/d64bfwf9268h2UdjjDn4SDa1/Jj6vqjdH7ndyZ+F8K2MO2OKazseum/MWaqMSbIGBNUvnx5e760Uio7ZQOtpewe/h386iBLn6btr/fweesoNjzXlXG312ZLWDRDp25lwJTNrDpwVoO9kMmPQD8JVEvzOMD2nFKqKApoBWOXwbAfwKTA7KFU/XkILzSLZ8sLd/LmwCZciEvkkZnB9P54A7/sPkVKqgZ7YZAfgb4YGGUb7dIOuJxd/7lSqpATseaBeXQr9PkfnDsIU7tQ4pcJPFBfWPPPLnw4tBnJqYYnZu/krg/WMTc4gqSUHE4cpgpETka5zAa6AP7AWeBVwB3AGPOFWJeZfQr0Aq4CY40x2X7bqV+KKlWEJFyBTR/Bls+s1Zm6vQwdniDVwIr9Z/jk91AOnL5CVd8STLijFoODquHlriNjCoJe+q+Uyh+XT8Kvz8PBX6Beb2t6Xu9yGGNY+2cUn/x+hB0nLlG+tCfjOtdieNvqlPTUK1Dzkwa6Uir/GGMNcVzxL2s5vcHTrX53wBjDlrBoPlsTyqbQaMp6u/Ngx5qM6hBImRLujq3bSWigK6XyX2QIzB0DMaehxxvWsnppJvraceIin/0eym+HzlHa041RHWrwYMea+JXSlZPyQgNdKVUwrl6ARY/C4eXQsB/0/xS8ytzUZP+py0xZc5Rl+07j5ebK/W2qM+GOWlTw8XJQ0UWbBrpSquAYA5s/gdX/Ad/qMGQGVG72t2ah52KZsjaUn3edwsPVhQc7BTLu9traFXOLNNCVUgXvxFaYOxauRkPvd6DVmJu6YK4LPx/HB6sOs3j3KcqUcOfRLrUZ3SFQR8XkkAa6Uso+4s7Dgkfg6O9w2xC4+0PwLJVh0/2nLvPeij9Z+2cUlXy8eLJ7XQa3CsDNVdeuz4oGulLKflJTrcm+1v4X/OrAkJlQoWGmzbeGRfPur4fYceIStfxL8s8e9elzWyVdSSkTBT05l1JK3eDiAndMgpGLIP4STO0Ku37ItHm7Wn7Mn9iBr0YF4eYqPPbDDvp9uomNR87bsWjnoGfoSqmCE3MW5j8E4RugxQjo/R54ZL4iUkqqYdHOk3yw6jAnL8XTobYfz/VqQPNqvnYsunDTLhellOOkJMO6t62FNCo0tkbB+NfNcpfE5BS+33qCz9aEEh13jV6NK/Fsz/rUqZBxf3xxooGulHK8I6utL0xTrkHLUVD3LqjREdwyv9AoNjGZbzYcY+r6o8QnpTC4VTWe7F6XKsV4oQ0NdKVU4XA5EpY9B6GrISUR3L2h5u1WuNe5C8rWyHC36NhEpqw9yqwtx0FgWOtqPNK5VrFc0FoDXSlVuFyLg/CNcGQlHFkFl45bz/vXg7o9oE53qNHhb2fvkRev8slvoSzYGUmqgX7NqjD+jlo0qOTjgF/CMTTQlVKFlzEQHXoj3I9vsrpl3EveOHuve5d1FarN6cvxfLPhGD9sP8HVaync2aACE7vUJiiwnAN/EfvQQFdKFR3X4uDYeivcQ1fBpRPW8/71b4R7QBvw8ObS1WvM3HKcbzcd4+LVJFoHlmVil9p0rV+h8I5jT0mC5MRML7jKjga6UqpoMgbOH74R7sc3W2fv4mIFfJXmULk5CRWaMjfSly82n+HkpXgaVCrNhDtqc3fTyoXjylNjIDIY9syB/Qug7URrrH4uaKArpZxDYqzVJRMZDKd3waldEHfO2iYuGP96nPCsx5Koivx2pSqXyzRg9B0NGeKoFZSij8LeuVaQXwgDNy+o3weCHoSanXN1yDwHuoj0Aj4GXIGvjTFvp9s+BniPG4tDf2qM+TqrY2qgK6XyzBhrPvZTO61wTxfyKbgQmlqFI661KVOrNS3adaNUjRZZXtyUZ3HR1ln4njkQ+QcgVng3HQYN7wGvvH2Bm6dAFxFX4DBwFxAJ/AHcb4w5kKbNGCDIGPN4TovSQFdKFYg0IW9O7eTS0T9wOb2bMqkXAUjFhZSyNXGv1AgqNLLmmanQCMrVAtdcTuWbFA9/Loc9P1ldQ6nJULEJNB0CTQZBmar59utlFeg5WeyvDRBqjAmzHexHoD9wIMu9lFLKEUTApwr4VEEa9KVsN8AYDoceZu3aVSQcD6He+Qiaxeyg0sGlCKnWfq4e1rDJ8g1uhHyFhuBbw5qfJr3UVDi+EXbPgQM/w7UYKF0F2j9mzTRZqYldf23IWaBXBSLSPI4E2mbQ7j4RuR3rbP5pY0xE+gYiMg4YB1C9evX0m5VSqmCIUK9uferVrU/kxavMDznJGyERRMVepqnXWQZXi+EO3/NUSAhDIrbDvnk39nX3hvL1bwR8udoQsRX2zoMrJ8GjNDTqb52NB3YCF8fN656TLpdBQC9jzMO2xyOBtmm7V0TED4g1xiSKyHhgqDGmW1bH1S4XpZQjpaYath6LZm5wJMv3nSYhKZW6FUoxJKgaAxv54B9/DM4dgHMHIeqg9TP2rLWzi5t18VPToVC/N7jbbyqCvPahtwf+Y4zpaXv8IoAx5q1M2rsCF4wxZTLafp0GulKqsLiSkMTSPaf5KTiCnScu4eYidG1QgcGtAujaoALu14c+xkVD9BFrnveS/g6pNa996H8AdUWkJtYolmHA8HQvUNkYc9r2sB9wMA/1KqWUXfl4uXN/m+rc36Y6oedimBscyYKdJ1l14Cz+pTwY2KIqg4OqUa+iH5T0c3S5mcrpsMU+wEdYwxanGWPeFJHXgWBjzGIReQsryJOBC8BEY8yhrI6pZ+hKqcIsOSWVdYej+Ck4gt8OniM51dCsmi+DWwXQvJov5Ut7Uq6kx42zdzvRC4uUUioPomMTWbjzJPNCIjl0Juambb7e7viX8sSvpAf+pT3xL+mBfylP/EvfeK58KU/8Snng7ZGTTpGsaaArpVQ+MMbw59kYws9f5XxsItGx1zgfm3jT/ajYRGISkjPc39vDFb9SHoxuH8jDnWvlqoa89qErpZQCRIQGlXyyna43MTnlr4CPjr1GVLrQL18680U98kIDXSml8pmnmytVfEvYfWWlQjANmVJKqfygga6UUk5CA10ppZyEBrpSSjkJDXSllHISGuhKKeUkNNCVUspJaKArpZSTcNil/yISBRx3yItnzx847+gislDY64PCX6PWlzdaX97kpb4axpjyGW1wWKAXZiISnNlcCYVBYa8PCn+NWl/eaH15U1D1aZeLUko5CQ10pZRyEhroGZvq6AKyUdjrg8Jfo9aXN1pf3hRIfdqHrpRSTkLP0JVSyklooCullJMotoEuItVEZI2IHBCR/SLyZAZtuojIZRHZZbu9Yucaw0Vkr+21/7Zen1gmi0ioiOwRkZZ2rK1+mvdll4hcEZGn0rWx+/snItNE5JyI7EvzXDkRWSUiR2w/y2ay72hbmyMiMtqO9b0nIods/w0XiohvJvtm+XkowPr+IyIn0/x37JPJvr1E5E/b5/EFO9Y3J01t4SKyK5N9C/T9yyxT7Pr5M8YUyxtQGWhpu18aOAw0StemC7DEgTWGA/5ZbO8DLAcEaAdsc1CdrsAZrAseHPr+AbcDLYF9aZ57F3jBdv8F4J0M9isHhNl+lrXdL2un+noAbrb772RUX04+DwVY33+AZ3PwGTgK1AI8gN3p/38qqPrSbX8feMUR719mmWLPz1+xPUM3xpw2xuyw3Y8BDgJVHVvVLesPzDSWrYCviFR2QB13AkeNMQ6/8tcYsx64kO7p/sAM2/0ZwIAMdu0JrDLGXDDGXARWAb3sUZ8xZqUx5vqqwluBgPx+3ZzK5P3LiTZAqDEmzBhzDfgR633PV1nVJyICDAFm5/fr5kQWmWK3z1+xDfS0RCQQaAFsy2BzexHZLSLLRaSxXQsDA6wUkRARGZfB9qpARJrHkTjmH6VhZP4/kSPfv+sqGmNO2+6fASpm0KawvJcPYv3VlZHsPg8F6XFbl9C0TLoMCsP71xk4a4w5ksl2u71/6TLFbp+/Yh/oIlIKmA88ZYy5km7zDqxuhGbAJ8AiO5fXyRjTEugNPCYit9v59bMlIh5AP2BuBpsd/f79jbH+vi2UY3VF5F9AMvB9Jk0c9Xn4HKgNNAdOY3VrFEb3k/XZuV3ev6wypaA/f8U60EXEHeuN/94YsyD9dmPMFWNMrO3+MsBdRPztVZ8x5qTt5zlgIdaftWmdBKqleRxge86eegM7jDFn029w9PuXxtnrXVG2n+cyaOPQ91JExgB3Aw/Y/qf/mxx8HgqEMeasMSbFGJMKfJXJ6zr6/XMD7gXmZNbGHu9fJplit89fsQ10W3/bN8BBY8wHmbSpZGuHiLTBer+i7VRfSREpff0+1hdn+9I1WwyMso12aQdcTvOnnb1kelbkyPcvncXA9VEDo4GfM2izAughImVtXQo9bM8VOBHpBTwH9DPGXM2kTU4+DwVVX9rvZQZm8rp/AHVFpKbtr7ZhWO+7vXQHDhljIjPaaI/3L4tMsd/nr6C+8S3sN6AT1p8+e4BdtlsfYAIwwdbmcWA/1jf2W4EOdqyvlu11d9tq+Jft+bT1CfAZ1uiCvUCQnd/DklgBXSbNcw59/7D+cTkNJGH1Qz4E+AG/AUeA1UA5W9sg4Os0+z4IhNpuY+1YXyhW/+n1z+EXtrZVgGVZfR7sVN8s2+drD1Y4VU5fn+1xH6yRHUftWZ/t+enXP3dp2tr1/csiU+z2+dNL/5VSykkU2y4XpZRyNhroSinlJDTQlVLKSWigK6WUk9BAV0opJ6GBrpRSTkIDXSmlnMT/A52isWtdNQE+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "history=pd.DataFrame(history, columns=['epochs','loss','val_loss'])\n",
    "from matplotlib import pyplot \n",
    "pyplot.plot(history['epochs'],history['loss'], label='train') \n",
    "pyplot.plot(history['epochs'],history['val_loss'], label='test') \n",
    "pyplot.legend() \n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 622
    },
    "colab_type": "code",
    "id": "MwDoSYgYMLcE",
    "outputId": "b6d4f1da-7a75-40b1-d5fc-c2cf6887a222"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epochs</th>\n",
       "      <th>loss</th>\n",
       "      <th>val_loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>4.040</td>\n",
       "      <td>4.130271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3.323</td>\n",
       "      <td>3.692981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3.094</td>\n",
       "      <td>3.383813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2.934</td>\n",
       "      <td>3.168670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2.824</td>\n",
       "      <td>2.984884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>2.630</td>\n",
       "      <td>2.759974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>2.477</td>\n",
       "      <td>2.477530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>2.347</td>\n",
       "      <td>2.196713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>2.006</td>\n",
       "      <td>1.951292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>1.747</td>\n",
       "      <td>1.632557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>1.533</td>\n",
       "      <td>1.303331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>1.403</td>\n",
       "      <td>1.219630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>1.140</td>\n",
       "      <td>1.486956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>1.094</td>\n",
       "      <td>1.070155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>0.963</td>\n",
       "      <td>0.901795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>0.816</td>\n",
       "      <td>0.670931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>0.654</td>\n",
       "      <td>0.583635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>0.511</td>\n",
       "      <td>0.522689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19</td>\n",
       "      <td>0.395</td>\n",
       "      <td>0.470531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>0.368</td>\n",
       "      <td>0.580331</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    epochs   loss  val_loss\n",
       "0        1  4.040  4.130271\n",
       "1        2  3.323  3.692981\n",
       "2        3  3.094  3.383813\n",
       "3        4  2.934  3.168670\n",
       "4        5  2.824  2.984884\n",
       "5        6  2.630  2.759974\n",
       "6        7  2.477  2.477530\n",
       "7        8  2.347  2.196713\n",
       "8        9  2.006  1.951292\n",
       "9       10  1.747  1.632557\n",
       "10      11  1.533  1.303331\n",
       "11      12  1.403  1.219630\n",
       "12      13  1.140  1.486956\n",
       "13      14  1.094  1.070155\n",
       "14      15  0.963  0.901795\n",
       "15      16  0.816  0.670931\n",
       "16      17  0.654  0.583635\n",
       "17      18  0.511  0.522689\n",
       "18      19  0.395  0.470531\n",
       "19      20  0.368  0.580331"
      ]
     },
     "execution_count": 65,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5cUmIlXWM7Uw"
   },
   "outputs": [],
   "source": [
    "#Evaluate results¶\n",
    "#Because our task is simple and the output is straight-forward, we will use MAE metric to evaluate the trained model during the epochs. Compute the value of the metric for the output from each epoch.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 342
    },
    "colab_type": "code",
    "id": "r5ZnQYTwLpXn",
    "outputId": "acf65b07-e1b0-4db9-d5a6-55062db9dd61"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, MAE: 42.640143, Invalid numbers: 96\n",
      "Epoch: 2, MAE: 53.935989, Invalid numbers: 95\n",
      "Epoch: 3, MAE: 53.935989, Invalid numbers: 95\n",
      "Epoch: 4, MAE: 53.935989, Invalid numbers: 95\n",
      "Epoch: 5, MAE: 67.419986, Invalid numbers: 95\n",
      "Epoch: 6, MAE: 60.302269, Invalid numbers: 95\n",
      "Epoch: 7, MAE: 67.419986, Invalid numbers: 87\n",
      "Epoch: 8, MAE: 85.280287, Invalid numbers: 89\n",
      "Epoch: 9, MAE: 79.772404, Invalid numbers: 89\n",
      "Epoch: 10, MAE: 73.854895, Invalid numbers: 84\n",
      "Epoch: 11, MAE: 85.280287, Invalid numbers: 74\n",
      "Epoch: 12, MAE: 81.818182, Invalid numbers: 76\n",
      "Epoch: 13, MAE: 90.909091, Invalid numbers: 79\n",
      "Epoch: 14, MAE: 100.000000, Invalid numbers: 81\n",
      "Epoch: 15, MAE: 100.000000, Invalid numbers: 80\n",
      "Epoch: 16, MAE: 100.000000, Invalid numbers: 73\n",
      "Epoch: 17, MAE: 95.346259, Invalid numbers: 78\n",
      "Epoch: 18, MAE: 100.000000, Invalid numbers: 62\n",
      "Epoch: 19, MAE: 95.346259, Invalid numbers: 71\n",
      "Epoch: 20, MAE: 95.346259, Invalid numbers: 62\n"
     ]
    }
   ],
   "source": [
    "for i, (gts, predictions, invalid_number_prediction_count) in enumerate(zip(all_ground_truth,\n",
    "                                                                            all_model_predictions,\n",
    "                                                                            invalid_number_prediction_counts), 1):\n",
    "       \n",
    "\n",
    "          mae=mean_absolute_error_texts(gts, predictions)\n",
    "          print(\"Epoch: %i, MAE: %f, Invalid numbers: %i\" % (i, mae, invalid_number_prediction_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fdc54de8b70>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeVyU5fr48c/NLogboIKouCsKoiJqmrlnpqZlpqZH22wz26ys08nOr+WbtndatUXLXLLUNNM0zSx3cMFdwRVBEVBEZOf+/fEMhAgywCws1/v1mtcMzzzz3BcjzjX3rrTWCCGEEAAO9g5ACCFExSFJQQghRD5JCkIIIfJJUhBCCJFPkoIQQoh8TvYOoDy8vb11QECAvcMQQohKJSIiIkFr7VPUc5U6KQQEBBAeHm7vMIQQolJRSp0q7jlpPhJCCJFPkoIQQoh8khSEEELkk6QghBAinyQFIYQQ+ayWFJRSXyul4pVS+wscq6eUWqeUOma6r2s6rpRSHymlopRSkUqpztaKSwghRPGsWVOYCwwudGw6sF5r3QpYb/oZ4Daglek2GfjMinEJIYQohtXmKWitNymlAgodvgPoY3o8D9gIvGA6/q021vHeppSqo5Ty1VrHWSs+IUTZZGbn8s3mE6RmZNstBldnR8aFNaGuh4vNy87KyeXHiBgGtGuAj6erzcu3NltPXmtQ4IP+HNDA9LgRcKbAeTGmY9clBaXUZIzaBE2aNLFepEKIIs3fdor/W30YpewXg9awcm8sCx7qTj0bJoasnFymLNjFbwfO86XPcRZO7k59TzeblW8LdpvRrLXWSqlS7/CjtZ4NzAYIDQ2VHYKEsKG0zBw+3RjNTS28WPBQd7vF8fexBB6Yt5Nxc7bZLDEUTAiTbgrgh/AzjJ29rcolBluPPjqvlPIFMN3Hm46fBRoXOM/fdEwIUYF8u/UkCVcyeGZga7vG0auVN19N7MqJhFTGzdlGUmqmVcsrmBBeHRbIq8Pb882krsQlpzN29jbiU9KtWr4t2ToprAAmmh5PBH4ucPxfplFI3YFk6U8QomK5kpHN539G07u1D6EB9ewdjs0SQ+GEMKlnMwC6NfeqkonBmkNSFwJbgTZKqRil1APAW8BApdQxYIDpZ4BfgeNAFDAHeMxacQkhymbu5hNcvJpl91pCQQUTw71fbrd4YiguIeQpmBjGzdleJRKDMgb8VE6hoaFaVkkVwvqS07K4eeYGwprV48uJXe0dznX+OnaBB+eF09ynJt8/2M0ifQxZObk8sWA3aw6cY8awQO4rlBAK2n48kUnf7KRR3RoseKhbhe9jUEpFaK1Di3pOZjQLIUr01d8nuJyezdMVqJZQ0M2tfPhyYijHL1yxSI2hNAkBjBrD3Pu6cvZiWqWvMUhSEELc0MXUTL7++wS3dWhIe7/a9g6nWIUTw8UyJobSJoQ8hRPDhZSMMpVvb5IUhBA3NOev46RmVtxaQkE3t/Jhzr+MxDCuDImhYEJ4Zaj5CSFPt+ZefGNKDGPnbKuUiUGSghCiWIlXMpi75STDgv1o3cDT3uGYpXfrsiWGwgnh/l6lSwh5ulfyxCBJQQhRrM//jCY9K4cnB7SydyilkpcYos1MDJZKCHkqc2KQpCCEKFL85XS+3XqKEZ0a0cKnpr3DKbXerX340ozEkJWTy9SFRkL4jwUSQp7uzb34elLlSwySFIQQRfp0YzTZuZon+1euWkJBBRNDUZ3PeQlh9X4jITxgoYSQp0eLfxLDuEqSGKpnUji0EhbcA3++DdEbID3Z3hEJUaHEXkpjwfbT3N3Fn6ZeHvYOp1zyEkNUocRg7YSQJy8xxFSSxFA9k0JmKiQdhz9eh+9GwltN4OOusOxR2PkVxO2FHPstCyyEvX38RxQazZR+Le0dikXk9THkJYYLKRn5CeHl29tZLSHkyUsMZy5erfCJoXrPaE67CGd3wdkIiAmHs+FwNdF4zqkG+IVAoy7gHwqNQqG2P3ZdL1gIGziTdJW+72xkbFgTXhvRwd7hWNSfRy/w0LfhODkormbm8PLt7Xjw5uY2K39rdCL3zd1B47rujOzcqFzX6t3Khw6NyjZv5EYzmqt3UihMa7h48tokERcJOaasXrOBkRz8u0D99uBot5XH7c/TFxq0t3cUwgqeW7KXn/fGsum5vjSsXbGXayiLP49e4KlFu5nSr5XVawhF2RqdyMPfhXM5vXytEa+P6MD47k3L9FpJCuWRnQnn90FMhJEkYsIhKdq6ZVYW45dCy/72jkJY0ImEVAa89ycTewTwyrBAe4djNVprlB1r/dk5uWTnlu+z18lB4eRYth6AGyWFavxV10xOLkYTUqMumDZ8g6tJkBgNVN6EWi5aw7KHYc2L8OhmcHS2d0TCQj5afwxnR8UjfWzXpGIP9kwIAE6ODjg52jWEYklSKAv3esatOrv1TVg0FnZ+Cd0ftXc0wgKi4lNYvucsk29uXuFX+RTWUz1HH4nya3MbtOgHf/wfpCbYOxphAe//fgx3Z0cevqWFvUMRdiRJQZSNUjD4Lci8Ahtes3c0opwOxV1mVWQc9/VsZpP9jkXFJUlBlJ1PGwibDBHzjFFaotJ6f91RPN2ceMiGwzNFxSRJQZRPn+lG/8rqF4wOaFHp7ItJZu3B8zzYqzm13WXQQHUnSUGUT4060O8/cHoLHFhq72hEGby37gh13J25v1eAvUMRFYAkBVF+nf8FDYNg7SuQedXe0YhSiDh1kT+OXGBy7+Z4ukktQUhSEJbg4Ai3zYLLMbD5Q3tHI0rh/XVH8fJwYWKPAHuHIioISQrCMpreBO3vhM0fwKXT9o5GmGH78UT+jkrgkVta4OEqU5aEQZKCsJyB/w9QsPY/9o5ElEBrzbvrjuLj6Vrm9XNE1SRJQVhOncbQ6yk4uBxO/m3vaMQNbIlOZMeJJB7v04IaLhV0vQVhF5IUhGXdNBVqN4bV0yE3x97RiCJorXl37RF8a7sxJqyJvcMRFYwkBWFZLu4w6DVjZdmIufaORhRh49EL7Dp9iSn9WuLmLLUEcS27JAWl1JNKqf1KqQNKqadMx+oppdYppY6Z7uvaIzZhAYEjoGkv2PC6sZGRqDC01ry/7ij+dWtwd5fG9g5HVEA2TwpKqQ7AQ0AY0BEYqpRqCUwH1mutWwHrTT+LykgpuO0tSL8EG9+ydzSigHUHzxMZk8zU/q1wcZKGAnE9e/xVtAO2a62vaq2zgT+BO4E7gHmmc+YBI+wQm7CUhkHQZRLsmAPxh+wdjQByczXvrTtKgJc7d3Yq31aQouqyR1LYD9yslPJSSrkDQ4DGQAOtdZzpnHNAg6JerJSarJQKV0qFX7hwwTYRi7Lp+zK41oQ102VdpApg9f5zHD6XwpMDWpV5xy5R9dn8L0NrfQiYCawF1gB7gJxC52iK2dZMaz1bax2qtQ718fGxdriiPDy8oM9LcHwjHPnV3tFUazm5mvd/P0rL+jUZ3lFqCaJ4dpnGqLX+CvgKQCn1JhADnFdK+Wqt45RSvkC8PWITFtb1AYj4Bn57CVr0B+dy7uilNRz/Aw7/CrlZZb+OWx24+Rlwq12+eCqJlXtjiYq/wsfjOuHoYN+tKEXFZpekoJSqr7WOV0o1wehP6A40AyYCb5nuf7ZHbMLCHJ2NzXi+GwHbPoGbny3bdTKuQOQi2D4bEo6AS01w8Sh7XKkJcGYHjP/JGEZbhWXn5PLh+mO0bejJkA6+9g5HVHD2WvDkJ6WUF5AFPK61vqSUegv4QSn1AHAKGG2n2ISltegLbW6HTe9Cx7FQy8/81148aXRW7/oOMpLBNwRGfgHtR4KTa9lj2r8UfrwflkyEMQuM5FVFLd19lhMJqXwxoQsOUksQJbBX89HNRRxLBPrbIRxhC7e+Dp90g9//C3d+ceNztYYTm2D7F0ZfhHKAwDug2yPQOMwY8lpeHe6E9GT45SlY9gjcOQccql7na1ZOLh+tP0ZQo9oMCixy7IYQ15ClEYVt1GsOPabA3+9B1wehcdfrz8m8CpGLjWRw4RC4exnNTaH3Q20rdI6G3mfMpfj9VWOzoCHvWCbhVCBLwmOIuZjGa3d0QFWx301YhyQFYTs3Pwt7FsDq5+HB9f98M790GnZ+aez1nH7JmONwx6fQ4a7yd0yXpNfTxqzrzR9CjbrQ72XrlmdDGdk5fLzhGJ2a1KFPGxmpJ8wjSUHYjmtNGPhfWPYw7F0AdQNg++dweBWgoN1Qo4moSQ/bfmMf8F8jMWx62xiVdNMU25VtRYt2nCE2OZ1ZozpKLUGYTZKCsK2g0UatYMUToHONb+c9n4TQB4ylt+1BKRj6AaRfhrX/NpqSOo23TywWkp6Vwyd/RBHWrB49W3rZOxxRiUhSELbl4GB8AP8+A9oNh+DR4FzD3lEZW4reORsyLhsJy602tBtm76jKbP62U8SnZPDR2E5SSxClUvWGW4iKr2EHY35Al4kVIyHkcXKFe+ZDo1BjuOrxjfaOqExSM7L5bGM0PVt60b251BJE6UhSEKIgFw+49wfwagULx0FMuL0jKrV5W0+SmJrJMwPb2DsUUQlJUhCisBp1YcJSqOkD34+qVKu8pqRnMXvTcfq08aFLU9mSRJSeJAUhiuLZECYsB0dX+G6kMbO6Evj675NcuprFMwNb2zsUUUlJUhCVzqnEVB7+LpzYS2nWLaheM5iwDLLS4NsRkHLeIpeNS07jse8jCD+ZZJHr5Um+msWXfx9nYGADgv3rWPTaovqQpCAqnZ/3xPLbgfM8OC+c1Ixs6xbWIBDu/RGuxMP8Oy2yveg3m0/y675z3DN7G59ujCI31zJ7Tcz56zgp6dlSSxDlIklBVDpbohPwrunC4XOXeXLRHnIs9KFarMZdYcz3kHAUFtwDmallvlR6Vg5Lws/Qt40Pg9s3ZNaaI9w3dyeJVzLKFWJSaibfbD7B7UG+tPOtVa5riepNkoKoVNIyc9h16hJ3dvZnxrD2/H7oPLPWHLZ+wS36wl1fQcxOWDwBsjPLdJk1+89x8WoWD97cnI/HdeK1ER3YejyRIR/9xbbjiWUO74tN0VzNyuGpAa3KfA0hQJKCqGTCTyWRmZNLjxZeTLwpgH/1aMoXm46zeOdp6xceOByGfQTR62HpQ5CbW+pLzN92imbeHvRo7oVSigndm7LssZtwd3Fi3JxtfLT+WKlrPhdSMvh2yynu6OhHqwaepY5JiIIkKYhKZUt0Ik4OirCAegC8MjSQm1t58+9l+9kaXfZv2mbrPMFYK+ngcjiwtFQvPXzuMuGnLjIurMk1+xq096vNyid6MayjH++tO8q/vt5OfEq62df9bGM0mTm5PDlA+hJE+UlSEJXKlqgEOjWpg4ersUKLk6MDn9zbmWbeHjwyP4LjF65YP4ibpoJ3G/j7A2PvBzMt2H4aFycH7urif91zNV2d+OCeEGbeFUT4yYsM+fBvNkcllHjNc8npzN9+ipGdGtHMuxw70QlhIklBVBrJaVnsO5tMjxbe1xyv5ebMVxO74uigeGBeOJeulq2932wODtBzKpzfZzQlmSE1I5ulu85ye5Av9TxcijxHKcU9XZuwYkov6rg7M/6r7by39sgNm5M++cMYvfRkf+lLEJYhSUFUGtuPJ5KroWeL69fzaeLlzhcTunD2YhqPzt9FZnbp2/tLJWg0ePoZtQUzrNwby5WMbO7t1qTEc9s09GTFlJ7c2cmfjzZEMW7ONs5fvr45KebiVRbtPM3doY1pXK9q7zMtbEeSgqg0tkQn4ubsQEiToidmdQ2ox1t3BbH1eCL/Wb4fXYqmnVJzcoEej8HJv+BsRImnf7/9NG0bepq99IS7ixPvju7IO3d3JDImmds+/Is/j1645pyPN0ShUDzRr2WZfgUhiiJJQVQaW6IT6BpQD1cnx2LPubOzP1P6tmRx+Bm+/OuEdQPqMslYYruE2kJkzCX2nU3m3m5NSr2M9agu/qx8oic+NV2Z+PUOZq45THZOLqcSU1kSEcPYsMb41alAK82KSk/2UxCVQnxKOkfPX+HOztd30hb2zMDWHE+4wpurDxHg7cFAa21Y7+pp7Df913uQEAXeRX9jn7/tFO4ujozoVLZ9plvW9+TnKT3578oDfLYxmp0nkqjj7oKTg+LxvlJLEJYlNQVRKeQNN72piP6EwhwcFO/eHUJQo9o8uWg3B2KTrRdYt0fA0QW2fFTk08lpWazYG8sdIX54ujmXuRg3Z0f+785gPhwTwqG4y/x+6DwTujelfi0r72Etqh1JCqJS2BKVSC03J9r71Tbr/Boujnz5r1Bq13DmwXnhxBfRUWsRNetDp3th70JIOXfd08t2xZCelcu4sKYWKe6OkEb8MvVmJvduzhTpSxBWIElBVAqboxPo0cILRwfz2+Tr13Ljy4mhJKdl8dC34aRl5lgnuJuegNxs2PbZNYe11ny//TQd/WsT5G9eMjNHM28PXhrSjjruRQ9ttQlrduILu5KkICq8M0lXibmYxk2F5ieYo71fbT64J4TIs8k8u2SPxVYkvUa95hB4B4R/Den/NFXtPHmRY/FXuLebZWoJdqc1nNkJPz4ArzeA/4XCskdgxxyI3Q05WfaOUFiAXTqalVJPAw8CGtgH3Af4AosALyACmKC1tvIsJFEZ5M3s7dmybPsND2rfkBdva8ubvx7mfZ+jPDvICttU9nwKDiyD8G+g11MAfL/9FJ5uTgzr6Gf58mwpOwMOLIftn0PsLnCtDR3HGMuJR/1uNJ0BOLmBb0fw7wqNuoB/KNRuDKUccSXsy+ZJQSnVCJgKBGqt05RSPwBjgCHA+1rrRUqpz4EHgM9ucClRTWyOTqS+pystfGqW+RoP3dyc6PhU/rchiuY+HozsVPIoplLxC4HmfYwmpO6PkpgOq/edY1y3JtRwKX4IbYWWct6o/YR/Danx4N0abn8XgseAq+nfQmu4dBrOhkNMhHG/80vY+rHxvEd9IznkJQm/zuAmS3tXZPYakuoE1FBKZQHuQBzQDxhnen4e8CqSFKo9rTVboxPo1dK71GP8C1JK8dqIDpxKSuWFH/fhX9edrqZF9Sym51Pw3QjYu4glV3qRmZNr1gzmCudsBGz/AvYvhdwsaHUrdHsYmvc1lvgoSCmo29S4dbjLOJaTBef3Q0y4ca2YcDjya94LwKcNNOkO/V4Bj7LV/oT12DwpaK3PKqXeAU4DacBajOaiS1rrvG20YoAiB3UrpSYDkwGaNKmE/+FEqRw9f4WEK5nc1LL0/QmFuTg58Pn4Loz8dAsPfxfBz4/3tOzyEM37gG9H9JaPWHS1EWHN6lWepayzM+HQCqOJKGYnuJjmYIQ9BF4tSnctR2fw62TceMg4lnbRlCBMtYnd30NqAtwzX5qXKhibdzQrpeoCdwDNAD/AAxhs7uu11rO11qFa61AfHx8rRSkqirz+BHPmJ5ijjrsLX00MJSsnl0e/jyA9y4IjkpSCnk+hEqNok/xX5aglXLkAf74NHwTBTw/A1SS47W149hDc9lbpE0JxatSFlgOgzwtw7xLo/x84/Avs/8ky1xcWY1ZSUEq5K6X+o5SaY/q5lVJqaBnLHACc0Fpf0FpnAUuBnkAdpVRezcUfOFvG64sqZEt0Ik293PGva7lv9M19avL+6BD2n71s+TWSAu8g3smPKS6/MLi9lWZSW0LsHlj2KLwfCH+8Dg07GHtRTwmHbpON2drW1GMKNAqFX58zOqxFhWFuTeEbIAPoYfr5LPB6Gcs8DXQ3JRoF9AcOAn8Ao0znTAR+LuP1RRWRnZPL9uOJZRqKWpIBgQ14ol9LlkTEsGCH5XZtO5eSxUfpgwkiCteYrRa7rsVcTYLlj8HsW+Dgz8b6TVPCYfxP0Grg9X0G1uLgCCM+Nfa7XvWMzHuoQMz9C2ihtZ4FZAFora8CZWoI1FpvB34EdmEMR3UAZgMvAM8opaIwhqV+VZbri6pjf+xlUjKyLdZ0VNhTA1rTu7UPr644wO7TFy1yzcU7z7Akuzc5Nbxhs3nLatuE1saQ2U+6QeRi6PWM0UQ05G3wttNeDD5toO+LcGilEZuoEMxNCplKqRoY8wpQSrXAqDmUidZ6hta6rda6g9Z6gtY6Q2t9XGsdprVuqbW+W2td5uuLqsHS/QmFOTooPhoTQoNabjw6fxcJV8r3J5edk8uinafp1roRjj0eNcbwn9tnoWjL4XIcLLoXlkyCWn4weSMMmGGs8GpvPZ4whqn+Os3o3xB2Z25SmAGsARorpb4H1gPPWy0qITCWym7b0BOvmq5WK6OOuwufj+/CxauZTFmwi+ycsm/Os+FwPHHJ6UYHc9cHwKUmbP7QgtGWUm6uMZnukzBjh7iBr8GD66FhkP1iKszRyWhGykgxEoMtXE2CNS8ZyVJcx6ykoLVeB9wJTAIWAqFa643WC0tUd+lZOYSfvGiV/oTCOjSqzRsjg9h2PIlZvx0p83W+336aBrVc6d+2vjHapsskY6z/xVOWC9ZcidHw7XD45SljlvGjW4wtRB0r4Gr59dtBn+lwcLn1m5EyUuD7UbDtE2NSnriOuaOPRgLZWutVWutfgGyl1Ajrhiaqs12nL5KRnVvmpS1Ka1QXfyZ0b8rsTcdZFVn6b5CnE6+y6dgFxnRtgpOj6b9Vj8dBOfwzu9cWcrKNTX8+uwniImHYRzBxpeWGllrLTU+CbwismmbMX7CGrHRYNM4YeeXpB8fWWqecSs7s5iOtdf5KX1rrSxhNSkJYxZaoRBwdFGHNLDzr+Ab+MzSQzk3q8NyPezl2PqVUr1248zQKGBPW+J+Dtfwg+B7Y9Z31PugKiouEL/vB7zOMOQGPb4cuEyvH5DBHJxjxmbGg4K/PWf76OdnGPIwTm4zmqrAHIW5PkcudV3fmJoWizquA9VBRVWyJTiDYv3a5NqYpLRcnBz69twvuLo48/F0El9PNW/UzMzuXH3aeoX+7BvjWLrQ1Zs+pkJ0GO2ZbIWKTrHT4/b8wu4/RTn73PGOmcC1f65VpDQ0CjcltB5Yaw2UtJTcXVk41JssNnmks5tdqkPFc1O+WK6eKMDcphCul3lNKtTDd3sNYmkIIi0tJz2JvTDI9bdCfUFjD2m58PK4zp5KuMu2HvWYttf3bgXMkpmYyvnsRS2T7tIE2txtJITPV8gGf2gKf94S/34OOY43aQfsRlaN2UJSepj6QVc9CamL5r6c1rP037Pke+rwI3R8xjjfoIE1IxTA3KTwBZAKLTbcM4HFrBSWqt50nk8jJ1VYbilqS7s29ePG2tqw9eJ7P/owu8fzvt5+icb0a3Fzc+ky9njLW/tn1reWCTL8MvzwN39wGOZkwYRmM+ATcbdfcZhWOznDHp5B2CVZbYIDjpndg26fGtqm3vPDPcaWMyXrRf8g+EIWYO/ooVWs9PW/NIa31i1prK3ztEQI2RyXi6uRA56Z17RbDA72aMTTYl3fXHuGvY8WPn4+Kv8K240mMC2uKQ3G7wjUOgyY3wdZPyv8BlJ5sLM/9STeImAvdH4fHtkGLfuW7bkXSsAPc8jzs/9GY2FZWO+YYS3gEj4Fb/+/62lOrQZBxGU5vK1+8VYy5o49aK6VmK6XWKqU25N2sHZyonjZHJRAaUBc3Z/vtQ6CUYuZdwbSsX5OpC3cTc/Fqked9v/0Uzo6Ku0NL2J+h11OQfKbsC8AlHDM6YN8LhDXToU5jeGAdDH4TXDzKds2KrNfTxnyKX54x5hWUVuQPxryHNkPgjo+LXr6jeR9wcIZjv5U32irF3OajJcBu4GXguQI3ISwq8UoGh8+l2GR+Qkk8XJ34YkIo2TmaR+fvum5F1bTMHH6KiGFwB1+8S5pg13Ig+LQzJrOZu85Pbi4cWwfz74KPQ42aQbth8NAf8MBaY9OaqsrR2RiNlJYEq18o+fyCjqwxtgkNuBlGfWNcqyiuNSGgp/Eei3zmJoVsrfVnWusdWuuIvJtVIxPV0tbjRueivfoTCmvm7cF794Sw72wyr/x87Yqqv0TGcjk927wlsh0coOeTEH+w5M7NjBRjk5uPQ42JVuf2Qd9/w9MHYOTn0KhzOX+rSqJhEPR+Dvb9AIdXmfeak5thyUTwDYaxC8HZ7cbntxoEFw7bZ4JhBWVuUliplHpMKeWrlKqXd7NqZKJa2hyViKerE0GNKsC6PCYDTSuq/hAew8IdZ/KPf7/9NC3r16SbuXMpgkZBLX9jcllREqNh9XR4t53RyVqjLtz1FTy132hjr1nfAr9NJdPrGWgQZHSql9SMFLsHFo6BOk3g3p/MW/671a3GvYxCymfuXIOJpvuCTUYaaG7ZcER1tzU6gW7N6/0zK7iCeGpAa/bGJPPqigME+tXCyUGx58wlZgwLNH+bUEdnY5bzby/CmR1GB7TWEL3BqBkcWwsOTtB+pLH9ZVVuHjKXk4sx2WxOX6Mv5c5i5nskHDOa2dxqw4Tl5m/z6dUC6jYzmpDCHrJc3JWYWUlBa93M2oEIcfZSGicTr/KvHgH2DuU6eSuqDv3f3zw6P4JOTerg5uzAnZ1K6GAurPO/4M+ZsOltaH2rkQwSjoKHj1EbCL0fPBta55eorHyD4eZnjfctcAS0HXLt88kx8K1pbsaE5VC7yJ18i6aU0YS061vISgPnGiW/pooz++uYUqqDUmq0UupfeTdrBiaqny2mpbJ7WmA/ZmvIW1E1KTWTX/edY1iwH7XdSznj2rUmhE02agWrngVndxj5hdFf0PclSQjFuXka1G9vLPBXsBkpNcFICBmXYfxS8G5Z+mu3HmTMOj/5t+XircTMHZI6A/if6dYXmAUMt2JcohraEp2Id00XWjeoae9QitWhUW3+784gPN2cmHhTQNkuctMU6P28MaR08kZj2QUn6y0PXiXkNSOlJsBvLxnH0i/D/DuNmsK4H4waRVk07WUkZ+lXAMyvKYzC2DbznNb6PqAjUHF6AkWlp7Vmc1QCPVp4m99Gbyd3dvZn938G0qGsneFutaHfv40+hQr+u1YofiFw8zOwdyEcWA4Lx8L5AzD6W2jao+TXF8fZDZrdAkd/k21BMT8ppGmtczGWzK4FxAONS3iNEGaLvpBKfEpGhRmKWpKK1hFebfR+DuoHGsNOT202mt5aDyr/dVsNhEunjA7raq40C+cuRLwAACAASURBVOLVAeZgLIS3C6iAu5KLympLtKk/oQJMWhMVmJOr0Yzk4QO3v2sM87WEvFVTpQnJ7NFHj5kefq6UWgPU0lpHWi8sUd1sjkrAv24Nmni52zsUUdH5dYJpxyzb9FansVEDOfab0edTjZVm9FGwUmo40BloqZS603phieokJ1ez7XhSpWk6EhWANfpiWg2EU1uNDuxqzNzRR18DXwN3AcNMt6FWjEtUIwdjL5OcllVhh6KKaqLVIMjNguMb7R2JXZk7o7m71jrQqpGIaiuvP6GH1BSEPTXuBq61jX6FwOo74t7c5qOtSilJCsIqNkcn0qp+Tep7lrB4mRDW5OgMLfoaS15U46Gp5iaFbzESwxGlVKRSap9SSjqaRbllZuey80SSNB2JiqH1rXDlHJyz4sdbbo6xhHrKOeuVUQ7mNh99BUwA9gG55SlQKdUGY0vPPM2BVzASz2IgADgJjNZaXyxPWaLi23PmEmlZOdLJLCqGlgOM+2Nrjb2irSFyMax7xVgS/a4vrVNGOZhbU7igtV6htT6htT6VdytLgVrrI1rrEK11CNAFuAosA6YD67XWrYD1pp9FFbc5KgEHBd2aS1IQFUDN+saQ16NWmq+QnQkb3wLlCPt+hPMHrVNOOZibFHYrpRYopcYqpe7Mu1mg/P5AtCnB3AHMMx2fB4ywwPVFBbclOoGgRrWpXaOUC8sJYS2tboWYnZCaaPlr75lvzJwe+YWx38PGNy1fRjmZmxRqABnAICw7JHUMsND0uIHWOs70+BzQwALXFxXY1cxsdp++RA+ZxSwqklaDANM+F5aUlQ5/vg2NuxszsXs8DodWQuxuy5ZTTiUmBaWUI5Cotb6v0O3+8hSslHLBWGl1SeHntLHnYZHd/0qpyUqpcKVU+IULF8oTgrCzHSeSyM7V9GwpTUeiAvHrBO7exuxmSwr/GlJiod/LxuS77o8au+tteMOy5ZRTiUlBa50D9LRC2bcBu7TW500/n1dK+QKY7uOLiWe21jpUax3q4+NjhbCErWyNTsTF0YHQprKzq6hAHByM2c1RvxsjhSwh4wr8/Z6xGmuzm41jbrWNfbuj1sHpbZYpxwLMbT7ao5RaoZSaYME+hbH803QEsIJ/tv2cCPxczuuLCm5zdAKdmtShhoujvUMR4lqtBkLaRTgbYZnr7ZgNqReg33+uPR42GTzqw4bXLVOOBZibFNyARKAfFuhTUEp5AAOBpQUOvwUMVEodAwaYfhZV1KWrmRyIvSzzE0TF1KKfMULoqAWakNKTjXkJrQdD467XPufiYWw1evKvCrO8hrmrpN5nyUK11qmAV6FjiRijkUQ1sO14Iloj/QmiYqpR11j24tha6P+fks+/ka2fQPolY7vVonSZBFs+MmoLzW6x+8ZL5i6I56+UWqaUijfdflJKlXLHciH+sTkqEQ8XR4L969g7FCGK1mqgMbP5clzJ5xYnNRG2fgqBdxQ/Gc7Zzdg8KGZnhdjPwdzmo28w2vz9TLeVpmNClMnm6ATCmtXDWXYwExVV61uN+6h1Zb/Glg8h8wr0KaaWkKfTeKgbYNQWcsu1aES5mfs/0kdr/Y3WOtt0mwvI0J9KKjdXcybpqt3KP5eczvELqdwk8xNERVY/EGo1Kvu395TzsH02BN8D9dve+FxHZ+jzolEzObyybOVZiLlJIVEpNV4p5Wi6jcfoeBaV0KKdZ+j99h9siUqwS/lLws8A0KuVJAVRgSllNCFFbzSWpyitv96FnEzo84J55wfdDd6tjXkLlhoKWwbmJoX7gdEYM43jgFGARTufhe0s3RWD1jB96T6uZmbbtOyo+BT+tyGK24N8aedby6ZlC1FqrW6FzBQ4Xcot6S+dgYhvjGahes3Ne42Do9EZnXDEWBfJTm6YFJRSM00Pw7TWw7XWPlrr+lrrEVrr0zaIT1jY2UtphJ+6SP+29TmddJV3fjtqs7JzcjXP/xiJh6sjrw5vb7NyhSizZr3B0aX0TUib3jbuez9Xute1uwMaBsHG/4OcrNK91kJKqikMUUop4EVbBCOsb1VkLACvDAtkfPcmfLPlBBGnbLNC+bwtJ9l1+hIzhrXHx9PVJmUKUS6uNaFpz9IlhcRo2D0futwHdRqXrjwHB+j7Mlw8AXsWlO61FlJSUlgDXASClVKXlVIpBe9tEJ+wsJV74+joX5umXh68MLgtvrXceOGnSDKyrduGeTrxKm//doR+betzR4ifVcsSwqJa3woJRyHphHnn/znTqF3c/GzZy2sUCn/OguyMsl2jHG6YFLTWz2mt6wCrtNa1tNaeBe9tFKOwkBMJqew7m8ywjsaHsqebM2/eGURU/BX+tz7KauVqrZm+NBInB8UbIzug7Dw5R4hSaTXIuD9mxtDU+MMQ+QN0mwyeZVzoWSlj0bzLMRAxt2zXKAdzV0mVBFAFrNwbi1IwNPifb+p92tTnrs7+fPZnNPvPJlul3EU7z7AlOpEXh7TDt3YNq5QhhNV4tYB6LcxrQtr4JrjUhJ5Pla/M5n2gaS/Y9A5k2nb4uLmrpOYqpWrbIB5hJVprVuyNpWtAPRrWdrvmuf8MbUdddxee/zGSrBzLTpyJS07jzVWH6NHci7FhpWxfFaKiaDXIWJ/oRh/QcXvh4M/Q4zFwL+fKv3m1hdR42DmnfNcqJXOHpF4B9imlvlJKfZR3s2ZgwrKOnE8hKv5KftNRQXXcXXh9RHsOxl1m9qbjFitTa83Ly/aTnat5664gaTYSlVergZCdbiSG4vzxJrjVMTbPsYSmPYw9o/9+H9Jt14VrblJYCvwH2AREFLiJSmLFnlgcHRRDOjQs8vnBHXwZEtSQD38/RlR8imXK3BvL+sPxTLu1DU29PCxyTSHsIqAXOLsX34R0ZiccXWPsj+BmwUaVvv82lvDe9pnlrlkCs5KC1noe8AOwTWs9L+9m3dCEpWitWRkZS8+W3njVLH4o6H+Hd8Dd1ZHnf4wkJ7fIje/MlnAlg1dXHKBzkzpMuimgXNcSwu6cXI12/qNrQRfxf2PDa+DhA90etmy5jTpD26Gw9WO4mmTZaxfD3FVShwF7MIaoopQKUUqtsGZgwnL2xiRzJimNYcG+NzzPx9OVGcMC2XX6EvO2nCxXmTNWHCA1I4dZo4JxdJBmI1EFtBoIyafhwpFrj5/YBCf+hF7PGPsjWFrflyAjBbb8z/LXLoK5zUevAmHAJQCt9R7AzLnbwt5W7o3FxdGBQe2LbjoqaERII/q28eHt345wOrFsox5+O3COVZFxTO3fkpb1Pct0DSEqnPyhqQWakLQ21iry9IPQcm1bX7wG7aHDXbD9c7hi/X3pzU0KWVrrwuMV7bu+qzBLTq7ml8hYbmnjQ+0aziWer5TijZFBODoopi+NRBdVVb6B5KtZvLx8P4G+tXj4lhZlDVuIiqe2P9Rvf21SiPodzmyD3tOMfRGspc+LRkf33+9ZrwwTc5PCAaXUOMBRKdVKKfU/YIsV4xIWsvNkEucvZzC8iFFHxfGrU4MXh7RlS3Qii3aeKVV5r686SFJqJrNGBcteCaLqaTXQWBwvPdlUS3gN6jSFThOsW653S+g4DnZ+BclnrVqUuf9rnwDaAxnAAiAZKOfsDGELK/fGUsPZkf7t6pfqdWO7NqFHcy/eXHWIuOQ0s16z6egFlkTE8MgtzenQSKa1iCqo9a2Qm23sp3z4F2NuQp/p4ORi/bJveR50Lvz1jlWLKWmVVDel1FPALOA00ENr3VVr/bLWOt2qkYlyy8rJZfX+cwwIbIC7i1nbcedzcFC8dVcQWbm5vLxsf4nNSFcysnlx6T5a+HjwRL9W5QlbiIrLP8wYcnpkjdGX4NUKgkbbpuy6TaHLRNj1LVw8abViSqopzANCgX3AbYB1U5SwqM1RCSSlZpY46qg4Tb08mDaoDesPx/PzntgbnjtrzWFik9OYNaojbs6OZSpPiArP0Qla9IfIRXDhEPR90ThmKzdPAwcn2Diz5HPLqKSkEKi1Hq+1/gJjY53eVotEWNzKvXF4ujlxS5uy75x6X89mdGpSh/+uPEDClaJXbNxxIolvt55i0k0BdGlat8xlCVEptBpkNOM06ACBI21bdi1f6PqgKSlZZy+UkpJC/i4PWmvbbtElyiU9K4e1B84xuH1DXJ3K/s3d0UEx665gUjNymLHiQJHlvPBTJI3r1eC5W9uUJ2QhKofWt4JPWxj0urH/ga31fApcPeG0dcb6lFTv6Vhg3wQF1DD9rAAty2dXXH8evUBKRnaRax2VVqsGnkzt35J31h5leMdz3FpgvsP7vx/lREIqCx7sVup+CyEqJfd68Ph2+5Vf0wee2g9u1vn4LWk/BUfT/gl5eyg4yX4KlcPKvbF4ebhwUwsvi1zv4VtaEOhbi5eX7yf5qlGB3HvmEnM2HWdsWGNuaultkXKEEGawUkIA84ekikokNSOb3w+dZ0iQL04Wmivg7OjArFHBJKVm8tqqg2Rm5/LCT5HU93TjxSHtLFKGEML+7FLfV0rVAb4EOgAauB84AiwGAoCTwGittW02D65ifj90nvSsXIs0HRXUoVFtHu7dnE83RpOUmsnhcyl8NTGUWm4lz5QWQlQO9qopfAis0Vq3BToCh4DpwHqtdStgvelnUQYr98bRsJYboVYYCTS1fyta+Hiw4XA8d4T40b9dGbccFEJUSDZPCqYd3HoDXwForTO11peAOzDmRWC6H2Hr2KqC5KtZ/Hk0nqHBvjhYYXVSN2dHPhzTiduDfZkxrL3Fry+EsC971BSaAReAb5RSu5VSXyqlPIAGWus40znngCK/giqlJiulwpVS4RcuWH/FwMrmtwPnyMrRDA+xbNNRQR0a1eaTcZ2p52GDqf1CCJuyR1JwAjoDn2mtOwGpFGoq0saaCkWuq6C1nq21DtVah/r4lH1SVlW1MjKWpl7uBMnaQ0KIMrBHUogBYrTWeQN9f8RIEueVUr4Apvt4O8RWqSVcyWBzVALDgv1kP2QhRJnYPClorc8BZ5RSedNf+wMHgRXARNOxicDPto6tslu9L45cjcVHHQkhqg97TUF9AvheKeUCHAfuw0hQPyilHgBOATZaerDqWLE3ljYNPGnTUHY7E0KUjV2Sgmk7z9Ainupv61iqithLaew8eZFpg1rbOxQhRCUmM5qriFWRxsCtocHSdCSEKDtJClXEyshYgv1rE+DtYe9QhBCVmCSFKuBEQiqRMcml2odZCCGKIkmhCvhlr7Er2u1l3GFNCCHySFKoAlZGxhIWUA/f2jXsHYoQopKTpFDJHTmXwtHzVxjWUWoJQojyk6RQya3YexZHB8VtQZIUhBDlJ0mhEtNas3JvHDe18MK7pqu9wxFCVAGSFCqxyJhkTiddlWUthBAWI0mhElu5NxYXRwdubd/Q3qEIIaoISQqVVG6u5pfIOG5p40PtGrIdphDCMiQpVFI7TyZx7nK6NB0JISxKkkIltTIylhrOjgxoV9/eoQghqhBJCpVQdk4uv+47R/929XF3sdfq50KIqkg+Ucpg1+mLzFx9mFxd5I6hVpeelUtSaqasdSSEsDhJCmXw4e/HOBh7mSB/++yD7OzowPCOftzSRvaoFkJYliSFUjqdeJVNxy7wRL9WPDNQNrQRQlQt0qdQSt/vOIWDUowNa2zvUIQQwuIkKZRCelYOP+w8w4B29WVFUiFElSRJoRRW74/j4tUsxndvau9QhBDCKiQplMJ3W0/RzNuDni287R2KEEJYhSQFMx2Mvcyu05e4t1sTHByUvcMRQgirkKRgpvnbT+Hq5MCoLv72DkUIIaxGkoIZUtKzWL77LMM6+lHH3cXe4QghhNVIUjDDst1nuZqZIx3MQogqzy6T15RSJ4EUIAfI1lqHKqXqAYuBAOAkMFprfdEe8RWktWb+tlMENapNRzvNYBZCCFux54zmvlrrhAI/TwfWa63fUkpNN/38gn1C+8eOE0kcPX+FmXcFoZR0MIvyycrKIiYmhvT0dHuHIqoBNzc3/P39cXY2f8+VirTMxR1AH9PjecBGKkBSmL/9NJ5uTrJvgbCImJgYPD09CQgIkC8Zwqq01iQmJhITE0OzZs3Mfp29+hQ0sFYpFaGUmmw61kBrHWd6fA5oYJ/Q/nEhJYM1++MY1cVflqgWFpGeno6Xl5ckBGF1Sim8vLxKXSu11yddL631WaVUfWCdUupwwSe11lopVeS61KYkMhmgSZMmVg3yh/AzZOVo7u0mHczCciQhCFspy9+aXWoKWuuzpvt4YBkQBpxXSvkCmO7ji3ntbK11qNY61MfHektH5+RqFmw/TY/mXrSsX9Nq5QghREVi86SglPJQSnnmPQYGAfuBFcBE02kTgZ9tHVtBG4/Ec/ZSGhN6SC1BVC1KKcaPH5//c3Z2Nj4+PgwdOhSA8+fPM3ToUDp27EhgYCBDhgwB4OTJk9SoUYOQkJD827fffltkGTt27KB37960adOGTp068eCDD3L16lUAli9fTnBwMO3atSMoKIjly5fnv27btm1069aNkJAQ2rVrx6uvvgrA3LlzmTJlCgCvvvoq7u7uxMf/872xZs1/vrg5OjpeE+Nbb711XXy5ublMnTqVDh06EBQURNeuXTlx4gQAAQEBJCQkmPVeFY7rnXfeKfL9SEhIwNnZmc8///ya4wEBAQQFBREcHMwtt9zCqVOnrvmd9u3bl/971KtXj2bNmhESEsKAAQNo27Yt+/btyz//7bff5uGHHy6y/NKwR/NRA2CZqVrjBCzQWq9RSu0EflBKPQCcAkbbIbZ83207RX1PVwYG2r1rQwiL8vDwYP/+/aSlpVGjRg3WrVtHo0aN8p9/5ZVXGDhwIE8++SQAkZGR+c+1aNGCPXv23PD658+f5+6772bRokX06NEDgB9//JGUlBSOHTvGtGnTWLduHc2aNePEiRMMHDiQ5s2bExwczMSJE/nhhx/o2LEjOTk5HDlypMgyvL29effdd5k5c+Z1z9WoUaPEGBcvXkxsbCyRkZE4ODgQExODh4dHqd8rcy1ZsoTu3buzcOFCHnnkkWue++OPP/D29mbGjBm8/vrrzJkzJ/+5oKCg/N9l0qRJDB06lFGjRgGwZs0aHnvsMTZt2kRsbCyff/454eHhpY6tMJsnBa31caBjEccTgf62jqcoZ5Ku8ufRCzzRtyXOjjK/T1jHf1ce4GDsZYteM9CvFjOGtS/xvCFDhrBq1SpGjRrFwoULGTt2LH/99RcAcXFxDBo0KP/c4ODgUsXwySefMHHixPyEAOR/kE2bNo2XXnopfzRMs2bNePHFF3n77bf57rvviI+Px9fXFzC+8QcGBhZZxv3338/cuXN54YUXqFevXqniA+N39PX1xcHB+P/t71/88jU3eq/MtXDhQt59913GjRtHTExMkeX16NGDjz76yOxrDh48mK+//ppvv/2WVatW8eqrr1K3bt1SxVUU+cQrwvfbTxsb6XSzbke2EPYyZswYFi1aRHp6OpGRkXTr1i3/uccff5wHHniAvn378sYbbxAbG5v/XHR09DVNM0V9OO7fv58uXboUWe6BAweuey40NJQDBw4A8PTTT9OmTRtGjhzJF198UezImZo1a3L//ffz4YcfXvdcWlraNTEuXrz4unNGjx7NypUrCQkJ4dlnn2X37t1FlgM3fq/McebMGeLi4ggLC2P06NFFxgPGN/8RI0aU6toffPAB//73v7lw4QITJkwo1WuLI+MsC8nIzuGH8DP0bysb6QjrMucbvbUEBwdz8uRJFi5cmN9nkOfWW2/l+PHjrFmzhtWrV9OpUyf2798PmNd8VB6vvPIK9957L2vXrmXBggUsXLiQjRs3Fnnu1KlTCQkJYdq0adccN6f5yN/fnyNHjrBhwwY2bNhA//79WbJkCf37X99YcaP3yhyLFy9m9GijNXzMmDHcf//9PPvss/nP9+3bl6SkJGrWrMlrr71Wqmv7+fnRr1+//D4OS5CaQiGr950jKTVT1jkSVd7w4cOZNm0aY8eOve65evXqMW7cOL777ju6du3Kpk2bir3OsmXL8r+Vh4eH0759eyIiIoo8NzAw8LrnIiIiaN/+nwTZokULHn30UdavX8/evXtJTEws8lp16tRh3LhxfPLJJyX+rtu3b8+PccWKFQC4urpy22238fbbb/PSSy9d0+Fd2I3eq5IsXLiQuXPnEhAQwPDhw4mMjOTYsWP5z//xxx+cOnWKkJAQZsyYUerrOzg45DeDWYLUFAqZv+0UAV7u9GopG+mIqu3++++nTp06BAUFXfNtfMOGDXTv3h13d3dSUlKIjo6+4ZygkSNHMnLkyPyfGzduTFhYGLfffnt+U8vSpUvp2bMn06ZN4+6776Zfv34EBARw8uRJ3nzzTX788UcAVq1axZAhQ1BKcezYMRwdHalTp06xZT/zzDN07dqV7OzsG/6u3bp1u6b2sGvXLho2bIifnx+5ublERkbesO+kuPeqJEePHuXKlSucPXs2/9iMGTNYuHAhr7zySv4xJycnPvjgA4KCgnj55ZfL1E9iKVJTKOBQ3GXCT13k3m5NZSMdUeX5+/szderU645HREQQGhpKcHAwPXr04MEHH6Rr167A9X0KRXWMNmjQgEWLFjFt2jTatGlDu3bt+O233/D09CQkJISZM2cybNgw2rZty7Bhw5g1axYhISEAfPfdd7Rp04aQkBAmTJjA999/j6OjY7G/g7e3NyNHjiQjIyP/WOE+henTp1/3uvj4eIYNG0aHDh0IDg7Gyckpf2hpad6rwl5//XX8/f3zbwsXLrwmYQLcddddLFy48LrX+vr6MnbsWLNqPtaktC5y4nClEBoaqi0xBCvPv5ftY0lEDNtf7E9dD9k3QVjeoUOHaNeunb3DENVIUX9zSqkIrXVoUedLTcEkfyOdYD9JCEKIakuSgsny3WdJzcxhfHcZhiqEqL4kKZC3kc5p2vvVIqRx8Z1aQghR1UlSAHaevMiR8ylM6N5UVrAUQlRrkhQwhqF6ujkxPEQ20hFCVG/VPikkXMlg9f447uosG+kIIUS1TwqLdxob6UgHs6guCi4zbSl9+vTJX6FzyJAhXLp06bpzbrS0tDVs3LjRoss/VBfV+qtx3kY63ZvXo2V9T3uHI0SV8Ouvv9o7BIvIycm54cS5qqpaJ4U/jxob6bw0RCYTCTtYPR3O7Sv5vNJoGAS3Xb+pTFE2btzIq6++ire3d/7KpvPnz+e3337jq6++YsmSJfnnvfPOO/zyyy88+uij7Ny5k7S0NEaNGsV///vf664bEBBAeHg43t7evPHGG8ybN4/69evTuHHjIldPnTRpErVq1SI8PJxz584xa9YsRo0adU25AFOmTCE0NJRJkyYREBDA2LFjWb16NU5OTsyePZsXX3yRqKgonnvuufw9Cy5fvsztt99OVFQUffv25dNPP8XBwYG1a9cyY8YMMjIyaNGiBd988w01a9YkICCAe+65h3Xr1vH8888THx/P559/jpOTE4GBgSxatKis/zKVRrVOCt9tPYWPpyuD2stGOqJ62r17NwcOHMDPz4+ePXuyefNmBgwYwOTJk0lNTcXDw4PFixczZswYAN544w3q1atHTk4O/fv3v+GaQRERESxatIg9e/aQnZ1N586di11SOy4ujr///pvDhw8zfPjw/P0XbqRJkybs2bOHp59+mkmTJrF582bS09Pp0KFDflLYsWMHBw8epGnTpgwePJilS5fSp08fXn/9dX7//Xc8PDyYOXMm7733Xv5aRF5eXuzatQswViE9ceIErq6uRTaJVUXVNimcSbrKxqMXmCIb6Qh7MfMbvTWFhYXlb/gSEhLCyZMn6dWrF4MHD2blypWMGjWKVatWMWvWLAB++OEHZs+eTXZ2NnFxcRw8eLDYpPDXX38xcuRI3N3dAWOl0eKMGDECBwcHAgMDOX/+vFmx510vKCiIK1eu4Onpiaen5zUf4GFhYTRv3hyAsWPH8vfff+Pm5sbBgwfp2bMnAJmZmddsCHTPPffkPw4ODubee+9lxIgRpd7roLKqtklhwY7TKGBsmHQwi+rL1dU1/7Gjo2P+aqNjxozh448/pl69eoSGhuLp6cmJEyd455132LlzJ3Xr1mXSpEnFboJTnjjy1mNzcnIiNzc3/3jhsvJe4+DgcM3rHRwc8n+PwvOOlFJorRk4cGCRi9IB12zLuWrVKjZt2sTKlSt544032LdvH05OVftjs1p+Rc7IzmHxzjP0b9cAvzqykY4Qhd1yyy3s2rWLOXPm5DcdXb58GQ8PD2rXrs358+dZvXr1Da/Ru3dvli9fTlpaGikpKaxcubJUMTRt2pSDBw+SkZHBpUuXWL9+fal/jx07dnDixAlyc3NZvHgxvXr1onv37mzevJmoqCgAUlNTOXr06HWvzc3N5cyZM/Tt25eZM2eSnJzMlStXSh1DZVO1U14x1uyXjXSEuBFHR0eGDh3K3LlzmTdvHgAdO3akU6dOtG3blsaNG+c3vxSnc+fO3HPPPXTs2JH69evnL79trsaNGzN69Gg6dOhAs2bN6NSpU6l/j65duzJlypT8juaRI0fi4ODA3LlzGTt2bP6S26+//jqtW7e+5rU5OTmMHz+e5ORktNZMnTr1hns7VBXVcuns3w+eZ3H4Gb4Y30X2TRA2JUtnC1sr7dLZ1bKmMCCwAQMCZcSREEIUVi37FIQQQhRNkoIQNlaZm2xF5VKWvzVJCkLYkJubG4mJiZIYhNVprUlMTMTNza1Ur6uWfQpC2Iu/vz8xMTFcuHDB3qGIasDNzS1/cqK57JYUlFKOQDhwVms9VCnVDFgEeAERwAStdaa94hPCGpydnWnWrJm9wxCiWPZsPnoSOFTg55nA+1rrlsBF4AG7RCWEENWYXZKCUsofuB340vSzAvoBP5pOmQdUj4VGhBCiArFXTeED4Hkgb2ETL+CS1jrb9HMM0KioFyqlJiuloyKGrAAAB7VJREFUwpVS4dIuK4QQlmXzPgWl1FAgXmsdoZTqU9rXa61nA7NN17qglDpVxlC8gYQyvtYWJL7ykfjKr6LHKPGVXbFr/Nijo7knMFwpNQRwA2oBHwJ1lFJOptqCP3C2pAtprX3KGoRSKry4ad4VgcRXPhJf+VX0GCU+67B585HW+kWttb/WOgAYA2zQWt8L/AHk7awxEfjZ1rEJIUR1V5Emr70APKOUisLoY/jKzvEIIUS1Y9fJa1rrjcBG0+PjQJgNi59tw7LKQuIrH4mv/Cp6jBKfFVTqpbOFEEJYVkVqPhJCCGFnkhSEEELkq/JJQSk1WCl1RCkVpZSaXsTzrkqpxabntyulAmwYW2Ol1B9KqYNKqQNKqSeLOKePUipZKbXHdHvFVvGZyj+plNpnKvu6be6U4SPT+xeplOpsw9jaFHhf9iilLiulnip0js3fP6XU10qpeKXU/gLH6iml1imljpnu6xbz2ommc44ppSbaKLa3lVKHTf9+y5RSRe45WdLfgpVjfFUpdbbAv+OQYl57w//vVoxvcYHYTiql9hTzWpu8h+Wita6yN8ARiAaaAy7AXiCw0DmPAZ+bHo8BFtswPl+gs+mxJ3C0iPj6AL/Y8T08CXjf4PkhwGpAAd2B7Xb8tz4HNLX3+wf0BjoD+wscmwVMNz2eDsws4nX1gOOm+7qmx3VtENsgwMn0eGZRsZnzt2DlGF8FppnxN3DD/+/Wiq/Q8+8Cr9jzPSzPrarXFMKAKK31cW2suLoIuKPQOXdgrLUExtpL/U1rMVmd1jpOa73L9DgFY4HAIpf3qMDuAL7Vhm0YkxB97RBHfyBaa13WGe4Wo7XeBCQVOlzw76y4tb1uBdZprZO01heBdcBga8emtV6r/1liZhvG5FG7Keb9M4c5/9/L7UbxmT47RgMLLV2urVT1pNAIOFPg56LWVMo/x/QfIxljnoRNmZqtOgHbi3i6h1Jqr1JqtVKqvU0DAw2sVUpFKKUmF/G8Oe+xLYyh+P+I9nz/8jTQWseZHp8DitokvCK8l/dj1PyKUtLfgrVNMTVxfV1M81tFeP9uBs5rrY8V87y938MSVfWkUCkopWoCPwFPaa0vF3p6F0aTSEfgf8ByG4f3/9u7txCrqjiO499faUUqk93oQlRaRAk1lEjo5IshGSElRhczsl4Ee6heejAopIceop6kpISs5iGUpCGkwAkGepAxRKcrNfQ0IjMQYUxh5PjvYa2z3ZzLeFLP2cP0+8Bhzuyzzt7rrFl7/+esvfd/9UXE3cAaYIuklV3e/hlJughYC+xu8nLV7dcg0jjCjLsWXNJW4CTQ36JIlX3hHWAx0AscIw3RzERPMP23hBm/P832oHAUuKH0e7OcSkUZSXOAHuC3rtQubXMuKSD0R8Sn9a9HxB8RMZmf7wPmSrqyW/WLiKP55wSwl8YbDNtp405bAxyKiPH6F6puv5Lx2rBa/jnRpExlbSnpGeAhYEMOWg3a6AsdExHjETEVEaeA91psu9K+mI8f64BPWpWpsg3bNduDwkHgVkk35/8mHwcG6soMkHItQcq99FWrneJ8y+OPO4EfI+KtFmWuqZ3jkLSM9DfrStCSNE/Sgtpz0gnJ7+qKDQBP56uQ7gWOl4ZJuqXlf2dVtl+dcj9rldvrS2C1pIV5eGR1XtZRkh4gpbJfGxF/tSjTTl/oZB3L56keabHtdvb3Trof+Ckixpq9WHUbtq3qM92dfpCujvmZdFXC1rxsG2kHgJSpdTcwCgwDi7pYtz7SMMIIcDg/HgQ2A5tzmeeB70lXUhwAlnexfovydo/kOtTar1w/Adtz+34LLO3y33ce6SDfU1pWafuRAtQx4B/SuPZzpPNUg8AvwH7g8lx2KfB+6b3P5r44CmzqUt1GSWPxtT5YuxrvOmDfdH2hi+33Ue5fI6QD/bX1dcy/N+zv3ahfXv5Brd+VylbShufycJoLMzMrzPbhIzMz+w8cFMzMrOCgYGZmBQcFMzMrOCiYmVnBQcGsi3LW1s+rrodZKw4KZmZWcFAwa0LSU5KGc977HZIulDQp6W2luS8GJV2Vy/ZKOlCaj2BhXn6LpP05Gd8hSYvz6udL2pPnMOgv3XH9htLcGiOS3qzoo9v/nIOCWR1JtwOPASsioheYAjaQ7p7+JiKWAEPAq/ktHwIvR8SdpLtua8v7ge2RkvEtJ90FCykb7gvAHaS7XFdIuoKUvmFJXs/rnf2UZs05KJg1WgXcAxzMM2itIh28T3E62dnHQJ+kHuCyiBjKy3cBK3OOm+sjYi9ARJyI03mFhiNiLFJyt8PATaSU7SeAnZLWAU1zEJl1moOCWSMBuyKiNz9ui4jXmpQ72xwxf5eeT5FmPTtJypi5h5St9IuzXLfZOXFQMGs0CKyXdDUU8yvfSNpf1ucyTwJfR8Rx4HdJ9+XlG4GhSDPpjUl6OK/jYkmXttpgnlOjJ1J67xeBuzrxwczOZE7VFTCbaSLiB0mvkGbIuoCUDXML8CewLL82QTrvACkV9rv5oP8rsCkv3wjskLQtr+PRaTa7APhM0iWkbyovneePZdYWZ0k1a5OkyYiYX3U9zDrJw0dmZlbwNwUzMyv4m4KZmRUcFMzMrOCgYGZmBQcFMzMrOCiYmVnhX2p/MvR3/Ny+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "msec={}\n",
    "msec['MSE-COSINE-SIMILARITY']=[42.640143,53.935989, 53.935989,53.935989, 67.419986, 60.302269,67.419986,85.280287, \n",
    "79.772404, 73.854895, 85.280287, 81.818182, 90.909091, 100.000000, 100.000000, 100.000000, 95.346259, 100.000000,\n",
    "95.346259, 95.346259]\n",
    "msec['Invalid numbers']=[96,95,95,95,95,95,87,89,89,84,74,76,79,81,80,73,78,62,71,62]\n",
    "import pandas as pd \n",
    "msec=pd.DataFrame.from_dict(msec)\n",
    "import matplotlib.pyplot as plt\n",
    "fig,ax = plt.subplots()\n",
    "\n",
    "for name in msec.columns:\n",
    "    ax.plot(msec[name].values,label=name)\n",
    "\n",
    "ax.set_xlabel(\"epochs\")\n",
    "ax.set_ylabel(\"Performance\")\n",
    "ax.legend(loc='best')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "_cell_guid": "265fd2f2-cd5f-590d-1fa3-cf6eeedf89fe",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "SRKF7S2a-fPe",
    "outputId": "6c30a2e9-594f-4427-a492-f4dd4b754d5f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "----Prepare evaluation\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 305, 47, 81, 618, 213, 73, 619, 7, 3, 3]\n",
      "  Input Words: tell me your plans for the future .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [692, 56, 123, 693, 694, 149]\n",
      "  Response Words: explica m els teus plans per\n",
      " Ground Truth: explica m els teus plans per al futur .\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 16, 15, 70, 644, 189, 16, 645, 49, 7, 3, 3]\n",
      "  Input Words: i can t remember where i bought it .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [82, 724, 210, 19, 71, 306]\n",
      "  Response Words: no recorde on el vaig comprar\n",
      " Ground Truth: no recordo on el vaig comprar .\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 632, 67, 633, 80, 634, 635, 7, 3, 3]\n",
      "  Input Words: bangkok is thailand s capital city .\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [712, 40, 98, 713]\n",
      "  Response Words: bangkok es la capital\n",
      " Ground Truth: bangkok es la capital de tailandia .\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 143, 14, 150, 138, 636, 637, 36, 37, 5, 3, 3]\n",
      "  Input Words: do you want to play tennis with us ?\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [715, 716, 27, 717]\n",
      "  Response Words: vols jugar a tennis\n",
      " Ground Truth: vols jugar a tennis amb nosaltres ?\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 143, 14, 150, 138, 636, 637, 36, 37, 5, 3, 3]\n",
      "  Input Words: do you want to play tennis with us ?\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [715, 716, 27, 717, 44]\n",
      "  Response Words: vols jugar a tennis amb\n",
      " Ground Truth: vols jugar a tennis amb nosaltres ?\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "\n",
      "...prepare evaluation finished.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#random = np.random.randint(0,3000)\n",
    "#text = news_texts_filtered[random]\n",
    "model_predictions = []\n",
    "ground_truth = []\n",
    "invalid_number_prediction_count = []\n",
    "checkpoint = logs_path + 'best_so_far_model.ckpt' \n",
    "print(\"\\n----Prepare evaluation\")\n",
    "for ii in range(99):\n",
    "    random = np.random.randint(410,513)\n",
    "    text = news_texts_filtered[random]\n",
    "\n",
    "    #random = np.random.randint(3000,len(news_texts_filtered))\n",
    "        \n",
    "    loaded_graph = tf.Graph()\n",
    "    with tf.Session(graph=loaded_graph) as sess:\n",
    "                  loader = tf.train.import_meta_graph(checkpoint + '.meta')\n",
    "                  loader.restore(sess, checkpoint)\n",
    "                  input_data = loaded_graph.get_tensor_by_name('input_data:0')\n",
    "                  logits = loaded_graph.get_tensor_by_name('predictions:0')\n",
    "                  text_length = loaded_graph.get_tensor_by_name('text_len:0')\n",
    "                  summary_length = loaded_graph.get_tensor_by_name('summary_len:0')\n",
    "                  keep_prob = loaded_graph.get_tensor_by_name('dropout_probs:0')\n",
    "                  result_logits = sess.run(logits, {input_data: [text]*batch_size, \n",
    "                  summary_length: [np.random.randint(5,8)], \n",
    "                  text_length: [len(text)]*batch_size,\n",
    "                  keep_prob: 1.0})[0] \n",
    "\n",
    "    \n",
    "                  \n",
    "                  O=\" \".join(int2word[i] for i in result_logits if i not in tokens_id)\n",
    "                  Y=\" \".join(int2word[i] for i in news_summaries_filtered[random] if i not in tokens_id)\n",
    "                  if mean_absolute_error_texts([Y],[O]) >90.00:\n",
    "                     print('        ')\n",
    "                     print('        ')\n",
    "                     #print('\\nOriginal Text:', input_sentence)\n",
    "\n",
    "                     print('\\nText')\n",
    "                     print('  Word Ids:    {}'.format([i for i in text]))\n",
    "                     print('  Input Words: {}'.format(\" \".join( int2wordx[i] for i in text if i not in tokens_id)))\n",
    "\n",
    "                     print('\\nSummary')\n",
    "                     print('  Word Ids:       {}'.format([i for i in result_logits if i not in tokens_id]))\n",
    "                     print('  Response Words: {}'.format(\" \".join(int2word[i] for i in result_logits if i not in tokens_id)))\n",
    "                     print(' Ground Truth: {}'.format(\" \".join(int2word[i] for i in news_summaries_filtered[random] if i not in tokens_id)))\n",
    "                     print('        ')\n",
    "                     print('        ')\n",
    " \n",
    "model_predictions.append(result_logits)\n",
    "ground_truth.append(news_summaries_filtered[random])         \n",
    "print('\\n...prepare evaluation finished.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "4UN0LwG3r7FP",
    "outputId": "cf95c914-a169-428f-d869-fb49e99101fa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finish \n"
     ]
    }
   ],
   "source": [
    "print(\"Finish \")"
   ]
  }
 ],
 "metadata": {
  "_change_revision": 0,
  "_is_fork": false,
  "colab": {
   "collapsed_sections": [],
   "name": "cat-eng-seq2seq-model.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}