05-02-amazon-reviews-seq2seq-colab-with-attetion.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GvEmSE6vTdTS"
   },
   "source": [
    "<h1>TEXT SUMMARIZTION WITH AMAZON-REVIEWS</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 358
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9505,
     "status": "ok",
     "timestamp": 1589541279980,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "O6YEZIHOYXqA",
    "outputId": "b94dbdde-6e6e-434f-fc67-0c1c733030b0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow==1.15 in /usr/local/lib/python3.6/dist-packages (1.15.0)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.12.1)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.28.1)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (3.10.0)\n",
      "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.18.4)\n",
      "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.2.0)\n",
      "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.12.0)\n",
      "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.8.1)\n",
      "Requirement already satisfied: tensorboard<1.16.0,>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.15.0)\n",
      "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.34.2)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.9.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.1.0)\n",
      "Requirement already satisfied: tensorflow-estimator==1.15.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.15.1)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (3.2.1)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.1.0)\n",
      "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (1.0.8)\n",
      "Requirement already satisfied: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.15) (0.2.2)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow==1.15) (46.1.3)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (1.0.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15) (3.2.1)\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow==1.15) (2.10.0)\n"
     ]
    }
   ],
   "source": [
    "from distutils.version import LooseVersion\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.layers.core import Dense\n",
    "\n",
    "\n",
    "# Check TensorFlow Version\n",
    "assert LooseVersion(tf.__version__) >= LooseVersion('1.1'), 'Please use TensorFlow version 1.1 or newer'\n",
    "!python3.6 -m pip install tensorflow==1.15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10104,
     "status": "ok",
     "timestamp": 1589541280646,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "Nu3dP2ypavXH",
    "outputId": "dc7e19ad-f2ee-45da-cc86-589769bbf03a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.15.0\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['AttentionCellWrapper',\n",
       " 'BasicLSTMCell',\n",
       " 'BasicRNNCell',\n",
       " 'BidirectionalGridLSTMCell',\n",
       " 'CompiledWrapper',\n",
       " 'Conv1DLSTMCell',\n",
       " 'Conv2DLSTMCell',\n",
       " 'Conv3DLSTMCell',\n",
       " 'ConvLSTMCell',\n",
       " 'CoupledInputForgetGateLSTMCell',\n",
       " 'DeviceWrapper',\n",
       " 'DropoutWrapper',\n",
       " 'EmbeddingWrapper',\n",
       " 'FusedRNNCell',\n",
       " 'FusedRNNCellAdaptor',\n",
       " 'GLSTMCell',\n",
       " 'GRUBlockCell',\n",
       " 'GRUBlockCellV2',\n",
       " 'GRUCell',\n",
       " 'GridLSTMCell',\n",
       " 'HighwayWrapper',\n",
       " 'IndRNNCell',\n",
       " 'IndyGRUCell',\n",
       " 'IndyLSTMCell',\n",
       " 'InputProjectionWrapper',\n",
       " 'IntersectionRNNCell',\n",
       " 'LSTMBlockCell',\n",
       " 'LSTMBlockFusedCell',\n",
       " 'LSTMBlockWrapper',\n",
       " 'LSTMCell',\n",
       " 'LSTMStateTuple',\n",
       " 'LayerNormBasicLSTMCell',\n",
       " 'LayerRNNCell',\n",
       " 'MultiRNNCell',\n",
       " 'NASCell',\n",
       " 'OutputProjectionWrapper',\n",
       " 'PhasedLSTMCell',\n",
       " 'RNNCell',\n",
       " 'ResidualWrapper',\n",
       " 'SRUCell',\n",
       " 'TimeFreqLSTMCell',\n",
       " 'TimeReversedFusedRNN',\n",
       " 'UGRNNCell',\n",
       " '__builtins__',\n",
       " '__cached__',\n",
       " '__doc__',\n",
       " '__file__',\n",
       " '__loader__',\n",
       " '__name__',\n",
       " '__package__',\n",
       " '__path__',\n",
       " '__spec__',\n",
       " 'best_effort_input_batch_size',\n",
       " 'stack_bidirectional_dynamic_rnn',\n",
       " 'stack_bidirectional_rnn',\n",
       " 'static_bidirectional_rnn',\n",
       " 'static_rnn',\n",
       " 'static_state_saving_rnn',\n",
       " 'transpose_batch_time']"
      ]
     },
     "execution_count": 2,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "print(tf.__version__)\n",
    "dir(tf.contrib.rnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "3aee62b9-47ce-e416-5816-8df7126fe690",
    "colab": {},
    "colab_type": "code",
    "id": "FQ7ytEbpTdTV"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import gzip\n",
    "import codecs\n",
    "import re\n",
    "import time\n",
    "from tensorflow.python.ops.rnn_cell_impl import _zero_state_tensors\n",
    "from tensorflow.python.layers.core import Dense\n",
    "from tensorflow.contrib.rnn import GRUCell, DropoutWrapper\n",
    "from tensorflow.contrib.seq2seq import TrainingHelper, GreedyEmbeddingHelper,BasicDecoder, dynamic_decode, \\\n",
    " BahdanauAttention, AttentionWrapper,sequence_loss\n",
    "import helper\n",
    "\n",
    "TOKEN_GO = '<GO>'\n",
    "TOKEN_EOS = '<EOS>'\n",
    "TOKEN_PAD = '<PAD>'\n",
    "TOKEN_UNK = '<UNK>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 82
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10362,
     "status": "ok",
     "timestamp": 1589541281002,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "flWSMJRkztR4",
    "outputId": "7ba5ddd7-2660-40f8-db34-aab7b26c6163"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords \n",
    "from nltk.tokenize import word_tokenize \n",
    "from numpy import linalg as LA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10288,
     "status": "ok",
     "timestamp": 1589541281004,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "O9dyYsyuVIAg",
    "outputId": "e0109d89-9bd1-463c-b0e2-081978f02e4a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/gdrive/; to attempt to forcibly remount, call drive.mount(\"/content/gdrive/\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 10227,
     "status": "ok",
     "timestamp": 1589541281007,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "vLDd9ViBV8aX",
    "outputId": "010491c0-4f6d-40ca-951d-068010eed3ad"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['NewsAggregatorDataset.zip',\n",
       " '02_example.ipynb',\n",
       " 'Colab Notebooks',\n",
       " 'IMG-20191005-WA0001.jpg',\n",
       " 'prepared_test_data.tsv',\n",
       " 'prepared_train_data.tsv',\n",
       " 'prepared_validation_data.tsv',\n",
       " 'starspace_stackoverflow_model_03',\n",
       " 'starspace_stackoverflow_model_03.tsv',\n",
       " 'starspace_stackoverflow_model_03.txt',\n",
       " 'test_embeddings.tsv',\n",
       " 'test.tsv',\n",
       " 'train.tsv',\n",
       " 'validation.tsv',\n",
       " 'Rebut.pdf',\n",
       " 'CONCURSO',\n",
       " 'aqu-cert-cong-01.zip',\n",
       " 'AQU-2016-17a.zip',\n",
       " 'AQU-2016-17b.zip',\n",
       " 'aqu-cert-cong-03.zip',\n",
       " 'aqu-cert-cong-02.zip',\n",
       " 'aqu-cert-cong-04.zip',\n",
       " 'aqu-cert-cong-07.zip',\n",
       " 'aqu-cert-cong-06.zip',\n",
       " 'aqu-cert-cong-05.zip',\n",
       " 'aqu-other-03.zip',\n",
       " 'aqu-other-01.zip',\n",
       " 'aqu-other-02.zip',\n",
       " 'aqu-publi-01.zip',\n",
       " 'aqu-publi-02.zip',\n",
       " 'COURSERA-CERTIFICATIONS.zip',\n",
       " 'aqu-publi-03.zip',\n",
       " 'aqu-publi-04.zip',\n",
       " 'docs-01.zip',\n",
       " 'docs-02.zip',\n",
       " 'docs-03.zip',\n",
       " 'docs-04.zip',\n",
       " 'docs-05.zip',\n",
       " 'docs-06.zip',\n",
       " 'docs-07.zip',\n",
       " 'docs-09.zip',\n",
       " 'docs-08.zip',\n",
       " 'docs-10.zip',\n",
       " 'docs-12.zip',\n",
       " 'docs-11.zip',\n",
       " 'SEMINAR.zip',\n",
       " 'Titols-02.zip',\n",
       " 'unsl1213.split.zip',\n",
       " 'Titols-01.zip',\n",
       " 'Report.docx',\n",
       " 'Report.gdoc',\n",
       " 'Colabs_data',\n",
       " 'Python-NLTK',\n",
       " 'data-protfolio.ppt',\n",
       " '20200417_220825.jpg',\n",
       " '20200417_214248.jpg',\n",
       " '20200417_223046.jpg',\n",
       " '20200420_163316.jpg',\n",
       " '20200420_165338.jpg',\n",
       " 'q2c 20-Apr-2020 18-13-28(2).pdf',\n",
       " 'docs_python_org_3_library_re_html.pdf',\n",
       " 'attention.py',\n",
       " 'news.txt.gz',\n",
       " 'summary.txt.gz',\n",
       " 'glove6b50dtxt.zip',\n",
       " 'cc.ca.300.vec.gz',\n",
       " 'cc.en.300.vec.gz',\n",
       " 'data_new.csv',\n",
       " 'best_so_far_model.ckpt.index',\n",
       " 'checkpoint',\n",
       " 'best_so_far_model.ckpt.data-00000-of-00001',\n",
       " 'best_so_far_model.ckpt.meta',\n",
       " 'events.out.tfevents.1589529082.2356474a121a',\n",
       " 'events.out.tfevents.1589533377.2356474a121a',\n",
       " 'events.out.tfevents.1589534224.2356474a121a',\n",
       " 'events.out.tfevents.1589535405.2356474a121a',\n",
       " 'events.out.tfevents.1589540456.2356474a121a']"
      ]
     },
     "execution_count": 6,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.listdir('/content/gdrive/My Drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "654175ff-07c7-455d-1b8f-d702cee211c4",
    "colab": {},
    "colab_type": "code",
    "id": "gRkJ05iHTdTh"
   },
   "outputs": [],
   "source": [
    "#titledata=[]\n",
    "#artdata=[]\n",
    "#with gzip.open('/content/gdrive/My Drive/news.txt.gz') as artfile:\n",
    "#    for li in artfile:\n",
    "#        artdata.append(li.decode())\n",
    "#with gzip.open('/content/gdrive/My Drive/summary.txt.gz') as titlefile:\n",
    "#    for li in titlefile:\n",
    "#        titledata.append(li.decode())\n",
    "#news = pd.DataFrame({'Text':artdata,'Summary':titledata})\n",
    "#news = news.sample(frac=0.5)\n",
    "#news['Text_len'] = news.Text.apply(lambda x: len(x.split()))\n",
    "#news['Summary_len'] = news.Summary.apply(lambda x: len(x.split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "F9pKvHHzGNv7"
   },
   "outputs": [],
   "source": [
    "data_new=pd.read_csv('/content/gdrive/My Drive/data_new.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 191
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11120,
     "status": "ok",
     "timestamp": 1589541282011,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "5fY2Sfu7Gs2H",
    "outputId": "d47b98b5-9fb1-4529-ee71-efb0923341d5"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Text_len</th>\n",
       "      <th>Summary_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bought several vitality canned dog food produc...</td>\n",
       "      <td>good quality dog food</td>\n",
       "      <td>23</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>product arrived labeled jumbo salted peanuts p...</td>\n",
       "      <td>not as advertised</td>\n",
       "      <td>18</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>confection around centuries light pillowy citr...</td>\n",
       "      <td>delight says it all</td>\n",
       "      <td>39</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>looking secret ingredient robitussin believe f...</td>\n",
       "      <td>cough medicine</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>great taffy great price wide assortment yummy ...</td>\n",
       "      <td>great taffy</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text  ... Summary_len\n",
       "0  bought several vitality canned dog food produc...  ...           4\n",
       "1  product arrived labeled jumbo salted peanuts p...  ...           3\n",
       "2  confection around centuries light pillowy citr...  ...           4\n",
       "3  looking secret ingredient robitussin believe f...  ...           2\n",
       "4  great taffy great price wide assortment yummy ...  ...           2\n",
       "\n",
       "[5 rows x 4 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news=data_new[['cleaned_text','cleaned_summary']]\n",
    "news=news.rename(columns={'cleaned_text':'Text','cleaned_summary':'Summary'})\n",
    "news['Text_len'] = news.Text.apply(lambda x: len(x.split()))\n",
    "news['Summary_len'] = news.Summary.apply(lambda x: len(x.split()))\n",
    "news.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 151
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11056,
     "status": "ok",
     "timestamp": 1589541282014,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "Rxa-JA_QTdT8",
    "outputId": "d39d5b5c-9cab-4f2d-b5de-ac3de34c0a3d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bought several vitality canned dog food products found good quality product looks like stew processed meat smells better labrador finicky appreciates product better'\n",
      " 'product arrived labeled jumbo salted peanuts peanuts actually small sized unsalted sure error vendor intended represent product jumbo'\n",
      " 'confection around centuries light pillowy citrus gelatin nuts case filberts cut tiny squares liberally coated powdered sugar tiny mouthful heaven chewy flavorful highly recommend yummy treat familiar story lewis lion witch wardrobe treat seduces edmund selling brother sisters witch'\n",
      " 'looking secret ingredient robitussin believe found got addition root beer extract ordered made cherry soda flavor medicinal'\n",
      " 'great taffy great price wide assortment yummy taffy delivery quick taffy lover deal']\n",
      "['good quality dog food ' 'not as advertised ' 'delight says it all '\n",
      " 'cough medicine ' 'great taffy ']\n"
     ]
    }
   ],
   "source": [
    "print(news['Text'].head(5).values)\n",
    "print(news['Summary'].head(5).values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "48f10124-4c1b-1f09-512d-352c068de1b4",
    "colab": {},
    "colab_type": "code",
    "id": "ZHaGwlh2TdUI"
   },
   "outputs": [],
   "source": [
    "news_summaries = []\n",
    "for summary in news.Summary:\n",
    "    news_summaries.append(summary)\n",
    "news_texts = []\n",
    "for text in news.Text:\n",
    "    news_texts.append(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "3f980247-3c32-240d-7d3d-7b0c3c6c13e5",
    "colab": {},
    "colab_type": "code",
    "id": "yHNDse2lTdUT"
   },
   "outputs": [],
   "source": [
    "def count_words(words_dict, text):\n",
    "    for sentence in text:\n",
    "        for word in sentence.split():\n",
    "            if word not in words_dict:\n",
    "                words_dict[word] = 1\n",
    "            else:\n",
    "                words_dict[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_cell_guid": "3e9ce130-88f4-8779-5b5f-86f2a23ab347",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11760,
     "status": "ok",
     "timestamp": 1589541282871,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "_P7WgumjTdUe",
    "outputId": "d3988b9f-9da1-45e0-f80c-612d97f8e948"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total words in Vocabulary: 56955\n"
     ]
    }
   ],
   "source": [
    "word_counts_dict = {}\n",
    "count_words(word_counts_dict, news_summaries)\n",
    "count_words(word_counts_dict, news_texts)\n",
    "            \n",
    "print(\"Total words in Vocabulary:\", len(word_counts_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QY8Sk4lWpYLC"
   },
   "outputs": [],
   "source": [
    "if 'glove.6B.50d.txt' not in os.listdir('/content/'):\n",
    "  !unzip /content/gdrive/My\\ Drive/glove6b50dtxt.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BGdGuM_NTdUm"
   },
   "outputs": [],
   "source": [
    "def build_word_vector_matrix(vector_file):\n",
    "    embedding_index = {}\n",
    "    with codecs.open(vector_file, 'r', 'utf-8') as f:\n",
    "        for i, line in enumerate(f):\n",
    "            sr = line.split()\n",
    "            if(len(sr)<26):\n",
    "                continue\n",
    "            word = sr[0]\n",
    "            embedding = np.asarray(sr[1:], dtype='float32')\n",
    "            embedding_index[word] = embedding\n",
    "    return embedding_index\n",
    "# Replace the path here to point to the glove.6B.50d.txt vectors file on your system\n",
    "embeddings_index = build_word_vector_matrix('/content/glove.6B.50d.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "0be42a13-70b2-e9cc-7468-1247b01f109c",
    "colab": {},
    "colab_type": "code",
    "id": "VdDgF69WTdUw"
   },
   "outputs": [],
   "source": [
    "word2int = {} \n",
    "count_threshold = 20\n",
    "value = 0\n",
    "for word, count in word_counts_dict.items():\n",
    "    if count >= count_threshold or word in embeddings_index:\n",
    "        word2int[word] = value\n",
    "        value += 1\n",
    "\n",
    "\n",
    "special_codes = [TOKEN_UNK,TOKEN_PAD,TOKEN_EOS,TOKEN_GO]   \n",
    "\n",
    "for code in special_codes:\n",
    "    word2int[code] = len(word2int)\n",
    "\n",
    "int2word = {}\n",
    "for word, value in word2int.items():\n",
    "    int2word[value] = word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_cell_guid": "4401990d-4baf-3f30-becc-3a6149716b56",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 21579,
     "status": "ok",
     "timestamp": 1589541292955,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "f4KKWkrTTdU3",
    "outputId": "5aea14c1-7415-456a-e6b8-d542f939823d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of word embeddings:  36805\n"
     ]
    }
   ],
   "source": [
    "embedding_dim = 50\n",
    "nwords = len(word2int)\n",
    "\n",
    "word_emb_matrix = np.zeros((nwords, embedding_dim), dtype=np.float32)\n",
    "for word, i in word2int.items():\n",
    "    if word in embeddings_index:\n",
    "        word_emb_matrix[i] = embeddings_index[word]\n",
    "    else:\n",
    "        new_embedding = np.array(np.random.uniform(-1.0, 1.0, embedding_dim))\n",
    "        word_emb_matrix[i] = new_embedding\n",
    "print(\"Length of word embeddings: \", len(word_emb_matrix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "25cfd0e3-ae3d-8728-1c82-1a61bb06aa0e",
    "colab": {},
    "colab_type": "code",
    "id": "x_n1GMkcTdU_"
   },
   "outputs": [],
   "source": [
    "def convert_sentence_to_ids(text, eos=False):\n",
    "    wordints = []\n",
    "    word_count = 0\n",
    "    for sentence in text:\n",
    "        sentence2ints = []\n",
    "        for word in sentence.split():\n",
    "            word_count += 1\n",
    "            if word in word2int:\n",
    "                sentence2ints.append(word2int[word])\n",
    "            else:\n",
    "                sentence2ints.append(word2int[TOKEN_UNK])\n",
    "        if eos:\n",
    "            sentence2ints.append(word2int[TOKEN_EOS])\n",
    "        wordints.append(sentence2ints)\n",
    "    return wordints, word_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "360cfdf4-ad4c-0316-56d3-70b6206e75e4",
    "colab": {},
    "colab_type": "code",
    "id": "umsLAGX2TdVJ"
   },
   "outputs": [],
   "source": [
    "id_summaries, word_count = convert_sentence_to_ids(news_summaries)\n",
    "id_texts, word_count = convert_sentence_to_ids(news_texts, eos=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "2a0ae7cd-a845-23dc-3563-ad133e2f02b4",
    "colab": {},
    "colab_type": "code",
    "id": "mPyYg9N3TdVV"
   },
   "outputs": [],
   "source": [
    "def unknown_tokens(sentence):\n",
    "    unk_token_count = 0\n",
    "    for word in sentence:\n",
    "        if word == word2int[TOKEN_UNK]:\n",
    "            unk_token_count += 1\n",
    "    return unk_token_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "50d631a2-fb5a-bb0d-6155-9cd15e70835b",
    "colab": {},
    "colab_type": "code",
    "id": "93RbMaQCTdVc"
   },
   "outputs": [],
   "source": [
    "news_summaries_filtered = []\n",
    "news_texts_filtered = []\n",
    "max_text_length = int(news.Text_len.mean() + news.Text_len.std())\n",
    "max_summary_length = int(int(news.Summary_len.mean() + news.Summary_len.std()))\n",
    "min_length = 4\n",
    "unknown_token_text_limit = 10\n",
    "unknown_token_summary_limit = 4\n",
    "\n",
    "for count,text in enumerate(id_texts):\n",
    "    unknown_token_text = unknown_tokens(id_texts[count])\n",
    "    unknown_token_summary = unknown_tokens(id_summaries[count])\n",
    "    text_len = len(id_texts[count])\n",
    "    summary_len = len(id_summaries[count])\n",
    "    if((unknown_token_text>unknown_token_text_limit) or (unknown_token_summary>unknown_token_summary_limit)):\n",
    "        continue\n",
    "    if(text_len<min_length or summary_len<min_length or text_len>max_text_length or summary_len>max_summary_length):\n",
    "        continue\n",
    "    news_summaries_filtered.append(id_summaries[count])\n",
    "    news_texts_filtered.append(id_texts[count])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "34d28c5f-8016-6b36-664e-3d5ee3db745d",
    "colab": {},
    "colab_type": "code",
    "id": "xuL2rU6mTdVk"
   },
   "outputs": [],
   "source": [
    "def model_inputs():\n",
    "    inputs_data = tf.placeholder(tf.int32, [None, None], name='input_data')\n",
    "    targets = tf.placeholder(tf.int32, [None, None], name='targets')\n",
    "    learning_rate = tf.placeholder(tf.float32, name='learning_rate')\n",
    "    dropout_probs = tf.placeholder(tf.float32, name='dropout_probs')\n",
    "    summary_len = tf.placeholder(tf.int32, (None,), name='summary_len')\n",
    "    max_summary_len = tf.reduce_max(summary_len, name='max_summary_len')\n",
    "    text_len = tf.placeholder(tf.int32, (None,), name='text_len')\n",
    "    return inputs_data, targets, learning_rate, dropout_probs, summary_len, max_summary_len, text_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "9c9b6087-3c28-478d-d311-4213e1c59654",
    "colab": {},
    "colab_type": "code",
    "id": "ANIGHjdQTdVr"
   },
   "outputs": [],
   "source": [
    "def process_encoding_input(target_data, word2int, batch_size):\n",
    "    ending = tf.strided_slice(target_data, [0, 0], [batch_size, -1], [1, 1])\n",
    "    decoding_input = tf.concat([tf.fill([batch_size, 1], word2int[TOKEN_GO]), ending], 1)\n",
    "    return decoding_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "d675562b-a9e0-df71-6979-a052fb78dcbc",
    "colab": {},
    "colab_type": "code",
    "id": "4-nmhhyfTdVz"
   },
   "outputs": [],
   "source": [
    "def get_cell(csize,dprob):\n",
    "    rnc = GRUCell(csize)\n",
    "    rnc = DropoutWrapper(rnc, input_keep_prob = dprob)\n",
    "    return rnc\n",
    "\n",
    "def encoding_layer(csize, len_s, nl, rinp, dprob):\n",
    "    for l in range(nl):\n",
    "        with tf.variable_scope('encoding_l_{}'.format(l)):\n",
    "            rnn_frnt = get_cell(csize,dprob)\n",
    "            rnn_bkwd = get_cell(csize,dprob)\n",
    "            eop, est = tf.nn.bidirectional_dynamic_rnn(rnn_frnt, rnn_bkwd, \n",
    "                                                                    rinp,\n",
    "                                                                    len_s,\n",
    "                                                                    dtype=tf.float32)\n",
    "    eop = tf.concat(eop,nl)\n",
    "    return eop, est"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "524d0246-ddae-b485-5ea4-11ad476447f4",
    "colab": {},
    "colab_type": "code",
    "id": "OBSH-yfcTdV-"
   },
   "outputs": [],
   "source": [
    "def trng_dec_layer(dec_emb_inp, summ_len, cell_dec, st_init, lyr_op, \n",
    "                            v_size, max_summ_len):\n",
    "    helper = TrainingHelper(inputs=dec_emb_inp,sequence_length=summ_len, time_major=False)\n",
    "    dec = BasicDecoder(cell_dec,helper,st_init,lyr_op) \n",
    "    logits, _, _ = dynamic_decode(dec,output_time_major=False,impute_finished=True, \n",
    "                                  maximum_iterations=max_summ_len)\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "6044b206-7f27-5304-4896-06d388af0949",
    "colab": {},
    "colab_type": "code",
    "id": "ExxIWv2YTdWK"
   },
   "outputs": [],
   "source": [
    "def infr_dec_layer(embeddings, start_token, end_token, decoding_cell, initial_state, op_layer,\n",
    "                             max_summary_len, batch_size):\n",
    "    \n",
    "    start_tokens = tf.tile(tf.constant([start_token], dtype=tf.int32), [batch_size], name='start_tokens')\n",
    "    inf_helper = GreedyEmbeddingHelper(embeddings,start_tokens,end_token)\n",
    "    inf_decoder = BasicDecoder(decoding_cell,inf_helper,initial_state,op_layer)       \n",
    "    inf_logits, _, _ = dynamic_decode(inf_decoder,output_time_major=False,impute_finished=True,\n",
    "                                                            maximum_iterations=max_summary_len)\n",
    "    return inf_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "4b50746f-8f78-0253-9178-56c62e4ac1bf",
    "colab": {},
    "colab_type": "code",
    "id": "IcMgwE8oTdWd"
   },
   "outputs": [],
   "source": [
    "def decoding_layer(dec_emb_op, embs, enc_op, enc_st, v_size, txt_len, \n",
    "                   summ_len,mx_summ_len, rnsize, word2int, dprob, batch_size, nlyrs):\n",
    "    \n",
    "    for l in range(nlyrs):\n",
    "        with tf.variable_scope('dec_rnn_layer_{}'.format(l)):\n",
    "            gru = tf.contrib.rnn.GRUCell(rnn_len)\n",
    "            cell_dec = tf.contrib.rnn.DropoutWrapper(gru,input_keep_prob = dprob)\n",
    "            \n",
    "    out_l = Dense(v_size, kernel_initializer = tf.truncated_normal_initializer(mean = 0.0, stddev=0.1))\n",
    "    \n",
    "    attention = BahdanauAttention(rnsize, enc_op,txt_len,\n",
    "                                                  normalize=False,\n",
    "                                                  name='BahdanauAttention')\n",
    "    cell_dec =  AttentionWrapper(cell_dec,attention,rnn_len)\n",
    "    attn_zstate = cell_dec.zero_state(batch_size , tf.float32 )\n",
    "    attn_zstate = attn_zstate.clone(cell_state = enc_st[0])\n",
    "    with tf.variable_scope(\"decoding_layer\"):\n",
    "        tr_dec_op = trng_dec_layer(dec_emb_op, \n",
    "                                                  summ_len, \n",
    "                                                  cell_dec, \n",
    "                                                  attn_zstate,\n",
    "                                                  out_l,\n",
    "                                                  v_size, \n",
    "                                                  mx_summ_len)\n",
    "    with tf.variable_scope(\"decoding_layer\", reuse=True):\n",
    "        inf_dec_op = infr_dec_layer(embs,  \n",
    "                                                    word2int[TOKEN_GO], \n",
    "                                                    word2int[TOKEN_EOS],\n",
    "                                                    cell_dec, \n",
    "                                                    attn_zstate, \n",
    "                                                    out_l,\n",
    "                                                    mx_summ_len,\n",
    "                                                    batch_size)\n",
    "\n",
    "    return tr_dec_op, inf_dec_op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "19ddcf22-4f6a-d531-071a-021b42b643e3",
    "colab": {},
    "colab_type": "code",
    "id": "pYVCMj7aTdWl"
   },
   "outputs": [],
   "source": [
    "def seq2seq_model(data_inp, data_summ_tgt, dprob, len_txt, len_summ, max_len_summ, \n",
    "                  v_size, rnsize, nlyrs, word2int, batch_size):\n",
    "    \n",
    "    inp_emb = word_emb_matrix\n",
    "    word_embs = tf.Variable(inp_emb, name=\"word_embs\")\n",
    "    inp_enc_emb = tf.nn.embedding_lookup(word_embs, data_inp)\n",
    "    op_enc, st_enc = encoding_layer(rnsize, len_txt, nlyrs, inp_enc_emb, dprob)\n",
    "    \n",
    "    inp_dec = process_encoding_input(data_summ_tgt, word2int, batch_size)\n",
    "    inp_dec_emb = tf.nn.embedding_lookup(inp_emb, inp_dec)\n",
    "    \n",
    "    op_tr, op_inf  = decoding_layer(inp_dec_emb, \n",
    "                                                        inp_emb,\n",
    "                                                        op_enc,\n",
    "                                                        st_enc, \n",
    "                                                        v_size, \n",
    "                                                        len_txt, \n",
    "                                                        len_summ, \n",
    "                                                        max_len_summ,\n",
    "                                                        rnsize, \n",
    "                                                        word2int, \n",
    "                                                        dprob, \n",
    "                                                        batch_size,\n",
    "                                                        nlyrs)\n",
    "    \n",
    "    return op_tr, op_inf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "725e92bf-2309-1a78-c771-641a42b440c6",
    "colab": {},
    "colab_type": "code",
    "id": "t6c1ac3qTdWs"
   },
   "outputs": [],
   "source": [
    "def pad_sentences(sentences_batch):\n",
    "\n",
    "    max_sentence = max([len(sentence) for sentence in sentences_batch])\n",
    "    return [sentence + [word2int[TOKEN_PAD]] * (max_sentence - len(sentence)) for sentence in sentences_batch]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "47e4f70a-6377-68dd-c06c-eed674b2bb3f",
    "colab": {},
    "colab_type": "code",
    "id": "usG7exzOTdWy"
   },
   "outputs": [],
   "source": [
    "def get_batches(summaries, texts, batch_size):\n",
    "    for batch_idx in range(0, len(texts)//batch_size):\n",
    "        start_idx = batch_idx * batch_size\n",
    "        summaries_batch = summaries[start_idx:start_idx + batch_size]\n",
    "        texts_batch = texts[start_idx:start_idx + batch_size]\n",
    "        pad_summaries_batch = np.array(pad_sentences(summaries_batch))\n",
    "        pad_texts_batch = np.array(pad_sentences(texts_batch))\n",
    "\n",
    "        pad_summaries_lens = []\n",
    "        for summary in pad_summaries_batch:\n",
    "            pad_summaries_lens.append(len(summary))\n",
    "        \n",
    "        pad_texts_lens = []\n",
    "        for text in pad_texts_batch:\n",
    "            pad_texts_lens.append(len(text))\n",
    "        \n",
    "        yield pad_summaries_batch, pad_texts_batch, pad_summaries_lens, pad_texts_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7SIkV3M9jYKl"
   },
   "outputs": [],
   "source": [
    "def train_on_batch(session, X, X_seq_len, Y, Y_seq_len, learning_rate, dropout_keep_probability):\n",
    "    feed_dict = {\n",
    "            data_inp: X,\n",
    "            tgts:Y,\n",
    "            lrt:learning_rate,\n",
    "            len_summ:Y_seq_len,\n",
    "            len_txt: X_seq_len,\n",
    "            dprobs: dropout_keep_probability}\n",
    "\n",
    "    _,pred,loss,summary = session.run([\n",
    "            train_op,\n",
    "            tr_cost,\n",
    "            inf_op,\n",
    "            merged_summary_op], feed_dict=feed_dict)\n",
    "    return _,pred, loss, summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6ejZsb2VqY7A"
   },
   "outputs": [],
   "source": [
    "def predict_for_batch(session, X, X_seq_len,Y, Y_seq_len,dropout_keep_probability):\n",
    "\n",
    "    feed_dict={data_inp:X, len_txt:X_seq_len,tgts:Y, len_summ: Y_seq_len,dprobs: dropout_keep_probability}\n",
    "\n",
    "    pred = session.run([inf_op], feed_dict=feed_dict)[0]\n",
    "\n",
    "    return pred\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xCltZfqbQpR1"
   },
   "outputs": [],
   "source": [
    "\n",
    "def predict_for_batch_with_loss(session, X, X_seq_len, Y, Y_seq_len,learning_rate,dropout_keep_probability):\n",
    "    \n",
    "    feed_dict={data_inp:X, len_txt:X_seq_len, tgts:Y, len_summ: Y_seq_len,lrt:learning_rate,dprobs: dropout_keep_probability}\n",
    "    \n",
    "    _,loss,pred = session.run([train_op,tr_cost,infer_predictions], feed_dict=feed_dict)\n",
    "    \n",
    "    return pred, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "77299c4b-a3cf-785b-981a-42a1bb3a2033",
    "colab": {},
    "colab_type": "code",
    "id": "HGM95d9MTdW5"
   },
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "rnn_len = 256\n",
    "n_layers = 2\n",
    "lr = 0.005\n",
    "dr_prob =0.75\n",
    "logs_path='/content/gdrive/My Drive/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "_cell_guid": "68781626-8bf4-0a23-4bb2-f24a5762fa1e",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 395
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 25714,
     "status": "ok",
     "timestamp": 1589541297963,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "AlrcY77-TdXF",
    "outputId": "d888a8ac-a8a0-46d4-8002-2df9bc83d8d8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-24-08a5f0b64492>:2: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-24-08a5f0b64492>:14: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn.py:464: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:559: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.add_weight` method instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:565: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:575: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn.py:244: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Graph created.\n"
     ]
    }
   ],
   "source": [
    "train_graph = tf.Graph()\n",
    "with train_graph.as_default():\n",
    "    \n",
    "    data_inp, tgts, lrt, dprobs, len_summ, max_len_summ, len_txt = model_inputs()\n",
    "\n",
    "    tr_opp, inf_opp = seq2seq_model(tf.reverse(data_inp, [-1]),\n",
    "                                                      tgts, \n",
    "                                                      dprobs,   \n",
    "                                                      len_txt,\n",
    "                                                      len_summ,\n",
    "                                                      max_len_summ,\n",
    "                                                      len(word2int)+1,\n",
    "                                                      rnn_len, \n",
    "                                                      n_layers, \n",
    "                                                      word2int,\n",
    "                                                      batch_size)\n",
    "    \n",
    "    tr_op = tf.identity(tr_opp.rnn_output, 'tr_op')\n",
    "    inf_op = tf.identity(inf_opp.sample_id, name='predictions')\n",
    "    \n",
    "    seq_masks = tf.sequence_mask(len_summ, max_len_summ, dtype=tf.float32, name='masks')\n",
    "\n",
    "    with tf.name_scope(\"optimizer\"):\n",
    "        tr_cost = sequence_loss(tr_op,tgts,seq_masks)\n",
    "        optzr = tf.train.AdamOptimizer(lrt)\n",
    "        grds = optzr.compute_gradients(tr_cost)\n",
    "        capped_grds = [(tf.clip_by_value(grd, -5., 5.), var) for grd, var in grds \n",
    "                        if grd is not None]\n",
    "        train_op = optzr.apply_gradients(capped_grds)\n",
    "    # Get predictions for evaluation.\n",
    "    train_predictions = tr_opp.sample_id\n",
    "    infer_predictions = inf_opp.sample_id\n",
    "    tf.summary.scalar(\"cost\", tr_cost)\n",
    "print(\"Graph created.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "_cell_guid": "989d2498-df4e-31e8-3133-56df524b8a28",
    "colab": {},
    "colab_type": "code",
    "id": "Tg98B19nTdXS"
   },
   "outputs": [],
   "source": [
    "def text_to_seq(text):\n",
    "    return [word2int.get(word, word2int[TOKEN_UNK]) for word in text.split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "s4afDgp2zidm"
   },
   "outputs": [],
   "source": [
    "# Program to measure similarity between  \n",
    "# two sentences using cosine similarity. \n",
    "def mean_absolute_error_texts(v1,v2):\n",
    "    meanerrorl2=[]\n",
    "    for X,Y in zip(v1,v2):\n",
    "        # tokenization \n",
    "        X_list = word_tokenize(X)  \n",
    "        Y_list = word_tokenize(Y) \n",
    "  \n",
    "        # sw contains the list of stopwords \n",
    "        sw = stopwords.words('english')  \n",
    "        l1 =[];l2 =[] \n",
    "  \n",
    "        # remove stop words from string \n",
    "        X_set = {w for w in X if not w in sw}  \n",
    "        Y_set = {w for w in Y if not w in sw} \n",
    "  \n",
    "        # form a set containing keywords of both strings  \n",
    "        rvector = X_set.union(Y_set)  \n",
    "        for w in rvector: \n",
    "            if w in X_set:\n",
    "               l1.append(1) # create a vector \n",
    "            else:\n",
    "               l1.append(0) \n",
    "            if w in Y_set:\n",
    "               l2.append(1) \n",
    "            else:\n",
    "               l2.append(0) \n",
    "        c = 0\n",
    "        # cosine formula  \n",
    "        for i in range(len(rvector)): \n",
    "                      c+= l1[i]*l2[i] \n",
    "        den=float((sum(l1)*sum(l2))**0.5)\n",
    "        try: \n",
    "           cosine = c /den\n",
    "        except:\n",
    "           cosine=0.0 \n",
    "        meanerrorl2.append(cosine) \n",
    "    return LA.norm(meanerrorl2,2)*100.0\n",
    "      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 25646,
     "status": "ok",
     "timestamp": 1589541297971,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "IXy1o-6INrQ1",
    "outputId": "11269dd9-070f-4cea-eb82-80243c101295"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27125"
      ]
     },
     "execution_count": 38,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(news_summaries_filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 25598,
     "status": "ok",
     "timestamp": 1589541297973,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "Lc3b_vCaNrtm",
    "outputId": "ce9d13ef-b935-4f2d-e0ce-30d694f79cec"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27125"
      ]
     },
     "execution_count": 39,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(news_texts_filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xALwgTyFOtLL"
   },
   "outputs": [],
   "source": [
    "epochs =5\n",
    "min_learning_rate = 0.0006\n",
    "display_step = 50\n",
    "early_stop_cnt = 0 \n",
    "early_stop_cnt_max = 3 \n",
    "per_epoch = 3 \n",
    "\n",
    "\n",
    "update_loss = 0 \n",
    "batch_loss = 0\n",
    "summary_update_loss = [] \n",
    "\n",
    "news_summaries_train = news_summaries_filtered[0:21700]\n",
    "news_texts_train = news_texts_filtered[0:21700]\n",
    "\n",
    "news_summaries_test =news_summaries_filtered[21700:]\n",
    "news_texts_test=news_texts_filtered[21700:]\n",
    "\n",
    "pad = word2int[TOKEN_PAD] \n",
    "eos = word2int[TOKEN_EOS]  \n",
    "unk = word2int[TOKEN_UNK]\n",
    "tokens_id=[eos,pad,unk]\n",
    "\n",
    "update_check = (len(news_texts_train)//batch_size//per_epoch)-1\n",
    "checkpoint = logs_path + 'best_so_far_model.ckpt' \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "_cell_guid": "6368ba0d-4083-e182-ca38-5356b307e09f",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3077500,
     "status": "ok",
     "timestamp": 1589544349957,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "9hbE8nF6TdXM",
    "outputId": "1fae6780-a547-4b81-d474-a1031609f401"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start training... \n",
      "\n",
      "** Epoch   1/5 Batch   50/423 - Batch Loss:  6.672, seconds: 84.59\n",
      "** Epoch   1/5 Batch  100/423 - Batch Loss:  5.414, seconds: 82.07\n",
      "Average loss: 5.982\n",
      "Saving model\n",
      "** Epoch   1/5 Batch  150/423 - Batch Loss:  5.269, seconds: 82.79\n",
      "** Epoch   1/5 Batch  200/423 - Batch Loss:  5.103, seconds: 82.21\n",
      "Average loss: 5.142\n",
      "Saving model\n",
      "** Epoch   1/5 Batch  250/423 - Batch Loss:  4.992, seconds: 81.48\n",
      "** Epoch   1/5 Batch  300/423 - Batch Loss:  4.878, seconds: 84.69\n",
      "Average loss: 4.909\n",
      "Saving model\n",
      "history: [1, 4.909, 4.832585]\n",
      " \n",
      "Test: epoch 1 loss: 4.832585\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 44, 24, 258]\n",
      "  Response Words: great snack and healthy snack\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 70, 17, 115]\n",
      "  Response Words: great snack for the go\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 44, 24, 258]\n",
      "  Response Words: great snack and healthy snack\n",
      " Ground Truth: these are the best\n",
      "\n",
      "** Epoch   2/5 Batch   50/423 - Batch Loss:  4.617, seconds: 85.45\n",
      "** Epoch   2/5 Batch  100/423 - Batch Loss:  4.340, seconds: 82.37\n",
      "Average loss: 4.476\n",
      "Saving model\n",
      "** Epoch   2/5 Batch  150/423 - Batch Loss:  4.377, seconds: 85.52\n",
      "** Epoch   2/5 Batch  200/423 - Batch Loss:  4.329, seconds: 82.81\n",
      "Average loss: 4.352\n",
      "Saving model\n",
      "** Epoch   2/5 Batch  250/423 - Batch Loss:  4.299, seconds: 82.94\n",
      "** Epoch   2/5 Batch  300/423 - Batch Loss:  4.237, seconds: 83.50\n",
      "Average loss: 4.251\n",
      "Saving model\n",
      "history: [2, 4.251, 4.3669915]\n",
      " \n",
      "Test: epoch 2 loss: 4.3669915\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [25, 81, 82, 258]\n",
      "  Response Words: best gluten free snack\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [60, 44, 60, 258]\n",
      "  Response Words: delicious and delicious snack\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 258, 70, 258]\n",
      "  Response Words: great snack for snack\n",
      " Ground Truth: these are the best\n",
      "\n",
      "** Epoch   3/5 Batch   50/423 - Batch Loss:  4.150, seconds: 83.82\n",
      "** Epoch   3/5 Batch  100/423 - Batch Loss:  3.879, seconds: 83.05\n",
      "Average loss: 4.013\n",
      "Saving model\n",
      "** Epoch   3/5 Batch  150/423 - Batch Loss:  3.957, seconds: 83.89\n",
      "** Epoch   3/5 Batch  200/423 - Batch Loss:  3.881, seconds: 83.54\n",
      "Average loss: 3.922\n",
      "Saving model\n",
      "** Epoch   3/5 Batch  250/423 - Batch Loss:  3.879, seconds: 82.26\n",
      "** Epoch   3/5 Batch  300/423 - Batch Loss:  3.834, seconds: 82.49\n",
      "Average loss: 3.834\n",
      "Saving model\n",
      "history: [3, 3.834, 3.7855482]\n",
      " \n",
      "Test: epoch 3 loss: 3.7855482\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [25, 258, 165, 453, 824]\n",
      "  Response Words: best snack ever there eaten\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [60, 44, 24, 258]\n",
      "  Response Words: delicious and healthy snack\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [48, 258, 1017, 1017]\n",
      "  Response Words: yummy snack bars bars\n",
      " Ground Truth: these are the best\n",
      "\n",
      "** Epoch   4/5 Batch   50/423 - Batch Loss:  3.881, seconds: 84.36\n",
      "** Epoch   4/5 Batch  100/423 - Batch Loss:  3.616, seconds: 84.18\n",
      "Average loss: 3.745\n",
      "Saving model\n",
      "** Epoch   4/5 Batch  150/423 - Batch Loss:  3.677, seconds: 83.89\n",
      "** Epoch   4/5 Batch  200/423 - Batch Loss:  3.612, seconds: 82.66\n",
      "Average loss: 3.655\n",
      "Saving model\n",
      "** Epoch   4/5 Batch  250/423 - Batch Loss:  3.639, seconds: 84.83\n",
      "** Epoch   4/5 Batch  300/423 - Batch Loss:  3.572, seconds: 81.46\n",
      "Average loss: 3.578\n",
      "Saving model\n",
      "history: [4, 3.578, 3.516473]\n",
      " \n",
      "Test: epoch 4 loss: 3.516473\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [25, 258, 462, 1128]\n",
      "  Response Words: best snack have tried\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [60, 44, 157, 1017]\n",
      "  Response Words: delicious and chocolate bars\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [48, 44, 157, 1017]\n",
      "  Response Words: yummy and chocolate bars\n",
      " Ground Truth: these are the best\n",
      "\n",
      "** Epoch   5/5 Batch   50/423 - Batch Loss:  3.600, seconds: 92.36\n",
      "** Epoch   5/5 Batch  100/423 - Batch Loss:  3.384, seconds: 84.66\n",
      "Average loss: 3.494\n",
      "Saving model\n",
      "** Epoch   5/5 Batch  150/423 - Batch Loss:  3.473, seconds: 83.99\n",
      "** Epoch   5/5 Batch  200/423 - Batch Loss:  3.386, seconds: 82.12\n",
      "Average loss: 3.437\n",
      "Saving model\n",
      "** Epoch   5/5 Batch  250/423 - Batch Loss:  3.421, seconds: 83.83\n",
      "** Epoch   5/5 Batch  300/423 - Batch Loss:  3.344, seconds: 85.23\n",
      "Average loss: 3.353\n",
      "Saving model\n",
      "history: [5, 3.353, 3.307459]\n",
      " \n",
      "Test: epoch 5 loss: 3.307459\n",
      " \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 2127, 258, 165, 1890, 156, 83, 2296, 496, 13, 145, 1032, 61, 181, 81, 82, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: best afternoon snack ever put bag way school work great source protein product natural gluten free\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 54, 44, 24]\n",
      "  Response Words: great taste and healthy\n",
      " Ground Truth: great gluten free snack\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [13305, 746, 845, 1017, 695, 1472, 1458, 1445, 1017, 3178, 2545, 60, 4367, 149, 1064, 771, 433, 1285, 36801, 1874, 1802, 1472, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: confess addicted kind bars first try blueberry pistachio bars sealed problem delicious fairly low calories sometimes use meal feel satisfied try\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [60, 44, 24, 258]\n",
      "  Response Words: delicious and healthy snack\n",
      " Ground Truth: addicted to kind bars\n",
      "\n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 1017, 1089, 1082, 1874, 1576, 258, 21, 684, 24, 36803, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802, 36802]\n",
      "  Input Words: love bars run quickly feel guilty snack tasty yet healthy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [60, 44, 24, 258]\n",
      "  Response Words: delicious and healthy snack\n",
      " Ground Truth: these are the best\n",
      "\n"
     ]
    }
   ],
   "source": [
    "invalid_number_prediction_counts = []\n",
    "all_model_predictions = []\n",
    "all_ground_truth = []\n",
    "history=[]\n",
    "print('Start training... \\n')\n",
    "\n",
    "with tf.Session(graph=train_graph) as sess:\n",
    "    tf_summary_writer = tf.summary.FileWriter(logs_path, graph=train_graph)\n",
    "    merged_summary_op = tf.summary.merge_all()\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for epoch_i in range(epochs):\n",
    "        update_loss = 0\n",
    "        batch_loss = 0\n",
    "        train_loss =0\n",
    "        for batch_i, (summaries_batch, texts_batch, summaries_len, texts_len) in enumerate(\n",
    "                get_batches(news_summaries_train, news_texts_train, batch_size)):\n",
    "            before = time.time()\n",
    "            _,loss,predictions,summary=train_on_batch(sess,texts_batch, texts_len, summaries_batch, summaries_len, lr, dr_prob)\n",
    "            #_,loss,summary = sess.run(\n",
    "            #    [train_op, tr_cost,merged_summary_op],\n",
    "            #    {data_inp: texts_batch,\n",
    "            #     tgts: summaries_batch,\n",
    "            #     lrt: lr,\n",
    "            #     len_summ: summaries_len,\n",
    "            #     len_txt: texts_len,\n",
    "            #     dprobs: dr_prob})\n",
    "            \n",
    "            batch_loss += loss\n",
    "            update_loss += loss\n",
    "            after = time.time()\n",
    "            batch_time = after - before\n",
    "            tf_summary_writer.add_summary(summary, epoch_i * batch_size + batch_i)\n",
    "            \n",
    "            if batch_i % display_step == 0 and batch_i > 0:\n",
    "                print('** Epoch {:>3}/{} Batch {:>4}/{} - Batch Loss: {:>6.3f}, seconds: {:>4.2f}'\n",
    "                      .format(epoch_i+1,\n",
    "                              epochs, \n",
    "                              batch_i, \n",
    "                              len(news_texts_filtered) // batch_size, \n",
    "                              batch_loss / display_step, \n",
    "                              batch_time*display_step))\n",
    "                batch_loss = 0\n",
    "            \n",
    "            if batch_i % update_check == 0 and batch_i > 0:\n",
    "                print(\"Average loss:\", round(update_loss/update_check,3))\n",
    "                train_loss=round(update_loss/update_check,3)\n",
    "                summary_update_loss.append(update_loss)\n",
    "                \n",
    "                if update_loss <= min(summary_update_loss):\n",
    "                    print('Saving model') \n",
    "                    early_stop_cnt = 0\n",
    "                    saver = tf.train.Saver() \n",
    "                    saver.save(sess, checkpoint)\n",
    "\n",
    "                else:\n",
    "                    print(\"No Improvement.\")\n",
    "                    early_stop_cnt += 1\n",
    "                    if early_stop_cnt == early_stop_cnt_max:\n",
    "                        break\n",
    "                update_loss = 0\n",
    "\n",
    "        summaries_test_batch, texts_test_batch, summaries_test_len, texts_test_len =\\\n",
    "         next(get_batches(news_summaries_test, news_texts_test, batch_size))\n",
    "    \n",
    "        predictions,test_loss = predict_for_batch_with_loss(sess,texts_test_batch, texts_test_len,summaries_test_batch,summaries_test_len,lr,1.0)\n",
    "        history.append([epoch_i+1,train_loss,test_loss])\n",
    "        print('history:',history[-1])\n",
    "        print(\" \")  \n",
    "        print('Test: epoch', epoch_i+1, 'loss:', test_loss,)\n",
    "        print(\" \")\n",
    "        for text, summary, result_logits  in list(zip(texts_test_batch, summaries_test_batch, predictions))[-3:]: \n",
    "                     print('\\nText')\n",
    "                     print('  Word Ids:    {}'.format([i for i in text]))\n",
    "                     print('  Input Words: {}'.format(\" \".join( int2word[i] for i in text if i not in tokens_id)))\n",
    "\n",
    "                     print('\\nSummary')\n",
    "                     print('  Word Ids:       {}'.format([i for i in result_logits if i not in tokens_id]))\n",
    "                     print('  Response Words: {}'.format(\" \".join(int2word[i] for i in result_logits if i not in tokens_id )))\n",
    "                     print(' Ground Truth: {}'.format(\" \".join(int2word[i] for i in summary if i not in tokens_id)))\n",
    "                     print('')\n",
    "        model_predictions = []\n",
    "        ground_truth = []\n",
    "        invalid_number_prediction_count = 0\n",
    "\n",
    "        ind=0\n",
    "        for summaries_test_batch, texts_test_batch, summaries_test_len, texts_test_len\\\n",
    "         in get_batches(news_summaries_test,news_texts_test, batch_size=batch_size):\n",
    "                 \n",
    "  \n",
    "                 #summaries_test_len=[np.random.randint(5,8)]\n",
    "                 pred=predict_for_batch(sess, texts_test_batch, texts_test_len,summaries_test_batch,summaries_test_len,1.0)   \n",
    "                 \n",
    "                 for y,p in list(zip(summaries_test_batch,pred)): \n",
    "                     Y=''.join(int2word[i] for i in y if i not in tokens_id)\n",
    "                     O=''.join(int2word[i] for i in p if i not in tokens_id)\n",
    "                     \n",
    "                     if mean_absolute_error_texts([Y],[O])<1e-15 : \n",
    "                             invalid_number_prediction_count += 1\n",
    "                     lmin=min(len(Y),len(O))+1\n",
    "                     OO=O[:lmin]\n",
    "                     YY=Y[:lmin] \n",
    "                     model_predictions.append(OO)\n",
    "                     ground_truth.append(YY)\n",
    "        all_model_predictions.append(model_predictions)\n",
    "        all_ground_truth.append(ground_truth)\n",
    "        invalid_number_prediction_counts.append(invalid_number_prediction_count)\n",
    "         \n",
    "        if early_stop_cnt == early_stop_cnt_max:\n",
    "            print(\"Stopping Training.\")\n",
    "            break        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D_sKJYKmkwDb"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 265
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3077408,
     "status": "ok",
     "timestamp": 1589544349969,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "wErDv9CAuhdy",
    "outputId": "e55fae5f-760d-405b-de83-cec1037b738a"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3xUVd7H8c8vmfSEAEloCQm91xAgCCiKhSZVsQCCoqjrKrq7tn1cd8V99lnXXUVdO0UERBEBaSKiIBYCJqGFXkMKEAiEECD9PH/cKIgpE8jMZMLv/XrxcjL33Du/XMk3l3PPPUeMMSillHJ/Hq4uQCmlVNXQQFdKqRpCA10ppWoIDXSllKohNNCVUqqGsLnqg0NDQ02TJk1c9fFKKeWWEhISThhjwkrb5rJAb9KkCfHx8a76eKWUcksiklzWNu1yUUqpGkIDXSmlaggNdKWUqiFc1oeulFKXo6CggNTUVHJzc11dikP5+voSERGBl5eX3ftooCul3EpqaipBQUE0adIEEXF1OQ5hjCEzM5PU1FSaNm1q937a5aKUciu5ubmEhITU2DAHEBFCQkIq/a8QDXSllNupyWH+s8v5Ht0u0A+dOMtLK3dRXKzT/iql1MXcLtBX7TjK22v38+LyHehc7kopZ8vKyuKtt96q9H6DBg0iKyvLARVd4HaB/kDfZtzXuykzfzjEa1/vdXU5SqmrTFmBXlhYWO5+K1asoHbt2o4qC3DDUS4iwnOD25KdW8DU1XsJ8vViYh/77wIrpdSVeOaZZ9i/fz9dunTBy8sLX19f6tSpw65du9izZw/Dhw8nJSWF3NxcJk+ezKRJk4AL053k5OQwcOBA+vTpw48//kh4eDiff/45fn5+V1yb2wU6gIeH8M+RHcnJLeTFZTsI8rUxOqaxq8tSSjnZC0u3syM9u0qP2a5RLf56a/syt//zn/8kKSmJzZs3s3btWgYPHkxSUtIvwwtnzJhB3bp1OX/+PN27d2fUqFGEhIT86hh79+5l3rx5vP/++4wePZrPPvuMsWPHXnHtbtfl8jObpwev3dWFvi1DeeazrXyx7YirS1JKXYV69Ojxq7Hir7/+Op07dyY2NpaUlBT27v1t13DTpk3p0qULAN26dePQoUNVUotbXqH/zMfmybvjujFu+kYe+3gT031sXNuq1FkllVI1UHlX0s4SEBDwy+u1a9eyevVq1q9fj7+/P/369St1LLmPj88vrz09PTl//nyV1OK2V+g/8/e2MWNCd1rUC+LB2QkkJJ90dUlKqRosKCiIM2fOlLrt9OnT1KlTB39/f3bt2kVcXJxTa3P7QAcI9vPiw/t60CDYlwkzf6ryPjWllPpZSEgIvXv3pkOHDjz55JO/2jZgwAAKCwtp27YtzzzzDLGxsU6tTVw1ljsmJsZc1gIXRYVw+jDUbfabTWlZ57nt7R8pKCpm/oO9aBYWWAWVKqWqk507d9K2bVtXl+EUpX2vIpJgjIkprb37XaF//yq83Qe2fPKbTeG1/Zg9sSfGwNhpG0jLqpp+KaWUcgfuF+hdx0DDzrBoEix5DAp+Hdot6gUy674enMktZNy0DZzIyXNRoUop5Vx2B7qIeIrIJhFZVsq2SBFZU7J9q4gMqtoyL1KrEYxfCn2egMRZMO0myNz/qyYdwoOZcW930k+f557pGzl9vsBh5SilVHVRmSv0ycDOMrY9B8w3xnQF7gQqP9FBZXja4Ma/wd2fQnYqvHsdJC38VZPuTeryzthu7M04w8QPfuJ8fpFDS1JKKVezK9BFJAIYDEwro4kBapW8DgbSr7w0O7S6GR78Duq1hQX3wvI/QeGFLpZ+resx9Y6uJB4+xYNzEsgvLHZKWUop5Qr2XqFPBZ4CykrEvwFjRSQVWAE8WlojEZkkIvEiEn/8+PHK1lq62o3h3hXQ6/fw0/sw/WY4efCXzYM7NeSfIzuxbs9xHv9kE0U67a5SqoaqMNBFZAiQYYxJKKfZXcAHxpgIYBAwW0R+c2xjzHvGmBhjTExYWBU+0enpBbf8L9z5EZw6aHXB7Fz6y+bR3Rvz3OC2rNh2lGcXbtVpd5VSl+1yp88FmDp1KufOnaviii6w5wq9NzBURA4BHwM3iMicS9pMBOYDGGPWA75AaBXWaZ82g+HBdRDSHD4ZCyufhcJ8AO7v24zHbmjB/PhU/r58p4a6UuqyVOdAr3AuF2PMs8CzACLSD/iTMebSacEOA/2BD0SkLVagV1GfSiXVaQL3rYRVf4G4tyBlI9w+E2pH8sRNrcjOLWT69wcJ9vPisf4tXVKiUsp9XTx97k033US9evWYP38+eXl5jBgxghdeeIGzZ88yevRoUlNTKSoq4i9/+QvHjh0jPT2d66+/ntDQUNasWVPltV325FwiMgWIN8YsAf4IvC8iT2DdIJ1gXHkJbPOBQf+CqGtgyaPwTl8Y8S7SegDPD2lHdm4Br3y1hyBfG/f21rnUlXJbXzwDR7dV7TEbdISB/yxz88XT565atYoFCxawceNGjDEMHTqUdevWcfz4cRo1asTy5csBa46X4OBgXnnlFdasWUNoqGM6MCoV6MaYtcDaktfPX/T+Dqyumeql/XDrf86n42HeHXDNY3j0f55/jepETm4hLyzdQZCvF7d1i3B1pUopN7Rq1SpWrVpF165dAcjJyWHv3r307duXP/7xjzz99NMMGTKEvn37OqUet54+1y4hzWHiavjyWfjxdUjZiO22Gbxxd1cmfhDPUwu2EOhjY0CHBq6uVClVWeVcSTuDMYZnn32WBx988DfbEhMTWbFiBc899xz9+/fn+eefL+UIVcv9Hv2/HF6+MORVGDUdjiXBu33xObiGd8d1o3Pj2jw2bxPf7z3h6iqVUm7g4ulzb7nlFmbMmEFOTg4AaWlpZGRkkJ6ejr+/P2PHjuXJJ58kMTHxN/s6wtUR6D/reBtMWguBDWDuKAK+/z9m3tOVZmEBTJodT0LyKVdXqJSq5i6ePverr77i7rvvplevXnTs2JHbbruNM2fOsG3bNnr06EGXLl144YUXeO655wCYNGkSAwYM4Prrr3dIbe43fW5VyD8HXzwFm2ZDVB9ODHiLUXMOcOpsPp882Iu2DWtVfAyllEvo9Lk1afrcquDtD8P+C8PfgfREQuf059Ob8/H3tjFu+kYOnjjr6gqVUqrSrs5A/1mXu+CBNeBXl3qL7mB55++huJCx0zaQrnOpK6XczNUd6AD12sCkNdDpDkJ+eoW1Dd/A6/xxxk7fQKbOpa5UtXQ1POl9Od+jBjqAdwCMeAeGvkHgsXhW+T9Ho1MJjJ+5kexcnUtdqerE19eXzMzMGh3qxhgyMzPx9fWt1H5X503R8hxNgk/HY04e4JWC29kYPp4PJsbi5+3p6sqUUkBBQQGpqank5ua6uhSH8vX1JSIiAi8vr1+9X95NUQ300uSdgaWTIekz1hZ3ZkHkX3hlQn+8bfoPGqWUa+kol8ryCbIeQhr8Cn1tO/hzyiTemDVb51JXSlVrGuhlEYHuE/G8fzWBfv5MPvw4K9/7M6ZYVz1SSlVPGugVadSFWo//yIGQfgw++hZ7X78Vc+6kq6tSSqnf0EC3h28wLX//GcsjHqfJqfWcee0aSK2m/f9KqauWBrqdxMODgff9jTebvkl2bgFF02+BuLehBg+dUkq5F7sDXUQ8RWSTiCwrY/toEdkhIttF5KOqK7H68PAQfj/uDl6Oep9vCjvBymdg/jg4n+Xq0pRSqlJX6JOBnaVtEJGWWMvU9TbGtAcer4LaqiUvTw9eGncdMyL+wT8Kx1C8awW8dx2kb3Z1aUqpq5xdgS4iEcBgYFoZTR4A3jTGnAIwxmRUTXnVk6+XJ+9P6M6GhmO4q+B58vJyYfpNsPF97YJRSrmMvVfoU4GngLLG7LUCWonIDyISJyIDSmskIpNEJF5E4o8fd80a0lUl0MfGrHu7kxUSzfU5L3K64TWw4k+w4D7rwSSllHKyCgNdRIYAGcaYhHKa2YCWQD/gLqwFo2tf2sgY854xJsYYExMWFnaZJVcftf29mT2xB7bAMK5LfZiMHk/DjsXwXj9rCgGllHIie67QewNDReQQ8DFwg4jMuaRNKrDEGFNgjDkI7MEK+BqvXi1f5t7fEx9vG4M39eDI8AWQlwPT+kPCLO2CUUo5TYWBbox51hgTYYxpAtwJfGOMGXtJs8VYV+eISChWF8yBqi21+mpc1585E3tSWFTM7SuFjDGrITIWlj4Gix6CfF0wQynleJc9Dl1EpojI0JIvvwQyRWQHsAZ40hiTWRUFuouW9YOYdV8Pss4VcPe8A5wc8TH0+zNs/QTeux4ySh0gpJRSVUZnW6xicQcyGT9jI63qB/HRAz0JSv8BPrvfukof/B/ocrerS1RKuTGdbdGJYpuF8PbYaHYeyWbirHhyG/eFh76H8G6w+GH4/BFrkWqllKpiGugOcEOb+vxndGd+OnSSh+ckkO9XD8YthmufhE1zrRumJ/a6ukylVA2jge4gw7qE8/fhHViz+zh/mL+ZIvGEG56DsQsg5xi8ex1s/dTVZSqlahANdAca0zOKpwe0YdnWIzy3OMlaA7HFjfDgd9CwEyy8H5Y+DgU1eyktpZRzaKA72MP9mvNwv+bM23iYl1butt4MDofxS6H3ZEiYCdNvhMz9ri1UKeX2NNCd4KlbWjOmZyTvfLuft9bus9709IKbpsBdn0BWitUFs32xawtVSrk1DXQnEBFeHNaBYV0a8a+Vu5kdl3xhY+sB1iiYem3g0/Gw4kkozHNdsUopt6WB7iQeHsK/b+9M/zb1eP7zJBZvSruwsXZjmLACYh+Bje/BjFvg1CGX1aqUck8a6E7k5enBm2Oi6dm0Ln/8dAurdxy7sNHmDQP+AXfMhcwD8O61sGu564pVSrkdDXQn8/XyZNr47rRvVIvffZTIj/tP/LpB2yHw4LdQpyl8fDd8+T9QVOCaYpVSbkUD3QUCfWx8cG8Pour688CseDanXLKEXd2mMHEV9JgE6/8LMwdaN06VUqocGuguUjfAm9kTe1I30JsJMzey59gli2LYfGDQy3D7B5CxC97tC3u+dEmtSin3oIHuQg2CfZk7MRZvTw/GTtvA4cxS5nhpP8LqggmOgI9Gw1d/haJC5xerlKr2NNBdLDLEn9kTe5JfVMyY6XEcyy7lqdGQ5jBxNXS7F36YCrOGQHa684tVSlVrGujVQOsGQXxwbw9O5uQzdtoGTp3N/20jL1+4dSqMnAZHtsI7fWDf184vVilVbWmgVxNdGtfm/fExJJ88x/iZGzmTW8bIlk63w6S1EFAP5oyCb/4OxUXOLFUpVU3ZHegi4ikim0RkWTltRomIEZFSJ19X5bumeShv3R3N9vRsHvgwntyCMoI6rBU88A10GQPrXoYPh8GZY6W3VUpdNSpzhT4ZKHMdNREJKmmz4UqLuprd2K4+/7m9MxsOnuT3HyVSUFRcekNvfxj+Jgx7C1LjrS6YA986t1ilVLViV6CLSAQwGJhWTrMXgZcAnQv2Cg3vGs6Uoe1ZvTODP326heLicpYJ7DrGulr3qw2zh8O3/9IuGKWuUvZeoU8FngJKvVwUkWigsTGm3GfVRWSSiMSLSPzx48crV+lVZlyvJjx5S2s+35zO80tK5lIvS/128MAa6HAbrPlfq289R8+vUlebCgNdRIYAGcaYhDK2ewCvAH+s6FjGmPeMMTHGmJiwsLBKF3u1+V2/5jx4XTPmxB3m5S93l9/YJxBGvge3vgbJP1oPIiX/6JxClVLVgj1X6L2BoSJyCPgYuEFE5ly0PQjoAKwtaRMLLNEbo1dORHhmQBvu6hHJW2v38/baChbBEIFuE+D+1eDlBx8Mge9fheIy+uGVUjVKhYFujHnWGBNhjGkC3Al8Y4wZe9H208aYUGNMk5I2ccBQY0y8o4q+mogIfx/egSGdGvLSyl3M3ZBc8U4NO8Gkb6HdUFj9N5h3J+SdqXA3pZR7u+xx6CIyRUSGVmUxqnSeHsIro7twfeswnlucxOeb0yreybcW3DYTBv0b9q22hjaeO+n4YpVSLiPl3mxzoJiYGBMfrxfxlXE+v4jxMzeSmHyK9+7pxg1t6tu3464V8OkEawqBcYsgqIFD61RKOY6IJBhjSu3S1idF3YiftyfTx8fQtmEtHp6TSNyBTPt2bDMIxsyHU8klU/EedmyhSimX0EB3M0G+Xsy6rweN6/pz/6x4tqZmVbwTQLN+cM/ncC4TZgyAE3sdWaZSygU00N1Q3QBv5kzsSW1/L8bP2MjeS+dSL0vj7jBhORTlW6F+ZKtjC1VKOZUGuptqEOzLnIk98fTwYOz0DaScLGUu9VJ37Aj3rgSbrzWsMWWjYwtVSjmNBrobaxIawJz7e5BbUMyYaRvIKG0u9dKEtoD7VkJACHw4HPavcWyhSimn0EB3c20a1GLmvd05kZPHuOkbyTpXylzqpand2LpSr9PEWglpV7mzNiil3IAGeg0QHVmH9++J4eCJs4yf+RM5eXYuURdUHyYsgwad4JNxsHW+YwtVSjmUBnoN0btFKG/c3ZWktNNMKm8u9Uv514V7FkPUNbBwEvw03bGFKqUcRgO9BrmlfQNevq0TP+7P5NF5m8qeS/1SPkEwZgG0ugWW/wG+n+rYQpVSDqGBXsOMjI7ghaHt+WrHMZ5asLX8udQv5uULd8yBDqNg9V/h6yngoqeIlVKXx+bqAlTVG39NE7LPF/Cfr/YQ5GvjhaHtEZGKd/T0gpHvg3cgfPcfa0KvAS+Bh/7eV8odaKDXUL+/oQXZuQW8/91Bgnxt/Onm1vaFuoenNae6by348Q3Iy4Ghb4Cn/lVRqrrTn9IaSkT486C2nMkt5M01+9l99Az/GNmRekG+9uwMN70IPsGw5u+QfwZGTQebj+MLV0pdNv23dA0mIvxjREeeG9yW7/ae4OZX17FkS3r5y9ld2BmuexIG/BN2LrXmVM8/6/iilVKXTQO9hvPwEO7v24zlj/WlSUgAj83bxCMfJZKZk2ffAWIfhqH/hQNrYfZIyD3t0HqVUpdPA/0q0aJeIAse6sVTA1rz1Y5j3DJ1HSuTjtq3c/Q4uG0GpCXArFvh7AnHFquUuix2B7qIeIrIJhFZVsq2P4jIDhHZKiJfi0hU1ZapqoLN04Pf9WvB0kf7UL+WLw/NSeCJTzZz+lxBxTu3HwF3zYPju2HmIMhOd3zBSqlKqcwV+mRgZxnbNgExxphOwALgX1damHKcNg1qsfiR3kzu35KlW9K5eeq3rNmdUfGOLW+CsQutMJ8xAE4edHyxSim72RXoIhIBDAamlbbdGLPGGPPz/K1xQETVlKccxcvTgyduasXiR3oT7OfFvTN/4ukFWzmTW8HVepPeMH4J5GVboZ5R1u94pZSz2XuFPhV4CrDnWfKJwBelbRCRSSISLyLxx48ft/OjlSN1CA9m6aN9eLhfcz5NSGHA1O/4YV8FfeTh0XBvyf/imYMgfZPjC1VKVajCQBeRIUCGMSbBjrZjgRjg5dK2G2PeM8bEGGNiwsLCKl2scgwfmydPD2jDgoevwcfmwZhpG3j+8yTO5Zcza2O9tnDfF+ATCB/cCod+cF7BSqlS2XOF3hsYKiKHgI+BG0RkzqWNRORG4H+AocYYO8fEqeokOrIOyx/ry329mzI7LpmBr33HT4dOlr1D3WbWnOq1GsKckbB3tfOKVUr9RoWBbox51hgTYYxpAtwJfGOMGXtxGxHpCryLFeZ23F1T1ZWftyfP39qOjx+IpdgYRr+7nr8v21H2dLzB4Vb3S2gr6+Gj7YudW7BS6heXPQ5dRKaIyNCSL18GAoFPRWSziCypkuqUy/RsFsLKydcypmck074/yODXv2NzSlbpjQNCYfxSCO8GC+6FTXOdW6xSCgCx6zFwB4iJiTHx8fEu+WxVOd/tPc5TC7ZyLDuXh/s157H+LfGxef62Yf5Z+GQs7P/GmqUx9iHnF6tUDSciCcaYmNK26ZOiqkJ9W4bx5RPXMio6gjfX7GfYf39ge3opUwB4B8BdH0ObIbDyafj2ZZ1TXSkn0kBXdqnl68XLt3dm+vgYMs/mM+y/P/Da6r2/XRXJ5gO3z4LOd1kzNX71Fw11pZxEA11VSv+29Vn1+LUM7tSQV1fvYeRbP7Ln2JlfN/K0wbC3oPsD1pzqy56AYjvXOFVKXTYNdFVpdQK8ee3Orrw9Jpq0rPMMef173vl2P0UXL3fn4QGDXoY+f4CEmdYC1EV2zBmjlLpsGujqsg3s2JBVT1zLDW3q8c8vdnH7Oz9y4HjOhQYicONf4ca/QdIC+GQcFOS6qlylajwNdHVFQgN9eHtsNK/d2YX9x88y6PXvmPH9wV8vTt3nCRj8H9izEj663VrWTilV5TTQ1RUTEYZ1CWfVE9dyTfNQpizbwV3vx5Fy8tyFRt3vhxHvWlMEzB4O50+5rmClaigNdFVl6tfyZfr4GP51Wye2p2dzy9R1zIlLvrDkXec7YPSHcGQLfDAEcvShYqWqkga6qlIiwuiYxnz5xLVER9bhucVJ3DNjI+lZ560GbYfA3fPh5AFr+t2sFNcWrFQNooGuHCK8th+zJ/bgxeEdSEg+xS2vruPT+BTrar359TBusbWU3YwBcGKfq8tVqkbQQFcOIyKMi41i5eRraduoFk8u2MoDH8aTkZ0LkT1hwlIozIWZA+BokqvLVcrtaaArh4sM8efjB2L5y5B2fLf3BDdPXceSLemYBp2smRo9veGDQZDyk6tLVcqtaaArp/DwECb2acqKyX1pEhLAY/M28chHiWT6RVmh7lcXPhwGB751dalKuS0NdOVUzcMCWfBQL54a0JrVOzK4+dV1rEzzgftWQp0omHs77C51BUOlVAU00JXT2Tw9+F2/Fix9tA8Ngn15aE4Cjy8/QvYdi6F+e2sK3m0LXF2mUm7H7kAXEU8R2SQiy0rZ5iMin4jIPhHZICJNqrJIVTO1bhDE4kd6M7l/S5ZtPcKN72xjXa/p0DgWPrsf4me6ukSl3EplrtAnAzvL2DYROGWMaQG8Crx0pYWpq4OXpwdP3NSKxY/0pra/F/fM3cn/BPyNwub9Ydnj8MPrri5RKbdhV6CLSAQwGJhWRpNhwKyS1wuA/iIiV16eulp0CA9m6aN9eLhfc+ZtyqB/6iRORA2y5lP/5n91TnWl7GDvFfpU4CmguIzt4UAKgDGmEDgNhFzaSEQmiUi8iMQfP378MspVNZmPzZOnB7RhwcPX4OnlQ4/ddxNfdwis+xesfAaKy/rrp5QCOwJdRIYAGcaYhCv9MGPMe8aYGGNMTFhY2JUeTtVQ0ZF1WP5YX+7t05zbj9zFJ7ahsOEdWPKoLpShVDlsdrTpDQwVkUGAL1BLROYYY8Ze1CYNaAykiogNCAYyq7xaddXw8/bkL0PacXO7+vzpUz/Sc714YvMcivLO4DlqGti8XV2iUtVOhVfoxphnjTERxpgmwJ3AN5eEOcASYHzJ69tK2minp7piPZuFsPLx68iMeYIXC8biufNzTn9wO+Sfq3hnpa4ylz0OXUSmiMjQki+nAyEisg/4A/BMVRSnFECAj42/D+/I9RNe4P9sDxOU8i0pbwwk76zOqa7UxcRVF9IxMTEmPj7eJZ+t3Fd2bgFL577B6MN/56CtKUV3L6Bt86auLksppxGRBGNMTGnb9ElR5VZq+XoxZuIf2HHdW0QWHcY2azDTlv9IQZGOgFFKA125pc433EnhnfNpbDvJTRvG8/AbC9lz7Iyry1LKpTTQldsKbHM9vvctp6FPHv/IepLJr3/C22v3U1Ss9+PV1UkDXbm3iG543/8Fof6ezPeZwrIvv+C2d37kwPEcV1emlNNpoCv3V789HhO/JDCoFosD/o/gjHgGvvYdM74/SLFerauriAa6qhlCmiP3rcQruAEzbf/HpPBDTFm2g7vej+Nwpo5ZV1cHDXRVcwRHwL0rkZAW/OHE88ztfYwd6dkMeG0dc+KS0WfdVE2nga5qlsAwmLAUadiF3ol/5NtbjhIdWYfnFidxz4yNpGedd3WFSjmMBrqqefzqwLhF0KQvdVc9xuyOW3hxeAcSkk9xy6vr+DQ+Ra/WVY2kga5qJp9AuHs+tB6MfPEk4woWsHLytbRtVIsnF2zl/lnxZGTnurpKpaqUBrqqubx8YfQs6Dgavp5CZOJLfHx/T/4ypB3f7zvBTa+u4/PNaXq1rmoMDXRVs3l6wYh3IeY++GEqHl/8iYnXRLFicl+ahQUw+ePN/G5uIpk5ea6uVKkrpoGuaj4PDxj8CvR+HOKnw+KHaF7Xl08f7MVTA1rz9c4Mbn51HSuTjri6UqWuiAa6ujqIwE0vQP/nYesnMP8ebMX5/K5fC5Y+2ocGwb48NCeRxz/exOlzBa6uVqnLooGuri59/wgDX4bdy2HeHZB/ltYNglj8SG8ev7Ely7Ye4cZXv+Xttfs5eTbf1dUqVSk6H7q6Om3+CD5/BMJjYMyn4FcbgKS007y4bAcbDp7E2+bBkI4NGdsriq6NayMiLi5aqfLnQ68w0EXEF1gH+GCtQbrAGPPXS9pEArOA2oAn8IwxZkV5x9VAVy63YwksuA/C2ljj1gMvLFy+59gZZq9PZmFiKmfzi+gQXotxsVEM7RyOn7enC4tWV7srDXQBAowxOSLiBXwPTDbGxF3U5j1gkzHmbRFpB6woWYO0TBroqlrYtxo+HmtNG3DPYuu/F8nJK2TRpjRmrz/EnmM51PK1cXtMY8b0jKRZWKBralZXtStaschYfp6L1Kvkz6W/BQxQq+R1MJB+mbUq5VwtbrSuznOOwYyBkLn/V5sDfWyMi43iy8ev5ZNJsVzbKoxZPx7ihv98y7jpG1i1/SiFulqSqibs6kMXEU8gAWgBvGmMefqS7Q2BVUAdIAC40RiTUMpxJgGTACIjI7slJydf8TegVJVI3wxzRoKHDcYthvrtymyacSaXjzem8NGGwxzNzqVRsC9394zkju6RhAX5OLFodTW6oi6XSw5UG1gEPGqMSbro/T+UHOs/ItILmA50MMaUeemiXS6q2jm+Gz4cBgXnoc/jED0e/OuW2bywqJjVOzOYHXeIH/Zl4uUpDOzQkHG9ooiJqqM3UYN+AaAAABR7SURBVJVDVFmglxzseeCcMebfF723HRhgjEkp+foAEGuMySjrOBroqlo6dQiWPAYHvwWbH3S5C3o+BGGty91tX0YOczcksyAhlTO5hbRpEMS4XlEM7xJOgI/NObWrq8KV3hQNAwqMMVki4ofVtfKSMWbZRW2+AD4xxnwgIm2Br4FwU87BNdBVtXZsO8S9DVvnQ1EeNO8Psb+D5jdYT56W4Vx+IZ9vTufD9cnsPJJNoI+NUdHhjOsVRYt6QU78BlRNdaWB3glrSKIn1k3U+caYKSIyBYg3xiwpGdnyPhCIdYP0KWPMqvKOq4Gu3MLZE5AwEzZOg5yjENoKej4Ine8C74AydzPGkHg4i9nrD7Fi21Hyi4rp1SyEcb2iuKldfbw89Zk+dXmqtMulqmigK7dSmA87Poe4NyF9E/gGQ7cJ0P0BqN243F1P5OQxPz6FuXGHScs6T/1aPtzVI5K7ekRSv5avc+pXNYYGulJVxRhI2Qhxb8HOpdZ7bW+1umMa97DmjClDUbFhza4MZscl8+2e43h6CLe0r8+42CbENqurN1GVXTTQlXKErBT46X1I+AByT0OjaIh9GNoNB5t3ubseOnGWuRuSmR+fyunzBbSoF8i42ChGRocT5OvlnPqVW9JAV8qR8s/ClnkQ9w5k7oXABtDjfuh2LwSElrtrbkERS7akM3t9MtvSTuPv7cmIrtZN1DYNapW7r7o6aaAr5QzFxbD/G6s7Zv/X4OkDnUZbV+3121e4+5aULGbHJbNkSzr5hcX0aFKXsb2iGNC+Ad42vYmqLBroSjnb8d2w4R3YPA8Kz0PT66xgb3lLucMeAU6dzefThBTmxB3m8MlzhAZ6c2f3SO7uGUmj2n5O+gZUdaWBrpSrnDsJibNg4/uQnQZ1m1kPKnW5G3zKH5deXGz4du9x5qxP5pvdGQhwY9v6jOsVRe/moXh46E3Uq5EGulKuVlRgjYqJextSN4JPLeg6DnpOgjpNKtw95eQ55m44zPz4FE6ezadZaABjYqO4LTqCYH+9iXo10UBXqjpJjbeCfcdiMMXQepA17DHqmnKHPYJ1E/WLpCPMXp9M4uEsfL08GN4lnLGxUXQID3bSN6BcSQNdqeooOx1+mgbxM+H8SWjQ0Qr2DqPAVvGsjUlpp5kTl8zizWnkFhTTNbI29/SKYmCHhvh66SIcNZUGulLVWcF5a86YuLfh+E4ICIPu90PMfRBYr8LdT58rYEFiKnPikjl44ix1A7wZXbIIR+O6/k74BpQzaaAr5Q6MgQNrrWDf+yV4ekOH2yD2IWjYucLdi4sNP+7PZHbcIb7acQwD3NC6HmN7RXFdyzC9iVpDaKAr5W5O7ION78KmuVBwFqJ6W8MeWw8Cj4q7U9KzzjNv42HmbUzhRE4ekXX9GRsbye3dGlMnoPynWFX1poGulLs6nwWbZsOG9+D0YagdaQ177DrWmiCsAvmFxazcfpQ565PZeOgk3jYPbu3UiHt6RdG5cW0nfAOqqmmgK+Xuigph9wqrO+bwj+AdCF3GWFP5hjS36xC7jmYze30yizalcS6/iE4RwYyNjWJo50Z6E9WNaKArVZOkb7aeQt22AIoLodUAq5+96XUVDnsEOJNbwKJNacxen8zejByC/bwYHRPBmJ5RNAkte453VT1ooCtVE505BvHT4afpcO4E1Gtn9bN3vB28Kp4iwBhD3IGTzIlL5svtRyksNlzbKox7YqO4vk09PPUmarV0pSsW+QLrAB/ABiwwxvy1lHajgb9hrVi0xRhzd3nH1UBXqooU5ELSZ1Z3zLFt4B9izfTY/X6o1dCuQxzLzi25iXqYY9l5hNf24+6ekdzRvTGhgRWPiVfOc6WBLkCAMSZHRLyA74HJxpi4i9q0BOYDNxhjTolIvfIWiAYNdKWqnDFw6Hsr2HevsEbDtB9pdceEd7PrEAVFxazecYwP1yez/kAm3p4eDOrYgHG9ooiOrKOLcFQDVdblIiL+WIH+sDFmw0Xv/wvYY4yZZu+xNNCVcqCTB6wJwRJnQ/4ZaNzT6o5pcyt42uw6xL6MM8yJO8xnCamcySukXcNajOsVxbAujfD3tu8YqupdcaCLiCeQALQA3jTGPH3J9sXAHqA31mLSfzPGrCzlOJOASQCRkZHdkpOTK/mtKKUqJTcbNs+1bqKeOgS1IqDHA9BtPPjVsesQZ/MKWbzZuom66+gZgnxt3NYtgrGxUTQPC3Rs/eo3qvIKvTawCHjUGJN00fvLgAJgNBCB1efe0RiTVdax9ApdKScqLoI9X1qLbxz6Drz8ofNd1pj2sFZ2HcIYQ0LyKT5cn8wXSUcoKDL0bhHCuNgo+retj5enLsLhDFU6ykVEngfOGWP+fdF77wAbjDEzS77+GnjGGPNTWcfRQFfKRY5us5bL2/YpFOVBixut7pjm/e0a9ghw/Ewe8+NTmBuXTPrpXOoGeDO0cyNGdA2nU0Sw9rU70JXeFA0DCowxWSLiB6wCXjLGLLuozQDgLmPMeBEJBTYBXYwxmWUdVwNdKRfLOQ4JM60ZH3OOQWhr60GlzneCt33j0QuLivl2z3EWJqbx1c5j5BcW0ywsgFHREQzr0oiIOjo5WFW70kDvBMzC6hv3AOYbY6aIyBQg3hizpGQkzH+AAUAR8L/GmI/LO64GulLVRGEebF9kdccc2QK+taHbBKuvPTjC7sOcPl/Aim1HWJSYxsZDJwGIbVaXkV0jGNixAUG+uhBHVdAHi5RSFTMGDsdZwb5rGSDQbqg1R3tEd7u7Y8BaYWnRpjQWbUrj4Imz+Ng8uLl9A0ZGh9O3RSg27W+/bBroSqnKOZUMP70PCR9C3mlrHHvs76DdMPC0/0rbGMOmlCwWJaaxdGs6WecKCA30YWjnRoyMDqd9o1ra315JGuhKqcuTlwNb5lnDHjP3QVBD6wnUbvdCQEilDpVfWMya3RksTEzlm10ZFBQZWtUPZGR0BMO7hNMg2NdB30TNooGulLoyxcWwb7XVHXNgDdh8odMd1uiYem0rfbisc/ks23qEhYmpJB7OQgR6Nw9lRNdwBnRoQICPPrhUFg10pVTVydhpXbFv+RgKc61ZHruMgdYDwbdWpQ938MTZkv72VFJOnsfPy5MBHRowoms4vVuE6iRhl9BAV0pVvbOZkPgB/DQDslPB0wda3gTtR1jhbufQx58ZY4hPPsXCxDSWb00nO7eQ+rV8GNYlnJHR4bRpUPlfFjWRBrpSynGKiyF1ozX0cftiyDkKNj9odQt0GAktb7ZrOt+L5RYU8c2uDBYmprF2dwaFxYa2DWsxsms4w7o0ol6tq7e/XQNdKeUcxUVweD0kLYQdn1vztHsFWFfsHUZaT6XaKjcdb2ZO3i/97VtST+Mh0KdlGKOiw7m5XQP8vK+u1ZY00JVSzldUCMnfW+G+cwmcPwU+taDNYKtbptn1YKvcgtX7MnJYtCmVxZvSScs6T4C3JwM7NmRk13Bim4XgcRX0t2ugK6Vcq6gADnxrdcvsWgq5p60nUtsOseZsb3ptpca3FxcbNh46ycLEVFZsO0pOXiGNgn0Z1jWckV3DaVk/yIHfjGtpoCulqo/CfNj/DWxfCLtWWPO1+9W1nkptPxKa9LEW57DT+fwivtp5jEWJqazbe4KiYkPH8GBGdA1naJdGNW7FJQ10pVT1VJBrjW/fvhB2r4SCsxBQz3oitf0IiOwFHvZPE3D8TB5LtqSzaFMqSWnZeHoI17UKY2R0ODe2rY+vl/v3t2ugK6Wqv/xzsPdLq1tmz5fWGPeghtBuuHVDtZLzyew5doaFiWks3pTG0excgnxsDO7UkBFdw+nepK7b9rdroCul3EteDuxZad1Q3fcVFOVDcGPryr3DSGgUbXe4FxUb4g5k8lliKiuTjnIuv4jw2n6MjA5nRNdwmrnZqksa6Eop95V7GnZ/YYX7/m+guADqNLG6ZNqPhAYd7Q73c/mFrNp+jM8SU/lh3wmKDXRpXJuR0eEM6dSIugGVG3XjChroSqma4fwp2LnM6nM/8C2YIghpcSHc67ez+1DHsnP5fHMaCxPT2HX0DDYP4fo29RjZNZwb2tbDx1Y9+9uvdIELX6w1Qn0AG7DAGPPXMtqOAhYA3Y0x5aa1BrpS6oqczbTGt29fCIe+B1MMYW2sYG8/wu61UgF2pGdb49s3p3P8TB7Bfl4M7tSQUdHhREfWqVZT/F5poAsQYIzJEREv4HtgsjEm7pJ2QcBywBv4vQa6Usppzhyzwj1pofWkKgbqdyi5ch8BIc3tOkxhUTE/7M9kUWIqK7cfJbegmKgQf0Z0tfrbo0IqNz+NI1RZl4uI+GMF+sPGmA2XbJsKfAU8CfxJA10p5RLZ6da0A0kLrTlmABp2vnDlXifKrsPk5BWyMukoCxNTWX8gE2MgJqoOI6LDGdKxEcH+rllS74oDXUQ8gQSgBfCmMebpS7ZHA/9jjBklImspI9BFZBIwCSAyMrJbcnJyZb8XpZSyX1YK7FhshXt6ovVeeIw1UqbdcAgOt+sw6VnnWbw5jUWJaezNyMHb04P+besxoms4/VrXw9vmvCX1qvIKvTawCHjUGJNU8p4H8A0wwRhzqLxAv5heoSulnOrkwZIZIRfB0a3We41jS8J9GAQ1qPAQxhiS0rJZuCmVpVvSOZGTTx1/L27t3IiR0RF0jgh2eH97lY5yEZHngXPGmH+XfB0M7AdySpo0AE4CQ8sLdQ10pZTLnNhXEu4LIWMHINaUA+2HQ9thEBhW4SEKior5bu9xFiamsWrHMfILi2kWGsDI6HCGdw0noo6/Q0q/0puiYUCBMSZLRPyAVcBLxphlZbRfi16hK6XcRcauC+F+Yg+IhzVZWPuR0PZW8K9b4SGycwv4YtsRPktMY+PBkwD0aFqXUdHhDOzYkFq+VdfffqWB3gmYBXgCHsB8Y8wUEZkCxBtjllzSfi0a6Eopd2MMHNtuBXvSQjh1EDxs0KyfFe5tBoNf7QoPk3Ly3C/j2w+cOIuPzYOb2tVnZHQ4fVuG4eV5Zf3t+mCRUkpVhjFwZEtJuC+C04fB0xua97+wxF4F66caY9iSepqFiVZ/+6lzBYQGenNr50aM6RlJi3qXN8WvBrpSSl0uYyAtwbpq374IzqRfWD+1w0hoNaDC9VPzC4tZuzuDRZvS+HpnBn8f0YHRMY0vqxwNdKWUqgo/r5+atNAaDplzrNLrp54+V4C3zeOyl87TQFdKqapWXATJP1pX7T+vn+odaHXHtB9xWeun2kMDXSmlHKmoEA59Z/W571x6yfqpI60bq5VcP7UsGuhKKeUsv6yfutCaGTLv0vVTrwNP22UfXgNdKaVcoTCvZP3URRfWT/UPgQEvQafbL+uQ5QX65f+aUEopVT6bj9Wn3nogFJy31k9NWgjBEY75OIccVSml1K95+VlPnra91WEf4bwpwpRSSjmUBrpSStUQGuhKKVVDaKArpVQNoYGulFI1hAa6UkrVEBroSilVQ2igK6VUDeGyR/9F5DiQfJm7hwInqrCcqqJ1VY7WVXnVtTatq3KupK4oY0ypi566LNCvhIjElzWXgStpXZWjdVVeda1N66ocR9WlXS5KKVVDaKArpVQN4a6B/p6rCyiD1lU5WlflVdfatK7KcUhdbtmHrpRS6rfc9QpdKaXUJTTQlVKqhqi2gS4iM0QkQ0SSytguIvK6iOwTka0iEl1N6uonIqdFZHPJn+edVFdjEVkjIjtEZLuITC6ljdPPmZ11Of2ciYiviGwUkS0ldb1QShsfEfmk5HxtEJEm1aSuCSJy/KLzdb+j67rosz1FZJOILCtlm9PPl511ufJ8HRKRbSWf+5s1N6v8Z9IYUy3/ANcC0UBSGdsHAV8AAsQCG6pJXf2AZS44Xw2B6JLXQcAeoJ2rz5mddTn9nJWcg8CS117ABiD2kja/A94peX0n8Ek1qWsC8F9n/x0r+ew/AB+V9v/LFefLzrpceb4OAaHlbK/Sn8lqe4VujFkHnCynyTDgQ2OJA2qLSMNqUJdLGGOOGGMSS16fAXYC4Zc0c/o5s7Mupys5BzklX3qV/Ll0hMAwYFbJ6wVAfxGRalCXS4hIBDAYmFZGE6efLzvrqs6q9Gey2ga6HcKBlIu+TqUaBEWJXiX/ZP5CRNo7+8NL/qnbFevq7mIuPWfl1AUuOGcl/0zfDGQAXxljyjxfxphC4DQQUg3qAhhV8k/0BSLS2NE1lZgKPAUUl7HdJefLjrrANecLrF/Gq0QkQUQmlbK9Sn8m3TnQq6tErLkWOgNvAIud+eEiEgh8BjxujMl25meXp4K6XHLOjDFFxpguQATQQ0Q6OONzK2JHXUuBJsaYTsBXXLgqdhgRGQJkGGMSHP1ZlWFnXU4/XxfpY4yJBgYCj4jItY78MHcO9DTg4t+0ESXvuZQxJvvnfzIbY1YAXiIS6ozPFhEvrNCca4xZWEoTl5yziupy5Tkr+cwsYA0w4JJNv5wvEbEBwUCmq+syxmQaY/JKvpwGdHNCOb2BoSJyCPgYuEFE5lzSxhXnq8K6XHS+fv7stJL/ZgCLgB6XNKnSn0l3DvQlwD0ld4ljgdPGmCOuLkpEGvzcbygiPbDOscNDoOQzpwM7jTGvlNHM6efMnrpccc5EJExEape89gNuAnZd0mwJML7k9W3AN6bkTpYr67qkj3Uo1n0JhzLGPGuMiTDGNMG64fmNMWbsJc2cfr7sqcsV56vkcwNEJOjn18DNwKWj46r0Z9J22dU6mIjMwxr9ECoiqcBfsW4QYYx5B1iBdYd4H3AOuLea1HUb8LCIFALngTsd/Ze6RG9gHLCtpP8V4M9A5EW1ueKc2VOXK85ZQ2CWiHhi/QKZb4xZJiJTgHhjzBKsX0SzRWQf1o3wOx1ck711PSYiQ4HCkromOKGuUlWD82VPXa46X/WBRSXXKjbgI2PMShF5CBzzM6mP/iulVA3hzl0uSimlLqKBrpRSNYQGulJK1RAa6EopVUNooCulVA2hga6UUjWEBrpSStUQ/w+flvNhXnsy/AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "history=pd.DataFrame(history, columns=['epochs','loss','val_loss'])\n",
    "from matplotlib import pyplot \n",
    "pyplot.plot(history['epochs'],history['loss'], label='train') \n",
    "pyplot.plot(history['epochs'],history['val_loss'], label='test') \n",
    "pyplot.legend() \n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 191
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3077388,
     "status": "ok",
     "timestamp": 1589544349972,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "MwDoSYgYMLcE",
    "outputId": "47655334-fd01-4528-fe02-b97119af14d0"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epochs</th>\n",
       "      <th>loss</th>\n",
       "      <th>val_loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>4.909</td>\n",
       "      <td>4.832585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>4.251</td>\n",
       "      <td>4.366992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3.834</td>\n",
       "      <td>3.785548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>3.578</td>\n",
       "      <td>3.516473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>3.353</td>\n",
       "      <td>3.307459</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epochs   loss  val_loss\n",
       "0       1  4.909  4.832585\n",
       "1       2  4.251  4.366992\n",
       "2       3  3.834  3.785548\n",
       "3       4  3.578  3.516473\n",
       "4       5  3.353  3.307459"
      ]
     },
     "execution_count": 43,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5cUmIlXWM7Uw"
   },
   "outputs": [],
   "source": [
    "#Evaluate results¶\n",
    "#Because our task is simple and the output is straight-forward, we will use MAE metric to evaluate the trained model during the epochs. Compute the value of the metric for the output from each epoch.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 98
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3077344,
     "status": "ok",
     "timestamp": 1589544349978,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "r5ZnQYTwLpXn",
    "outputId": "901d3e5d-f1b3-4b25-c933-0b1e797c7109"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, MAE: 42.857143, Invalid numbers: 42\n",
      "Epoch: 2, MAE: 42.857143, Invalid numbers: 34\n",
      "Epoch: 3, MAE: 61.721340, Invalid numbers: 58\n",
      "Epoch: 4, MAE: 73.029674, Invalid numbers: 40\n",
      "Epoch: 5, MAE: 83.333333, Invalid numbers: 43\n"
     ]
    }
   ],
   "source": [
    "for i, (gts, predictions, invalid_number_prediction_count) in enumerate(zip(all_ground_truth,\n",
    "                                                                            all_model_predictions,\n",
    "                                                                            invalid_number_prediction_counts), 1):\n",
    "       \n",
    "\n",
    "          mae=mean_absolute_error_texts(gts, predictions)\n",
    "          print(\"Epoch: %i, MAE: %f, Invalid numbers: %i\" % (i, mae, invalid_number_prediction_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ifAvzROshiqL"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "_cell_guid": "265fd2f2-cd5f-590d-1fa3-cf6eeedf89fe",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3261071,
     "status": "ok",
     "timestamp": 1589544533749,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "SRKF7S2a-fPe",
    "outputId": "350a7263-a6c7-48c6-e91d-b251ada74861",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "----Prepare evaluation\n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [2063, 318, 1636, 433, 112, 2261, 12370, 3165, 12150, 1782, 1636, 2068, 36803]\n",
      "  Input Words: tender light popcorn use back basics stainless steel stove top popcorn popper\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [1476, 2791, 1106, 1401]\n",
      "  Response Words: cutter advanced corn topping\n",
      " Ground Truth: fresh and light popcorn\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [10590, 36801, 3, 781, 1723, 2, 1129, 2901, 14597, 1957, 5892, 1274, 1129, 879, 872, 1106, 33862, 1578, 865, 626, 719, 1216, 517, 1256, 15, 468, 1127, 215, 374, 985, 24, 469, 316, 643, 298, 2407, 1961, 155, 14217, 2881, 6470, 373, 1816, 2022, 4913, 843, 1355, 672, 35, 469, 1871, 719, 4270, 91, 1674, 38, 446, 36803]\n",
      "  Input Words: length <UNK> food well recieved dog however closer inspection beef main ingredient however soy wheat corn bumping amount meat quite bit dried beans potatoes nice artificial color shame really fine healthy ingredients make cost something folks willing pay corners cut occasional mix might honestly reading label wish used better ingredients charged bit certainly would feed regular basis\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [4, 1136, 17, 418]\n",
      "  Response Words: not worth the money\n",
      " Ground Truth: after researching the ingredients found this\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [5980, 579, 1156, 71, 108, 25, 474, 539, 323, 1082, 519, 36803]\n",
      "  Input Words: looked around decent price stuff best could find arrived quickly expected\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 71, 70, 6031]\n",
      "  Response Words: great price for caveats\n",
      " Ground Truth: it is what it is\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [13, 83, 92, 1108, 83, 309, 1806, 1203, 2951, 149, 11954, 92, 3, 495, 5230, 57, 3, 400, 174, 5569, 175, 202, 19336, 400, 1113, 401, 92, 5510, 2055, 295, 4582, 496, 12383, 12128, 5510, 36803]\n",
      "  Input Words: great way buy cereal way help mother lives fixed low income buy food shipped directly home food get cold colorado weather bad roads get grocery store buy groceries hope amazon continues work companies expand groceries\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 1108, 429, 71]\n",
      "  Response Words: great cereal terrible price\n",
      " Ground Truth: great way to help people\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [13, 248, 2802, 1646, 13, 68, 1329, 454, 179, 36803]\n",
      "  Input Words: great value subscribe save great flavor superb organic coffee\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 248, 13, 71]\n",
      "  Response Words: great value great price\n",
      " Ground Truth: lbs of dark roast\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [374, 8284, 4389, 454, 1774, 12998, 2389, 474, 1450, 104, 1215, 1079, 102, 6113, 369, 128, 466, 565, 321, 322, 359, 374, 1450, 1136, 418, 1463, 1742, 251, 18679, 353, 5933, 61, 1129, 54, 0, 35, 695, 2443, 26, 7003, 724, 174, 480, 1388, 36803]\n",
      "  Input Words: really ceremony grade organic think critic probably could tell order items online time wonder real deal drink mostly health benefits cannot really tell worth money company website looks amateur makes question product however taste good better first brewed hot refrigerated added cold foods drinks\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [25, 271, 462, 1128]\n",
      "  Response Words: best tasting have tried\n",
      " Ground Truth: pricey but tastes good\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [36801, 1216, 4750, 60, 24, 530, 1899, 294, 55, 2899, 649, 758, 298, 0, 508, 61, 655, 677, 2683, 170, 36801, 1216, 4750, 1050, 11542, 11542, 1216, 4750, 2980, 2603, 61, 2862, 10251, 6111, 36801, 1216, 4750, 811, 1770, 156, 2920, 1211, 4699, 15410, 36682, 824, 1585, 156, 827, 508, 56, 226, 676, 758, 36801, 363, 1216, 4750, 36803]\n",
      "  Input Words: <UNK> dried mangoes delicious healthy treat comes handy sweet tooth want eat something good eating product become habit knew nothing <UNK> dried mangoes son introduced introduced dried mangoes iraq said product popular whoever supply <UNK> dried mangoes found empty bag upon return living quarters comrades eaten entire bag instead eating candy cookies prefer eat <UNK> brand dried mangoes\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [17, 6914, 1191, 293]\n",
      "  Response Words: the bounty cherry juice\n",
      " Ground Truth: <UNK> brand naturally delicious dried mangoes\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [87, 345, 401, 3576, 26, 1341, 3144, 1216, 331, 7547, 759, 2603, 4443, 1355, 474, 539, 7547, 474, 1890, 1341, 781, 656, 816, 518, 110, 7547, 539, 2697, 1133, 1341, 110, 7260, 1141, 7757, 719, 36801, 26, 1341, 1711, 7547, 20915, 5951, 1109, 112, 2031, 174, 2092, 324, 26, 1341, 19418, 36803]\n",
      "  Input Words: wife loves store purchased hot cocoa freeze dried mini marshmallows years said gee wish could find marshmallows could put cocoa well finally available exactly like marshmallows find pre packaged cocoa like growing may moved bit <UNK> hot cocoa add marshmallows haute cuisine bring back memories cold winter days hot cocoa sledding\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [4271, 1720, 3065, 157]\n",
      "  Response Words: yo christmas childhood chocolate\n",
      " Ground Truth: thought these were impossible to find\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [778, 1039, 87, 829, 133, 409, 316, 0, 54, 2089, 6093, 133, 8317, 110, 135, 1061, 221, 719, 87, 5370, 1481, 846, 91, 16769, 1650, 36801, 567, 781, 1948, 3405, 2577, 659, 1183, 945, 1285, 36803]\n",
      "  Input Words: made dinner wife three kids easy make good taste opinion although kids act like loved ate every bit wife underwhelmed thought bland would prefered larger <UNK> pieces well definitely beats mac cheese vegetable quick meal\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 70, 133, 9]\n",
      "  Response Words: good for kids it\n",
      " Ground Truth: quick and easy personally liked it\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 157, 32, 1213, 2109, 32, 542, 1017, 171, 542, 1017, 157, 1213, 2109, 7072, 110, 1230, 1472, 1017, 19095, 157, 1781, 6223, 1922, 542, 1781, 914, 4812, 1782, 331, 157, 1213, 2109, 440, 16957, 157, 3168, 2002, 1032, 54, 0, 48, 450, 2226, 649, 758, 110, 56, 538, 793, 298, 253, 758, 945, 89, 1089, 719, 2127, 258, 36803]\n",
      "  Input Words: love chocolate love peanut butter love granola bars special granola bars chocolate peanut butter sounded like must try bars consist chocolate layer bottom chewy granola layer making middle top mini chocolate peanut butter chips drizzled chocolate appreciate fiber protein taste good yummy enough going want eat like candy bar still something happy eat quick breakfast run bit afternoon snack\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [1213, 2109, 1213, 2109, 1017]\n",
      "  Response Words: peanut butter peanut butter bars\n",
      " Ground Truth: peanut butter chocolate and of protein\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [906, 4041, 1059, 1901, 3, 1797, 110, 1129, 1523, 1172, 16756, 2043, 825, 3240, 474, 758, 3808, 184, 189, 9734, 91, 793, 758, 4359, 7448, 5430, 13445, 2296, 61, 980, 13, 2718, 2406, 777, 945, 1039, 1547, 5139, 3808, 479, 573, 1904, 5177, 906, 1022, 5067, 5430, 1465, 10764, 452, 91, 777, 4694, 1426, 184, 1120, 317, 510, 353, 447, 945, 1285, 1698, 1855, 2584, 21, 246, 2020, 1856, 3292, 36803]\n",
      "  Input Words: family generally likes indian food surprisingly like however year old announced today got bus could eat lentils rice stomach ache would still eat sent warmed left overs school product cheap great stand ideas need quick dinner two pouches lentils one chick peas young family easily servings left feeding teenagers sure would need heck lot rice salad fruit veggie makes fabulous quick meal minutes keep several tasty bite products hand times\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 88, 345, 9]\n",
      "  Response Words: my favorite loves it\n",
      " Ground Truth: great product but pricey\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1465, 1523, 1172, 5260, 1763, 1172, 5052, 373, 1217, 3, 1567, 1426, 13, 128, 716, 295, 1048, 716, 2841, 401, 853, 3, 1217, 12380, 1698, 36803]\n",
      "  Input Words: feeding year old maltese month old chihuahua mix dry food enjoy lot great deal buying amazon compared buying pet store side food dry leaving minutes\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 2, 345, 9]\n",
      "  Response Words: my dog loves it\n",
      " Ground Truth: great food topper for my dogs\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [3576, 205, 11644, 1824, 10465, 2583, 1140, 155, 4353, 1608, 205, 155, 4353, 1211, 262, 205, 170, 104, 36803]\n",
      "  Input Words: purchased item ace photo digital broke received pay postage receive item pay postage return broken item nothing order\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [1646, 112, 17, 418]\n",
      "  Response Words: save back the money\n",
      " Ground Truth: cup holder ace photo digital\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [6535, 9594, 2218, 2339, 1128, 2047, 1332, 300, 1332, 54, 0, 433, 1278, 34, 300, 2786, 13052, 13, 7295, 2038, 546, 1803, 36803]\n",
      "  Input Words: peak blew away expectations tried powdered milk never milk taste good use staple diet never worry spoilage great survival stash highly recommend\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 61, 70, 231]\n",
      "  Response Words: great product for older\n",
      " Ground Truth: better than fresh no waste\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [36801, 36458, 36801, 184, 1106, 1330, 353, 25, 271, 874, 2020, 81, 82, 34, 2383, 36803]\n",
      "  Input Words: <UNK> ris <UNK> rice corn pasta makes best tasting macaroni products gluten free diet market\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [5218, 44, 5218, 44, 1438]\n",
      "  Response Words: tastier and tastier and vegan\n",
      " Ground Truth: excellent product for macaroni and cheese\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [338, 990, 3603, 4116, 1064, 165, 466, 989, 34, 1388, 110, 34, 990, 54, 433, 1379, 426, 57, 445, 1581, 442, 280, 1327, 1480, 1607, 1472, 3406, 5164, 721, 34, 466, 54, 2040, 1866, 1244, 110, 468, 3158, 1641, 3815, 54, 1388, 1141, 181, 151, 766, 52, 654, 54, 110, 885, 34, 990, 36803]\n",
      "  Input Words: huge soda drinker carbs calories ever drink traditional diet drinks like diet soda taste use stevia raw home daily enjoyed lack calorie sweetness gives lead try sodas wrap strong diet drink taste trying avoid begin like artificial sweeteners concept mind taste drinks may natural alternative dont expect flavors taste like anything diet soda\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [3409, 151, 990, 990]\n",
      "  Response Words: fizzy alternative soda soda\n",
      " Ground Truth: tastes like diet soda\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [474, 300, 375, 2, 885, 10616, 110, 288, 91, 1895, 588, 4958, 2192, 16447, 6167, 1176, 4013, 588, 679, 2584, 910, 1271, 1661, 1176, 4154, 91, 0, 2886, 588, 277, 2218, 6256, 4267, 88, 398, 6603, 777, 1895, 375, 3793, 1522, 1698, 2886, 3429, 6769, 2589, 409, 224, 1089, 26, 1712, 1725, 590, 13, 6603, 186, 374, 6231, 6190, 744, 400, 1125, 11371, 1368, 2226, 2878, 896, 793, 13, 1895, 2886, 1014, 36803]\n",
      "  Input Words: could never give dog anything cloth like soft would chew take stuffing bed blanket toys bought kong take even several treats stuffed inside bought hoping would good toy take right away eventually became favorite anytime bored need chew give busy ten minutes toy tough flexible needed easy clean run hot water couple seconds great bored dogs really aggressive chewers thing get pick dinosaur darn going affect rating still great chew toy overall\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 2, 345, 64]\n",
      "  Response Words: my dog loves these\n",
      " Ground Truth: best for my dog\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [54, 660, 433, 466, 373, 999, 221, 86, 3972, 2316, 474, 400, 54, 0, 36803]\n",
      "  Input Words: taste sour use drink mix sticks every day matter mixed could get taste good\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [4, 110, 17, 54]\n",
      "  Response Words: not like the taste\n",
      " Ground Truth: global brands grape mix sticks\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [6045, 110, 6045, 36664, 36665, 12215, 4155, 4562, 693, 2902, 13, 36665, 3440, 957, 4869, 1243, 2955, 68, 883, 474, 2621, 3035, 61, 1258, 1012, 433, 18136, 5970, 2621, 962, 68, 1136, 3561, 91, 1803, 36803]\n",
      "  Input Words: prepared like prepared buddhist monastery cleaned sliced thinly salted peppered great monastery funky stronger odor slightly moldy flavor though could see mold product frozen ones use thaw later see helps flavor worth returning would recommend\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [4, 110, 17, 6303]\n",
      "  Response Words: not like the burned\n",
      " Ground Truth: not as fresh as expected\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [407, 1047, 646, 1332, 54, 277, 3809, 102, 357, 476, 1332, 373, 805, 94, 0, 1780, 695, 8319, 805, 3870, 251, 1045, 1812, 106, 68, 737, 13, 1359, 1339, 479, 263, 0, 108, 36803]\n",
      "  Input Words: super picky flavored milk taste right longest time looking vanilla milk mix syrup tastes good surprised first opened syrup clear looks thin boy pack flavor punch great sorry ordered one bottle good stuff\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [94, 110, 1191, 466]\n",
      "  Response Words: tastes like cherry drink\n",
      " Ground Truth: finally great vanilla milk syrup\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [374, 1567, 68, 13, 512, 1945, 154, 1528, 425, 4116, 38, 293, 777, 2881, 4116, 425, 1064, 630, 151, 2342, 293, 1176, 3901, 18, 3852, 1450, 0, 1, 61, 36803]\n",
      "  Input Words: really enjoy flavor great ice refreshing much less sugar carbs regular juice need cut carbs sugar calories unique alternative orange juice bought sale expensive otherwise tell good quality product\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 151, 990, 990]\n",
      "  Response Words: great alternative soda soda\n",
      " Ground Truth: flavorful and clean taste\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1547, 36801, 486, 135, 391, 3147, 772, 1355, 71, 3096, 1547, 186, 479, 1953, 245, 1228, 479, 86, 36803]\n",
      "  Input Words: two <UNK> absolutely loved gave four stars wish price lower two dogs one package last long one day\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 186, 32, 64]\n",
      "  Response Words: my dogs love these\n",
      " Ground Truth: my dogs loved these\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [663, 1386, 373, 2584, 14369, 466, 777, 433, 1585, 5138, 2376, 466, 2965, 849, 1133, 1969, 585, 4945, 402, 684, 753, 9592, 4056, 295, 36803]\n",
      "  Input Words: box packets mix several glasses drink need use entire packet per drink prompt delivery packaged safely shipping damage disappointed yet service orders placed amazon\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 61, 53, 4755]\n",
      "  Response Words: great product poor packing\n",
      " Ground Truth: white grape drink mix\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [133, 374, 110, 914, 512, 661, 91, 7283, 799, 640, 661, 827, 307, 307, 316, 512, 661, 9950, 36803]\n",
      "  Input Words: kids really like making ice cream would suggest using heavy cream instead half half make ice cream richer\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 108, 100, 512]\n",
      "  Response Words: good stuff but ice\n",
      " Ground Truth: very good ice cream\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [60, 503, 716, 321, 3, 401, 5415, 418, 2546, 3290, 36803]\n",
      "  Input Words: delicious seasoning buying health food store alot money stopped selling\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [5218, 44, 5218, 36]\n",
      "  Response Words: tastier and tastier than\n",
      " Ground Truth: delicious for pasta and pizza\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1128, 539, 0, 157, 2686, 1748, 1749, 1228, 5177, 5053, 539, 1022, 157, 2686, 1748, 1749, 9543, 6630, 54, 374, 0, 157, 374, 0, 54, 1748, 1749, 17310, 2603, 5053, 374, 1450, 1288, 17439, 672, 1748, 1749, 54, 1890, 327, 101, 2389, 758, 509, 156, 324, 2617, 36803]\n",
      "  Input Words: tried find good chocolate covered sunflower seeds long young couldnt find easily chocolate covered sunflower seeds yankee traders taste really good chocolate really good taste sunflower seeds coworker said couldnt really tell guess becasue used sunflower seeds taste put little next probably eat whole bag days thank\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [25, 1749, 462, 1128, 405]\n",
      "  Response Words: best seeds have tried out\n",
      " Ground Truth: chocolate covered sunflower seeds yum\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [87, 1243, 358, 36801, 179, 110, 1708, 6393, 110, 3874, 179, 479, 1129, 277, 466, 445, 467, 661, 425, 995, 1205, 156, 1355, 811, 179, 3095, 13, 36803]\n",
      "  Input Words: wife slightly different <UNK> coffee like mine darker like milder coffee one however right drink daily without cream sugar second pound bag wish found coffee sooner great\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 179, 70, 17, 115]\n",
      "  Response Words: good coffee for the go\n",
      " Ground Truth: great mild medium flavored coffee\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [402, 331, 594, 486, 54, 1636, 374, 357, 5817, 331, 594, 1371, 479, 3352, 4175, 846, 54, 1650, 594, 626, 21, 36803]\n",
      "  Input Words: disappointed mini bags absolutely taste popcorn really looking forward mini bags solution one person understand bland taste larger bags quite tasty\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 100, 4, 17, 25]\n",
      "  Response Words: good but not the best\n",
      " Ground Truth: act ii butter mini bags\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1547, 186, 479, 640, 6191, 186, 32, 2886, 588, 5302, 6538, 9295, 781, 679, 640, 6191, 881, 12695, 452, 91, 6250, 781, 719, 1045, 567, 2513, 110, 5953, 3312, 9295, 1797, 781, 36803]\n",
      "  Input Words: two dogs one heavy chewer dogs love toy take turns stealing held well even heavy chewer gets ahold sure would hold well bit thin pieces stick like foot head held surprisingly well\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 186, 32, 9]\n",
      "  Response Words: my dogs love it\n",
      " Ground Truth: great for heavy chewer\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [2, 3167, 3797, 2020, 4576, 1702, 12955, 1472, 3, 5632, 156, 663, 2, 11567, 13637, 2287, 5038, 1783, 8319, 12240, 8317, 110, 400, 2, 910, 24711, 579, 5014, 474, 1006, 3041, 1056, 6539, 277, 6786, 1634, 2, 4991, 5430, 758, 5325, 23659, 374, 1774, 91, 831, 710, 827, 4210, 1782, 38, 3, 8138, 110, 530, 190, 5987, 2675, 5285, 6786, 3797, 11592, 1687, 651, 464, 4179, 2, 2669, 2295, 3, 1122, 127, 36803]\n",
      "  Input Words: dog grew purina products trust name eager try food removed bag box dog eagerly sniffing seemed excited contents opened began act like get dog treats hopping around begging could wait poured dish gobbled right unlike normal dog chow left eat hungry rationed really think would stop gone instead sprinkle top regular food dole like treat problems digesting chef michael unlike purina beneful original pounds tested ago dog point view food totally awesome\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 2, 345, 9]\n",
      "  Response Words: my dog loves it\n",
      " Ground Truth: my dog loves chef michael\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [3561, 61, 307, 1771, 1784, 36801, 11267, 1128, 479, 1784, 186, 515, 36803]\n",
      "  Input Words: returning product half cans dented <UNK> falling tried one dented dogs impressed\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [17, 1771, 73, 429]\n",
      "  Response Words: the cans is terrible\n",
      " Ground Truth: my dogs were not impressed\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [256, 61, 687, 273, 439, 0, 1855, 43, 3056, 508, 5415, 1735, 571, 6128, 1216, 4071, 108, 110, 253, 61, 1803, 546, 36803]\n",
      "  Input Words: excellent product touch salt packaging good keep fresh plan eating alot crackers near future dried fruits stuff like happy product recommend highly\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [907, 73, 17, 25]\n",
      "  Response Words: where is the best\n",
      " Ground Truth: back to nature crispy wheats cracker\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [8042, 1469, 944, 1693, 3, 811, 1804, 2020, 13, 1046, 92, 61, 436, 36803]\n",
      "  Input Words: citric acid seems basic food found many products great place buy product thanks\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 151, 84, 990]\n",
      "  Response Words: great alternative to soda\n",
      " Ground Truth: basic food great price\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [226, 140, 1580, 307, 8074, 1386, 1243, 1163, 262, 3256, 388, 567, 32, 363, 228, 2342, 8297, 68, 816, 92, 4996, 104, 1079, 2074, 2513, 716, 1005, 145, 36803]\n",
      "  Input Words: cookies stale least half foil packets slightly open broken million small pieces love brand cookie orange essence flavor available buy decided order online advice stick buying local source\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [17, 25, 418, 418]\n",
      "  Response Words: the best money money\n",
      " Ground Truth: old broken and stale\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [61, 94, 0, 1093, 0, 4191, 390, 9999, 6438, 454, 1141, 13876, 2584, 321, 322, 71, 277, 295, 2117, 849, 626, 489, 546, 547, 36803]\n",
      "  Input Words: product tastes good smells good stable high temp frying organic may possess several health benefits price right amazon shipment delivery quite fast highly recommended\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 100, 4, 24]\n",
      "  Response Words: good but not healthy\n",
      " Ground Truth: an excellent oil for cooking\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [133, 374, 1581, 8346, 1735, 3576, 409, 258, 2839, 15453, 156, 8788, 13444, 3793, 1722, 324, 3168, 433, 1378, 425, 509, 872, 133, 32, 11664, 54, 54, 626, 0, 9295, 781, 695, 102, 1472, 11762, 3759, 363, 359, 1006, 1472, 3363, 36803]\n",
      "  Input Words: kids really enjoyed animal crackers purchased easy snack throw zoo bag park backpack busy summer days appreciate use refined sugar whole wheat kids love shapes taste taste quite good held well first time try barbara bakery brand cannot wait try varieties\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 151, 84, 990]\n",
      "  Response Words: great alternative to soda\n",
      " Ground Truth: yummy without the bad stuff\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [25, 172, 165, 463, 700, 6046, 596, 3052, 1615, 1899, 3178, 1040, 3286, 68, 2715, 3052, 4724, 4902, 537, 1851, 2837, 481, 1385, 742, 68, 847, 172, 634, 3873, 10188, 388, 933, 1782, 2481, 172, 359, 2443, 5430, 1383, 1698, 1725, 3292, 300, 4267, 532, 2699, 59, 54, 123, 847, 172, 4902, 721, 300, 5159, 1385, 110, 4723, 1005, 4884, 4189, 827, 34392, 566, 2911, 26740, 110, 783, 784, 400, 172, 402, 36803]\n",
      "  Input Words: best tea ever tasted period canister double lid plus comes sealed plastic preserve flavor opening lid rush bergamot aroma hits nose wow also smell flavor black tea leaves indeed prized small buds top plant tea cannot brewed left longer minutes couple times never became bitter acidic always taste smooth black tea bergamot strong never powering also like supporting local grower workers instead syphoned large mega corporation like earl grey get tea disappointed\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [25, 172, 462, 662, 662]\n",
      "  Response Words: best tea have had had\n",
      " Ground Truth: the best earl grey\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [2291, 552, 59, 298, 2449, 13145, 179, 59, 463, 327, 997, 374, 1774, 1040, 15334, 179, 400, 154, 433, 2291, 811, 179, 8032, 2043, 36803]\n",
      "  Input Words: keurig cups always something missing reminded coffee always tasted little funny really think plastic surrounding coffee get much use keurig found coffee reordering today\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [10310, 4, 496, 399]\n",
      "  Response Words: hopefully not work cup\n",
      " Ground Truth: real coffee less plastic\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [59, 2058, 433, 295, 104, 1539, 1215, 1339, 3382, 11367, 59, 2058, 12829, 25075, 1140, 104, 4420, 2617, 36803]\n",
      "  Input Words: always pleased use amazon order various items ordered dental dinosaurs always pleased promptness quickness received order supplier thank\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 61, 13, 753]\n",
      "  Response Words: great product great service\n",
      " Ground Truth: another pleased amazon customer\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [13, 61, 489, 82, 585, 13, 71, 61, 2389, 1862, 1229, 3, 1585, 2080, 322, 5182, 5693, 970, 1511, 6281, 6866, 5072, 11029, 184, 6974, 400, 1027, 322, 4402, 1385, 181, 145, 34483, 36801, 546, 1145, 866, 867, 36803]\n",
      "  Input Words: great product fast free shipping great price product probably nutrient dense food entire planet benefits sub systems human body endless web search stabilized rice bran get nutritional benefits details also natural source gamma <UNK> highly potent anti oxidant\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 61, 13, 71]\n",
      "  Response Words: great product great price\n",
      " Ground Truth: bob red mill stabilized rice bran\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [776, 822, 489, 83, 154, 369, 409, 1540, 2240, 13, 4024, 36803]\n",
      "  Input Words: things grow fast way much real easy care produce great kefir\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 151, 84, 38]\n",
      "  Response Words: great alternative to regular\n",
      " Ground Truth: coming out of my ears\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1797, 450, 1579, 925, 905, 1039, 409, 1501, 1738, 556, 11186, 0, 54, 2973, 7880, 154, 4225, 7123, 985, 6439, 2581, 1481, 0, 128, 36803]\n",
      "  Input Words: surprisingly enough everyone house liked dinner easy fix especially since preparation good taste consistency reservation much broccoli included fine chop issue thought good deal\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 100, 4, 153, 0]\n",
      "  Response Words: good but not so good\n",
      " Ground Truth: quick and easy dinner fix\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [695, 102, 3576, 845, 1017, 6093, 695, 102, 1339, 4026, 68, 2517, 32, 1017, 1565, 2018, 68, 3478, 60, 2365, 367, 2018, 542, 538, 55, 1213, 2919, 35, 154, 1589, 1948, 3556, 36803]\n",
      "  Input Words: first time purchased kind bars although first time ordered particular flavor combination love bars wonderfully nutty flavor delightfully delicious reminds blend nutty granola bar sweet peanut brittle better much healthier definitely ordering\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [60, 44, 55, 258]\n",
      "  Response Words: delicious and sweet snack\n",
      " Ground Truth: wonderfully nutty and delightfully delicious\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1523, 1172, 230, 758, 885, 1088, 2, 3, 7123, 881, 354, 1128, 221, 845, 230, 3, 2383, 479, 758, 445, 400, 354, 1385, 20, 147, 3844, 102, 539, 470, 354, 1217, 3, 2595, 3825, 5827, 1774, 36801, 489, 2040, 4349, 154, 3, 3852, 229, 25, 3, 1385, 433, 3939, 3, 375, 388, 12812, 278, 1134, 530, 2, 400, 89, 1039, 13163, 1217, 3, 86, 2706, 1228, 375, 1472, 474, 739, 617, 391, 415, 36803]\n",
      "  Input Words: year old cat eat anything people dog food included gets sick tried every kind cat food market one eat daily get sick also wonderful preventing hairballs time find getting sick dry food hours pigs refill think <UNK> fast trying jam much food otherwise far best food also use wet food give small tablespoon size serving treat dog get breakfast dinner access dry food day night long give try could believe glad gave shot\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 230, 345, 9, 3]\n",
      "  Response Words: my cat loves it food\n",
      " Ground Truth: the best food out there\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [3031, 1125, 2621, 57, 4594, 11590, 339, 707, 771, 5126, 771, 2023, 3772, 934, 4154, 1088, 4855, 13, 212, 1707, 822, 1255, 5126, 1005, 831, 1431, 0, 5072, 92, 3138, 359, 539, 1677, 13, 1056, 5560, 841, 524, 212, 1774, 110, 841, 524, 212, 799, 6047, 19, 767, 841, 524, 212, 6278, 1255, 54, 524, 1062, 1580, 316, 253, 1490, 38, 446, 16887, 25292, 1561, 35450, 212, 277, 433, 316, 2469, 841, 8670, 36803]\n",
      "  Input Words: usually pick see home goods maxx hit miss sometimes grab sometimes goes months none hoping people catch great vinegar popularity grow able grab local stop shop good search buy internet cannot find condiment great dish calling red wine vinegar think like red wine vinegar using substandard brands know red wine vinegar shoud able taste wine watery least make happy cook regular basis basting cornish game hens vinegar right use make sausages red cabbage\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [17, 25, 2673, 165]\n",
      "  Response Words: the best taco ever\n",
      " Ground Truth: the best red wine vinegar\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [4400, 1317, 20, 507, 71, 1738, 110, 6615, 454, 1355, 756, 757, 36803]\n",
      "  Input Words: buttery rich wonderful oil price especially like fact organic wish expiration date\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [20, 61, 100, 18]\n",
      "  Response Words: wonderful product but expensive\n",
      " Ground Truth: good flavor good buy\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [3576, 1804, 358, 552, 245, 1523, 2543, 36801, 59, 357, 123, 13, 399, 179, 438, 532, 817, 7416, 925, 367, 123, 1317, 179, 2484, 36801, 353, 13, 128, 5233, 316, 452, 300, 1089, 32, 36803]\n",
      "  Input Words: purchased many different cups last year owned <UNK> always looking smooth great cup coffee weak bitter smokey tullys house blend smooth rich coffee morning <UNK> makes great deal purchasing make sure never run love\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [123, 550, 399, 179]\n",
      "  Response Words: smooth mountain cup coffee\n",
      " Ground Truth: one great cup of coffee\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [479, 18105, 545, 86, 31, 91, 687, 1780, 479, 4026, 3586, 14680, 359, 1731, 479, 1093, 15, 1630, 13341, 1279, 3454, 139, 18, 1057, 781, 24, 230, 5896, 5252, 1129, 2603, 1540, 2100, 4650, 5943, 3586, 105, 1906, 452, 3190, 3323, 3586, 659, 110, 12741, 452, 1803, 1472, 2621, 230, 1059, 688, 2039, 36803]\n",
      "  Input Words: one reject surprise day cats would touch surprised one particular turkey junky cannot figure one smells nice strange odors detect anyway pretty expensive toss well healthy cat pate chunks however said care chunky fancy feast turkey variety either sure gobble friskies turkey cheese like tomorrow sure recommend try see cat likes maybe boring\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 1206, 91, 4, 110, 9]\n",
      "  Response Words: my kitty would not like it\n",
      " Ground Truth: not their best flavor\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [2, 1260, 905, 1217, 3, 851, 300, 1059, 1217, 2, 3, 5767, 3492, 61, 825, 54, 277, 1948, 298, 649, 1674, 2, 445, 6047, 1899, 1027, 248, 2621, 1875, 1725, 3292, 3787, 1021, 452, 883, 1014, 110, 1047, 2, 1899, 1217, 2, 3, 375, 1472, 766, 1803, 1875, 445, 474, 15, 6313, 186, 38, 3, 36803]\n",
      "  Input Words: dog actually liked dry food surprising never likes dry dog food clearly makers product got taste right definitely something want feed dog daily substandard comes nutritional value see giving couple times week break sure though overall like picky dog comes dry dog food give try dont recommend giving daily could nice complement dogs regular food\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 2, 345, 9]\n",
      "  Response Words: my dog loves it\n",
      " Ground Truth: the dog likes it\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [374, 94, 110, 1497, 632, 466, 26, 174, 110, 26, 172, 1050, 1059, 174, 26, 1397, 1855, 716, 1228, 729, 32, 36803]\n",
      "  Input Words: really tastes like ginger honey drink hot cold like hot tea son likes cold hot inexpensive keep buying long sold love\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [25, 271, 466, 165]\n",
      "  Response Words: best tasting drink ever\n",
      " Ground Truth: this tastes so good\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [654, 1128, 256, 479, 91, 7283, 36803]\n",
      "  Input Words: flavors tried excellent one would suggest\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 54, 100, 4, 154]\n",
      "  Response Words: good taste but not much\n",
      " Ground Truth: not as good as the others\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1872, 0, 1182, 1796, 811, 12892, 1062, 20533, 706, 205, 1210, 1211, 295, 3184, 2630, 268, 11856, 2291, 552, 5466, 5467, 608, 83, 36803]\n",
      "  Input Words: shocked good reviews sludge found akin watery mud horrible item eligible return amazon unfortunately total rip splurge keurig cups lesson learned hard way\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [689, 689, 8141, 462, 165, 662, 7130]\n",
      "  Response Words: worst worst gopher have ever had monster\n",
      " Ground Truth: money down the drain\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [544, 746, 798, 608, 102, 5498, 1005, 1114, 3278, 1255, 104, 520, 295, 1472, 110, 721, 982, 88, 172, 36803]\n",
      "  Input Words: husband addicted tips hard time finding local stores thrilled able order line amazon try like strong flavorful favorite tea\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [25, 172, 165, 662, 539]\n",
      "  Response Words: best tea ever had find\n",
      " Ground Truth: the very best british tea\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1205, 6011, 59, 18582, 508, 1128, 16056, 2020, 3579, 1053, 89, 221, 86, 2201, 639, 6233, 3052, 36801, 590, 2, 1315, 3576, 58, 221, 1547, 3772, 2802, 1646, 13, 128, 205, 36803]\n",
      "  Input Words: pound poodle always resistant eating tried sunrise products cesar eats breakfast every day additional goodness remove lid <UNK> seconds dog crazy purchased delivered every two months subscribe save great deal item\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 2, 345, 9]\n",
      "  Response Words: my dog loves it\n",
      " Ground Truth: fantastic size for small dogs\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [21, 4444, 258, 3, 149, 346, 2132, 793, 80, 641, 91, 8116, 781, 659, 5628, 3969, 36803]\n",
      "  Input Words: tasty healty snack food low fat content still satisfying crisp would pair well cheese dips parties\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [21, 100, 4, 24]\n",
      "  Response Words: tasty but not healthy\n",
      " Ground Truth: tasty and healthy snack\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1339, 179, 1116, 556, 466, 1486, 486, 345, 1385, 1128, 374, 515, 68, 1565, 123, 1385, 1093, 244, 91, 546, 1803, 1486, 179, 2305, 71, 2308, 36803]\n",
      "  Input Words: ordered coffee mom since drink decaf absolutely loves also tried really impressed flavor wonderfully smooth also smells amazing would highly recommend decaf coffee anyone price reasonable\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 5356, 345, 9]\n",
      "  Response Words: my girlfriend loves it\n",
      " Ground Truth: great flavor and price\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [2, 345, 3, 2923, 1674, 16244, 91, 5023, 1172, 3, 6256, 400, 579, 508, 3, 486, 32, 1948, 716, 36803]\n",
      "  Input Words: dog loves food soon feed scarfs would sniff old food eventually get around eating food absolutely love definitely buying\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 2, 345, 33, 3]\n",
      "  Response Words: my dog loves this food\n",
      " Ground Truth: blue buffalo weight control\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1156, 1490, 3077, 3354, 0, 469, 316, 60, 3, 3948, 346, 781, 1136, 418, 1355, 585, 3498, 1288, 7818, 556, 1804, 2366, 1574, 384, 3948, 346, 36803]\n",
      "  Input Words: decent cook knows takes good ingredients make delicious food duck fat well worth money wish shipping costly guess suck since many places purchase quantity duck fat\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [21, 100, 4, 17, 25]\n",
      "  Response Words: tasty but not the best\n",
      " Ground Truth: the most delicious thanks giving ever\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1128, 179, 52, 21, 781, 32, 32, 32, 1463, 736, 36801, 4088, 179, 20, 1040, 156, 390, 1, 104, 13309, 2584, 594, 4330, 2366, 179, 1040, 1855, 43, 179, 358, 6223, 2291, 179, 1129, 3421, 2291, 9463, 11044, 179, 525, 1344, 42, 2291, 17941, 3503, 179, 1433, 2439, 72, 1433, 672, 2439, 42, 72, 5438, 63, 36801, 5477, 72, 672, 36803]\n",
      "  Input Words: tried coffee expect tasty well love love love company packs <UNK> individual coffee wonderful plastic bag high quality order consisted several bags manufacturer places coffee plastic keep fresh coffee different bottom keurig coffee however fits keurig platinum series coffee maker aware new keurig vue brewer coffee pods fit machine pods used fit new machine gimmick sell <UNK> replace machine used\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 179, 202, 71]\n",
      "  Response Words: good coffee bad price\n",
      " Ground Truth: san francisco bay excellent coffee\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 399, 179, 318, 181, 476, 68, 123, 532, 635, 229, 1107, 5579, 8444, 592, 476, 91, 5037, 772, 476, 68, 2078, 957, 1883, 1326, 476, 36803]\n",
      "  Input Words: good cup coffee light natural vanilla flavor smooth bitter aftertaste far superior gloria jean french vanilla would given stars vanilla flavor tad stronger truly hint vanilla\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [721, 399, 399, 399]\n",
      "  Response Words: strong cup cup cup\n",
      " Ground Truth: pleasant cup of coffee\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [2, 1059, 2675, 11587, 1289, 3, 1481, 91, 1472, 1217, 3, 781, 1890, 4700, 172, 12974, 273, 82, 553, 5351, 3038, 116, 135, 0, 2071, 3145, 579, 860, 8911, 1039, 36803]\n",
      "  Input Words: dog likes chef michaels canned food thought would try dry food well put bowl tea spoons salt free chicken broth went nuts loved good teeth dances around kitchen appear dinner\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 2, 345, 9]\n",
      "  Response Words: my dog loves it\n",
      " Ground Truth: my dog loves this stuff\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1440, 945, 316, 626, 21, 1899, 1216, 2248, 240, 503, 778, 1528, 1698, 8010, 1698, 1712, 9052, 1698, 1490, 529, 240, 480, 1440, 327, 240, 433, 307, 5138, 2881, 112, 7622, 751, 777, 298, 945, 409, 21, 316, 1285, 1948, 36803]\n",
      "  Input Words: noodles quick make quite tasty comes dried vegetables spicy seasoning made less minutes stovetop minutes water boil minutes cook fan spicy foods noodles little spicy use half packet cut back spiciness sodium need something quick easy tasty make meal definitely\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [21, 100, 4, 17, 25]\n",
      "  Response Words: tasty but not the best\n",
      " Ground Truth: quick and tasty best noodles\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [110, 1426, 659, 68, 672, 2173, 3586, 827, 1957, 374, 1581, 92, 36803]\n",
      "  Input Words: like lot cheese flavor used ground turkey instead beef really enjoyed buy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 54, 100, 4, 13]\n",
      "  Response Words: great taste but not great\n",
      " Ground Truth: if you like cheesy\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [9022, 13, 36801, 32, 149, 751, 976, 62, 1711, 10341, 2617, 36803]\n",
      "  Input Words: bouillon great <UNK> love low sodium version please add inventory thank\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 61, 100, 5630]\n",
      "  Response Words: great product but fatty\n",
      " Ground Truth: please carry the low sodium version\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [4400, 1317, 20, 507, 71, 1738, 110, 6615, 454, 1355, 756, 757, 36803]\n",
      "  Input Words: buttery rich wonderful oil price especially like fact organic wish expiration date\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [20, 61, 100, 18]\n",
      "  Response Words: wonderful product but expensive\n",
      " Ground Truth: good flavor good buy\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 60, 21103, 8071, 5972, 6483, 94, 3157, 13, 4175, 202, 1182, 29409, 982, 110, 659, 4151, 1088, 1603, 613, 1106, 1285, 646, 273, 1549, 0, 258, 3454, 2035, 0, 1629, 13, 2608, 659, 666, 957, 654, 91, 154, 2259, 35, 654, 854, 1711, 3363, 54, 35, 1930, 479, 1106, 12003, 12949, 1106, 12003, 1687, 91, 92, 654, 1629, 1, 479, 719, 846, 1014, 1660, 6612, 24, 318, 409, 1511, 36803]\n",
      "  Input Words: good delicious utilitarian fills hunger void tastes pleasant great understand bad reviews hammering flavorful like cheese curls people expecting baked corn meal flavored salt yeast good snack anyway reasonably good texture great intolerant cheese spice stronger flavors would much improved better flavors perhaps add varieties taste better call one corn tings unflavored corn tings original would buy flavors texture quality one bit bland overall job designed healthy light easy body\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 100, 4, 17, 25]\n",
      "  Response Words: good but not the best\n",
      " Ground Truth: decent taste healthy but very basic\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [9380, 1128, 479, 2706, 35, 1481, 1088, 2603, 463, 110, 11, 805, 1141, 4707, 1129, 309, 9007, 22643, 672, 2802, 1646, 6771, 825, 1122, 1136, 36803]\n",
      "  Input Words: buddies tried one night better thought people said tasted like cough syrup may everybody however help headaches partying used subscribe save promotion got totally worth\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 5356, 32, 9]\n",
      "  Response Words: my girlfriend love it\n",
      " Ground Truth: good for what it is\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [94, 5508, 29781, 11633, 1749, 316, 1120, 7336, 15213, 1355, 401, 793, 6721, 36803]\n",
      "  Input Words: tastes rye pumpernickel caraway seeds make salad event anticipate wish store still stocked\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 54, 100, 153, 110]\n",
      "  Response Words: great taste but so like\n",
      " Ground Truth: combination of breads winner\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [3474, 36801, 2109, 1551, 13, 625, 2795, 2702, 2906, 2959, 627, 659, 4349, 479, 277, 7765, 88, 1, 512, 661, 295, 3460, 13, 71, 806, 228, 68, 36803]\n",
      "  Input Words: jules <UNK> butter crisps great thick dalmatian style fig spread salty cheese jam one right scoop favorite quality ice cream amazon offers great price substitute cookie flavor\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 70, 17, 71]\n",
      "  Response Words: great for the price\n",
      " Ground Truth: natural buttery crisp taste\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [793, 2040, 1731, 4106, 19979, 374, 0, 1887, 8292, 1385, 2003, 0, 12881, 1458, 317, 4106, 538, 15, 530, 353, 4106, 1749, 1022, 35, 56, 538, 36803]\n",
      "  Input Words: still trying figure chia seeks really good joints supposedly also omega good regardless blueberry fruit chia bar nice treat makes chia seeds easily better candy bar\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [24, 70, 417, 1511, 345]\n",
      "  Response Words: healthy for your body loves\n",
      " Ground Truth: tasty snack easy omega\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1176, 2226, 1182, 1079, 1484, 298, 1995, 1288, 418, 479, 359, 1774, 541, 416, 418, 767, 608, 539, 776, 467, 2688, 1136, 36803]\n",
      "  Input Words: bought going reviews online boxes something inedible guess money one cannot think else waste money know hard find things without allergens worth\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [109, 416, 417, 418]\n",
      "  Response Words: don waste your money\n",
      " Ground Truth: good if you like cardboard\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [0, 793, 0, 4976, 925, 476, 110, 721, 399, 179, 36803]\n",
      "  Input Words: good still good maxwell house vanilla like strong cup coffee\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 399, 41, 892]\n",
      "  Response Words: good cup of joe\n",
      " Ground Truth: melitta french vanilla coffee\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1687, 4178, 1045, 1958, 59, 627, 1958, 1349, 627, 0, 108, 36803]\n",
      "  Input Words: original oberto thin jerky always salty jerky supposed salty good stuff\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [1161, 627, 44, 6507]\n",
      "  Response Words: sooo salty and primal\n",
      " Ground Truth: what else would you expect\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [3293, 586, 1, 172, 54, 1145, 774, 679, 883, 1890, 310, 466, 1698, 2192, 539, 470, 8852, 36803]\n",
      "  Input Words: calming soothing quality tea taste potent robust even though put sleep drink minutes bed find getting sleepy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 54, 44, 1945]\n",
      "  Response Words: great taste and refreshing\n",
      " Ground Truth: it works for me\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [61, 13, 387, 190, 189, 297, 885, 3840, 2839, 3860, 1875, 61, 2488, 253, 2885, 36801, 9852, 204, 36803]\n",
      "  Input Words: product great puppy problems stomach almost anything causes throw started giving product man happy pup <UNK> cloud star\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [10877, 4722, 460, 781]\n",
      "  Response Words: tuff dent works well\n",
      " Ground Truth: cloud star itty bitty buddy biscuits\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [767, 767, 767, 466, 1712, 2545, 2344, 1712, 1840, 2545, 181, 3158, 467, 425, 468, 1187, 318, 46, 68, 110, 8789, 56, 433, 327, 1528, 373, 547, 32, 1260, 601, 5817, 533, 340, 86, 353, 1712, 1073, 6464, 2362, 353, 221, 479, 814, 533, 36803]\n",
      "  Input Words: know know know drink water problem hate water solves problem natural sweeteners without sugar artificial colors light strawberry flavor like cotton candy use little less mix recommended love actually look forward drinking office day makes water lovely pale pink makes every one ask drinking\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [139, 55, 44, 196]\n",
      "  Response Words: pretty sweet and convenient\n",
      " Ground Truth: sugar free liquid cotton candy\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1128, 539, 3382, 530, 1674, 1523, 1172, 1902, 3986, 2032, 3772, 934, 547, 2884, 910, 5097, 1618, 2165, 2920, 695, 5023, 3565, 3921, 3645, 1842, 945, 1668, 3028, 452, 617, 486, 10165, 102, 1705, 1123, 1047, 1624, 13326, 455, 1574, 36803]\n",
      "  Input Words: tried find dental treat feed year old yellow lab past months none recommended chews treats trick plain refused upon first sniff took chance greenies based quick dirty research sure glad absolutely devours time flat somewhat picky eater proved fantastic purchase\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 2, 345, 9]\n",
      "  Response Words: my dog loves it\n",
      " Ground Truth: my yellow lab loves them\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [12467, 1024, 4154, 202, 1182, 34625, 585, 54, 13, 59, 479, 1484, 731, 585, 5201, 9233, 4874, 12738, 4882, 400, 11226, 14805, 1503, 778, 16401, 895, 9529, 1024, 104, 775, 278, 1355, 91, 11889, 585, 4755, 2545, 11497, 2194, 19929, 914, 21708, 14579, 4755, 585, 812, 1727, 35, 3056, 3666, 602, 36803]\n",
      "  Input Words: signed subscription hoping bad reviews flukes shipping taste great always one boxes damaged shipping leaked laundry room contacted retailer get replaced mopped mess made mental note cancel subscription order smaller size wish would address shipping packing problem mentioned site whomever making executive decisions packing shipping needs come better plan customers elsewhere\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 61, 44, 753]\n",
      "  Response Words: great product and service\n",
      " Ground Truth: tastes great but packaging needs improvement\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1481, 632, 154, 10521, 632, 5396, 154, 68, 679, 133, 1540, 0, 71, 883, 110, 2264, 632, 111, 36803]\n",
      "  Input Words: thought honey much thinner honey gotten much flavor even kids care good price though like bottles honey came\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [4, 202, 100, 676]\n",
      "  Response Words: not bad but prefer\n",
      " Ground Truth: would not buy again\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1339, 626, 374, 21, 719, 18, 853, 400, 155, 980, 1958, 2133, 0, 57, 778, 1958, 672, 400, 1417, 4270, 755, 1385, 1463, 13660, 3518, 479, 9592, 827, 470, 594, 825, 479, 10164, 1463, 4359, 829, 42, 594, 277, 2218, 1774, 1687, 88, 68, 327, 1327, 825, 2865, 68, 781, 327, 55, 654, 327, 240, 36803]\n",
      "  Input Words: ordered quite really tasty bit expensive side get pay cheap jerky gross good home made jerky used get aunt certainly close also company responsive helpful one orders instead getting bags got one emailed company sent three new bags right away think original favorite flavor little sweetness got teriyaki flavor well little sweet flavors little spicy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 100, 4, 360, 153]\n",
      "  Response Words: good but not be so\n",
      " Ground Truth: little pricey but delicious\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [71, 0, 1048, 2109, 1612, 226, 729, 1382, 626, 519, 7834, 357, 2109, 228, 54, 54, 110, 2109, 226, 54, 297, 518, 110, 34612, 900, 202, 4415, 1010, 1953, 1103, 479, 10031, 4367, 566, 226, 278, 3122, 1192, 1629, 15, 3355, 25, 2065, 36803]\n",
      "  Input Words: price good compared butter type cookies sold live quite expected primarily looking butter cookie taste taste like butter cookies taste almost exactly like nila wafers bad course wanted package contains one row fairly large cookies size palm say texture nice crumbly best part\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 100, 4, 17, 25]\n",
      "  Response Words: good but not the best\n",
      " Ground Truth: meh these are just ok\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [71, 179, 15570, 746, 363, 179, 106, 1855, 179, 13, 71, 36803]\n",
      "  Input Words: price coffee rising addicted brand coffee pack keep coffee great price\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [179, 2326, 2326, 2326]\n",
      "  Response Words: coffee pricing pricing pricing\n",
      " Ground Truth: wake up to this deal\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [61, 441, 4367, 849, 489, 2033, 165, 561, 975, 5770, 518, 110, 358, 4018, 15436, 1774, 778, 55, 2653, 558, 2109, 2653, 373, 5770, 43, 1289, 1106, 327, 622, 10445, 1538, 198, 1106, 5770, 1722, 853, 1056, 60, 36803]\n",
      "  Input Words: product priced fairly delivery fast affordable ever heinz india relish exactly like different pickle relishes think made sweet pickles bread butter pickles mix relish fresh canned corn little onion bell pepper perfect corn relish summer side dish delicious\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [907, 73, 17, 25]\n",
      "  Response Words: where is the best\n",
      " Ground Truth: great base for corn relish\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1629, 1663, 15, 3184, 54, 474, 472, 54, 1663, 721, 54, 507, 9672, 1663, 1061, 567, 3130, 2320, 36803]\n",
      "  Input Words: texture pineapple nice unfortunately taste could barely taste pineapple strong taste oil overpowered pineapple ate pieces threw rest\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [4, 202, 100, 4, 202]\n",
      "  Response Words: not bad but not bad\n",
      " Ground Truth: will not buy again\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1679, 4026, 205, 1385, 8054, 0, 6203, 2584, 3292, 1523, 1654, 1782, 36801, 5786, 1039, 126, 4894, 7263, 1129, 8054, 101, 3787, 126, 0, 36803]\n",
      "  Input Words: disapointed particular item also tenderloin good entertain several times year hopes top <UNK> suitable dinner guests sadly mistaken however tenderloin next week guests good\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [17, 25, 28, 17, 2320]\n",
      "  Response Words: the best in the rest\n",
      " Ground Truth: very chewy and hard to eat\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [4684, 218, 375, 2, 530, 3455, 3318, 1547, 1348, 646, 3401, 110, 588, 7326, 263, 566, 6085, 3461, 4647, 11115, 6085, 2889, 2, 13807, 9527, 1101, 4647, 11115, 807, 34673, 61, 2545, 6085, 12098, 3855, 2, 359, 316, 6085, 2346, 696, 4647, 4032, 400, 1744, 1146, 3401, 574, 2664, 979, 896, 61, 218, 0, 807, 496, 781, 36803]\n",
      "  Input Words: clever idea give dog treat offering lick two bacon flavored liquid like take cap bottle large ball similar roll deodorant ball offer dog lap appears actual roll deodorant container repurposed product problem ball rotate freely dog cannot make ball turn tongue roll thumb get wash hands liquid brown sticky mediocre rating product idea good container work well\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 2, 501, 9]\n",
      "  Response Words: my dog does it\n",
      " Ground Truth: dispenser does not work well\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [32, 798, 1079, 1114, 9871, 4359, 1484, 479, 1763, 2218, 756, 757, 4359, 112, 5241, 7762, 375, 986, 7762, 1871, 663, 8319, 1472, 172, 1871, 4353, 20843, 736, 1211, 4353, 10742, 8498, 3663, 986, 7762, 579, 13, 7381, 1328, 91, 92, 36803]\n",
      "  Input Words: love tips online stores inc sent boxes one month away expiration date sent back asked refund give full refund charged box opened try tea charged postage reimburse packs return postage respond wrote receiving full refund around great merchant experience would buy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [295, 73, 17, 71]\n",
      "  Response Words: amazon is the price\n",
      " Ground Truth: good tea not good company\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1433, 496, 13, 3031, 466, 36801, 596, 847, 4016, 4503, 2798, 701, 1928, 1071, 124, 4470, 438, 179, 1511, 68, 552, 1948, 1136, 2040, 402, 111, 13, 36803]\n",
      "  Input Words: pods work great usually drink <UNK> double black diamond recently newmans extra bold dark brew concerned weak coffee body flavor cups definitely worth trying disappointed came great\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 721, 399, 41, 892]\n",
      "  Response Words: good strong cup of joe\n",
      " Ground Truth: just as good for less\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [452, 1804, 1088, 4175, 695, 54, 512, 3425, 1712, 3462, 3065, 20879, 533, 42, 740, 8700, 1712, 2624, 2484, 964, 8700, 556, 2188, 10070, 4179, 277, 29, 36803]\n",
      "  Input Words: sure many people understand first taste ice sparkling water flashback childhood swore drinking new york seltzer water ive morning loss seltzer since discontinued decade ago right world\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [9, 73, 1191, 1191, 1191]\n",
      "  Response Words: it is cherry cherry cherry\n",
      " Ground Truth: greatest tasting water since\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [825, 902, 106, 695, 1128, 539, 479, 905, 25, 374, 110, 1629, 1958, 110, 369, 744, 1385, 110, 6507, 1958, 7179, 479, 358, 6507, 5207, 36687, 514, 758, 1089, 1953, 608, 1163, 1958, 409, 2839, 11219, 2740, 1580, 825, 0, 68, 374, 1629, 36803]\n",
      "  Input Words: got sampler pack first tried find one liked best really like texture jerky like real thing also like primal jerky tend one different primal softer oozy difficult eat run package hard open jerky easy throw ziploc messy least got good flavor really texture\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [17, 25, 1958, 462, 1128]\n",
      "  Response Words: the best jerky have tried\n",
      " Ground Truth: best of the vegan dream jerky\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [357, 61, 11064, 12114, 253, 539, 295, 1114, 3036, 71, 436, 914, 816, 36803]\n",
      "  Input Words: looking product relatives overseas happy find amazon stores retail price thanks making available\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [295, 73, 17, 71]\n",
      "  Response Words: amazon is the price\n",
      " Ground Truth: best values on amazon stores\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [13, 128, 11324, 186, 32, 245, 1228, 102, 11779, 1082, 6, 36803]\n",
      "  Input Words: great deal antlers dogs love last long time arrive quickly advertised\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 186, 32, 341]\n",
      "  Response Words: my dogs love em\n",
      " Ground Truth: great deal on natural antlers\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1523, 1172, 1052, 4573, 81, 2608, 4659, 539, 1634, 3, 1330, 1948, 3421, 3283, 94, 13, 110, 20617, 1330, 253, 5778, 2174, 1330, 1291, 36803]\n",
      "  Input Words: year old daughter diagnosed gluten intolerant quest find normal food pasta definitely fits bill tastes great like semolina pasta happy sister told pasta choice\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [13, 81, 82, 1330]\n",
      "  Response Words: great gluten free pasta\n",
      " Ground Truth: the best gf pasta\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [186, 32, 11103, 11104, 3565, 479, 719, 1383, 1731, 8294, 6085, 2769, 32, 1738, 110, 3899, 23933, 530, 102, 5892, 433, 433, 1875, 12, 460, 13, 3119, 251, 5817, 3663, 21, 8334, 5658, 12, 13, 11107, 61, 0, 71, 36803]\n",
      "  Input Words: dogs love lickety stik took one bit longer figure roller ball done love especially like personal interaction treat time main use use giving medicine works great girl looks forward receiving tasty licks taking medicine great innovative product good price\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 88, 345, 9]\n",
      "  Response Words: my favorite loves it\n",
      " Ground Truth: both dogs love it\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [758, 2426, 4367, 3854, 32, 1366, 5607, 172, 1491, 5980, 298, 110, 811, 1240, 3571, 1530, 7054, 172, 36803]\n",
      "  Input Words: eat changs fairly often love jasmine blackberry tea serve looked something like found china mist delighted exact tea\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 88, 172, 594]\n",
      "  Response Words: my favorite tea bags\n",
      " Ground Truth: exactly pf changs tea\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [695, 338, 529, 74, 1429, 2603, 771, 305, 298, 945, 409, 205, 1158, 316, 327, 35, 2916, 2201, 2964, 374, 962, 4225, 36002, 110, 1212, 1216, 1990, 574, 2762, 43, 2964, 91, 2434, 5138, 4225, 36002, 713, 36803]\n",
      "  Input Words: first huge fan instant meals said sometimes mood something quick easy item okay make little better adding additional veggies really helps broccoli florets like tiny dried bits brown paper fresh veggies would leave packet broccoli florets completely\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 108, 100, 153, 154]\n",
      "  Response Words: good stuff but so much\n",
      " Ground Truth: ok but certainly not gourmet\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [2, 345, 2, 3, 3395, 543, 4319, 3693, 3, 28890, 1044, 1307, 345, 36803]\n",
      "  Input Words: dog loves dog food switch another joint supplement food unavailability issues lucy loves\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [2, 345, 345, 3]\n",
      "  Response Words: dog loves loves food\n",
      " Ground Truth: happy hips makes my dog happy\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [1574, 2802, 1646, 1484, 1763, 110, 327, 3489, 827, 1433, 71, 13, 544, 3129, 179, 957, 2819, 592, 593, 1129, 374, 110, 774, 68, 71, 3622, 327, 661, 3080, 156, 91, 375, 3147, 772, 92, 566, 4868, 3032, 36803]\n",
      "  Input Words: purchase subscribe save boxes month like little filters instead pods price great husband thinks coffee stronger previous french roast however really like robust flavor price adds little cream resealable bag would give four stars buy large airtight jar\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [0, 179, 202, 71]\n",
      "  Response Words: good coffee bad price\n",
      " Ground Truth: robust taste packaging could be better\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [20, 3811, 13769, 553, 555, 2, 667, 2298, 591, 7188, 515, 248, 470, 6292, 9381, 4459, 345, 61, 3811, 695, 825, 15175, 112, 321, 24, 782, 5212, 2065, 191, 142, 21570, 2767, 186, 321, 7761, 511, 2071, 1116, 36803]\n",
      "  Input Words: wonderful rescue thrived chicken soup dog lover soul read nutrients impressed value getting gsd glossy coat loves product rescue first got nurse back health healthy balanced joy part life nutrition vitally important dogs health learn brush teeth mom\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [25, 530, 70, 17, 2383]\n",
      "  Response Words: best treat for the market\n",
      " Ground Truth: nutritional value for your dog\n",
      "        \n",
      "        \n",
      "INFO:tensorflow:Restoring parameters from /content/gdrive/My Drive/best_so_far_model.ckpt\n",
      "        \n",
      "        \n",
      "\n",
      "Text\n",
      "  Word Ids:    [479, 3162, 5514, 2248, 1047, 2249, 345, 5139, 375, 1015, 479, 4255, 102, 452, 375, 1528, 55, 4255, 695, 1141, 18105, 995, 479, 1855, 1759, 156, 59, 24, 258, 1856, 327, 599, 36803]\n",
      "  Input Words: one ways sneak vegetables picky toddler loves pouches give child one pouch time sure give less sweet pouch first may reject second one keep diaper bag always healthy snack hand little guy\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [30, 133, 32, 9]\n",
      "  Response Words: my kids love it\n",
      " Ground Truth: sneaks veggies into toddlers\n",
      "        \n",
      "        \n",
      "\n",
      "...prepare evaluation finished.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#random = np.random.randint(0,3000)\n",
    "#text = news_texts_filtered[random]\n",
    "model_predictions = []\n",
    "ground_truth = []\n",
    "invalid_number_prediction_count = 0\n",
    "checkpoint = logs_path + 'best_so_far_model.ckpt' \n",
    "print(\"\\n----Prepare evaluation\")\n",
    "for ii in range(100):\n",
    "    random = np.random.randint(21700,27125)\n",
    "    text = news_texts_filtered[random]\n",
    "\n",
    "    #random = np.random.randint(3000,len(news_texts_filtered))\n",
    "        \n",
    "    loaded_graph = tf.Graph()\n",
    "    with tf.Session(graph=loaded_graph) as sess:\n",
    "                  loader = tf.train.import_meta_graph(checkpoint + '.meta')\n",
    "                  loader.restore(sess, checkpoint)\n",
    "                  input_data = loaded_graph.get_tensor_by_name('input_data:0')\n",
    "                  logits = loaded_graph.get_tensor_by_name('predictions:0')\n",
    "                  text_length = loaded_graph.get_tensor_by_name('text_len:0')\n",
    "                  summary_length = loaded_graph.get_tensor_by_name('summary_len:0')\n",
    "                  keep_prob = loaded_graph.get_tensor_by_name('dropout_probs:0')\n",
    "                  result_logits = sess.run(logits, {input_data: [text]*batch_size, \n",
    "                  summary_length: [np.random.randint(5,8)], \n",
    "                  text_length: [len(text)]*batch_size,\n",
    "                  keep_prob: 1.0})[0] \n",
    "\n",
    "                  pad = word2int[TOKEN_PAD] \n",
    "                  eos = word2int[TOKEN_EOS] \n",
    "                  Y=''.join(int2word[i] for i in result_logits if i !=pad)\n",
    "                  O=''.join(int2word[i] for i in news_summaries_filtered[random] if i!=pad)\n",
    "                     \n",
    "                  if mean_absolute_error_texts([Y],[O]) > 90:\n",
    "                  \n",
    "                     print('        ')\n",
    "                     print('        ')\n",
    "                     #print('\\nOriginal Text:', input_sentence)\n",
    "\n",
    "                     print('\\nText')\n",
    "                     print('  Word Ids:    {}'.format([i for i in text]))\n",
    "                     print('  Input Words: {}'.format(\" \".join( int2word[i] for i in text if i!=eos)))\n",
    "\n",
    "                     print('\\nSummary')\n",
    "                     print('  Word Ids:       {}'.format([i for i in result_logits if i != pad]))\n",
    "                     print('  Response Words: {}'.format(\" \".join(int2word[i] for i in result_logits if i!=pad )))\n",
    "                     print(' Ground Truth: {}'.format(\" \".join(int2word[i] for i in news_summaries_filtered[random] if i!=pad)))\n",
    "                     print('        ')\n",
    "                     print('        ')\n",
    " \n",
    "model_predictions.append(result_logits)\n",
    "ground_truth.append(news_summaries_filtered[random])         \n",
    "print('\\n...prepare evaluation finished.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3261056,
     "status": "ok",
     "timestamp": 1589544533755,
     "user": {
      "displayName": "Ruperto Pedro Bonet Chaple",
      "photoUrl": "",
      "userId": "16360856078137771581"
     },
     "user_tz": -60
    },
    "id": "4UN0LwG3r7FP",
    "outputId": "bf5c659c-d3e1-47ee-c108-3e42b1068f64"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finish \n"
     ]
    }
   ],
   "source": [
    "print(\"Finish \")"
   ]
  }
 ],
 "metadata": {
  "_change_revision": 0,
  "_is_fork": false,
  "colab": {
   "collapsed_sections": [],
   "name": "02_example-amazon-reviews-seq2seq-colab-with-attetion.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}