NMT_Project.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NMT-Project.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Wangsherpa/english_to_nepali_translation/blob/main/NMT_Project.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kd6Q4EyYxZAf"
      },
      "source": [
        "# Neural Machine Translation System\n",
        "## English to Nepali"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fZUMnxFcZMuN"
      },
      "source": [
        "## Import Necessary Packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "47tHQTQnkeQj"
      },
      "source": [
        "import os\n",
        "import time\n",
        "import pickle\n",
        "import unicodedata\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import string\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.layers import LSTM, Dense, Embedding, Input, GRU\n",
        "from tensorflow.keras.layers import TimeDistributed, RepeatVector\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n",
        "from keras.utils.vis_utils import plot_model\n",
        "\n",
        "from nltk.translate.bleu_score import corpus_bleu, sentence_bleu\n",
        "from nltk.translate.bleu_score import SmoothingFunction\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from IPython.display import clear_output"
      ],
      "execution_count": 91,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fTaKPpAfx13y",
        "outputId": "0fa752f3-adea-454f-8a90-21370111f1e6"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LKXB3VNlypdP"
      },
      "source": [
        "### Define paths"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0JVq_3LN8FdB"
      },
      "source": [
        "# model checkpoint paths\n",
        "checkpoint_1 = \"/content/drive/MyDrive/english-to-nepali-translation/Checkpoints/model_1\"\n",
        "checkpoint_2 = \"/content/drive/MyDrive/english-to-nepali-translation/Checkpoints/model_2\"\n",
        "checkpoint_3 = \"/content/drive/MyDrive/english-to-nepali-translation/Checkpoints/model_3\"\n",
        "checkpoint_4 = \"/content/drive/MyDrive/english-to-nepali-translation/Checkpoints/model_4\"\n",
        "\n",
        "# paths to save predictions\n",
        "prediction_dir = \"/content/drive/MyDrive/english-to-nepali-translation/predictions\""
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cu3pK-rck-xV"
      },
      "source": [
        "### Load the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WQBvKuTyo_Ks",
        "outputId": "a542f549-2296-49e9-afdc-fed4c8155fb5"
      },
      "source": [
        "# Download the data\n",
        "!wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1pHhyR00L4Pi_ezg-p_jzVjhEMR9AoGlN' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\\1\\n/p')&id=1pHhyR00L4Pi_ezg-p_jzVjhEMR9AoGlN\" -O eng_nepali.csv && rm -rf /tmp/cookies.txt\n",
        "clear_output()\n",
        "print(\"Dataset downloaded successfully!\")"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Dataset downloaded successfully!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "LlEWLiVAlAbi",
        "outputId": "afa8db74-4e06-4b00-9c76-a069d2f539ba"
      },
      "source": [
        "# set datapath\n",
        "datapath = \"/content/eng_nepali.csv\"\n",
        "# load the data\n",
        "dataset = pd.read_csv(datapath)\n",
        "dataset.head()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>English</th>\n",
              "      <th>Nepali</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>go</td>\n",
              "      <td>जानु</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>hi</td>\n",
              "      <td>हाई</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>run</td>\n",
              "      <td>दगुर्ने काम</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>fire</td>\n",
              "      <td>आगो</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>help</td>\n",
              "      <td>सहायता</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  English       Nepali\n",
              "0      go         जानु\n",
              "1      hi          हाई\n",
              "2     run  दगुर्ने काम\n",
              "3    fire          आगो\n",
              "4    help       सहायता"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4hsNGQ8oAVO1",
        "outputId": "2e824b72-f73e-48d6-8acc-8ecaa93e4180"
      },
      "source": [
        "# check the shape of the dataframe\n",
        "dataset.shape"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(9886, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "77CEmVeXEJHe"
      },
      "source": [
        "**Preprocessing functions used for cleaning the dataset**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_gQmeYEwEGQl"
      },
      "source": [
        "# create a dictionary for subwords\n",
        "dic = {\"won't\":\"will not\", \"ain't\":\"am not\", \"t\": \"not\", \"re\": \"are\", \"s\": \"is\", \"can't\": \"can not\", \n",
        "       \"ll\": \"will\", \"m\": \"am\", \"ve\": \"have\", \"let's\": \"let us\", \"d\": \"had\"}\n",
        "\n",
        "# function to create subwords\n",
        "def get_subwords(string):\n",
        "    if string in dic.keys():\n",
        "        return dic[string]\n",
        "    \n",
        "    string = string.split(\"'\")\n",
        "    if string[-1] in dic.keys():\n",
        "        last = dic[string[1]]\n",
        "    else:\n",
        "        last = string[-1]\n",
        "    if string[-1] == 't':\n",
        "        first = string[0][:-1]\n",
        "    else:\n",
        "        first = string[0]\n",
        "    return ' '.join([first, last])\n",
        "\n",
        "def add_subwords(sentence):\n",
        "    subword_map = {}\n",
        "    for word in sentence.split():\n",
        "        if \"'\" in word:\n",
        "            subwords = get_subwords(word)\n",
        "            subword_map[word] = subwords\n",
        "    for word, subword in subword_map.items():\n",
        "        sentence = sentence.replace(word, subword)\n",
        "    return sentence\n",
        "\n",
        "# clean english sentences\n",
        "def preprocess_english(sent):\n",
        "    # get list of punctuations but not \"'\"\n",
        "    punctuations = string.punctuation.replace(\"'\", \"\")\n",
        "    digits = string.digits\n",
        "    \n",
        "    remove_digits = str.maketrans('', '', digits)\n",
        "    # convert to lower case\n",
        "    sent = sent.lower()\n",
        "    sent = \"\".join([c for c in sent if c not in punctuations])\n",
        "    # remove numbers\n",
        "    sent = sent.translate(remove_digits)\n",
        "    # remove extra spaces \n",
        "    sent = sent.strip()\n",
        "    # replace words like don't, we've with do not, we have...\n",
        "    sent = add_subwords(sent)\n",
        "    return sent"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LDn4nzo3FW5t"
      },
      "source": [
        "## Prepare data for English to Nepali Translation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9jI9bh6t0UL_"
      },
      "source": [
        "# add start and end tokens\n",
        "dataset['English'] = 'start_ ' + dataset.English + ' end_'\n",
        "dataset['Nepali'] = 'start_ ' + dataset.Nepali + ' end_'"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oHh24MoZljEw"
      },
      "source": [
        "## Tokenize the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_OEzFT9mEY3T"
      },
      "source": [
        "# create a function to build tokenizer\n",
        "def tokenize(sentences, vocab_size=None):\n",
        "  # create a tokenizer\n",
        "  tokenizer = Tokenizer(filters='', num_words=vocab_size, oov_token='UNK')\n",
        "  # fit the tokenizer on sentences\n",
        "  tokenizer.fit_on_texts(sentences)\n",
        "  if vocab_size is None:\n",
        "    # calculate vocabulary size\n",
        "    vocab_size = len(tokenizer.word_index)\n",
        "\n",
        "  # calculate max sentence length in a sentence\n",
        "  max_sent_len = max([len(sentence.split()) for sentence in sentences])\n",
        "  return tokenizer, vocab_size, max_sent_len"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fnbxhUrcHrYY",
        "outputId": "9b0514de-8348-460d-fc15-f7b96b800e64"
      },
      "source": [
        "# english sentence tokenizer\n",
        "english_tokenizer, english_vocabsize, english_maxlen = tokenize(dataset.English, vocab_size=None)\n",
        "\n",
        "# nepali sentence tokenizer\n",
        "nepali_tokenizer, nepali_vocabsize, nepali_maxlen = tokenize(dataset.Nepali, vocab_size=None)\n",
        "\n",
        "print(\"English Vocab Size:\", english_vocabsize)\n",
        "print(\"Nepali Vocab Size:\", nepali_vocabsize)\n",
        "print(\"Max English Sentence Length:\", english_maxlen)\n",
        "print(\"Max Nepali Sentence Length:\", nepali_maxlen)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "English Vocab Size: 2825\n",
            "Nepali Vocab Size: 4353\n",
            "Max English Sentence Length: 8\n",
            "Max Nepali Sentence Length: 11\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KlOvhB5omVfT"
      },
      "source": [
        "# Split the dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bgbFe1hymQKQ",
        "outputId": "9a18befb-af5d-4366-b4c7-64452c629909"
      },
      "source": [
        "# Split the data into train test and validation\n",
        "X_train, X_valid, y_train, y_valid = train_test_split(dataset.English, dataset.Nepali, test_size=0.2,\n",
        "                                                    random_state=42)\n",
        "\n",
        "X_valid, X_test, y_valid, y_test = train_test_split(X_valid, y_valid, test_size=0.5, random_state=40)\n",
        "\n",
        "X_train.shape, X_valid.shape, X_test.shape"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((7908,), (989,), (989,))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WZqzm4qXmdFk"
      },
      "source": [
        "## Convert to TensorFlow Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n2YgIDatI1fe"
      },
      "source": [
        "# write a function to create dataset\n",
        "def create_dataset(source, target, src_maxlen=english_maxlen,\n",
        "                   tar_maxlen=nepali_maxlen,\n",
        "                   src_tokenizer=english_tokenizer,\n",
        "                   tar_tokenizer=nepali_tokenizer, teacher_forcing=False):\n",
        "  # tokenize the sentences\n",
        "  source = src_tokenizer.texts_to_sequences(source)\n",
        "  target = tar_tokenizer.texts_to_sequences(target)\n",
        "  \n",
        "  if not teacher_forcing:\n",
        "    # pad sequences to max length\n",
        "    source_padded = pad_sequences(source, maxlen=src_maxlen, padding='post')\n",
        "    target_padded = pad_sequences(target, maxlen=tar_maxlen, padding='post')\n",
        "\n",
        "    # create tensorflow dataset\n",
        "    dataset = tf.data.Dataset.from_tensor_slices((source_padded, target_padded))\n",
        "    dataset = dataset.batch(32, drop_remainder=True)\n",
        "    dataset = dataset.prefetch(1)\n",
        "    return dataset\n",
        "\n",
        "  # if teacher_forcing is True\n",
        "  target_in = [sentence[:-1] for sentence in target] # end token excluded\n",
        "  target_out = [sentence[1:] for sentence in target] # start token excluded\n",
        "\n",
        "  # pad sequences to max length\n",
        "  source_padded = pad_sequences(source, maxlen=src_maxlen, padding='post')\n",
        "  target_in_padded = pad_sequences(target_in, maxlen=tar_maxlen, padding='post')\n",
        "  target_out_padded = pad_sequences(target_out, maxlen=tar_maxlen, padding='post')\n",
        "\n",
        "  # create a tensorflow datasest\n",
        "  dataset = tf.data.Dataset.from_tensor_slices(((source_padded, target_in_padded), target_out_padded))\n",
        "  # batch and prefetch dataset\n",
        "  dataset = dataset.batch(32, drop_remainder=True)\n",
        "  dataset = dataset.prefetch(1)\n",
        "  return dataset"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KGad1rgJmgtm"
      },
      "source": [
        "# create tensorflow datasets for first model\n",
        "train_dataset = create_dataset(X_train, y_train)\n",
        "valid_dataset = create_dataset(X_valid, y_valid)\n",
        "test_dataset = create_dataset(X_test, y_test)"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UPt6HHa8MfYS",
        "outputId": "79e03e9a-ce9a-44c5-9ae5-6e1afa5e40f9"
      },
      "source": [
        "# look at the data in training set\n",
        "for row in train_dataset:\n",
        "  print(english_tokenizer.sequences_to_texts([row[0][0].numpy()]))\n",
        "  print(nepali_tokenizer.sequences_to_texts([row[1][0].numpy()]))\n",
        "  break"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['start_ i like puzzles end_ UNK UNK UNK']\n",
            "['start_ मलाई पजलहरू मन पर्छ end_ UNK UNK UNK UNK UNK']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KGEqEeEQNNmz"
      },
      "source": [
        "## Build the model (Simple Model)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TtSaZUU0NMlk"
      },
      "source": [
        "def build_model(src_vocab, tar_vocab, src_maxlen, tar_maxlen, n_units):\n",
        "  # initialize sequential model\n",
        "  src_input = Input(shape=[None])\n",
        "\n",
        "  # embedding layer\n",
        "  src_embedding = Embedding(src_vocab+1, n_units, input_length=src_maxlen, mask_zero=True)(src_input)\n",
        "\n",
        "  # source LSTM layer\n",
        "  src_lstm_out = LSTM(n_units)(src_embedding)\n",
        "  # repeat vector\n",
        "  rv = RepeatVector(tar_maxlen)(src_lstm_out)\n",
        "  # target LSTM layer\n",
        "  tar_lstm_out = LSTM(n_units, return_sequences=True)(rv)\n",
        "\n",
        "  # output layer\n",
        "  output = TimeDistributed(Dense(tar_vocab+1, activation='softmax'))(tar_lstm_out)\n",
        "\n",
        "  model = Model(inputs=[(src_input)], outputs=[output])\n",
        "\n",
        "  # compile the model\n",
        "  model.compile(loss='sparse_categorical_crossentropy', optimizer='adam')\n",
        "  return model"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l8vKzw9_NMZ9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "934df4ac-a473-46fe-a115-9781ba7f3397"
      },
      "source": [
        "# build model\n",
        "model_1 = build_model(english_vocabsize, nepali_vocabsize, english_maxlen, nepali_maxlen, 128)\n",
        "model_1.summary()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         [(None, None)]            0         \n",
            "_________________________________________________________________\n",
            "embedding (Embedding)        (None, None, 128)         361728    \n",
            "_________________________________________________________________\n",
            "lstm (LSTM)                  (None, 128)               131584    \n",
            "_________________________________________________________________\n",
            "repeat_vector (RepeatVector) (None, 11, 128)           0         \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                (None, 11, 128)           131584    \n",
            "_________________________________________________________________\n",
            "time_distributed (TimeDistri (None, 11, 4354)          561666    \n",
            "=================================================================\n",
            "Total params: 1,186,562\n",
            "Trainable params: 1,186,562\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 564
        },
        "id": "9a_owTWhNMRq",
        "outputId": "f94a4a79-bbae-481f-aaf4-df2803eda375"
      },
      "source": [
        "# plot model\n",
        "plot_model(model_1)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAASMAAAIjCAYAAABIyH2wAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3de1xUdf4/8NcBBmYGYUBFSAUV1PBumqmI5mXbtTRNUcEb6v4qL+uqeS0vrV9XNk0NNtP6urp2/Sogrq2mrd82b5uXLE1LRU1DM1RMEQwQBnj//ujr7E6AzMAw8xl8PR+P+WPOOfP5vM/nHF7MOWfmjCYiAiIi10r1cHUFREQAwDAiIiUwjIhICQwjIlKCl6sLqGnDhw93dQlE1da9e3fMnDnT1WXUqFofRlu2bEG3bt3QuHFjV5dCVCWHDx92dQlOUevDCABeeOEFjBgxwtVlEFXJg/LunueMiEgJDCMiUgLDiIiUwDAiIiUwjIhICQwjIlICw4iIlMAwIiIlMIyISAkMIyJSAsOIiJTAMCIiJTCMiEgJDCMiUgLD6Bd27twJk8mE7du3u7oUhygtLUViYiKioqKq3Mbhw4fRqlUreHh4QNM0BAcHY+nSpQ6ssvrS0tIQHh4OTdOgaRpCQkIwZswYV5dFdngg7mdkj9r0y03nz5/HhAkT8Nlnn6FDhw5Vbqdbt244c+YM+vfvj3/84x84e/YsAgICHFhp9cXExCAmJgbNmzfHjz/+iGvXrrm6JLIT3xn9woABA5CTk4Onn37a1aWgoKCgyu9oTpw4gRdffBGTJ09Gx44dHVyZ61VnbEhNDCOFbdiwAVlZWVV6bYcOHZCWlobRo0fDx8fHwZW5XnXGhtTEMPoP//rXvxAWFgZN0/DGG28AANauXQtfX18YjUZ8+OGHePLJJ+Hv74/GjRtj06ZNlte+/vrr0Ov1aNCgASZNmoSHHnoIer0eUVFROHLkiGW5adOmwdvbGyEhIZZpv/vd7+Dr6wtN0/Djjz8CAGbMmIFZs2bhwoUL0DQNzZs3r5F1/vjjj+Hv74+EhAS7X+vuY3PgwAG0bt0aJpMJer0e7dq1wz/+8Q8AwLPPPms5/xQREYHjx48DACZMmACj0QiTyYS///3vAICSkhK8/PLLCAsLg8FgQPv27ZGcnAwAePXVV2E0GuHn54esrCzMmjULjRo1wtmzZ6tUc60mtRwASU5Otnn577//XgDI6tWrLdMWLFggAOSf//yn5OTkSFZWlvTs2VN8fX2lqKjIstzEiRPF19dXTp8+LXfv3pVTp05Jly5dxM/PTy5fvmxZbvTo0RIcHGzV74oVKwSA3LhxwzItJiZGIiIiqrLaVrp27SodOnQod96OHTvEz89PlixZUmk7v/nNbwSAZGdnW6apNjYRERFiMpkqXRcRkdTUVFm8eLHcunVLbt68Kd26dZN69epZ9eHp6Sk//PCD1etGjRolf//73y3PZ8+eLT4+PrJlyxbJzs6W+fPni4eHhxw9etRqjKZPny6rV6+WoUOHypkzZ2yqUURk2LBhMmzYMJuXd1MpfGdkh6ioKPj7+yMoKAhxcXHIy8vD5cuXrZbx8vJCq1at4OPjg9atW2Pt2rW4c+cONm7c6KKq72/AgAHIzc3FokWLqtWOO47NsGHD8Ic//AGBgYGoW7cuBg0ahJs3b+LGjRsAgMmTJ6OkpMSqvtzcXBw9ehRPPfUUAODu3btYu3YthgwZgpiYGAQEBGDhwoXQ6XRl1mvZsmWYOnUq0tLSEBkZ6bwVdRMMoyry9vYGAJjN5vsu9+ijj8JoNCI9Pd0ZZSnBXcdGp9MB+PmwCwD69u2Lli1b4q9//avlKuvmzZsRFxcHT09PAMDZs2eRn5+Ptm3bWtoxGAwICQlRZr3cBcPICXx8fCz/bcmaK8fmo48+Qu/evREUFAQfHx/MnTvXar6maZg0aRIuXryIf/7znwCAd999F//v//0/yzJ5eXkAgIULF1rOMWmahkuXLiE/P995K1MLMIxqmNlsxu3bt/kjkuVw9tjs378fiYmJAIDLly9jyJAhCAkJwZEjR5CTk4Ply5eXec348eOh1+uxfv16nD17Fv7+/mjSpIllflBQEAAgMTERImL1OHTokFPWq7bghx5r2N69eyEi6Natm2Wal5dXpYcwDwJnj82XX34JX19fAMDXX38Ns9mMKVOmIDw8HMDP74R+KTAwELGxsdi8eTP8/Pzw3HPPWc0PDQ2FXq/HV199VSM1P0j4zsjBSktLkZ2djeLiYpw8eRIzZsxAWFgYxo8fb1mmefPmuHXrFrZt2waz2YwbN27g0qVLZdqqW7cuMjMzkZGRgTt37tTIH+muXbuqfGnfXq4aG7PZjOvXr2Pv3r2WMAoLCwMAfPLJJ7h79y7Onz9v9TGD/zR58mQUFhZix44dZT4Mq9frMWHCBGzatAlr165Fbm4uSkpKcOXKFVy9etXeIXqwufBSnlPAjkv7q1evlpCQEAEgRqNRBg0aJGvWrBGj0SgApEWLFnLhwgVZt26d+Pv7CwBp0qSJnDt3TkR+vnyt0+mkUaNG4uXlJf7+/vLMM8/IhQsXrPq5efOm9OnTR/R6vTRr1kx+//vfy5w5cwSANG/e3HKp+9ixY9KkSRMxGAwSHR0t165ds3m9Dx06JD169JCHHnpIAAgACQkJkaioKNm3b59luZ07d4qfn58sXbq0wrYOHz4sbdq0EQ8PD0s7CQkJSo3Nm2++KREREZZ1reixdetWS1/z5s2TunXrSkBAgAwfPlzeeOMNASARERFWHzcQEXnkkUfkpZdeKnd8CgsLZd68eRIWFiZeXl4SFBQkMTExcurUKVm+fLkYDAYBIKGhofLee+/ZvA3veVAu7WsitejLWOXQNA3JyckYMWJEjfc1adIkpKam4ubNmzXel7tx97EZMGAA3njjDTRr1szpfQ8fPhwAkJqa6vS+nSiVh2kOdu+yMJXlTmPzn4d9J0+ehF6vd0kQPUgYRm4iPT3d6tJxRY+4uDhXl1orzJs3D+fPn8e5c+cwYcIE/PGPf3R1SbUew8hB5s+fj40bNyInJwfNmjXDli1bHNp+ZGRkmUvH5T02b97s0H4doabHpiYYjUZERkbiV7/6FRYvXozWrVu7uqRaj+eMiBTHc0ZERE7EMCIiJTCMiEgJDCMiUgLDiIiUwDAiIiUwjIhICQwjIlICw4iIlMAwIiIlMIyISAkMIyJSAsOIiJTwQNyQPzExsbZ/45lqscOHD1v9aEFtVevfGQ0bNow/E1SDMjMzLb85TzWjW7du6N69u6vLqHG1/n5GVLNSUlIQGxsL7kZUTbyfERGpgWFEREpgGBGREhhGRKQEhhERKYFhRERKYBgRkRIYRkSkBIYRESmBYURESmAYEZESGEZEpASGEREpgWFEREpgGBGREhhGRKQEhhERKYFhRERKYBgRkRIYRkSkBIYRESmBYURESmAYEZESGEZEpASGEREpgWFEREpgGBGREhhGRKQEhhERKYFhRERKYBgRkRIYRkSkBIYRESnBy9UFkPv44Ycf8PTTT8NsNlum5eXloU6dOmjXrp3Vsh07dsR7773n7BLJjTGMyGaNGjXC3bt3cebMmTLzvvnmG6vnsbGxziqLagkeppFd4uPj4eVV+f8whhHZi2FEdhk1ahRKSkoqnK9pGjp16oQWLVo4sSqqDRhGZJewsDB06dIFHh7l7zqenp6Ij493clVUGzCMyG7x8fHQNK3ceSUlJRg+fLiTK6LagGFEdhsxYkS50z09PfH444+jYcOGTq6IagOGEdktKCgIvXv3hqenZ5l5Y8eOdUFFVBswjKhKxo4dCxGxmubh4YGhQ4e6qCJydwwjqpKhQ4daXeL38vLCk08+iYCAABdWRe6MYURV4ufnh4EDB0Kn0wH4+cT1mDFjXFwVuTOGEVXZ6NGjUVxcDADQ6/UYOHCgiysid8Ywoip76qmnYDQaAQAxMTEwGAwurojcmfLfTbty5QoOHjzo6jKoAl26dMHevXsRGhqKlJQUV5dDFajo4xgq0eSXl0QUk5KSwu85EVWT4n/mAJDqNodpIsKHgo/i4mIsWbLE5XXwUf4jOTnZ1X+6NnObMCI1eXp64qWXXnJ1GVQLMIyo2my5pQhRZRhGRKQEhhERKYFhRERKYBgRkRIYRkSkBIYRESmBYURESmAYEZESGEZEpASGEREpgWFEREpgGBGREhhGdujSpQs8PT3RsWNHh7f97LPPws/PD5qm4auvvrJ7uZ07d8JkMmH79u0Or80eaWlpCA8Ph6ZpFT6aNm3qkL64PWoXhpEdjh49ij59+tRI2+vXr8df/vKXKi8nosbNs2JiYnDx4kVERETAZDJZ3fcoPz8f169ft9yqtrq4PWoX3vuhCir6aWdXGjBgAHJyclxdRoU8PT1hMBhgMBjQsmVLh7bN7VE78J1RFdz7eR5Hs/WPyhl/fCKC1NRUrFu3zuFtb9u2zaHtcXvUDrUyjEpKSvDyyy8jLCwMBoMB7du3t9x+MykpCb6+vvDw8EDnzp0RHBwMnU4HX19fdOrUCT179kRoaCj0ej0CAgIwd+7cMu1/++23iIyMhK+vLwwGA3r27Il//etfNtcA/LxzrVixAg8//DB8fHxgMpkwZ86cMn3Zsty//vUvhIWFQdM0vPHGGwCAtWvXwtfXF0ajER9++CGefPJJ+Pv7o3Hjxti0aVOZWv/0pz/h4YcfhsFgQP369dGsWTP86U9/srqR+8cffwx/f38kJCTYuUUqxu1R9e1R64jikpOTxd4yZ8+eLT4+PrJlyxbJzs6W+fPni4eHhxw9elRERP7whz8IADly5Ijk5eXJjz/+KP379xcA8tFHH8mNGzckLy9Ppk2bJgDkq6++srTdr18/CQ8Pl++++07MZrN888030rVrV9Hr9XLu3Dmba1iwYIFomiarVq2S7Oxsyc/PlzVr1ggAOX78uKUdW5f7/vvvBYCsXr3a6rUA5J///Kfk5ORIVlaW9OzZU3x9faWoqMiyXEJCgnh6esqHH34o+fn58uWXX0pwcLD07t3balx37Nghfn5+smTJkkq3QUREhJhMJqtp06dPl6+//rrMstweVdsetqjK34+LpChfpb2DWVBQIEajUeLi4izT8vPzxcfHR6ZMmSIi/97579y5Y1nmnXfeEQBWfyyff/65AJDNmzdbpvXr1086dOhg1efJkycFgMyePdumGvLz88VoNMoTTzxh1c6mTZusdmpblxO5/85fUFBgmXbvD+fbb7+1TOvSpYs89thjVn08//zz4uHhIYWFhVIVERERAqDM435hxO3xM0duD3cKo1p3mHb27Fnk5+ejbdu2lmkGgwEhISFIT0+v8HXe3t4AYPmFVODf5yLMZvN9+2zXrh1MJhNOnjxpUw3ffvst8vPz0a9fv/u2a+ty9ri3nv+5Tnfv3i1z9aekpAQ6nQ6enp5V7us/r6aJCKZPn253ndweP3PE9lBdrQujvLw8AMDChQutPtty6dIl5Ofn11i/Op3OskNVVsOVK1cAAEFBQfdt09blquupp57Cl19+iQ8//BAFBQX44osvsG3bNgwcONChO39SUpJVINQkbg/3U+vC6N6OkpiYWOY3pA4dOlQjfRYXF+PWrVsICwuzqQa9Xg8AKCwsvG+7ti5XXYsXL0bfvn0xfvx4+Pv7Y+jQoRgxYoRNn7NREbeHe6p1YXTvysv9PjXraHv27EFpaSk6depkUw1t27aFh4cH9u3bd992bV2uuk6dOoULFy7gxo0bMJvNuHz5MtauXYvAwMAa6e/q1auYMGFCjbQNcHu4q1oXRnq9HhMmTMCmTZuwdu1a5ObmoqSkBFeuXMHVq1cd0kdRURFycnJQXFyMY8eOYdq0aWjSpAnGjx9vUw1BQUGIiYnBli1bsGHDBuTm5uLkyZNlPkNi63LVNXXqVISFheGnn36673K7du2q1qV9EUFBQQHS0tLg7+9fpTbK86Buj1rH2afM7VWVqwGFhYUyb948CQsLEy8vLwkKCpKYmBg5deqUJCUlidFoFADStGlTOXDggCxbtkxMJpMAkODgYPnggw9k8+bNEhwcLAAkMDBQNm3aJCIiGzdulD59+kiDBg3Ey8tL6tWrJyNHjpRLly7ZXIOIyJ07d+TZZ5+VevXqSZ06dSQ6OlpefvllASCNGzeWEydO2Lzc6tWrJSQkRACI0WiUQYMGyZo1ayzr2aJFC7lw4YKsW7dO/P39BYA0adLEcun7008/lXr16lld9dLpdNKqVStJS0uzrNPOnTvFz89Pli5dWuHYb926tcIraf/5WLhwoYgIt0c1toct3OlqmvJVutFguq01a9bIjBkzrKYVFhbKCy+8ID4+PpKfn++iyh5MjtwebvT3k8Lvpj3grl27hmnTppU5n+Lt7Y2wsDCYzWaYzWYYDAYXVfhgeZC3R607Z0T2MRgM0Ol02LBhA65fvw6z2YzMzEysX78eL7/8MuLi4hx6fofu70HeHgyjB5zJZMLu3bvxzTffoGXLljAYDGjdujU2btyIZcuW4Z133nF1iQ+UB3l78DCN0LNnT/zv//6vq8ug//Ogbg++MyIiJTCMiEgJDCMiUgLDiIiUwDAiIiUwjIhICQwjIlICw4iIlMAwIiIlMIyISAkMIyJSAsOIiJTAMCIiJbjNt/ZTUlJcXQKR26mpX8SpCW4TRrGxsa4ugYhqkCbyi5+uJLJDSkoKYmNjy/wCKpGdUnnOiIiUwDAiIiUwjIhICQwjIlICw4iIlMAwIiIlMIyISAkMIyJSAsOIiJTAMCIiJTCMiEgJDCMiUgLDiIiUwDAiIiUwjIhICQwjIlICw4iIlMAwIiIlMIyISAkMIyJSAsOIiJTAMCIiJTCMiEgJDCMiUgLDiIiUwDAiIiUwjIhICQwjIlICw4iIlMAwIiIlMIyISAkMIyJSAsOIiJTg5eoCyH1cv34db7/9ttW0kydPAgCWL19uNT0wMBDPP/+8s0qjWkATEXF1EeQeiouLERwcjJycHHh5/fv/mIhA0zTL88LCQjz33HNYt26dK8ok95TKwzSymZeXF+Li4uDh4YHCwkLLo6ioyOo5AIwaNcrF1ZK7YRiRXUaOHAmz2XzfZYKCgtCzZ08nVUS1BcOI7NKjRw80bNiwwvne3t6Ij4+Hp6enE6ui2oBhRHbRNA1jxoyBTqcrd35RURFGjhzp5KqoNmAYkd3ud6jWpEkTdO7c2ckVUW3AMCK7dezYES1atCgz3dvbG+PHj3d+QVQrMIyoSuLj48scqhUVFSE2NtZFFZG7YxhRlYwcORLFxcWW55qmoX379mjVqpULqyJ3xjCiKomIiEDHjh3h4fHzLuTl5YX4+HgXV0XujGFEVRYfH28Jo+LiYh6iUbUwjKjKYmNjUVpaCgDo3r07Gjdu7OKKyJ0xjKjKHnroIcsnrceNG+fiasjd8YuyNSQlJYWHLbUQ/1xqTCpvIVLDkpOTXV1CjcrLy8O6devwwgsvuLqUGnXo0CEkJSW5uoxajWFUw0aMGOHqEmrcE0888UCcL2IY1SyeM6JqexCCiGoew4iIlMAwIiIlMIyISAkMIyJSAsOIiJTAMCIiJTCMiEgJDCMiUgLDiIiUwDAiIiUwjIhICQwjIlICw4iIlMAwUsTKlSvRoEEDaJqGt956y9XlVCgtLQ3h4eHQNA2apiEkJARjxoyp9HUnTpxAXFwcmjVrBh8fH9SvXx8dOnTA0qVLLcvExcVZ2q3ssWPHjjK1LFq06L41vPbaa9A0DR4eHoiMjMT+/furPR7kOAwjRcyePRsHDx50dRmViomJwcWLFxEREQGTyYRr167h/fffv+9rvv76a0RFRSEkJAR79uxBTk4ODh48iP79+2Pv3r1Wy+7evRu3b9+G2WzG1atXAQCDBg1CUVER8vLykJWVheeee65MLQCwfv36Cn/ptqSkBK+//joAoG/fvkhPT0evXr2qMxTkYAwjN1ZQUICoqChXl1GplStXIiAgAElJSWjatCn0ej1atmyJP/7xjzAYDJblNE1Djx49YDKZ4OXlZTVdp9PBaDQiKCio3J/P7ty5M65du4Zt27aVW0NaWhoaNWrk+JUjh2EYubENGzYgKyvL1WVU6ubNm8jJycGtW7espnt7e2P79u2W55s2bYLRaKy0vYkTJ2LgwIFW06ZMmQIAePPNN8t9zWuvvYZZs2bZWzo5EcNIcfv27cNjjz0Go9EIf39/tGvXDrm5uZgxYwZmzZqFCxcuQNM0NG/eHElJSfD19YWHhwc6d+6M4OBg6HQ6+Pr6olOnTujZsydCQ0Oh1+sREBCAuXPnWvX18ccfw9/fHwkJCQ5dhy5duiAvLw99+/bFZ5995tC27+nbty9atWqFPXv24OzZs1bzPvvsM+Tn5+PXv/51jfRNjsEwUlheXh4GDRqEYcOG4datWzh//jxatmyJoqIiJCUl4emnn0ZERAREBN9++y1mzJiBOXPmQETw5ptv4rvvvsO1a9fQq1cvHD9+HC+99BKOHz+OW7duYdy4cVixYgVOnDhh6a+kpAQALL+F5ihz587Fo48+ihMnTiA6Ohpt2rTBq6++WuadUnVNmjQJAMpcAFi1ahVmzpzp0L7I8RhGCsvIyEBubi7atGkDvV6P4OBgpKWloX79+pW+tnXr1jAajahXrx5GjhwJAAgLC0P9+vVhNBotV8DS09MtrxkwYAByc3MrvSplL4PBgIMHD+LPf/4zIiMjcfr0acybNw+tWrXCvn37HNbPuHHj4Ovri3feeQcFBQUAgIsXL+Lo0aMYNWqUw/qhmsEwUlh4eDgaNGiAMWPGYPHixcjIyKhSO97e3gB+/gnqe3Q6HQBUePXJ0XQ6HaZNm4YzZ87g8OHDeOaZZ5CVlYXhw4cjOzvbIX2YTCaMGjUK2dnZ2Lx5MwAgMTERU6ZMsYwBqYthpDCDwYBPP/0U0dHRSEhIQHh4OOLi4iz/9d1V165d8be//Q2TJ0/GjRs3sGfPHoe1fe9E9ltvvYXbt28jNTXVcvhGamMYKa5NmzbYvn07MjMzMW/ePCQnJ2PlypWuLuu+9u/fj8TERMvzmJgYq3dl94wdOxYAkJ+f77C+O3bsiG7duuHzzz/HxIkTMXz4cAQGBjqsfao5DCOFZWZm4vTp0wCAoKAgvPLKK+jUqZNlmqq+/PJL+Pr6Wp4XFhaWW/O9q17t27d3aP/33h1t2bKl1v/SbW3CMFJYZmYmJk2ahPT0dBQVFeH48eO4dOkSunXrBgCoW7cuMjMzkZGRgTt37lT7/M+uXbuqdWnfbDbj+vXr2Lt3r1UYAcCQIUOQkpKC27dvIycnBx9++CFefPFFDB482OFhNGLECNSvXx9DhgxBeHi4Q9umGiRUI5KTk8We4V21apUEBwcLAPH19ZWhQ4dKRkaGREVFSWBgoHh6ekrDhg1lwYIFUlxcLCIix44dkyZNmojBYJDo6Gh56aWXxGg0CgBp2rSpHDhwQJYtWyYmk0kASHBwsHzwwQeyefNmS1+BgYGyadMmERHZuXOn+Pn5ydKlSyusc+vWrRIRESEA7vvYunWr5TW7d++W2NhYiYiIEB8fH/H29paHH35YFi9eLHfv3i3TR25urvTq1Uvq1q0rAMTDw0OaN28uCQkJFdZSv359mTp1qmXe3Llz5eDBg5bnCxculJCQEEt7rVu3lgMHDti8fezdnmS3FE1ExOkJ+ABISUlBbGwsOLy1A7dnjUvlYRoRKYFhRERKYBgRkRIYRkSkBIYRESmBYURESmAYEZESGEZEpASGEREpgWFEREpgGBGREhhGRKQEhhERKYFhRERKYBgRkRIYRkSkBIYRESnBy9UF1Haaprm6BCK3wDCqIVFRUUhOTnZ1GTXu0KFDSEpKeiDWlWoW74FN1cJ7Q5OD8B7YRKQGhhERKYFhRERKYBgRkRIYRkSkBIYRESmBYURESmAYEZESGEZEpASGEREpgWFEREpgGBGREhhGRKQEhhERKYFhRERKYBgRkRIYRkSkBIYRESmBYURESmAYEZESGEZEpASGEREpgWFEREpgGBGREhhGRKQEhhERKYFhRERKYBgRkRIYRkSkBIYRESmBYURESmAYEZESvFxdALmPgoICXL161Wra9evXAQAXL160mu7p6YkmTZo4rTZyf5qIiKuLIPdw8+ZNhISEoLi4uNJl+/fvj127djmhKqolUnmYRjarV68ennjiCXh43H+30TQNcXFxTqqKaguGEdllzJgxqOzNtJeXF5555hknVUS1BcOI7DJ48GD4+PhUON/LywuDBg2CyWRyYlVUGzCMyC6+vr4YPHgwdDpdufNLSkowevRoJ1dFtQHDiOw2evRomM3mcucZDAY8+eSTTq6IagOGEdmtf//+8Pf3LzNdp9MhNjYWer3eBVWRu2MYkd10Oh1GjBhR5lDNbDZj1KhRLqqK3B3DiKpk1KhRZQ7V6tWrhz59+rioInJ3DCOqkscffxwNGjSwPPf29saYMWPg6enpwqrInTGMqEo8PDwwZswYeHt7AwCKioowcuRIF1dF7oxhRFU2cuRIFBUVAQAaN26Mxx57zMUVkTtjGFGVPfroo2jWrBkAYPz48dA0zcUVkTtz6rf2X3vtNRw6dMiZXVINMxgMAIDPP/8cw4cPd3E15EgzZ85E9+7dndafU98ZHTp0CIcPH3Zml1TDQkNDYTKZyv3cEbmvLVu24Pvvv3dqn06/n1G3bt2Qmprq7G6pBv3jH//Ab37zG1eXQQ7kikNunjOiamMQkSMwjIhICQwjIlICw4iIlMAwIiIlMIyISAkMIyJSAsOIiJTAMCIiJTCMiEgJDCMiUgLDiIiUwDAiIiUwjIhICQwjqjFpaWkIDw+HpmlWD71ej2bNmuG3v/0tvvvuO1eXabOJEyfC19cXmqZBp9OhQ4cOOHPmjNUyf/3rXxEWFgZN0xAcHIy3337bNcW6I3GiYcOGybBhw5zZJSkgIiJCTCaTiIiUlJTI9evX5d133xWj0SgNGjSQH3/80cUV2u7EiRMCQLp27VrhMhkZGdKwYUMpKipyYmWOBUCSk5Od2WUK3xlVU0FBAaKiolxdxmpcwRQAAB4/SURBVH2pVKOHhwcaNGiAsWPHYurUqcjKysInn3zi6rLKVd64tW/fHtHR0Thy5AiOHTtW7uveeust/Pa3vy3zI5eOqqG2YhhV04YNG5CVleXqMu5L1RqbN28OALh27ZqLKylfReM2depUAMCaNWvKzCsqKsK7776LiRMn1mgNtZIz34fZe5i2fPlyMRgMUqdOHbl+/brMnDlTGjZsKOnp6VJcXCyLFi2S0NBQ0ev10q5dO9m8ebOIiPz5z38WHx8fCQoKkokTJ0pISIj4+PhI9+7d5fDhw1Z93K8dEZH9+/dLq1atxN/fX3x8fKRt27by8ccfi4jI9OnTxdvbWwAIAImIiLB53SIjIwWAaJomnTp1kry8PBERmTNnjqWvjRs32lSjiMi7774rnTt3Fh8fHzEajdKkSRNZsmRJhTWWlpbKqlWrJDIyUry9vSUgIEAGDx4sZ86csWn8d+3aJX5+frJ06dJK1/U/D9P+05w5cwSA7Nmzx2q66tvWbDZLw4YNxWg0SnZ2tlWfH3zwgQwZMsTmGlyx7WwBFxymKR1GIiILFiwQADJ9+nRZvXq1DB06VM6cOSOzZ88WHx8f2bJli2RnZ8v8+fPFw8NDjh49KiIiEydOFF9fXzl9+rTcvXtXTp06JV26dBE/Pz+5fPmypf3K2klNTZXFixfLrVu35ObNm9KtWzepV6+e5fUxMTF2hdA9xcXF0rRpUwkLC5Pi4mKreS+88IIkJibaXGNiYqIAkFdeeUVu3rwpt27dkv/+7/+W0aNHV1jjyy+/LN7e3vLee+/J7du35eTJk9KpUyepX7++XLt2rdLx37Fjh/j5+cmSJUsqXddfhlF2dra8/fbbYjQaZcCAAWWWd4dtu3jxYgEgr732mtX06Oho+eSTT2yuwRXbzhYMo3LcG9CCggLLtIKCAjEajRIXF2eZlp+fLz4+PjJlyhQR+XmH/eV/46NHjwoA+a//+i+b2/mlP/3pTwJAsrKyRKTqYSTy7x0xJSXFMi0vL0/CwsIkJyfHphqLiookICBA+vTpY9V2cXGxJCUllVtjfn6+1KlTx6pNEZHPP/9cAFgFTHnjb6+IiAjLf/d7D03TZOnSpWVO8rrLtr169arodDpp2bKllJaWiojIyZMnJTIy0uZ1UXnbuSKM3PKc0dmzZ5Gfn4+2bdtaphkMBoSEhCA9Pb3C1z366KMwGo2WZarSzr2TkiUlJdVej2effRYmkwlJSUmWae+//z6eeeYZy0//VFbjyZMncfv27TI3xff09MT06dPL7ffUqVP46aef8Oijj1pN79KlC7y9vXHkyJFqr9svmUwmiAhEBHPmzIGIwGQylTnJ6y7bNiQkBDExMTh37pzlBPybb76JyZMn27wu7rLtnMUtwygvLw8AsHDhQqvPr1y6dAn5+fn3fa2Pjw9u3LhhczsfffQRevfujaCgIPj4+GDu3LkOW486derg+eefx8GDB/H5558D+HmHnjZtms3rmpubCwAICAiwud/bt29b+v+lgIAA3Llzp8rrZItFixYhJCQE8+fPL/PbXO60be+dyF67di3u3LmDv/3tbxg3bpzN6+KO264muWUYBQUFAQASExMt/23vPe73i7Vmsxm3b99G48aNbWrn8uXLGDJkCEJCQnDkyBHk5ORg+fLlDl2XadOmQafTITExEfv370doaCgiIiJsXteGDRsCAH788Ueb+7y385e34/7n+NQUPz8/LFu2DHfu3MGUKVOs5rnTtu3RowceeeQRbN++Ha+88goGDx4Mk8lk87q447arSW4ZRqGhodDr9fjqq6/set3evXshIujWrZtN7Xz99dcwm82YMmUKwsPDodfrHf7jdo0bN8aIESOwZcsWLFq0CDNmzLCaX1mNTZs2Rd26dbF7926b+2zbti3q1KmDL774wmr6kSNHUFRUhM6dO9u/InaKj49H165dsWPHDqSkpFimu9u2/d3vfoeSkhIsW7asTLDW1m1XU9wyjPR6PSZMmIBNmzZh7dq1yM3NRUlJCa5cuYKrV69alistLUV2djaKi4tx8uRJzJgxA2FhYRg/frxN7YSFhQEAPvnkE9y9exfnz58vc0xet25dZGZmIiMjA3fu3IHZbLZ7fWbNmoXi4mJkZ2ejb9++dq2rj48P5s+fj/3792PatGn44YcfUFpaijt37uD06dPl1ujp6YlZs2Zh69ateP/995Gbm4uvv/4akydPxkMPPWTTZ2R27doFf39/JCQk2L2+wM+/WPr6669D0zRMmzYN2dnZNq3vPaps21GjRqFu3bro0aMH2rdvbzVP1W2nLKecJ/8/Vf2cEQAJDQ2V9957zzKvsLBQ5s2bJ2FhYeLl5SVBQUESExMjp06dEpGfr7jodDpp1KiReHl5ib+/vzzzzDNy4cIFqz4qa2fevHlSt25dCQgIkOHDh8sbb7xh+czH5cuX5dixY9KkSRMxGAwSHR1tdWnVHn369JH169eXO6+yGkVE3njjDWnXrp3o9XrR6/XyyCOPyJo1a0REyq2xtLRUVqxYIS1atBCdTieBgYEyZMgQOXv2rE3jv3Pnzko/Z/TZZ59Jy5YtLVfQGjZsKJMmTbJaZvz48QJAAgIC5JVXXrFpfVXbtnPmzJH/+Z//cZttZwu44Gqa9n8dO8Xw4cMBAKmpqTXe16RJk5CamoqbN2/WeF/kXNy2NU/TNCQnJ2PEiBHO6jLVLQ/TbOWIy++kJm7b2qdWh5Gzpaenl7ldRnmPuLg4V5dKpJxaGUbz58/Hxo0bkZOTg2bNmmHLli1O6TcyMrLMJdzyHps3b3ZKPbWRq7Yt1bxae86IiKqO54yI6IHFMCIiJTCMiEgJDCMiUgLDiIiUwDAiIiUwjIhICQwjIlICw4iIlMAwIiIlMIyISAkMIyJSAsOIiJTg5ewODx8+bPn2PhHRPU4No+7duzuzO3KCzMxMfPHFFxg0aJCrSyEHGjZsGEJDQ53ap1PvZ0S1T0pKCmJjY8HdiKqJ9zMiIjUwjIhICQwjIlICw4iIlMAwIiIlMIyISAkMIyJSAsOIiJTAMCIiJTCMiEgJDCMiUgLDiIiUwDAiIiUwjIhICQwjIlICw4iIlMAwIiIlMIyISAkMIyJSAsOIiJTAMCIiJTCMiEgJDCMiUgLDiIiUwDAiIiUwjIhICQwjIlICw4iIlMAwIiIlMIyISAkMIyJSAsOIiJTAMCIiJXi5ugByHz/88AOefvppmM1my7S8vDzUqVMH7dq1s1q2Y8eOeO+995xdIrkxhhHZrFGjRrh79y7OnDlTZt4333xj9Tw2NtZZZVEtwcM0skt8fDy8vCr/H8YwInsxjMguo0aNQklJSYXzNU1Dp06d0KJFCydWRbUBw4jsEhYWhi5dusDDo/xdx9PTE/Hx8U6uimoDhhHZLT4+HpqmlTuvpKQEw4cPd3JFVBswjMhuI0aMKHe6p6cnHn/8cTRs2NDJFVFtwDAiuwUFBaF3797w9PQsM2/s2LEuqIhqA4YRVcnYsWMhIlbTPDw8MHToUBdVRO6OYURVMnToUKtL/F5eXnjyyScREBDgwqrInTGMqEr8/PwwcOBA6HQ6AD+fuB4zZoyLqyJ3xjCiKhs9ejSKi4sBAHq9HgMHDnRxReTOGEZUZU899RSMRiMAICYmBgaDwcUVkTvjd9PsdOXKFRw8eNDVZSijS5cu2Lt3L0JDQ5GSkuLqcpRR0ccfqGKa/PKSCN1XSkoKv3dFleKfld1SeZhWRSLChwiKi4uxZMkSl9ehyiM5OdnVu6bbYhhRtXh6euKll15ydRlUCzCMqNpsuaUIUWUYRkSkBIYRESmBYURESmAYEZESGEZEpASGEREpgWFEREpgGBGREhhGRKQEhhERKYFhRERKYBgRkRIYRjVs5cqVaNCgATRNw1tvveXqcmxSWlqKxMREREVFVbmNtLQ0hIeHQ9M0aJqGkJAQm+6RfeLECcTFxaFZs2bw8fFB/fr10aFDByxdutSyTFxcnKXdyh47duwoU8uiRYvuW8Nrr70GTdPg4eGByMhI7N+/v8rjQLZjGNWw2bNnu9WdIc+fP49evXph5syZyM/Pr3I7MTExuHjxIiIiImAymXDt2jW8//77933N119/jaioKISEhGDPnj3IycnBwYMH0b9/f+zdu9dq2d27d+P27dswm824evUqAGDQoEEoKipCXl4esrKy8Nxzz5WpBQDWr18Ps9lcbg0lJSV4/fXXAQB9+/ZFeno6evXqVeVxINsxjBRUUFBQrXclVXXixAm8+OKLmDx5Mjp27Oj0/leuXImAgAAkJSWhadOm0Ov1aNmyJf74xz9a3V9b0zT06NEDJpPJ6vYlmqZBp9PBaDQiKCgInTt3LtNH586dce3aNWzbtq3cGtLS0tCoUSPHrxxVimGkoA0bNiArK8vp/Xbo0AFpaWkYPXo0fHx8nN7/zZs3kZOTg1u3bllN9/b2xvbt2y3PN23aZPkhgPuZOHFimV8smTJlCgDgzTffLPc1r732GmbNmmVv6eQADCMX2bdvHx577DEYjUb4+/ujXbt2yM3NxYwZMzBr1ixcuHABmqahefPmSEpKgq+vLzw8PNC5c2cEBwdDp9PB19cXnTp1Qs+ePREaGgq9Xo+AgADMnTu3Rmv/+OOP4e/vj4SEBIe226VLF+Tl5aFv37747LPPHNr2PX379kWrVq2wZ88enD171mreZ599hvz8fPz617+ukb7p/hhGLpCXl4dBgwZh2LBhuHXrFs6fP4+WLVuiqKgISUlJePrppxEREQERwbfffosZM2Zgzpw5EBG8+eab+O6773Dt2jX06tULx48fx0svvYTjx4/j1q1bGDduHFasWIETJ07UWP0lJSUAfj7R7Uhz587Fo48+ihMnTiA6Ohpt2rTBq6++WuadUnVNmjQJAMpcUFi1ahVmzpzp0L7IdgwjF8jIyEBubi7atGkDvV6P4OBgpKWloX79+pW+tnXr1jAajahXrx5GjhwJAAgLC0P9+vVhNBotV6zS09NrrP4BAwYgNze30qtS9jIYDDh48CD+/Oc/IzIyEqdPn8a8efPQqlUr7Nu3z2H9jBs3Dr6+vnjnnXdQUFAAALh48SKOHj2KUaNGOawfsg/DyAXCw8PRoEEDjBkzBosXL0ZGRkaV2vH29gYAy6+6ArD83HRFV4tUp9PpMG3aNJw5cwaHDx/GM888g6ysLAwfPhzZ2dkO6cNkMmHUqFHIzs7G5s2bAQCJiYmYMmWKZUzJ+RhGLmAwGPDpp58iOjoaCQkJCA8PR1xcnOW/NP2sa9eu+Nvf/obJkyfjxo0b2LNnj8Pavnci+6233sLt27eRmppqOXwj12AYuUibNm2wfft2ZGZmYt68eUhOTsbKlStdXZZT7d+/H4mJiZbnMTExVu/y7hk7diwAVOtzT7/UsWNHdOvWDZ9//jkmTpyI4cOHIzAw0GHtk/0YRi6QmZmJ06dPAwCCgoLwyiuvoFOnTpZpD4ovv/wSvr6+lueFhYXljsG9q17t27d3aP/33h1t2bIFL7zwgkPbJvsxjFwgMzMTkyZNQnp6OoqKinD8+HFcunQJ3bp1AwDUrVsXmZmZyMjIwJ07d5Q7/7Nr165qXdo3m824fv069u7daxVGADBkyBCkpKTg9u3byMnJwYcffogXX3wRgwcPdngYjRgxAvXr18eQIUMQHh7u0LapCoTskpycLPYM26pVqyQ4OFgAiK+vrwwdOlQyMjIkKipKAgMDxdPTUxo2bCgLFiyQ4uJiERE5duyYNGnSRAwGg0RHR8tLL70kRqNRAEjTpk3lwIEDsmzZMjGZTAJAgoOD5YMPPpDNmzdb+goMDJRNmzbZtW6HDh2SHj16yEMPPSQABICEhIRIVFSU7Nu3z7Lczp07xc/PT5YuXVphW1u3bpWIiAhLOxU9tm7dannN7t27JTY2ViIiIsTHx0e8vb3l4YcflsWLF8vdu3fL9JGbmyu9evWSunXrCgDx8PCQ5s2bS0JCQoW11K9fX6ZOnWqZN3fuXDl48KDl+cKFCyUkJMTSXuvWreXAgQM2j6G9+wdZpGgiIk7OP7eWkpKC2NhYcNioPNw/qiyVh2lEpASGUS2Wnp5u02024uLiXF0qEbwqX4TcVWRkJA8XyG3wnRERKYFhRERKYBgRkRIYRkSkBIYRESmBYURESmAYEZESGEZEpASGEREpgWFEREpgGBGREhhGRKQEhhERKYFhRERK4C1EqiglJcXVJZCCDh065OoS3BbDqIpiY2NdXQJRrcJ7YFO18J7P5CC8BzYRqYFhRERKYBgRkRIYRkSkBIYRESmBYURESmAYEZESGEZEpASGEREpgWFEREpgGBGREhhGRKQEhhERKYFhRERKYBgRkRIYRkSkBIYRESmBYURESmAYEZESGEZEpASGEREpgWFEREpgGBGREhhGRKQEhhERKYFhRERKYBgRkRIYRkSkBIYRESmBYURESmAYEZESGEZEpASGEREpwcvVBZD7uH79Ot5++22raSdPngQALF++3Gp6YGAgnn/+eWeVRrWAJiLi6iLIPRQXFyM4OBg5OTnw8vr3/zERgaZplueFhYV47rnnsG7dOleUSe4plYdpZDMvLy/ExcXBw8MDhYWFlkdRUZHVcwAYNWqUi6sld8MwIruMHDkSZrP5vssEBQWhZ8+eTqqIaguGEdmlR48eaNiwYYXzvb29ER8fD09PTydWRbUBw4jsomkaxowZA51OV+78oqIijBw50slVUW3AMCK73e9QrUmTJujcubOTK6LagGFEduvYsSNatGhRZrq3tzfGjx/v/IKoVmAYUZXEx8eXOVQrKipCbGysiyoid8cwoioZOXIkiouLLc81TUP79u3RqlUrF1ZF7oxhRFUSERGBjh07wsPj513Iy8sL8fHxLq6K3BnDiKosPj7eEkbFxcU8RKNqYRhRlcXGxqK0tBQA0L17dzRu3NjFFZE7YxhRlT300EOWT1qPGzfOxdWQuyvzRdmUlBS+3SaiGlXO9/NTK7yFSHJycs1WQ7VCXl4e1q1bhxdeeMHVpZAbOHToEJKSksqdV2EYjRgxosYKotrliSee4PkisllFYcRzRlRtDCJyBIYRESmBYURESmAYEZESGEZEpASGEREpgWFEREpgGBGREhhGRKQEhhERKYFhRERKYBgRkRIYRkSkBIYRESmhxsJo586dMJlM2L59e0114RDPPvss/Pz8oGkavvrqK8v0mqz/l2136dIFnp6e6Nixo8P7qo6KxsYWcXFx0DTNpseOHTucur+kpaUhPDy8TB3e3t5o0KABevfujRUrViA7O7vMa7lfVG+/uJ8aC6Ny7uSmpPXr1+Mvf/lLmek1Wf8v2z569Cj69OlTY/1VVUVjY6vdu3fj9u3bMJvNuHr1KgBg0KBBKCoqQl5eHrKysvDcc88BcO7+EhMTg4sXLyIiIgImkwkigtLSUmRlZSElJQXNmjXDvHnz0KZNG3zxxRdWr+V+Uf39oiIV3lzNHgUFBejXrx8OHjxomTZgwADk5OQ4onmXsLf+8sbA3rY1TbOrRkfVUxM0TUOPHj1gNBrLTNfpdNDpdDAajZafwnb1/qJpGgICAtC7d2/07t0bAwYMQGxsLAYMGIBz587BZDJVqU7uF7ZzyDujDRs2ICsryxFNuYQjNrYjxuCXv9BaHY7aJlUdm02bNpUJovJMnDgRAwcOrFIfNWnYsGEYP348srKy8NZbb1W5He4XdpBfSE5OlnImV2j69Oni7e0tAASAREREyIEDByQ0NFQAyOrVq0VEJDExUYxGo2iaJp06dZIGDRqIl5eXGI1GeeSRRyQ6OloaN24sPj4+YjKZZM6cOVb9FBcXy6JFiyQ0NFT0er20a9dONm/ebHOd95SWlsqrr74qLVu2FG9vb/H397fUevz4cRGRcusXEdm7d6906dJFDAaD+Pn5Sdu2bSUnJ6fcMVi+fLkYDAapU6eOXL9+XWbOnCkNGzaU9evXl9t2v379JDAwUB5++GExGo2i1+slOjpaDhw4YFnm97//veh0OgkODrZMmzJlihiNRgEgN27cqHCb2DKGtoyNiMiuXbvEz89Pli5davO4X716VQDI4MGDy8xz1f4SEREhJpOpwpr3798vAOTxxx+vsE4R7hf2uE++pFQ7jEREYmJiLCt2z/fff19mYP/whz8IADly5Ijk5eXJjz/+KP379xcA8tFHH8mNGzckLy9Ppk2bJgDkq6++srx29uzZ4uPjI1u2bJHs7GyZP3++eHh4yNGjR+2qdcGCBaJpmqxatUqys7MlPz9f1qxZU2Zgf1n/Tz/9JP7+/rJ8+XIpKCiQa9euydChQy0burwxWLBggQCQ6dOny+rVq2Xo0KFy5syZcsemX79+Eh4eLt99952YzWb55ptvpGvXrqLX6+XcuXOW5UaPHm2104mIrFixwmqnq6ieysbQ1rHZsWOH+Pn5yZIlS2we9/uFUXnjLVLz+0tlYZSbmysAJDQ0tMI6uV84Loxccmm/devWMBqNqFevHkaOHAkACAsLQ/369WE0GjFmzBgAQHp6OgDg7t27WLt2LYYMGYKYmBgEBARg4cKF0Ol02Lhxo839FhQUIDExEb/61a8wc+ZMBAQEwGAwoG7dupW+NiMjA7m5uWjTpg30ej2Cg4ORlpaG+vXrV/raZcuWYerUqUhLS0NkZGSFy/n5+aFp06bw8vJCmzZt8Je//AV3797FunXrbF7HilQ2hvaMzYABA5Cbm4tFixZVuy5buGp/uXfF6M6dOxUuw/3CcVz+OSNvb28AP/888j33jpHNZjMA4OzZs8jPz0fbtm0tyxgMBoSEhFh2QFt8++23yM/PR79+/eyuMzw8HA0aNMCYMWOwePFiZGRk2N2Gvdq1aweTyYSTJ09Wu63KxrA6Y+NMztxf8vLyICLw9/evcBnuF47j8jCyRV5eHgBg4cKFVp8LuXTpEvLz821u58qVKwCAoKAgu2swGAz49NNPER0djYSEBISHhyMuLg4FBQV2t2UPnU5n+SOrjsrGsDpjoxpH7S/nzp0DgPu+a+F+4ThuEUb3BiIxMREiYvU4dOiQze3o9XoAQGFhYZXqaNOmDbZv347MzEzMmzcPycnJWLlyZZXaskVxcTFu3bqFsLCwardV2RhWd2xU4qj95eOPPwYAPPnkk/ddjvuFY7hFGIWGhkKv11f7055t27aFh4cH9u3bZ/drMzMzcfr0aQA/b8BXXnkFnTp1skyrCXv27EFpaSk6depkmebl5VWl/4iVjWF1xkY1jthfrl27hsTERDRu3Bi//e1vK1yO+4XjOCSM6tati8zMTGRkZODOnTsOefv4n/R6PSZMmIBNmzZh7dq1yM3NRUlJCa5cuWL5ZK8tgoKCEBMTgy1btmDDhg3Izc3FyZMnbToRmJmZiUmTJiE9PR1FRUU4fvw4Ll26hG7dugFwzBgUFRUhJycHxcXFOHbsGKZNm4YmTZpg/PjxlmWaN2+OW7duYdu2bTCbzbhx4wYuXbpUpq1f1uPp6XnfMbRnbHbt2gV/f38kJCTYvY7OYM/+IiL46aefUFpaChHBjRs3kJycjB49esDT0xPbtm277zkj7hcOZMeltwodO3ZMmjRpIgaDQaKjo2XhwoUSEhIiAMRoNMqgQYMkKSnJ8rmHpk2byoEDB2TZsmViMpkEgAQHB8sHH3wgmzdvluDgYAEggYGBsmnTJhERKSwslHnz5klYWJh4eXlJUFCQxMTEyKlTp+yq9c6dO/Lss89KvXr1pE6dOhIdHS0vv/yyAJDGjRvLiRMnZPXq1WXqz8jIkKioKAkMDBRPT09p2LChLFiwQIqLi8sdg5kzZ4rBYLBcGn7vvfdERMptW0Rk48aN0qdPH8vnaerVqycjR46US5cuWdV/8+ZN6dOnj+j1emnWrJn8/ve/lzlz5ggAad68uVy+fLnceq5du1bpGNoyNiIiO3futPlzRrm5udKrVy+pW7euABAPDw9p3ry5JCQkWJYpb0xqcn/5+9//Lu3btxej0Sje3t7i4eEhAETTNAkICJDHHntMlixZIjdv3rRaF+4X998vbHG/S/uaiPUXYlJSUhAbG+s23y0jIvdxn3xJdYtzRkRU+7l9GKWnp9t0m4q4uDhXl0pE9+GQb+27UmRkJA8piWoBt39nRES1A8OIiJTAMCIiJTCMiEgJDCMiUgLDiIiUwDAiIiUwjIhICQwjIlICw4iIlMAwIiIlMIyISAkMIyJSAsOIiJRQ4S1EauS3tImIKlAmjKKiopCcnOyKWojoAVbmHthERC7Ae2ATkRoYRkSkBIYRESnBC0Cqq4sgogfe4f8PUN02mWsOJ94AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4kL_waIxNMPf",
        "outputId": "6f5ad2b7-1bd4-4e28-9daf-1e4d1d7432f8"
      },
      "source": [
        "# create callbacks to save model and early stop model training \n",
        "checkpoint_dir = os.path.join(checkpoint_1, \"model_1.h5\")\n",
        "model_checkpoint = ModelCheckpoint(checkpoint_dir, monitor='val_loss', save_best_only=True)\n",
        "early_stopping = EarlyStopping(patience=7, monitor='val_loss', restore_best_weights=True)\n",
        "\n",
        "history = model_1.fit(train_dataset, epochs=100, validation_data=valid_dataset,\n",
        "                    callbacks=[model_checkpoint, early_stopping])"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/100\n",
            "247/247 [==============================] - 10s 16ms/step - loss: 3.2559 - val_loss: 2.4855\n",
            "Epoch 2/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 2.2561 - val_loss: 2.2170\n",
            "Epoch 3/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 2.0528 - val_loss: 2.1391\n",
            "Epoch 4/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 1.9635 - val_loss: 2.0969\n",
            "Epoch 5/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 1.8977 - val_loss: 2.0408\n",
            "Epoch 6/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 1.8426 - val_loss: 2.0071\n",
            "Epoch 7/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 1.8087 - val_loss: 1.9894\n",
            "Epoch 8/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 1.7672 - val_loss: 1.9749\n",
            "Epoch 9/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 1.7194 - val_loss: 1.9078\n",
            "Epoch 10/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 1.6542 - val_loss: 1.8512\n",
            "Epoch 11/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 1.5870 - val_loss: 1.8212\n",
            "Epoch 12/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 1.5294 - val_loss: 1.7840\n",
            "Epoch 13/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 1.4790 - val_loss: 1.7546\n",
            "Epoch 14/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 1.4322 - val_loss: 1.7625\n",
            "Epoch 15/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 1.3826 - val_loss: 1.7282\n",
            "Epoch 16/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 1.3328 - val_loss: 1.7016\n",
            "Epoch 17/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 1.2854 - val_loss: 1.6922\n",
            "Epoch 18/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 1.2414 - val_loss: 1.6724\n",
            "Epoch 19/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 1.1969 - val_loss: 1.6526\n",
            "Epoch 20/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 1.1525 - val_loss: 1.6356\n",
            "Epoch 21/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 1.1102 - val_loss: 1.6402\n",
            "Epoch 22/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 1.0721 - val_loss: 1.6281\n",
            "Epoch 23/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 1.0333 - val_loss: 1.6049\n",
            "Epoch 24/100\n",
            "247/247 [==============================] - 3s 10ms/step - loss: 0.9913 - val_loss: 1.6181\n",
            "Epoch 25/100\n",
            "247/247 [==============================] - 3s 10ms/step - loss: 0.9506 - val_loss: 1.6286\n",
            "Epoch 26/100\n",
            "247/247 [==============================] - 3s 10ms/step - loss: 0.9116 - val_loss: 1.6168\n",
            "Epoch 27/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 0.8766 - val_loss: 1.5859\n",
            "Epoch 28/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 0.8421 - val_loss: 1.5848\n",
            "Epoch 29/100\n",
            "247/247 [==============================] - 3s 10ms/step - loss: 0.8090 - val_loss: 1.5913\n",
            "Epoch 30/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 0.7776 - val_loss: 1.5832\n",
            "Epoch 31/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 0.7510 - val_loss: 1.5852\n",
            "Epoch 32/100\n",
            "247/247 [==============================] - 2s 10ms/step - loss: 0.7181 - val_loss: 1.5926\n",
            "Epoch 33/100\n",
            "247/247 [==============================] - 3s 10ms/step - loss: 0.6845 - val_loss: 1.5845\n",
            "Epoch 34/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 0.6517 - val_loss: 1.5909\n",
            "Epoch 35/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 0.6196 - val_loss: 1.5931\n",
            "Epoch 36/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 0.5897 - val_loss: 1.5894\n",
            "Epoch 37/100\n",
            "247/247 [==============================] - 3s 11ms/step - loss: 0.5603 - val_loss: 1.6019\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AycB_mb9NMNR",
        "outputId": "4ee10b44-5dbb-40fc-f98d-e687198c51d1"
      },
      "source": [
        "# evaluate on a test set\n",
        "print(\"Test set loss: {:.4f}\".format(model_1.evaluate(test_dataset, verbose=0)))"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test set loss: 1.5545\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 404
        },
        "id": "YGqRc2qTNMKx",
        "outputId": "08ae4d1e-d88b-4f21-e06a-69c51c26271c"
      },
      "source": [
        "# plot loss and accuracy\n",
        "def plot_loss(history):\n",
        "  fig, ax = plt.subplots(figsize=(8, 6))\n",
        "  pd.DataFrame(history.history)[[\"loss\", \"val_loss\"]].plot(ax=ax)\n",
        "  ax.set_title(\"Training and Validation Loss\")\n",
        "  ax.set_xlabel(\"Epoch\")\n",
        "  ax.set_ylabel(\"Loss\")\n",
        "  plt.show()\n",
        "\n",
        "# plot model 1 losses\n",
        "plot_loss(history)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfEAAAGDCAYAAAA72Cm3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxU9b3/8dcnO4EkQEIWlhA22VVkk0VE6oK4Va11r9JWb22r3Zd7u2h77W1v7a/b1dba1rVapbiUqohaQVwQ2fdVIJCQPZANss7398eZSAwJBJLJzCTv5+Mxj5k5c+bM50we8J7v93zP95hzDhEREQk/EcEuQERERE6PQlxERCRMKcRFRETClEJcREQkTCnERUREwpRCXEREJEwpxEXawMwWm9ltHb1uMJnZPjO7MADbXWZmX/Q/vtnMXm/LuqfxOZlmVmlmkadbq0i4U4hLl+X/D77x5jOzo02e33wq23LOXeqce6Kj1w1FZvZ9M1vewvIUM6s1s3Ft3ZZz7mnn3MUdVNcnfnQ45/Y753o55xo6YvvNPsuZ2fCO3q5IR1OIS5fl/w++l3OuF7AfuKLJsqcb1zOzqOBVGZL+Bkw3syHNlt8AbHLObQ5CTSLSAoW4dDtmNtvMcszse2aWDzxmZn3M7GUzKzKzQ/7HA5u8p2kX8e1m9q6Z/cq/7l4zu/Q01x1iZsvNrMLM3jSzh8zsb63U3ZYa/9vM3vNv73UzS2ny+q1mlm1mJWb2g9a+H+dcDvAWcGuzlz4HPHmyOprVfLuZvdvk+UVmtt3MyszsQcCavDbMzN7y11dsZk+bWW//a08BmcC//D0p3zWzLH+LOcq/Tn8zW2RmpWa228zuaLLt+8xsgZk96f9utpjZpNa+g9aYWZJ/G0X+7/KHZhbhf224mb3t37diM3vOv9zM7DdmVmhm5Wa26VR6M0RORCEu3VU60BcYDNyJ92/hMf/zTOAo8OAJ3j8V2AGkAL8E/mpmdhrrPgN8CCQD93F8cDbVlhpvAuYDqUAM8G0AMxsD/NG//f7+z2sxeP2eaFqLmY0EzvbXe6rfVeM2UoAXgB/ifRcfATOargL83F/faGAQ3neCc+5WPtmb8ssWPuJZIMf//s8A/2Nmc5q8fqV/nd7AorbU3IL/A5KAocD5eD9s5vtf+2/gdaAP3nf7f/7lFwOzgDP87/0sUHIany1yHIW4dFc+4F7nXI1z7qhzrsQ597xz7ohzrgL4Gd5/0q3Jds792X889gkgA0g7lXXNLBOYDPzYOVfrnHsXL1xa1MYaH3PO7XTOHQUW4AUveKH2snNuuXOuBviR/ztozYv+Gqf7n38OWOycKzqN76rRPGCLc26hc64O+C2Q32T/djvn3vD/TYqAX7dxu5jZILwfBN9zzlU759YDf/HX3ehd59yr/r/DU8BZbdl2k8+IxDuk8J/OuQrn3D7g/3Hsx04d3g+b/v4a3m2yPAEYBZhzbptzLu9UPlukNQpx6a6KnHPVjU/MLN7M/uTvIi0HlgO9rfWRz03D54j/Ya9TXLc/UNpkGcCB1gpuY435TR4faVJT/6bbds5VcYLWoL+mfwCf8/ca3Aw8eQp1tKR5Da7pczNLM7NnzSzXv92/4bXY26Lxu6xosiwbGNDkefPvJs5ObTxEChDt325Ln/FdvN6ED/3d9Z8HcM69hdfqfwgoNLNHzCzxFD5XpFUKcemuml++71vASGCqcy4Rr/sTmhyzDYA8oK+ZxTdZNugE67enxrym2/Z/ZvJJ3vMEXtfvRXgtyX+1s47mNRif3N//wfu7jPdv95Zm2zzRJRcP4n2XCU2WZQK5J6npVBRzrLV93Gc45/Kdc3c45/oD/wH8wfwj3J1zv3fOTQTG4HWrf6cD65JuTCEu4knAO7Z72Mz6AvcG+gOdc9nAauA+M4sxs2nAFQGqcSFwuZnNNLMY4Kec/N//O8Bh4BHgWedcbTvreAUYa2bX+FvA9+CNTWiUAFQCZWY2gOODrgDvWPRxnHMHgPeBn5tZnJmdCXwBrzV/umL824ozszj/sgXAz8wswcwGA99s/Awzu67JAL9DeD86fGY22cymmlk0UAVUc+JDGSJtphAX8fwW6IHX2voAeK2TPvdmYBpe1/b9wHNATSvrnnaNzrktwFfwBqbl4YVMzkne4/C60Af779tVh3OuGLgO+AXe/o4A3muyyk+Ac4AyvMB/odkmfg780MwOm9m3W/iIG4EsvFb5i3hjHt5sS22t2IL3Y6XxNh+4Gy+I9wDv4n2fj/rXnwysNLNKvLENX3PO7QESgT/jfefZePv+QDvqEvmYef9ORSQU+E9L2u6cC3hPgIiEP7XERYLI39U6zMwizGwucBXwUrDrEpHwoJmqRIIrHa/bOBmve/su59y64JYkIuFC3ekiIiJhSt3pIiIiYUohLiIiEqbC7ph4SkqKy8rKCnYZIiIinWbNmjXFzrl+zZeHXYhnZWWxevXqYJchIiLSacwsu6Xl6k4XEREJUwpxERGRMKUQFxERCVNhd0xcRETCS11dHTk5OVRXV5985W4uLi6OgQMHEh0d3ab1FeIiIhJQOTk5JCQkkJWVhXcFWmmJc46SkhJycnIYMmRIm96j7nQREQmo6upqkpOTFeAnYWYkJyefUo+FQlxERAJOAd42p/o9KcRFRKTL69WrV7BLCAiFuIiISJhSiIuISLfhnOM73/kO48aNY/z48Tz33HMA5OXlMWvWLM4++2zGjRvHO++8Q0NDA7fffvvH6/7mN78JcvXH0+h0ERHpND/51xa2Hizv0G2O6Z/IvVeMbdO6L7zwAuvXr2fDhg0UFxczefJkZs2axTPPPMMll1zCD37wAxoaGjhy5Ajr168nNzeXzZs3A3D48OEOrbsjdOuW+KGqWpZuL6TsaF2wSxERkU7w7rvvcuONNxIZGUlaWhrnn38+q1atYvLkyTz22GPcd999bNq0iYSEBIYOHcqePXu4++67ee2110hMTAx2+cfp1i3xrXnlzH98Fc/eeS7nDk0OdjkiIl1eW1vMnW3WrFksX76cV155hdtvv51vfvObfO5zn2PDhg0sWbKEhx9+mAULFvDoo48Gu9RP6NYt8bTEWAAKyjWLkIhId3Deeefx3HPP0dDQQFFREcuXL2fKlClkZ2eTlpbGHXfcwRe/+EXWrl1LcXExPp+Pa6+9lvvvv5+1a9cGu/zjdOuWeGpiHKAQFxHpLq6++mpWrFjBWWedhZnxy1/+kvT0dJ544gkeeOABoqOj6dWrF08++SS5ubnMnz8fn88HwM9//vMgV3+8bh3iCbFRxMdEUlBeE+xSREQkgCorKwFvMpUHHniABx544BOv33bbbdx2223HvS8UW99NdevudDMjLTFOLXEREQlL3TrEAVITYilUS1xERMJQtw/xtMQ48tUSFxGRMKQQT4yloLwa51ywSxERETklCvHEOGrqfZQfrQ92KSIiIqdEId54mlmFutRFRCS8KMR1rriIiIQphbh/1rb8MoW4iIh4TnT98X379jFu3LhOrKZ1CnF/S7ywQqeZiYhIeOnWM7YBxEVHktQjWt3pIiKdYfH3IX9Tx24zfTxc+osTrvL973+fQYMG8ZWvfAWA++67j6ioKJYuXcqhQ4eoq6vj/vvv56qrrjqlj66uruauu+5i9erVREVF8etf/5oLLriALVu2MH/+fGpra/H5fDz//PP079+fz372s+Tk5NDQ0MCPfvQjrr/++tPebVCIA8dOMxMRka7p+uuv5+tf//rHIb5gwQKWLFnCPffcQ2JiIsXFxZx77rlceeWVmFmbt/vQQw9hZmzatInt27dz8cUXs3PnTh5++GG+9rWvcfPNN1NbW0tDQwOvvvoq/fv355VXXgGgrKys3fulEKdxwhd1p4uIBNxJWsyBMmHCBAoLCzl48CBFRUX06dOH9PR0vvGNb7B8+XIiIiLIzc2loKCA9PT0Nm/33Xff5e677wZg1KhRDB48mJ07dzJt2jR+9rOfkZOTwzXXXMOIESMYP3483/rWt/je977H5Zdfznnnndfu/er2x8TBC/FCtcRFRLq06667joULF/Lcc89x/fXX8/TTT1NUVMSaNWtYv349aWlpVFd3TBbcdNNNLFq0iB49ejBv3jzeeustzjjjDNauXcv48eP54Q9/yE9/+tN2f45a4njd6YUVNfh8joiItnejiIhI+Lj++uu54447KC4u5u2332bBggWkpqYSHR3N0qVLyc7OPuVtnnfeeTz99NPMmTOHnTt3sn//fkaOHMmePXsYOnQo99xzD/v372fjxo2MGjWKvn37csstt9C7d2/+8pe/tHufFOJ4LfEGn6OkqpZ+CbHBLkdERAJg7NixVFRUMGDAADIyMrj55pu54oorGD9+PJMmTWLUqFGnvM0vf/nL3HXXXYwfP56oqCgef/xxYmNjWbBgAU899RTR0dGkp6fzX//1X6xatYrvfOc7REREEB0dzR//+Md275OF25zhkyZNcqtXr+7Qbb62OZ8v/W0NL989k3EDkjp02yIi3d22bdsYPXp0sMsIGy19X2a2xjk3qfm6OiYOpCdp1jYREQk/6k7n2KxtBRqhLiIifps2beLWW2/9xLLY2FhWrlwZpIqOpxAHUnrFYqaWuIiIHDN+/HjWr18f7DJOSN3pQHRkBMk9NeGLiEighNv4q2A51e9JIe6XnqQQFxEJhLi4OEpKShTkJ+Gco6SkhLi4uDa/J2Dd6WYWBywHYv2fs9A5d2+zdWKBJ4GJQAlwvXNuX6BqOpG0hDjydCUzEZEON3DgQHJycigqKgp2KSEvLi6OgQMHtnn9QB4TrwHmOOcqzSwaeNfMFjvnPmiyzheAQ8654WZ2A/C/QPtmgz9NqYlxbMg5HIyPFhHp0qKjoxkyZEiwy+iSAtad7jyV/qfR/lvzvpSrgCf8jxcCn7JTmXm+A6UlxlJcWUttvS8YHy8iInLKAnpM3MwizWw9UAi84ZxrPi5/AHAAwDlXD5QByYGsqTWN1xUvqtRpZiIiEh4CGuLOuQbn3NnAQGCKmY07ne2Y2Z1mttrMVgfqmEp6oiZ8ERGR8NIpo9Odc4eBpcDcZi/lAoMAzCwKSMIb4Nb8/Y845yY55yb169cvIDWm+id80dXMREQkXAQsxM2sn5n19j/uAVwEbG+22iLgNv/jzwBvuSCdg9DYnZ6vEeoiIhImAjk6PQN4wswi8X4sLHDOvWxmPwVWO+cWAX8FnjKz3UApcEMA6zmhvvExREcaBRU6Ji4iIuEhYCHunNsITGhh+Y+bPK4GrgtUDaciIsJITYjTMXEREQkbmrGtidTEWAp1ERQREQkTCvEm0hLiyFdLXEREwoRCvIm0RM2fLiIi4UMh3kRaUhwV1fUcqa0PdikiIiInpRBvIi3BO81Mx8VFRCQcKMSb+PhccXWpi4hIGFCIN5Hmn7VNx8VFRCQcKMSbSEtSd7qIiIQPhXgTCbFR9IiOVEtcRETCgkK8CTPzTjPT1KsiIhIGFOLNpCbGUaCLoIiISBhQiDeTlhhHQYVCXEREQp9CvJl0/6xtQboiqoiISJspxJtJS4yjus5HebVmbRMRkdCmEG8m1T/hi0aoi4hIqFOIN5OWoAlfREQkPCjEm0lPamyJ6zQzEREJbQrxZlIT1J0uIiLhQSHeTI+YSBLjohTiIiIS8hTiLUhLjFOIi4hIyFOItyA9KU7HxEVEJOQpxFuQmhBHoVriIiIS4hTiLUhLjKWwogafT7O2iYhI6FKItyAtMY56n6OkqjbYpYiIiLRKId6CNM3aJiIiYUAh3oK0RG/WtkJdzUxEREKYQrwFjS3x/DKNUBcRkdClEG9Bv4RYzNSdLiIioU0h3oLoyAiSe8aqO11EREKaQrwVaYmxmvBFRERCmkK8FWmJceSXqSUuIiKhSyHeCm/CF4W4iIiELoV4K1IT4iiurKWuwRfsUkRERFqkEG9FepJ3mllRhY6Li4hIaFKIt6Jxwpd8nWYmIiIhSiHeitQEryWuq5mJiEioUoi34tj86epOFxGR0KQQb0VyzxiiIkyztomISMhSiLciIsJITdCELyIiEroU4ieQmhinlriIiIQshfgJeFOvKsRFRCQ0KcRPIF0tcRERCWEK8RNITYyjvLqeo7UNwS5FRETkOArxEzh2mpla4yIiEnoU4ifQOGubQlxEREKRQvwE0htb4po/XUREQlDAQtzMBpnZUjPbamZbzOxrLawz28zKzGy9//bjQNVzOlITNfWqiIiErqgAbrse+JZzbq2ZJQBrzOwN59zWZuu945y7PIB1nLbEuCjioiPIL1OIi4hI6AlYS9w5l+ecW+t/XAFsAwYE6vMCwcxIS4xTd7qIiISkTjkmbmZZwARgZQsvTzOzDWa22MzGdkY9pyItQeeKi4hIaAp4iJtZL+B54OvOufJmL68FBjvnzgL+D3iplW3caWarzWx1UVFRxxVX8hG8/A04erjVVdKS4nRMXEREQlJAQ9zMovEC/Gnn3AvNX3fOlTvnKv2PXwWizSylhfUecc5Ncs5N6tevX8cVWF0Gqx+Fzc+3ukpaQiz55dU45zruc0VERDpAIEenG/BXYJtz7tetrJPuXw8zm+KvpyRQNR2n/wRIHQvrn251lbTEOKrrfJRX13daWSIiIm0RyJb4DOBWYE6TU8jmmdmXzOxL/nU+A2w2sw3A74EbXGc2ec1gws2QuwYKmg+a96T6J3xRl7qIiISagJ1i5px7F7CTrPMg8GCgamiTM6+HN37stcYv+dlxL3884Ut5DSPSEjq7OhERkVZpxraeKTDyUtjwLDTUHfdy4/zp+WqJi4hIiFGIA5x9Cxwphp1LjnspVfOni4hIiFKIAwy/EHqlwbq/HfdSfEwUCXFROiYuIiIhRyEOEBkFZ90Iu16HioLjXk5PjKOgXLO2iYhIaFGIN5pwC7gG2PjscS+lJcbpmLiIiIQchXijlBEwaKrXpd7sLLfUxFh1p4uISMhRiDc14RYo3gk5qz+xOC0xjsKKGnw+zdomIiKhQyHe1NirIToe1j31icXpiXHU+xylR2qDVJiIiMjxFOJNxSbAmE/D5hegturjxWn+08x0XXEREQklCvHmJtwCtRWw7V8fL0r1T/hSWKEQFxGR0KEQb27wdOg79BPnjKc1mXpVREQkVCjEmzODs2+Gfe9A6V4AUhM0a5uIiIQehXhLzroRLALWPwNAdGQEKb1iFOIiIhJSFOItSRoAw+Z4Ie5rACA1QbO2iYhIaFGIt2bCLVCeA3uWAd4IdbXERUQklCjEWzNyHvTo8/EAtzTNny4iIiFGId6aqFgY/1nY/gocKSUtMY6SqhrqGnzBrkxERARQiJ/YhFugoQY2P09aYhzOQVGFWuMiIhIaFOInknEmpI+HdU99PGubjouLiEioUIifzIRbIW8Dg+v2AJrwRUREQodC/GTGXweRMfTf9wKgqVdFRCR0KMRPJr4vjLqMHtsWkhTjWJN9KNgViYiIAArxtjn7FuxoKT8akc2/Nhzko6LKYFckIiKiEG+TYRdAQn+u9L1FbFQk//fvXcGuSERERCHeJhGRcPZNxOx7iy9P7MGiDQfZXajWuIiIBJdCvK3Ovgmcjzur/kTf6Dp+r9a4iIgEmUK8rZKHwYX3EbvrVV6Lv5dtmz5kV0FFsKsSEZFuTCF+KmZ+Az73En0jqvhn9I94/8U/BLsiERHpxhTip2robCK+9A4liaO5Lf9/OLzgy1Cnc8dFRKTzKcRPR2IGve5czF/cVfTe+jT89UIo3RPsqkREpJtRiJ+mPgnxHJ7+Az5f+20aDu2HP50PWxcFuywREelGFOLt8MXzhrAqego/Tv8jJA+HBbfCa/8J9bXBLk1ERLoBhXg79I6PYf6MLJ7eAdsuXQBT/gM++AM8fhmU5QS7PBER6eIU4u30hZlDSYiL4rfLsmHeL+Ezj0HhNnj4PNj8AjTUB7tEERHpohTi7ZQUH83nZwxhyZYCthwsg3HXwJ3LILE/LJwPvzsL3n4AKgqCXaqIiHQxCvEO8PmZQ7zW+Jv+WdxShntB/tmnvMdL74ffjIF/3A573wHnglitiIh0FQrxDpDUI5ovzhzKG1sL2Jxb5i2MjIYxV8Ln/glfXeMdL/9oKTxxOTw0FVb+CarLglu4iIiENYV4B5k/M4vEuCh+++bO419MGQ5z/we+uQ2ueghiesLi78L/GwWL7oG8DZ1fsIiIhD2FeAdJjIvmjvOG8ua2QjbmHG55pZh4mHAL3LnU624fdy1sXAB/mgV/vdjrahcREWkjhXgHun1GFr3jo48dGz+R/hPgqgfhW9tg7i+gLNfran/mem90u4iIyEkoxDtQgr81/tb2QtbtP9S2N/XoA+feBXevhgt/Atkr4I/TYdHdUJ4X2IJFRCSsKcQ72G3Ts+jT1tZ4U9E9YObX4WvrYeqXYP3f4fcT4K37obo8MMWKiEhYU4h3sF6xUdwxayhv7yxiTXYbW+NNxfeFuT+Hr66CUfNg+QNemH/4Z2io6/iCRUQkbCnEA+C2aVn07RnDT/61hcNHTnMe9b5D4DOPwh1vQb9R8Oq3vVPTti7SeeYiIgIoxAOiZ2wU/3P1eLbnVXDNH99nf8mR09/YgIlw+8tw0wLv3PMFt3oj2Tc8q/PMRUS6OXNh1qqbNGmSW716dbDLaJOVe0q486k1REcaf7ltMmcP6t2+DTbUw/qnvS72sgMQEQ3D5sCYq2DkpV5XvIiIdDlmtsY5N+m45YEKcTMbBDwJpAEOeMQ597tm6xjwO2AecAS43Tm39kTbDacQB9hdWMn8xz+kqKKG398wgYvHprd/oz4f5K6BrS953etl+yEiCoac7wX6qMuhZ3L7P0dEREJCMEI8A8hwzq01swRgDfBp59zWJuvMA+7GC/GpwO+cc1NPtN1wC3GAoooavvjkajbmHObHl49h/owhHbdx5+DgOtj6Ty/UD+0Di4SsmV6gj74CeqV23OeJiEin6/QQb6GAfwIPOufeaLLsT8Ay59zf/c93ALOdc62eIB2OIQ5wtLaBe55dxxtbC/j8jCH84LLRREZYx36Ic5C/6Vigl+wGDFJH+29j/LfR0HswRGhIhIhIOGgtxKM66cOzgAnAymYvDQAONHme41/W5WY56RETycO3TOS/X97Ko+/tJffwEX57/QR6xER23IeYQcaZ3m3OD72Z37Yt8lrqOatg8/PH1o3uCamj/OE+9ljI90r1tiMiIiEv4CFuZr2A54GvO+dOa9YSM7sTuBMgMzOzA6vrXJERxn1XjmVQ33juf2UrN/75A/5y2yRSesV2/IeZQdoY79aouhyKdkDh1mO3nUtg3d+OrRPXG5KHQd+h/luTx/F9FfAiIiGkTd3pZtYTOOqc85nZGcAoYLFz7oSzj5hZNPAysMQ59+sWXu823enNvbY5j689u560xDgenz+Zof16Ba+YyiJ/qG+D4h1Quse7leWA8x1bLzYJkoceC/XUMTD6SojslA4dEZFuq13HxM1sDXAe0Ad4D1gF1Drnbj7Bewx4Aih1zn29lXUuA77KsYFtv3fOTTlRLV0lxAHW7j/EF59Yjc85/vy5SUzOCrFTxOpr4PB+KPnoWLCX7oHSj7zlzue/kMtDkDY22NWKiHRZ7Q3xtc65c8zsbqCHc+6XZrbeOXf2Cd4zE3gH2AQ0Nuf+C8gEcM497A/6B4G5eKeYzXfOnTChu1KIA2SXVHH7Y6vIOXSEL88ezpcvGEZsVAceJw+U+lrvePvi73mTzsz6Nsz8JkTFBLsyEZEup70hvg74MvAb4AvOuS1mtsk5N77jSz2xrhbiAIeP1HLfoi28tP4gI1J78Ytrz2Ti4D7BLqttqkrgte/Bpn94A+Q+/ZDXOhcRkQ7TWoi39RyjrwP/CbzoD/ChwNKOLLA76x0fw29vmMBjt0+mqqaezzz8Pvf+czOVNfXBLu3keibDtX+BG/4OR0rgz5+CN+6FuupgVyYi0uWd8nniZhYB9Drdkebt1RVb4k1V1tTzqyU7eGLFPjIS47j/6nHMGZUW7LLa5uhheP2HsO4pSB4BVz0ImecGuyoRkbDXrpa4mT1jZon+Ueqbga1m9p2OLlK8S5ned+VYFn5pOj1jo/j846u55+/rKK6sCXZpJ9ejtxfct77oDYp7dC4s/j7UVgW7MhGRLqmt3elj/C3vTwOLgSHArQGrSpg4uA8v3zOTr184gsWb87jw12/z/JocwuKCNcPmwJdXwJQ7YOUf4Q/TYM/bwa5KRKTLaWuIR/vP+f40sMh/fngYpEl4i42K5OsXnsGr95zH0JSefOsfG/jcox9yoLQdlzbtLLG9YN4DMH+xd3GWJ6+E578Ih7KDXZmISJfR1hD/E7AP6AksN7PBQFCOiXdHI9ISWPil6fzkyrGszT7Exb9Zzm/f3BkeA98GT4e73oPzvg3b/gUPToLXf+QdPxcRkXY57QugmFmUc67TU6SrD2w7mdzDR7n/5a0s3pxPSq8Y7p4zghunZBITFQYXMynLhaU/g/XPeMfPZ30XJn9R55aLiJxEe88TTwLuBWb5F70N/NQ5V9ahVbZBdw/xRmv3H+J/F29n5d5SMvvG8+1LRnL5+AwiOvrKaIGQv8lrje9ZCn2GwIX3wphPa152EZFWtDfEn8cblf6Ef9GtwFnOuWs6tMo2UIgf45xj2c4i/nfxdrbnVzC2fyLfv3QU543oF+zS2mb3m16YF26FgVPg4vsh84SXkxcR6ZbaG+LHTbF6smlXA0Uhfjyfz/HPDbn8aslOcg8fZebwFL43dxTjByYFu7ST8zV43etLfwYVeTD6CrjwJ96V1EREBGh/iK8AvuOce9f/fAbwK+fctA6v9CQU4q2rqW/gbx/s58G3dnHoSB2Xn5nBty8eSVZKz2CXdnK1VbDiIXj3t9BQ43Wvj7zUO10tPsQuDCMi0snaG+JnAU8CjU27Q8BtzrmNHVplGyjET668uo4/L9/DX97ZS12Dj7nj0rlsfAazR6bSIybEL65SWQjLH4DNz3vTuFoEDJoKIy6GMy7xLn+qY+ci0s20K8SbbCQRwDlXbmZfd879tgNrbBOFeNsVllfzh2UfsWjDQUqraomPiVfU5L4AACAASURBVOSCUanMG5fBBaP6ER8TwtcB9zVA7lrYtQR2LoF8/+/FxIFwxsUw4hIYMgti4oNbp4hIJ+iQEG+2wf3Oucx2V3aKFOKnrr7Bx4d7S3llUx5LtuRTXFlLj+hILhjVj0vHZTBnVCo9Y0M40AHK82DX697to6VQVwVRcZB1HoycC2Ou9i7GIiLSBQUixA845wa1u7JTpBBvnwafY+XeEhZvymfx5nyKK2uIjYpg9sh+zBufwadGp9Er1AO9vgay34Odr3st9dI93qxwIy6GM6+HM+ZCdFywqxQR6TBqictxGnyOVftKeXVTHos351NUUUN8TCRXnd2fm6cOZtyAMBjdDt555xuehU0LoTIfYpNg7KfhrBtg0LkQEQYT4YiInMBphbiZVdDyHOkG9HDOdXqTTSEeGA0+x5rsQyxcc4BFGw5SXefjrIFJ3DQ1kyvO6h/ax88b+RpgzzLYuMCb4rWuCnpnwvjPeoGeMiLYFYqInJYOb4kHi0I88MqO1vHi2hyeXrmfXYWVJMRGcc05A7hp6mBGpicEu7y2qamE7a/Axme9YHc+6H+O190+8lLoMzjYFYqItJlCXE6Zc47V2Yd4+oNsXt2UT22Dj8lZfbhpaiaXjssgLjrET1drVJ4HmxfChuegYJO3LGUkjLjIu2VO1/ztIhLSFOLSLqVVtSxcc4BnVu5nX8kResdHc93Egdw+YwgDevcIdnltV7zr2Cj37PehoRZiesHQ2TD8Qi/Ukwae+nYb/x3pHHYRCQCFuHQIn8+xYk8Jz6zcz5It+QBcdfYAvnT+UEakhUlXe6OaSti7HHa/AbvegLID3vLUMV6YD7/QC/ijpXCk1Jt85uP7kuOX90yB0VfC2Kth4GQNqBORDqMQlw6Xe/gof3lnD89+eICjdQ1cODqVu2YPY+LgMJwm1Tko2u6F+a7XYf8K8LV0pV2DHn0gPtl/6+vdevSFkt3eRV0aaiFxAIy5ygv0AZMU6CLSLgpxCZhDVbU8sWIfT7y/j0NH6pic1Ye7Zg/jgpGpWLh2L1eXe93tzucPan9oxyVBxAnGAlSXwY7XYOtLzQL90/4W+iR1uYvIKVOIS8Adqa1nwaoD/PmdveQePsrItAT+4/yhXHFWf6Iju2FLtDHQt7wIH/3bH+gDvXPYR18JAyZCZBicuiciQacQl05T1+Dj5Y0HeXjZHnYUVDCgdw++eN4QbpySGT4j2jtadRnsWAxbXjoW6LGJMHiGNwf8kFnesXh1u4tICxTi0umccyzdUcjDy/bw4b5ShqT05H+uHs+0Yd18jvPqMtj9b29Q3d7lUPqRtzw+BYacB0PO90K971B1vYsIoBCXIHt3VzE/eGkT2SVHuH7SIP5z3ih6x+vcbAAOH4B978Cet2Hv21CR5y1PGuSFedZM6DMEeqV6t5heCneRbkYhLkFXXdfAb9/cxZ/f2UOf+GjuvWIsl5+ZEb6D3wLBOSj5CPYu87fU3/FOZWsqOt4L856px4K9V9qxZcnDIHmEjreLdCEKcQkZWw6W8Z8vbGJjThlzRqXy358eF14TxnQmnw+Kd0LFQags9N8KvPuqJs+PlHzyfVE9IG0sZJwJ6WdCxlneMXdd3U0kLCnEJaQ0+ByPv7+P//f6DgC+ffFIbpueRWSEWuWnpaEOqoq9q7gV7YC8jZC/0buvKfPWiYjyppvNOOtYuKePh7jE4NYuIielEJeQlHPoCD98aTPLdhRx1qDe/OKa8YzOUKh0GOfg0D5/oG/wQj1vg9eKb9R3qBfs6Wd64Z5xtjf7nIiEDIW4hCznHIs2HOSn/9pK2dE67pw1lHs+NaL7no7WGSry/a31DcfC/XD2sdcT+h9rsTcGfNJADagTCRKFuIS8Q1W1/OzVbSxck8OQlJ788jNnMjkrDKdwDVdHD0H+pk+22Et2ebPWgTfdbL/RkDrKu+83ElJHQ89+CneRAFOIS9h4b3cx33t+I7mHj/K5cwfz3bmj6BmrkdZBUVsFBVu8QM/f5B1vL9rmneveqDHcG0O930jv2LtFQF2Vt43aI1BbCXVH/M/9tzr/8uieXks/aaB3al3SAIjWYEeRRgpxCStVNfU8sGQHT6zYx4DePfjFNWcyc4SO04YE57wR8YXbjoV64fbjw72tonpAfTXQ7P+i+BToPahJsPtDvu9Q70dDME6hq6v2Bg9W5HvfQ9JASMjQ6XwScApxCUur9pXyvYUb2VNcxQ2TB/Ffl40mMS462GVJSxrDvWi7d912M29imuh4iOl57BYd7y2PifceR0R6o+vLD3qXgy3LaXLvvx0+4LXqG8X08uaeHzTVuw2cBD16n37tPp8XzuV53mQ7H9/yP3l/9NDx77UI6JXu9R4kDfQueNP0Pmmg94NEU+pKOyjEJWxV1zXwmzd38ufle0hNiONnV4/jU6PTgl2WdCbnvAAty/Fa/wdWereCzf5j9gb9RsGgKV6oZ577yWlrG99/aJ83gO9Q9ifvDx+AhppPfqZFepPoJKR7re2m94kZXsdBeQ6U5Xp1NT4uz/X3LDQRkwDDPwUjL4URF3tXxhM5BQpxCXsbDhzmuws3sqOggqsnDODHl4+hT09N3dqt1VRC7ho48KEX6jkfHuvSj0/2zoOvKvGCuqb8k++N6w19BkPvwcfuG7vHEzK80+xOdNnZ1jjnTb5TluMFelmON55g1+teT4VFwKBzYeRcGDkPUka0/3uQ4Kuv9f+9D3g/5s66oUMHfCrEpUuorffx0NLdPLR0N73jo/nvq8Zx6fiMYJcloaJxhrsDH3jBXrjVm4q2eVj3GexdG76za8tb512edsdiKNjkLe87zGuhj7zUC/eOPr7eUAfV5VB92PuBEd/X+wHTWd37vgbvh9WRUq835Ogh74dMz2Tvh1Z8sndYpT2B11Dv/UirLvP2t6EWfHX+x43P6737xmW+OoiI9mYxjOrhDaT8+HGcV1NUnLc8MsbrXTl8AMr2e/eH93uBffiAd19+kE+M6/ju3g7tcVGIS5ey9WA5331+A5tzy7lkbBo/vGwMg/rGB7sskbY7fAB2vubd9i73AiauNwyb4wVbRKTXpR8R4YWeRTZZFumFnkV64XL0sBdg1WVeWH/8uMwb/d+cRUCPvl5vQ3yyFzaNgRrvXxaXBK7BH3z1x0LRV9fsuT8cq8uOhfTHgV3atsGOUXHN6khpEvBx/h8hZcffGoO7pX3sSBZx7FTLRhFRkNjf34MzyD8IcxD0zvQe9x58ej05rZWgEJeupr7BxyPv7OH//r2bBp9j/swsvnLBcA18k/BTUwEfLfUCfc/b3iA+5/Na767Ba8023jcfxY95U+fGJflvvZvdN7lZhNfV/4lbKRwpPvbYNZzePsQleacb9ujr3cf775svi+vt7dtxdbRwa/wBYBHH70tcEsQ2f54AUbEQGe21niOi/Y8bn0d595HR3mNfPdQd9X4I1R31Pz7qnYVQf7TJsmpvu0mNAZ3pHXLpwJA+GYW4dFn5ZdX86vUdPL82hz7xMXzjojO4cfIgoiI1Gli6IOf8Ae8P9sjYjusa9/m8ufarSrz7iKhjQRgR5b9v4Xljz0BHa6jzAlSX31WIS9e3ObeM/355Kyv3ljI8tRc/uGw0s8/op0udikjYay3E1VSRLmPcgCSevfNcHrl1IvUNPuY/torPPfoh2/PLT/5mEZEwpBCXLsXMuHhsOq9/43x+fPkYNuaUMe937/CfL2yiqKLm5BsQEQkjCnHpkmKiIvj8zCG8/Z3Z3DY9i3+sPsDsB5by0NLdHKmtD3Z5IiIdImAhbmaPmlmhmW1u5fXZZlZmZuv9tx8HqhbpvnrHx3DvFWN5/RuzmD48hQeW7GDm/y7lD8t2U1mjMBeR8BawgW1mNguoBJ50zo1r4fXZwLedc5efynY1sE3aY012Kb//927e3llE7/hovjBjCLfNyNJpaSIS0jp9YJtzbjlQGqjti5yOiYP78sTnp/DSV2YwaXAf/t8bO5nxi7f4zRs7KTtSF+zyREROSbCPiU8zsw1mttjMxga5FulGzh7Um7/cNpmX757J9GHJ/O7fu5jxv2/xqyU7OFRVG+zyRETaJKDniZtZFvByK93piYDPOVdpZvOA3znnWrwSgJndCdwJkJmZOTE7OztgNUv3tC2vnAff2s2rm/PoER3JrdMGc8d5Q0npFRvs0kREgjPZy4lCvIV19wGTnHPFJ1pPx8QlkHYWVPDgW7t5eeNBYqIiuGFyJl+YOUTzsotIUIXcZC9mlm7+qbTMbIq/lpJg1SMCcEZaAr+/cQJvfPN8Lhvfn799kM3sXy3j7r+vY3NuGy7kICLSiQI5Ov3vwGwgBSgA7gWiAZxzD5vZV4G7gHrgKPBN59z7J9uuWuLSmfLKjvLYe/t4ZuV+KmvqmT4smTtmDdV0riLSqTR3ukg7lFfX8feV+3n0vb0UlNcwMi2BO2YN5cqz+hMTFezxoSLS1SnERTpAbb2PRRsO8ufle9hRUEF6YhzzZ2Rx49RMnWsuIgGjEBfpQM45lu0s4pG397BiTwkJsVHcNDWTL5w3hNSEuGCXJyJdjEJcJEA25ZTxp+Uf8eqmPKIjI7hxSiZ3zhpK/949gl2aiHQRCnGRANtbXMUfl+3mhbW5mMFnJg7krvOHk5ms09NEpH0U4iKdJOfQEf709h6eW32ABp/jqrP68+ULhjE8NSHYpYlImFKIi3SywvJq/vzOHv72wX6q6xu4dFw6X7lgOGP7JwW7NBEJMwpxkSAprarl0Xf38sT7+6ioqedTo1L5ypzhnJPZJ9iliUiYUIiLBFnZ0TqeWrGPv767l0NH6pg5PIWvzhnOuUOTg12aiIQ4hbhIiKiqqefpldk8snwvxZU1TMnqy92fGs7M4SmaBU5EWqQQFwkx1XUNPPvhfh5+ew/55dWcPag3d88ZzpxRqQpzEfkEhbhIiKqpb+D5Nbn8Ydlucg4dZUxGInfPGc4lY9OJiFCYi4hCXCTk1TX4+Of6g/xh6W72FFcxIrUXX50znMvGZxAVqfnZRbozhbhImGjwOV7ZlMeDb+1iZ0ElWcnx/Mf5w7h6wgDioiODXZ6IBIFCXCTM+HyO17cW8NDS3WzKLaNfQizzZ2Rx89TBJPXQxVZEuhOFuEiYcs6x4qMSHl6+h+U7i+gZE8lNUzP5/MwhZCRpfnaR7kAhLtIFbDlYxiPL9/DyxjwiDK46ewB3zhrKGWma0lWkK1OIi3QhB0qP8Nd39/Lsqv1U1/n41KhU/uP8YUzO6qPT00S6IIW4SBdUWlXLUyuyeWLFPkqrajknszd3nDeUi8akaUS7SBeiEBfpwo7WNrBwzQEeeWcPB0qP0j8pjlunZXHD5EH06RkT7PJEpJ0U4iLdQIPP8e9tBTz+/j7e/6iE2KgIrp4wgNumZzE6IzHY5YnIaVKIi3QzO/IrePz9fby4LofqOh9Th/Rl/owsLhytrnaRcKMQF+mmDh+p5blVB3hyRTa5h48yoHcPbp02mBsmD6J3vLraRcKBQlykm6tv8PHmtkIef38vH+wpJS7a62q/ffoQRqbrFDWRUKYQF5GPbcsr54n39/Hiulxq6n3MGJ7M52cM4YKRqbroikgIUoiLyHFKq2r5+4f7eWpFNvnl1WQlx3Pb9CyumzSIXrFRwS5PRPwU4iLSqroGH4s35/PYe3tZt/8wCbFRXDdpELdPzyIzOT7Y5Yl0ewpxEWmT9QcO89h7e3llYx4NznHh6DTmz8hi2tBkzQYnEiQKcRE5JQXl1Ty1IptnPtxPaVUto9ITmD8ji6vO1iVRRTqbQlxETkt1XQOL1h/k0ff2sj2/gj7x0dw0NZNbzh2sq6iJdBKFuIi0i3OOD/aU8th7e3ljWwGRZswdl878GUM4J7O3utpFAqi1ENfwUxFpEzNj2rBkpg1L5kDpEZ54fx/PrT7AyxvzOGtgEvNnDGHe+AxiojQbnEhnUUtcRE5bVU09z6/N4fH39rGnuIp+CbHcMnUwN03NpF9CbLDLE+ky1J0uIgHj8zmW7yrisff28fbOImIiI7jszAxuOXewutpFOoC600UkYCIijNkjU5k9MpXdhZU8uWIfL6zN5cV1uYzJSOTWaYO56uz+xMfovxyRjqSWuIgERFVNPS+tz+WpFdlsz68gITaKaycO5JZzMxmeqrnaRU6FutNFJCicc6zdf4inVmTz6qZ8aht8nDu0L7eem8XFY9OI1mVRRU5KIS4iQVdSWcOC1Tk8vTKbnENHSU2I5YbJg7hhSib9e+ucc5HWKMRFJGQ0+BzLdxbx1AfZLN1RCMD0Yclce85A5o5L17FzkWYU4iISkg6UHuH5tTm8sDaX/aVH6BkTybzxGVw7cSBTsvrq0qgiKMRFJMQ551i17xDPr8nhlU15VNbUM7BPD645ZyDXnjOAwck9g12iSNAoxEUkbBytbWDJlnyeX5vDu7uLcQ4mZ/Xh2nMGMu/MDBLjooNdokinUoiLSFjKKzvKi+tyeX5NDh8VVREXHcHcselcN2kQ04Ymq7tdugWFuIiENeccG3LKWLjmAIvWH6S8up4BvXtw7cSBXDdxIIP6xge7RJGAUYiLSJdRXdfA61sL+MfqAx93t587tC/XTRzEpeM1ul26HoW4iHRJuYeP8sKaHBauzSG75Ai9YqO4bHwG100ayMTBfTRvu3QJCnER6dKcc3y4t5R/rMnh1U15HKltYGhKT26ckslnJg6kT8+YYJcocto6PcTN7FHgcqDQOTeuhdcN+B0wDzgC3O6cW3uy7SrEReRkqmrqeWVTHgtWHWB19iFioiK43H9VtQmDdFU1CT/BCPFZQCXwZCshPg+4Gy/EpwK/c85NPdl2FeIiciq25ZXz9MpsXlybS1VtA2MyErnlXO+qaj1jdexcwkNQutPNLAt4uZUQ/xOwzDn3d//zHcBs51zeibapEBeR01FZU89L63L52wfeVdV6xUZxzTkDuHnqYEam66pqEtpC8XriA4ADTZ7n+JedMMRFRE5Hr9gobjl3MDdPzWTt/sM8/UE2z646wJMrspmc1Ydbzh3M3HHpxEZFBrtUkTYLi74kM7sTuBMgMzMzyNWISDgzMyYO7sPEwX344eVjWLjmAE+v3M/Xnl1Pn/hoPjNxIDdMyWRYv17BLlXkpNSdLiLdns/neHd3Mc+u2s/rWwqo9zmmDOnLTVMymTsunbhotc4luEKxO30R8FUzexZvYFvZyQJcRCQQIiKMWWf0Y9YZ/SiqqGHhmhyeXbWfrz+3nqRF0VxzzgBunJLJGWk6di6hJZCj0/8OzAZSgALgXiAawDn3sP8UsweBuXinmM13zp20ia2WuIh0Bp/P8cGeEp75cD9LtuRT1+CYNLgPN07JZN74DHrEqHUunUeTvYiInKaSyhqeX5vDsx8eYE9xFQlxUVw9YQDXTRzEuAGJOu9cAk4hLiLSTs45Vu4t5e8f7mfx5nxq632MSk/gukmD+PTZ/UnuFRvsEqWLUoiLiHSgsiN1LNp4kIWrD7Ahp4zoSGPOqFQ+O2kQ55/Rj6jIiGCXKF2IQlxEJEB25Ffwj9UHeHFdLiVVtfRLiOWaCQO4btJAhqdqMJy0n0JcRCTA6hp8vLW9kH+szmHpjkIafI4Jmb35zMSBzBuXoYuwyGlTiIuIdKLCimpeWpfLP1bnsKuwkqgIY+aIFC4/sz8Xj00jMS462CVKGFGIi4gEgXOOLQfL+dfGg7y8IY/cw0eJiYzg/JH9uPzMDC4cnaYLschJKcRFRILMOce6A4f514aDvLopj4LyGuKiI/jUqDQuPzODC0alanY4aZFCXEQkhPh8jlX7SvnXxoMs3pRPSVUtPWMiuWhMGpef2Z/zzkjRxVjkYwpxEZEQVd/g44M9pby88SCLN+dTdrSOhNgoLhqbxhVn9mfG8BRionTKWnemEBcRCQO19T7e+6iYVzbmsWRLPhXV9STGRXHJ2HQuOzODGcNTiNY56N2OQlxEJMzU1Dfw7i4v0F/fWkBlTT2946OZ6w/0aUOTNalMN6EQFxEJY9V1Dbyzq5iXNx7kza0FVNU20LdnDJeMTWPuOC/Q1eXedSnERUS6iOq6BpbtKOLljQdZur2QqtoGEuOiuHB0GnPHpTPrjH4a5d7FKMRFRLqg6jqvy33x5nze3FZA2dE64mMiuWBUKnPHpnPBqFR66Tz0sNdaiOsvKyISxuKiI7lwTBoXjkmjrsHHio9KeG1LPq9vyeeVjXnEREUwa0Q/Lh2XzoWj00iK10xxXYla4iIiXVCDz7F6XymvbclnyeZ8DpZVExlhTMnqy0Vj0rhoTBqD+sYHu0xpI3Wni4h0U845NuSU8foWr8t9Z0ElAKPSE7hwtBfo4wckERFhQa5UWqMQFxERALJLqnhjawFvbC1g1b5SfA7SEmP51Og0LhqdxrRhyRoYF2IU4iIicpxDVbUs21nIG1sLeHtHEVW1DcTHRHLeiBRmjujH9GHJDE3piZla6cGkEBcRkROqqW9gxUclvLmtgKXbi8g9fBSA9MQ4pg9LZtqwZKYPT2FA7x5BrrT70eh0ERE5odioSGaPTGX2yFScc+wvPcJ7u0t4/6Ni3t5ZxAvrcgEYnBzP9GHJTB+WwrRhyaT0ig1y5d2XWuIiInJSzjl2FlTy3u5i3v+ohJV7SqioqQdgZFoCc0ancum4dMYPSFLXewCoO11ERDpMfYOPLQfLee+jYt7dVczKvaU0+Bz9k+K4ZFw6l47LYOLgPkRqxHuHUIiLiEjAHD5Sy5vbCnltcx7LdxVTW+8jpVcsF49NY+7YdKYNS9bV19pBIS4iIp2isqaeZTsKWbw5n6XbCzmiud3bTSEuIiKd7kRzu88bl8EFo/oRH6Mx1iej0ekiItLpms/t/sGeEl7bnM8S/9zucdERzD4jlUvHpzNnVCoJcZrb/VSoJS4iIp2uwedYta+UxZvyWLw5n8KKmk9erGVMGkk9FOiN1J0uIiIhyedzrN1/iFc35bN4cx55ZdVERxozhqdw6bh0LhqTTt+eMcEuM6gU4iIiEvJ8PseGnMMs3pzPq5vyyDl0lAiDqUOSmTsunYvHppGR1P1mjFOIi4hIWHHOseVgOa9tzue1LfnsLvSuvnb2oN7MHZfO3LHpZKX0DHKVnUMhLiIiYW13YQVLthTw2uZ8NuWWAd7lVC8Zm86l49MZmZbQZWeLU4iLiEiXkXPoCEu2FLBkcz6rsktxDrKS47l4bDoXj0ljQmbXmi1OIS4iIl1SUUUNb2wt4LUt+az4qJi6BkdKrxguHJ3GJf7Z4sJ9chmFuIiIdHnl1XUs21HE61vyWbajiMqaenrGeFdnu3hsGrNHpoblqWsKcRER6VYar4/++tYC3thaQFFFDVERxrRhyVw8Jo1PjU6jf5hcG10hLiIi3ZbP51h34DCvb83n9S0F7C2uAryBcReMSuWCkamck9mbqBC9SItCXEREBO/UtY+KKnlreyFvbS9k9b5D1PscST2imXVGPy4Y2Y/ZI1NDaoIZhbiIiEgLyqvreGdnMUt3FLJsRyHFlbWYeeejzxmZygWjUhnbPzGop68pxEVERE7C53Nsyi1j6Y5Clm4vZEOOdz56akIs55/RjwtGpTJjeEqnD45TiIuIiJyioooalu0oZNnOIt7ZWUR5dT2REcbEzD6cP7IfF4xMZXRG4CeZUYiLiIi0Q32Dj3UHDnuhvqOILQfLAUhL9LfSR6YyY0QKiQG4nKpCXEREpAMVllezbGcRb+8oYvmuIiqq64mKMM4Z3IcHb5xAamJch31WayEe1WGfICIi0o2kJsbx2UmD+OykQdQ3+Fi732ulr8k+RHKv2E6pQSEuIiLSTlGREUwZ0pcpQ/p26ueG5lntIiIiclIBDXEzm2tmO8xst5l9v4XXbzezIjNb7799MZD1iIiIdCUB6043s0jgIeAiIAdYZWaLnHNbm636nHPuq4GqQ0REpKsKZEt8CrDbObfHOVcLPAtcFcDPExER6VYCGeIDgANNnuf4lzV3rZltNLOFZjYogPWIiIh0KcEe2PYvIMs5dybwBvBESyuZ2Z1mttrMVhcVFXVqgSIiIqEqkCGeCzRtWQ/0L/uYc67EOVfjf/oXYGJLG3LOPeKcm+Scm9SvX7+AFCsiIhJuAhniq4ARZjbEzGKAG4BFTVcws4wmT68EtgWwHhERkS4lYKPTnXP1ZvZVYAkQCTzqnNtiZj8FVjvnFgH3mNmVQD1QCtweqHpERES6Gs2dLiIiEuJamzs92APbRERE5DQpxEVERMKUQlxERCRMhd0xcTMrArI7cJMpQHEHbi8UdfV97Or7B11/H7V/4a+r72Ow92+wc+64c6zDLsQ7mpmtbmmwQFfS1fexq+8fdP191P6Fv66+j6G6f+pOFxERCVMKcRERkTClEIdHgl1AJ+jq+9jV9w+6/j5q/8JfV9/HkNy/bn9MXEREJFypJS4iIhKmunWIm9lcM9thZrvN7PvBrqejmdk+M9tkZuvNrEvMVWtmj5pZoZltbrKsr5m9YWa7/Pd9gllje7Syf/eZWa7/77jezOYFs8b2MLNBZrbUzLaa2RYz+5p/eVf6G7a2j13i72hmcWb2oZlt8O/fT/zLh5jZSv//p8/5L3wVdk6wf4+b2d4mf7+zg10rdOPudDOLBHYCFwE5eFddu9E5tzWohXUgM9sHTHLOdZlzN81sFlAJPOmcG+df9kug1Dn3C/+PsT7Oue8Fs87T1cr+3QdUOud+FczaOoL/yoUZzrm1ZpYA/7+9uwuxqgrDOP5/GC1EIfsQCU2kEoLKTCoovBAvoq4sCk0KJIJSKowgom6CyJugD6wwkhIvLJHUki5C0eiDoqIarbCLEqFk/LqwEqIPfbrYa2QzeMaymc7sfZ4fDLP3OmeG9fIy+z1rrzV78QVwC9XmR23JYacYF9GCPEoSMNH2MUnjgY+AFcDDwGbbGyS9vm1e4wAABCZJREFUDOyyvbqbfT0Tw8S3DHjH9ptd7eAQvTwSvw743vZe238AG4CFXe5TnIbtD6h2vKtbCKwrx+uoLpiN1CG+1rA9YPvLcvwr1fbD02hXDjvF2AquHCun48uXgQXAYIFrbA6HiW9M6uUiPg34sXb+Ey36QysMbJP0haR7u92ZUTTV9kA5PgBM7WZnRskDknaX2+2NvdVcJ2kmcDXwKS3N4ZAYoSV5lNQnqR84BGwHfgCO2v6rvKXR19Oh8dkezN/Kkr/nJJ3dxS6e1MtFvBfMsz0XuBm4v9yqbTVX80Nj9lPzGVoNXALMAQaAZ7rbnf9O0iRgE/CQ7V/qr7Ulh6eIsTV5tH3c9hxgOtVdzcu63KURNTQ+SVcAj1HFeS1wHjAmpnt6uYjvBy6qnU8vba1he3/5fgjYQvXH1kYHyzzk4HzkoS73Z0TZPlguKieANTQ8j2WecROw3vbm0tyqHJ4qxrblEcD2UeA94HpgsqRx5aVWXE9r8d1Upkls+3dgLWMkf71cxD8HZpUVlWcBdwBbu9ynESNpYllUg6SJwI3AN8P/VGNtBZaW46XA213sy4gbLG7FrTQ4j2XR0KvAHtvP1l5qTQ47xdiWPEqaImlyOZ5AtTh4D1Wxu728rbE57BDfd7UPmaKa7x8T+evZ1ekA5V88ngf6gNdsr+xyl0aMpIupRt8A44DX2xCfpDeA+VQ7Ch0EngDeAjYCM6h2uFtku5GLwzrEN5/qFqyBfcB9tfnjRpE0D/gQ+Bo4UZofp5ozbksOO8W4hBbkUdJsqoVrfVQDwY22nyzXnA1Ut5q/Au4qo9ZGGSa+ncAUQEA/sKy2AK5rerqIR0RENFkv306PiIhotBTxiIiIhkoRj4iIaKgU8YiIiIZKEY+IiGioFPGIHiPpeG0npn6N4A5+kmaqtgNbRIyucad/S0S0zG/lkZIR0XAZiUcEcHL/+adV7UH/maRLS/tMSTvLxg87JM0o7VMlbSn7Lu+SdEP5VX2S1pS9mLeVp15FxChIEY/oPROG3E5fXHvtZ9tXAi9SPc0Q4AVgne3ZwHpgVWlfBbxv+ypgLvBtaZ8FvGT7cuAocNsoxxPRs/LEtogeI+mY7UmnaN8HLLC9t2zgccD2+ZKOABfa/rO0D9i+QNJhYHr90Zpl683ttmeV80eB8bafGv3IInpPRuIRUecOx/9G/XnZx8nam4hRkyIeEXWLa98/KccfU+3yB3An1eYeADuA5QCS+iSd8391MiIq+YQc0XsmSOqvnb9re/DfzM6VtJtqNL2ktD0IrJX0CHAYuLu0rwBekXQP1Yh7OdC4Xbkimixz4hEBnJwTv8b2kW73JSL+mdxOj4iIaKiMxCMiIhoqI/GIiIiGShGPiIhoqBTxiIiIhkoRj4iIaKgU8YiIiIZKEY+IiGiovwGDOtgfdhzKmAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 576x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fbrY-d6jNMIZ"
      },
      "source": [
        "# prepare data for translation\n",
        "def prepare_data(sent):\n",
        "    sent = preprocess_english(sent)\n",
        "    sent_tokens = english_tokenizer.texts_to_sequences([sent])\n",
        "    sent_padded = pad_sequences(sent_tokens, maxlen=english_maxlen, padding='post')\n",
        "    return sent_padded\n",
        "\n",
        "# remove start_, end_ tokens\n",
        "def remove_tokens(sentence):\n",
        "  # sometime predicted texts contains UNK after end_ tokens\n",
        "  sentence = sentence[:sentence.find('end_')].replace('start_', '').strip()\n",
        "  return sentence\n",
        "\n",
        "# translate english to nepali\n",
        "def translate(eng_sent, model):\n",
        "  sent = prepare_data(eng_sent)\n",
        "  preds = model.predict(sent)\n",
        "  translation = nepali_tokenizer.sequences_to_texts(preds.argmax(-1))[0]\n",
        "  translation = remove_tokens(translation)\n",
        "  return translation\n",
        "\n",
        "# create a dataframe\n",
        "def create_df(english, nepali, predicted):\n",
        "  df = pd.DataFrame()\n",
        "  df['enlgish'] = english\n",
        "  df['nepali'] = nepali\n",
        "  df['predicted'] = predicted\n",
        "  return df"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qh4BW-CuYFpg"
      },
      "source": [
        "def predict_and_save_as_dataframe(model, save_path, translate_fn, X_test=X_test, y_test=y_test):\n",
        "  \n",
        "  english, nepali, predicted = [], [], []\n",
        "  for i, sent in enumerate(X_test.values):\n",
        "    # translate english to nepali\n",
        "    translation = translate_fn(sent, model)\n",
        "    # remove start and end tokens\n",
        "    eng = remove_tokens(sent)\n",
        "    nep = remove_tokens(y_test.values[i])\n",
        "\n",
        "    # store sentences to create a dataframe\n",
        "    english.append(eng)\n",
        "    nepali.append(nep)\n",
        "    predicted.append(translation)\n",
        "\n",
        "  # save results to a dataframe\n",
        "  df = create_df(english, nepali, predicted)\n",
        "  # save dataframe to disk\n",
        "  df.to_csv(save_path, index=False)\n",
        "  return df\n",
        "\n",
        "# create a dataframe from above lists\n",
        "save_path = os.path.join(prediction_dir, 'model_1_predictions.csv')\n",
        "model_1_predictions = predict_and_save_as_dataframe(model=model_1, save_path=save_path,\n",
        "                                                    translate_fn=translate)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "P0O3Xqq2ivhZ",
        "outputId": "06031297-a734-44b0-d1cc-fe18140638c8"
      },
      "source": [
        "# display some predictions\n",
        "model_1_predictions.sample(5)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>enlgish</th>\n",
              "      <th>nepali</th>\n",
              "      <th>predicted</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>683</th>\n",
              "      <td>i mean that</td>\n",
              "      <td>मेरो मतलब त्यो हो</td>\n",
              "      <td>मलाई मेरो काम काम गर्दछ</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>229</th>\n",
              "      <td>he is skinny</td>\n",
              "      <td>ऊ पातलो छ</td>\n",
              "      <td>उनीसँग पहिले दु खी छ</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>484</th>\n",
              "      <td>nothing happened</td>\n",
              "      <td>केहि पनि थिएन</td>\n",
              "      <td>यहाँ लागि काम पर्छ</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>347</th>\n",
              "      <td>tom was quick</td>\n",
              "      <td>टम छिटो थियो</td>\n",
              "      <td>हामी लागि काम काम</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>796</th>\n",
              "      <td>i sell cars</td>\n",
              "      <td>म कारहरू बेच्छु</td>\n",
              "      <td>मलाई यो प्रयास मन पर्छ</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              enlgish             nepali                predicted\n",
              "683       i mean that  मेरो मतलब त्यो हो  मलाई मेरो काम काम गर्दछ\n",
              "229      he is skinny          ऊ पातलो छ     उनीसँग पहिले दु खी छ\n",
              "484  nothing happened      केहि पनि थिएन       यहाँ लागि काम पर्छ\n",
              "347     tom was quick       टम छिटो थियो        हामी लागि काम काम\n",
              "796       i sell cars    म कारहरू बेच्छु   मलाई यो प्रयास मन पर्छ"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g72Dls4MeX-p"
      },
      "source": [
        "## With Teacher Forcing Method"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6SJvi0Bhmy-B"
      },
      "source": [
        "### Build the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y7134N3HeXF5",
        "outputId": "b585da92-f9c8-427b-8d3d-f4edac07b033"
      },
      "source": [
        "# define embedding size\n",
        "src_embed = 256\n",
        "target_embed = 256\n",
        "units = 256\n",
        "\n",
        "# define inputs\n",
        "encoder_inputs = Input(shape=[None], dtype=tf.int32)\n",
        "decoder_inputs = Input(shape=[None], dtype=tf.int32)\n",
        "\n",
        "# encoder embeddings\n",
        "encoder_embeddings = Embedding(english_vocabsize+1, src_embed, mask_zero=True)(encoder_inputs)\n",
        "# define encoder lstm layers\n",
        "encoder_lstm_layer1 = LSTM(units, return_sequences=True, dropout=0.4)\n",
        "encoder_lstm_layer2 = LSTM(units, return_state=True, dropout=0.4)\n",
        "# get encoder states\n",
        "layer1_out = encoder_lstm_layer1(encoder_embeddings)\n",
        "_, en_state_h, en_state_c = encoder_lstm_layer2(layer1_out)\n",
        "encoder_states = [en_state_h, en_state_c]\n",
        "\n",
        "# Decoder embeddings\n",
        "decoder_embeddings = Embedding(nepali_vocabsize+1, target_embed, mask_zero=True)(decoder_inputs)\n",
        "decoder_lstm_layer = LSTM(units, return_sequences=True)\n",
        "decoder_lstm_output = decoder_lstm_layer(decoder_embeddings, initial_state=encoder_states)\n",
        "# get decoder final output\n",
        "decoder_output = Dense(nepali_vocabsize+1, activation=\"softmax\")(decoder_lstm_output)\n",
        "\n",
        "# Create a model\n",
        "model_2 = Model(inputs=[encoder_inputs, decoder_inputs], outputs=[decoder_output])\n",
        "\n",
        "# compile model\n",
        "model_2.compile(loss='sparse_categorical_crossentropy', optimizer='adam')\n",
        "# display model summary\n",
        "model_2.summary()"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_2 (InputLayer)            [(None, None)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_1 (Embedding)         (None, None, 256)    723456      input_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "input_3 (InputLayer)            [(None, None)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "lstm_2 (LSTM)                   (None, None, 256)    525312      embedding_1[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "embedding_2 (Embedding)         (None, None, 256)    1114624     input_3[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lstm_3 (LSTM)                   [(None, 256), (None, 525312      lstm_2[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "lstm_4 (LSTM)                   (None, None, 256)    525312      embedding_2[0][0]                \n",
            "                                                                 lstm_3[0][1]                     \n",
            "                                                                 lstm_3[0][2]                     \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, None, 4354)   1118978     lstm_4[0][0]                     \n",
            "==================================================================================================\n",
            "Total params: 4,532,994\n",
            "Trainable params: 4,532,994\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 564
        },
        "id": "9IlDyPxsmW0F",
        "outputId": "8bb74cb0-624f-442b-c080-44471f7394c9"
      },
      "source": [
        "# plot model\n",
        "plot_model(model_2)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAb0AAAIjCAYAAACXqsXqAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdeXiM994/8PckM8ksMhFEgkgQ1Qi6qVatRbW2IglJrOVUbfWgVVJL1aXlUFrakPZnOY5uWYhSauuxa1FVyqE0qK1BiEgiiWQSn98ffcxjGksiM3Mnc79f15U/cs93vvfn3uY99zoaEREQERGpgJvSBRARETkLQ4+IiFSDoUdERKrB0CMiItXQKjXi3r17KzVqIrt57rnn8OabbypdBhGVkGKht3LlSjRv3hwBAQFKlUBUJnv37lW6BCIqJcVCDwDeeOMNREZGKlkC0UPj0Qqiiofn9IiISDUYekREpBoMPSIiUg2GHhERqQZDj4iIVIOhR0REqsHQIyIi1WDoERGRajD0iIhINRh6RESkGgw9IiJSDYYeERGpBkOPiIhUg6FHRESqUWFCb/369fD29sbatWuVLqVMpk+fjtDQUJjNZnh6eqJ+/fqYMGECbty4Ueq+9u7di4YNG8LNzQ0ajQZ+fn54//33HVD1w0tOTka9evWg0Wig0Wjg7++P/v37K10WEamUor+nVxoionQJdrF161aMGjUK0dHR0Ol02LBhA/r3748jR45gw4YNpeqrefPm+O2339CpUyds2rQJJ06cQOXKlR1U+cOJiIhAREQE6tevj6tXr+LSpUtKl0REKlZh9vS6du2KzMxMvPzyy0qXgry8PLRo0eKh3lupUiUMGzYMVapUgZeXFyIjIxEWFoaNGzfi/Pnzdq7U+coyb4iIHK3C7OmVJ0uXLkVaWtpDvXfdunXFhlWrVg0AkJubW6a6yoOyzBsiIkerEHt6u3fvRmBgIDQaDRYsWAAAiIuLg8lkgtFoxJo1a9C5c2eYzWYEBAQgPj7e+t5PPvkEer0e1atXx/Dhw1GjRg3o9Xq0aNEC+/bts7YbPXo0PDw84O/vbx32+uuvw2QyQaPR4OrVqwCAsWPHYty4cTh16hQ0Gg3q169f5un7888/YTAYULduXeuwjRs3wmw2Y8aMGaXur6LPm127diE0NBTe3t7Q6/Vo0qQJNm3aBAAYMmSI9fxgcHAwDh48CAAYPHgwjEYjvL298e233wIAioqKMHXqVAQGBsJgMOCxxx5DYmIiAOCDDz6A0WiEl5cX0tLSMG7cONSqVQsnTpx4qJqJqIIQhQCQxMTEErc/f/68AJDY2FjrsMmTJwsA2bJli2RmZkpaWpq0bt1aTCaTFBQUWNsNGzZMTCaTHDt2TG7evClHjx6VZs2aiZeXl5w7d87arl+/fuLn52cz3jlz5ggAuXLlinVYRESEBAcHP8xkF5OTkyNeXl4yevRom+Hr1q0TLy8vmT59+gP7eOmllwSAZGRkWIeVt3kTHBws3t7eD54hIrJixQqZNm2aXLt2TdLT06V58+ZStWpVm3G4u7vLn3/+afO+vn37yrfffmv9/6233hJPT09ZuXKlZGRkyKRJk8TNzU32799vM4/GjBkjsbGxEh4eLr/99luJahQR6dWrl/Tq1avE7YlIeRViT+9BWrRoAbPZDF9fX0RHRyMnJwfnzp2zaaPVatGwYUN4enoiNDQUcXFxyM7OxrJlyxSq+i8zZ85EjRo1il112bVrV2RlZeGdd94pU/8Vcd706tUL7777Lnx8fFClShV0794d6enpuHLlCgBgxIgRKCoqsqkvKysL+/fvR5cuXQAAN2/eRFxcHMLCwhAREYHKlStjypQp0Ol0xaZr1qxZGDVqFJKTkxESEuK8CSUip3OJ0LuTh4cHAMBisdy33dNPPw2j0Yjjx487o6y7WrVqFZKSkrBp0yZ4eXk5fHwVad7cSafTAfjrcCUAtG/fHg0aNMC//vUv61W9CQkJiI6Ohru7OwDgxIkTyM3NRePGja39GAwG+Pv7l5vpIiLnc7nQKw1PT0/r3oOzJSQkYNasWdi+fTvq1KmjSA33o+S8+e677/D888/D19cXnp6emDBhgs3rGo0Gw4cPx+nTp7FlyxYAwOeff45XX33V2iYnJwcAMGXKFOs5QI1Gg7Nnz7rEBUNE9HBUG3oWiwXXr19HQECA08cdGxuLL7/8Elu3bkXNmjWdPv4Hcfa82blzJ+bNmwcAOHfuHMLCwuDv7499+/YhMzMTs2fPLvaeQYMGQa/XY8mSJThx4gTMZjOCgoKsr/v6+gIA5s2bBxGx+duzZ49TpouIyh/V3rKwfft2iAiaN29uHabVah946K8sRARvv/02MjIysHr1ami15XP2O3veHDhwACaTCQBw5MgRWCwWjBw5EvXq1QPw157d3/n4+CAqKgoJCQnw8vLCa6+9ZvN67dq1odfrcejQIYfUTEQVk2r29G7duoWMjAwUFhbi8OHDGDt2LAIDAzFo0CBrm/r16+PatWtYvXo1LBYLrly5grNnzxbrq0qVKkhNTcWZM2eQnZ1d4jA4duwYPvjgAyxevBg6nc7msJtGo8HcuXOtbTds2PDQtyyUllLzxmKx4PLly9i+fbs19AIDAwEA//nPf3Dz5k2kpKTY3D5xpxEjRiA/Px/r1q0r9tACvV6PwYMHIz4+HnFxccjKykJRUREuXLiAixcvlnYWEZGrUOqyUZTiloXY2Fjx9/cXAGI0GqV79+6ycOFCMRqNAkAeeeQROXXqlCxatEjMZrMAkKCgIPn9999F5K/L8nU6ndSqVUu0Wq2YzWbp2bOnnDp1ymY86enp0q5dO9Hr9VK3bl35n//5Hxk/frwAkPr161sv4f/ll18kKChIDAaDtGrVSi5dulSi6Thy5IgAuOffnDlzrG3Xr18vXl5e8v7779+zv71790qjRo3Ezc1NAIi/v7/MmDGjXM2bTz/9VIKDg+873QBk1apV1nHFxMRIlSpVpHLlytK7d29ZsGCBAJDg4GCb2yhERJ588kmZOHHiXedPfn6+xMTESGBgoGi1WvH19ZWIiAg5evSozJ49WwwGgwCQ2rVryxdffFGiZXgn3rJAVPFoRJR5qKVGo0FiYiIiIyMdPq7hw4djxYoVSE9Pd/i4KpqKPm+6du2KBQsW2NzY7yy9e/cGAKxYscLp4yaih6Oaw5u3L3en4irSvLnzcOnhw4eh1+sVCTwiqphUE3qOcvz48WLn5u72Fx0drXSpLiEmJgYpKSn4/fffMXjwYLz33ntKl0REFYjLh96kSZOwbNkyZGZmom7duli5cqVd+w8JCSl2Sfzd/hISEuw6Xntw9LxxBKPRiJCQELzwwguYNm0aQkNDlS6JiCoQVZzTI3IEntMjqnhcfk+PiIjoNoYeERGpBkOPiIhUg6FHRESqwdAjIiLVYOgREZFqMPSIiEg1GHpERKQaDD0iIlINhh4REakGQ4+IiFSDoUdERKrB0CMiItVQ9FcWmjdvjoCAACVGT1Rme/fuRfPmzfkrC0QViGJ7er169WLgOVBqaiq+/fZbpctwac2bN8dzzz2ndBlEVAqK7emRYyUlJSEqKgpcvERE/4fn9IiISDUYekREpBoMPSIiUg2GHhERqQZDj4iIVIOhR0REqsHQIyIi1WDoERGRajD0iIhINRh6RESkGgw9IiJSDYYeERGpBkOPiIhUg6FHRESqwdAjIiLVYOgREZFqMPSIiEg1GHpERKQaDD0iIlINhh4REakGQ4+IiFSDoUdERKrB0CMiItVg6BERkWow9IiISDUYekREpBoMPSIiUg2GHhERqQZDj4iIVIOhR0REqsHQIyIi1WDoERGRajD0iIhINTQiIkoXQWXz559/4uWXX4bFYrEOy8nJwZUrV1CnTh2btk888QS++OILJ1dIRFQ+aJUugMquVq1auHnzJn777bdir/33v/+1+T8qKspZZRERlTs8vOkiBg4cCK32wd9hGHpEpGY8vOkizp07hzp16uBei1Oj0eDJJ5/EgQMHnFwZEVH5wT09FxEYGIhmzZrBze3ui9Td3R0DBw50clVEROULQ8+FDBw4EBqN5q6vFRUVoXfv3k6uiIiofGHouZDIyMi7Dnd3d0fbtm1Rs2ZNJ1dERFS+MPRciK+vL55//nm4u7sXe23AgAEKVEREVL4w9FzMgAEDil3M4ubmhvDwcIUqIiIqPxh6LiY8PNzm1gWtVovOnTujcuXKClZFRFQ+MPRcjJeXF7p16wadTgfgrwtY+vfvr3BVRETlA0PPBfXr1w+FhYUAAL1ej27duilcERFR+cDQc0FdunSB0WgEAERERMBgMChcERFR+VDsuVUXLlzAjz/+qEQtZEfNmjXD9u3bUbt2bSQlJSldDpXRvW5HIaLSKfYYsqSkJD6fkaic4dMCiezjnk8o5kZWsRUVFWHmzJl45513lC6FyoBfQonsi+f0XJS7uzsmTpyodBlEROUKQ8+FleSnhoiI1IShR0REqsHQIyIi1WDoERGRajD0iIhINRh6RESkGgw9IiJSDYYeERGpBkOPiIhUg6FHRESqwdAjIiLVYOgREZFqMPSIiEg1ynXoNWvWDO7u7njiiSfs3veQIUPg5eUFjUaDQ4cOlbrd+vXr4e3tjbVr19q9tod169YtzJs3Dy1atHjoPpKTk1GvXj1oNJp7/tWpU8cu9XL5EpGzlevQ279/P9q1a+eQvpcsWYLFixc/dLvy9nuDKSkpaNOmDd58803k5uY+dD8RERE4ffo0goOD4e3tDRGBiKCwsBC5ubm4fPkyjEajXWrm8iUiZ6sQvz2j0WiULqGYrl27IjMzU+kyAAC//vorpk+fjhEjRiAnJ8chH9ju7u4wGAwwGAxo0KCBXfvm8iUiZynXe3q36XQ6h/Rb0g9bZ3woiwhWrFiBRYsWlfq9jz/+OJKTk9GvXz94eno6oDpbq1evtmt/XL5E5Cx2Cb2ioiJMnToVgYGBMBgMeOyxx5CYmAgAmD9/PkwmE9zc3NC0aVP4+flBp9PBZDLhqaeeQuvWrVG7dm3o9XpUrlwZEyZMKNb/yZMnERISApPJBIPBgNatW2P37t0lrgH460Nnzpw5ePTRR+Hp6Qlvb2+MHz++2LhK0m737t0IDAyERqPBggULAABxcXEwmUwwGo1Ys2YNOnfuDLPZjICAAMTHxxerdebMmXj00UdhMBhQrVo11K1bFzNnzkRkZOTDLYQS2LhxI8xmM2bMmGG3Prl8y8/yJaISkL9JTEyUuwy+r7feeks8PT1l5cqVkpGRIZMmTRI3NzfZv3+/iIi8++67AkD27dsnOTk5cvXqVenUqZMAkO+++06uXLkiOTk5Mnr0aAEghw4dsvbdoUMHqVevnvzxxx9isVjkv//9rzz77LOi1+vl999/L3ENkydPFo1GIx9++KFkZGRIbm6uLFy4UADIwYMHrf2UtN358+cFgMTGxtq8F4Bs2bJFMjMzJS0tTVq3bi0mk0kKCgqs7WbMmCHu7u6yZs0ayc3NlQMHDoifn588//zzpZrvd/Pss8/K448/ftfX1q1bJ15eXjJ9+vQH9hMcHCze3t42w8aMGSNHjhwp1pbL13HL92G2RyK6tzKHXl5enhiNRomOjrYOy83NFU9PTxk5cqSI/N+HYnZ2trXN8uXLBYDNh+hPP/0kACQhIcE6rEOHDsU+xA8fPiwA5K233ipRDbm5uWI0GqVjx442/cTHx9t82JW0ncj9PxTz8vKsw25/oJ48edI6rFmzZvLMM8/YjGPo0KHi5uYm+fn5Uhb3C73SCA4OFgDF/u4Xely+f7Hn8mXoEdlXmQ9vnjhxArm5uWjcuLF1mMFggL+/P44fP37P93l4eAAACgsLrcNun9uxWCz3HWeTJk3g7e2Nw4cPl6iGkydPIjc3Fx06dLhvvyVtVxq3p/POabp582axi02Kioqg0+ng7u5ut3GX1Z1Xb4oIxowZU+L3cvmW/+VLpEZlDr2cnBwAwJQpU2zu5Tp79myZLp1/EJ1OZ/2geVANFy5cAAD4+vret8+StiurLl264MCBA1izZg3y8vLw888/Y/Xq1ejWrVu5/lCcP3++TfA4EpcvETlCmUPv9gfIvHnzbPYKRAR79uwpc4F3U1hYiGvXriEwMLBENej1egBAfn7+ffstabuymjZtGtq3b49BgwbBbDYjPDwckZGRJbqvTA24fInIUcocerevzLvfUy/sbdu2bbh16xaeeuqpEtXQuHFjuLm5YceOHfftt6Ttyuro0aM4deoUrly5AovFgnPnziEuLg4+Pj4OHa+9XLx4EYMHD3ZY/1y+ROQoZQ49vV6PwYMHIz4+HnFxccjKykJRUREuXLiAixcv2qNGFBQUIDMzE4WFhfjll18wevRoBAUFYdCgQSWqwdfXFxEREVi5ciWWLl2KrKwsHD58uNg9UyVtV1ajRo1CYGAgbty4Ydd+H2TDhg1lumVBRJCXl4fk5GSYzWa71cXlS0RO8/crWx7marH8/HyJiYmRwMBA0Wq14uvrKxEREXL06FGZP3++GI1GASB16tSRXbt2yaxZs8Tb21sAiJ+fn3z11VeSkJAgfn5+AkB8fHwkPj5eRESWLVsm7dq1k+rVq4tWq5WqVatKnz595OzZsyWuQUQkOztbhgwZIlWrVpVKlSpJq1atZOrUqQJAAgIC5Ndffy1xu9jYWPH39xcAYjQapXv37rJw4ULrdD7yyCNy6tQpWbRokZjNZgEgQUFB1kvwt27dKlWrVrW5KlKn00nDhg0lOTm5VPNeRGTPnj3SsmVLqVGjhrU/f39/adGihezYscPabv369eLl5SXvv//+PftatWrVPa/cvPNvypQpIiJcvg5evrx6k8i+NCK2l5klJSUhKiqKzx50oLi4OKSkpGDevHnWYQUFBXj77bcRFxeHjIwMGAwGBSuksrDn8uX2SGRfFeLZm67k0qVLGD16dLHzUx4eHggMDITFYoHFYmHoVVBcvkTlW4V49qYrMRgM0Ol0WLp0KS5fvgyLxYLU1FQsWbIEU6dORXR0NFJTU+/70z63/6Kjo5WeHPqbkixfe54PJaLS4Z6ek3l7e2Pz5s2YPn06GjRogJycHFSqVAmNGjXCrFmzMHToUGi1Wh7OqqBKsnyJSDkMPQW0bt0a33//vdJlkINw+RKVXzy8SUREqsHQIyIi1WDoERGRajD0iIhINRh6RESkGgw9IiJSDYYeERGpBkOPiIhUg6FHRESqwdAjIiLVYOgREZFqMPSIiEg1GHpERKQa9/yVhaSkJGfWQUR3sWfPHqVLIHIp9wy9qKgoZ9ZBRETkcBrhr5W6pKSkJERFRfHHaImI7sBzekREpBoMPSIiUg2GHhERqQZDj4iIVIOhR0REqsHQIyIi1WDoERGRajD0iIhINRh6RESkGgw9IiJSDYYeERGpBkOPiIhUg6FHRESqwdAjIiLVYOgREZFqMPSIiEg1GHpERKQaDD0iIlINhh4REakGQ4+IiFSDoUdERKrB0CMiItVg6BERkWow9IiISDUYekREpBoMPSIiUg2GHhERqQZDj4iIVIOhR0REqsHQIyIi1WDoERGRajD0iIhINRh6RESkGlqlC6Cyu3z5Mv7973/bDDt8+DAAYPbs2TbDfXx8MHToUGeVRkRUrmhERJQugsqmsLAQfn5+yMzMhFb7f99jRAQajcb6f35+Pl577TUsWrRIiTKJiBTHw5suQKvVIjo6Gm5ubsjPz7f+FRQU2PwPAH379lW4WiIi5XBPz0Xs3r0brVu3vm8bX19fXLx4Ee7u7k6qioiofOGenoto2bIlatasec/XPTw8MHDgQAYeEakaQ89FaDQa9O/fHzqd7q6vFxQUoE+fPk6uioiofOHhTRdy6NAhPPnkk3d9LSgoCGfOnHFuQURE5Qz39FzIE088gUceeaTYcA8PDwwaNMj5BRERlTMMPRczcODAYoc4CwoKEBUVpVBFRETlBw9vuphTp07hkUcewe3FqtFo0KRJE/z6668KV0ZEpDzu6bmY4OBgPPHEE3Bz+2vRarVaDBw4UOGqiIjKB4aeCxo4cKA19AoLC3lok4jof/Hwpgu6ePEiAgICcOvWLbRo0QI//PCD0iUREZUL3NNzQTVq1LA+neWVV15RuBoiovLDZfb0kpKSeBiPHqiir+5cz8lVJCYmIjIy0unjdbmfFkpMTFS6hHIhJycHixYtwhtvvKF0KeXCnj17MH/+fKXLsBuu51SRKfnFzeVCT4lvDuVVx44dERAQoHQZ5YYrhR7Xc6rIlAw9ntNzYQw8IiJbDD0iIlINhh4REakGQ4+IiFSDoUdERKrB0CMiItVg6BERkWow9IiISDUYekREpBoMPSIiUg2GHhERqQZDj4iIVIOhR0REqsHQIyIi1VBt6M2dOxfVq1eHRqPBZ599pnQ59zV9+nSEhobCbDbD09MT9evXx4QJE3Djxo1S95WcnIx69epBo9FAo9HA398f/fv3f+D7fv31V0RHR6Nu3brw9PREtWrV8Pjjj+P999+3tomOjrb2+6C/devWFavlnXfeuW8NH330ETQaDdzc3BASEoKdO3eWevrVaP369fD29sbatWuVLqVMZs+ejZCQEBgMBphMJoSEhOCdd95BVlZWqfvau3cvGjZsCDc3N2g0Gvj5+dmsy+XBw26r9ADiIhITE6W0k5OSkiIA5NNPP3VQVfbRtm1bWbhwoaSnp0tWVpYkJiaKTqeTTp06PXSfwcHB4u3tXaK2hw8fFqPRKGPGjJE//vhD8vLy5MSJEzJhwgTp0KGDtV1UVJRs3rxZrl+/LhaLRS5evCgApHv37lJQUCA5OTmSlpYmr732mqxdu9amFgDi7+8vBQUFd62hsLBQgoKCBIDNOEvqYdaP8uhhpmPdunViNpvl22+/dVBVztG1a1eZO3eupKWlSXZ2tiQlJYlOp5OOHTs+dJ8vvfSSAJCMjAw7VmpfpdlWKwoAkpiYqMi4Vbun9zDy8vLQokULp4+3UqVKGDZsGKpUqQIvLy9ERkYiLCwMGzduxPnz5x0+/rlz56Jy5cqYP38+6tSpA71ejwYNGuC9996DwWCwttNoNGjZsiW8vb2h1Wpthut0OhiNRvj6+qJp06bFxtG0aVNcunQJq1evvmsNycnJqFWrlv0nTgW6du2KzMxMvPzyy0qXUqZtyMPDA6+//jp8fX1RqVIl9O7dGz179sT333+Pixcv2rlS51Pq80VtGHqlsHTpUqSlpTl9vOvWrYO7u7vNsGrVqgEAcnNzHT7+9PR0ZGZm4tq1azbDPTw8bA6ZxcfHw2g0PrC/YcOGoVu3bjbDRo4cCQD49NNP7/qejz76COPGjStt6VTOlGUbWrVqFfR6vc2w21+EHuZQf3mj1OeL2jD0/mbHjh145plnYDQaYTab0aRJE2RlZWHs2LEYN24cTp06BY1Gg/r162P+/PkwmUxwc3ND06ZN4efnB51OB5PJhKeeegqtW7dG7dq1odfrUblyZUyYMMFudf75558wGAyoW7euddjGjRthNpsxY8YMu40HAJo1a4acnBy0b98eP/zwg137vq19+/Zo2LAhtm3bhhMnTti89sMPPyA3NxcvvviiQ8btynbv3o3AwEBoNBosWLAAABAXFweTyQSj0Yg1a9agc+fOMJvNCAgIQHx8vPW9n3zyCfR6PapXr47hw4ejRo0a0Ov1aNGiBfbt22dtN3r0aHh4eMDf39867PXXX4fJZIJGo8HVq1cB4K7bUFmlpKSgcuXKCAoKsg4ry3ZQ0efNrl27EBoaCm9vb+j1ejRp0gSbNm0CAAwZMsR6fjA4OBgHDx4EAAwePBhGoxHe3t749ttvAQBFRUWYOnUqAgMDYTAY8NhjjyExMREA8MEHH8BoNMLLywtpaWkYN24catWqVWy7LbcUOajqAPY4p3fjxg0xm80ye/ZsycvLk0uXLkl4eLhcuXJFREQiIiIkODjYpo93331XAMi+ffskJydHrl69Kp06dRIA8t1338mVK1ckJydHRo8eLQDk0KFDZZ7WnJwc8fLyktGjR9sMX7dunXh5ecn06dMf2EdpzhPk5ubK008/LQAEgISGhsrs2bMlPT39vu+7fU6vR48eD6zljz/+kI8//lgAyNixY21eDwsLk2XLlkl2djbP6T3EdJw/f14ASGxsrHXY5MmTBYBs2bJFMjMzJS0tTVq3bi0mk8nmvOqwYcPEZDLJsWPH5ObNm3L06FFp1qyZeHl5yblz56zt+vXrJ35+fjbjnTNnjgCwbj8id9+GSqugoEAuXLggsbGx4unpKV988YXN66XZDu52Tq+8zZvSbKsrVqyQadOmybVr1yQ9PV2aN28uVatWtRmHu7u7/Pnnnzbv69u3r80537feeks8PT1l5cqVkpGRIZMmTRI3NzfZv3+/zTwaM2aMxMbGSnh4uPz2228lqlGE5/TKjTNnziArKwuNGjWCXq+Hn58fkpOTrYcS7yc0NBRGoxFVq1ZFnz59AACBgYGoVq0ajEaj9aqr48ePl7nOmTNnokaNGsWuNuvatSuysrIeeBVkaRkMBvz444/4+OOPERISgmPHjiEmJgYNGzbEjh077DaeV155BSaTCcuXL0deXh4A4PTp09i/fz/69u1rt/HQ/2nRogXMZjN8fX0RHR2NnJwcnDt3zqaNVqtFw4YN4enpidDQUMTFxSE7OxvLli1TpObatWsjICAA06ZNwwcffICoqCib1+21HVTEedOrVy+8++678PHxQZUqVdC9e3ekp6fjypUrAIARI0agqKjIpr6srCzs378fXbp0AQDcvHkTcXFxCAsLQ0REBCpXrowpU6ZAp9MVm65Zs2Zh1KhRSE5ORkhIiPMmtAwYeneoV68eqlevjv79+2PatGk4c+bMQ/Xj4eEBACgsLLQO0+l0AACLxVKmGletWoWkpCRs2rQJXl5eZeqrNHQ6HUaPHo3ffvsNe/fuRc+ePZGWlobevXsjIyPDLuPw9vZG3759kZGRgYSEBADAvHnzMHLkSOs8Jce5PY8ftI4+/fTTMBqNdvkC9zDOnz+PtLQ0fP3111i+fDmefPJJhylmopMAACAASURBVJ8Lqyjz5u9uf+4UFRUB+Os0QoMGDfCvf/0Lf+1wAQkJCYiOjrZeN3DixAnk5uaicePG1n4MBgP8/f3LzXSVBUPvDgaDAVu3bkWrVq0wY8YM1KtXD9HR0da9DqUlJCRg1qxZ2L59O+rUqaNYHc8++yy++eYbjBgxAleuXMG2bdvs1vftC1o+++wzXL9+HStWrMDw4cPt1j/Zh6enp3Xvwdl0Oh18fX3x4osvIiEhAUePHsXMmTMVqeVulJw33333HZ5//nn4+vrC09Oz2HUEGo0Gw4cPx+nTp7FlyxYAwOeff45XX33V2iYnJwcAMGXKFJt7a8+ePeuUC+ccjaH3N40aNcLatWuRmpqKmJgYJCYmYu7cuUqXhdjYWHz55ZfYunUratas6dBx7dy5E/PmzbP+HxERYbPXetuAAQMA2PcK0ieeeALNmzfHTz/9hGHDhqF3797w8fGxW/9UdhaLBdevX0dAQIDSpaB+/fpwd3fH0aNHlS4FgPPnzZ3b6rlz5xAWFgZ/f3/s27cPmZmZmD17drH3DBo0CHq9HkuWLMGJEydgNpttLgTy9fUF8NdRFhGx+duzZ49TpsuRGHp3SE1NxbFjxwD8teD/+c9/4qmnnrIOU4KIICYmBkeOHMHq1atRqVIlh4/zwIEDMJlM1v/z8/PvOg9uX6312GOP2XX8t/f2Vq5ciTfeeMOufVPZbd++HSKC5s2bW4dptdoyH7q/n/T09Lue101JSUFRURFq167tsHGXhrPnzZ3b6pEjR2CxWDBy5EjUq1cPer0eGo2m2Ht8fHwQFRWF1atXY+7cuXjttddsXr99xfmhQ4ccUrPSGHp3SE1NxfDhw3H8+HEUFBTg4MGDOHv2rHUFrlKlClJTU3HmzBlkZ2c7dCO/7dixY/jggw+wePFi6HS6Yo/zunMvdMOGDWW6ZcFiseDy5cvYvn27TegBQFhYGJKSknD9+nVkZmZizZo1ePvtt9GjRw+7h15kZCSqVauGsLAw1KtXz659U+ndunULGRkZKCwsxOHDhzF27FgEBgZi0KBB1jb169fHtWvXsHr1algsFly5cgVnz54t1tfDbkMmkwmbN2/G1q1bkZWVBYvFgoMHD1ovfnrzzTetbcu6HZSGUvPmbttqYGAgAOA///kPbt68iZSUFJvbJ+40YsQI5OfnY926dcUeWqDX6zF48GDEx8cjLi4OWVlZKCoqwoULF1ziIQAV/xru/1XaS7k//PBD8fPzEwBiMpkkPDxczpw5Iy1atBAfHx9xd3eXmjVryuTJk6WwsFBERH755RcJCgoSg8EgrVq1kokTJ4rRaBQAUqdOHdm1a5fMmjVLvL29BYD4+fnJV199JQkJCdZx+fj4SHx8fInrPHLkiPVWgbv9zZkzx9p2/fr14uXlJe+///49+1u1apX1sV/3+1u1apX1PZs3b5aoqCgJDg4WT09P8fDwkEcffVSmTZsmN2/eLDaOrKwsadOmjVSpUkUAiJubm9SvX19mzJhxz1qqVasmo0aNsr42YcIE+fHHH63/T5kyRfz9/a39hYaGyq5du0o8H9V6y0JsbKx1vhmNRunevbssXLjQut4+8sgjcurUKVm0aJGYzWYBIEFBQfL777+LyF+X5et0OqlVq5ZotVoxm83Ss2dPOXXqlM140tPTpV27dqLX66Vu3bryP//zPzJ+/HgBIPXr17dewv/3bejSpUslnpbu3btL3bp1pVKlSuLp6SnBwcESHR0tR44csWlXku1g79690qhRI3Fzc7M+Am/GjBnlat58+umnpd5WY2JipEqVKlK5cmXp3bu3LFiwQABIcHCwzW0UIiJPPvmkTJw48a7zJz8/X2JiYiQwMFC0Wq34+vpKRESEHD16VGbPni0Gg0EASO3atYvdMlISUPCWhYr/KfC/XOVDjRzDVdYPZ0/HsGHDpEqVKk4bX0VS0edNly5d5PTp04qMW8nQ4+FNIrqv25e7U3EVad7cebj08OHD0Ov1Nk90UguGngKOHz9eop/fiY6OVrpUIofhduBcMTExSElJwe+//47BgwfjvffeU7okRWgf3ITsLSQkxHpjKFF5NWnSJCxbtgwFBQWoW7cu5syZg169etmt/4q8HTh63jiC0WhESEgIatWqhYULFyI0NFTpkhShkYq61v1NUlISoqKiKuxGRI7lKuuHq0wHqZtGo0FiYiIiIyOdPm4e3iQiItVg6BERkWow9IiISDUYekREpBoMPSIiUg2GHhERqQZDj4iIVIOhR0REqsHQIyIi1WDoERGRajD0iIhINRh6RESkGgw9IiJSDZf7aSGNRqN0CUQOx/Wc6OG4zE8LXbhwAT/++KPSZZQbe/bswfz585GYmKh0KeWKEj9lYk9czx1v3rx5AIA33nhD4UpcW4sWLRAQEOD08bpM6JEt/u4a0cO5/cUoKSlJ4UrIEXhOj4iIVIOhR0REqsHQIyIi1WDoERGRajD0iIhINRh6RESkGgw9IiJSDYYeERGpBkOPiIhUg6FHRESqwdAjIiLVYOgREZFqMPSIiEg1GHpERKQaDD0iIlINhh4REakGQ4+IiFSDoUdERKrB0CMiItVg6BERkWow9IiISDUYekREpBoMPSIiUg2GHhERqQZDj4iIVIOhR0REqsHQIyIi1WDoERGRajD0iIhINRh6RESkGgw9IiJSDYYeERGphlbpAqjs8vLycPHiRZthly9fBgCcPn3aZri7uzuCgoKcVhtReXb16lVkZWXZDMvJyQFQfNsxm82oVq2a02ojx9CIiChdBJVNeno6/P39UVhY+MC2nTp1woYNG5xQFVH5t3TpUgwZMqREbZcsWYJXX33VwRWRo/HwpguoWrUqOnbsCDe3+y9OjUaD6OhoJ1VFVP6Fh4dDp9M9sJ1Op0N4eLgTKiJHY+i5iP79++NBO+1arRY9e/Z0UkVE5Z+Pjw86deoErfbeZ3q0Wi06d+4MHx8fJ1ZGjsLQcxE9evSAp6fnPV/XarXo3r07vL29nVgVUfnXv39/FBUV3fP1oqIi9O/f34kVkSMx9FyEyWRCjx497nmopqioCP369XNyVUTlX/fu3WEwGO75ul6vR9euXZ1YETkSQ8+F9OvXDxaL5a6vGQwGdO7c2ckVEZV/er0eYWFhd/3CqNPpEBERAaPRqEBl5AgMPRfSqVMnmM3mYsN1Oh2ioqKg1+sVqIqo/Ovbt+9dvzBaLBb07dtXgYrIURh6LkSn0yEyMrLYN1ZuuET39+KLL971QpXKlSvjhRdeUKAichSGnou52zfWqlWrol27dgpVRFT+abVaREdHw8PDwzpMp9Ohb9++JbqlgSoOhp6Ladu2LapXr27938PDA/3794e7u7uCVRGVf3369EFBQYH1f4vFgj59+ihYETkCQ8/FuLm5oX///tZvrAUFBdxwiUqgVatWqFmzpvV/f39/tGzZUsGKyBEYei7ozm+sAQEBeOaZZxSuiKj802g01i+MOp0OAwcOhEajUbossjOGngt6+umnUbduXQDAoEGDuOESldDtL4y8+Mt1ucyvLOzZswcfffSR0mWUG7dvtv3pp5/Qu3dvhaspP1asWOGwvj/66CPs2bPHYf2Tc1SqVAkA8P777ytcCZXVc889hzfffNNmmMvs6Z0/fx4rV65Uuoxyo3bt2vD29r7rfXtqdOHCBYevH3v27MHevXsdOg5yvKCgIP78lgvYu3fvXb+Eusye3m2O/CZf0WzatAkvvfSS0mWUC0lJSYiKinL4eJo3b851sII7deoUACA4OFjhSqgs7nWEy+VCj/4PA4+o9Bh2rs1lDm8SERE9CEOPiIhUg6FHRESqwdAjIiLVYOgREZFqMPSIiEg1GHpERKQaDD0iIlINhh4REakGQ4+IiFSDoUdERKrB0CMiItVg6BERkWqoNvTmzp2L6tWrQ6PR4LPPPlO6nPuaPXs2QkJCYDAYYDKZEBISgnfeeQdZWVml7is5ORn16tWDRqOBRqOBv78/+vfv/8D3/frrr4iOjkbdunXh6emJatWq4fHHH7f5oc3o6Ghrvw/6W7duXbFa3nnnnfvW8NFHH0Gj0cDNzQ0hISHYuXNnqadfLZo1awZ3d3c88cQTdu97yJAh8PLygkajwaFDh0rdbv369fD29sbatWvtXltpTZ8+HaGhoTCbzfD09ET9+vUxYcIE3Lhxo9R9/X19vttfnTp17FI3l+/DU23ovfXWW/jxxx+VLqNEdu3ahddeew3nzp3D5cuX8d5772H27Nno1atXqfuKiIjA6dOnERwcDG9vb1y6dAlffvnlfd9z5MgRtGjRAv7+/ti2bRsyMzPx448/olOnTti+fbtN282bN+P69euwWCy4ePEiAKB79+4oKChATk4O0tLS8NprrxWrBQCWLFkCi8Vy1xqKiorwySefAADat2+P48ePo02bNqWefrXYv38/2rVr55C+lyxZgsWLFz90OxFxRFkPZevWrRg1ahTOnDmDq1evYubMmZg/f/49f4vtfv6+bYkIRASFhYXIzc3F5cuXYTQa7VI3l+/DU23oPYy8vDy0aNHC6eP18PDA66+/Dl9fX1SqVAm9e/dGz5498f3331uDxZHmzp2LypUrY/78+ahTpw70ej0aNGiA9957DwaDwdpOo9GgZcuW8Pb2hlartRmu0+lgNBrh6+uLpk2bFhtH06ZNcenSJaxevfquNSQnJ6NWrVr2nzgXp9FolC6hmK5duyIzMxMvv/yy0qWgUqVKGDZsGKpUqQIvLy9ERkYiLCwMGzduxPnz5+0yDnd3dxgMBlSvXh0NGjSwS5+3cfmWHkOvFJYuXYq0tDSnj3fVqlXQ6/U2w24HwMMchimt9PR0ZGZm4tq1azbDPTw8bA5hxMfHl+ib7LBhw9CtWzebYSNHjgQAfPrpp3d9z0cffYRx48aVtnTV0+l0Dum3pB+2zvhQFhGsWLECixYtKvV7161bB3d3d5th1apVAwDk5ubapb473etL3cPi8i09ht7f7NixA8888wyMRiPMZjOaNGmCrKwsjB07FuPGjcOpU6eg0WhQv359zJ8/HyaTCW5ubmjatCn8/Pyg0+lgMpnw1FNPoXXr1qhduzb0ej0qV66MCRMm2K3OlJQUVK5cGUFBQdZhGzduhNlsxowZM+w2HuCv8wc5OTlo3749fvjhB7v2fVv79u3RsGFDbNu2DSdOnLB57YcffkBubi5efPFFh4xbSUVFRZg6dSoCAwNhMBjw2GOPITExEQDssn6dPHkSISEhMJlMMBgMaN26NXbv3l3iGoC/PnTmzJmDRx99FJ6envD29sb48eOLjask7Xbv3o3AwEBoNBosWLAAABAXFweTyQSj0Yg1a9agc+fOMJvNCAgIQHx8fLFaZ86ciUcffRQGgwHVqlVD3bp1MXPmTERGRj7cQvibP//8EwaDAXXr1rUOc8S2xeWrzPKFuIjExEQp7eSkpKQIAPn0009FROTGjRtiNptl9uzZkpeXJ5cuXZLw8HC5cuWKiIhERERIcHCwTR/vvvuuAJB9+/ZJTk6OXL16VTp16iQA5LvvvpMrV65ITk6OjB49WgDIoUOHHnoaCwoK5MKFCxIbGyuenp7yxRdf2Ly+bt068fLykunTpz+wr+DgYPH29i7ReHNzc+Xpp58WAAJAQkNDZfbs2ZKenn7f9128eFEASI8ePR5Yyx9//CEff/yxAJCxY8favB4WFibLli2T7OxsASAdOnQoUd13epj1o7R69eolvXr1KtV73nrrLfH09JSVK1dKRkaGTJo0Sdzc3GT//v0iUrb1q0OHDlKvXj35448/xGKxyH//+1959tlnRa/Xy++//17iGiZPniwajUY+/PBDycjIkNzcXFm4cKEAkIMHD1r7KWm78+fPCwCJjY21eS8A2bJli2RmZkpaWpq0bt1aTCaTFBQUWNvNmDFD3N3dZc2aNZKbmysHDhwQPz8/ef7550s13+8lJydHvLy8ZPTo0TbDy7ptjRkzRo4cOVKsLZev45bvvbZHht4dofff//5XAMi6devu2v5+oZednW0dtnz5cgFgs5L/9NNPAkASEhJKVeOd/Pz8BIBUrVpVPv74Y5uVpbRKE3oifwXuxx9/LCEhIdbwq169umzfvv2e7ylt6F2/fl1MJpP4+PhIbm6uiIicOnVKAgICJD8/3+VCLy8vT4xGo0RHR1uH5ebmiqenp4wcOVJEyrZ+dejQQR5//HGbcR4+fFgAyFtvvVWiGnJzc8VoNErHjh1t+omPj7f5sCtpO5H7fyjm5eVZh93+QD158qR1WLNmzeSZZ56xGcfQoUPFzc1N8vPzpawmT54sDRo0kKysrIfuIzg42LqN3Pl3v9Dj8v2LPZfvvbZHHt68Q7169VC9enX0798f06ZNw5kzZx6qHw8PDwBAYWGhddjtY+/3ujqxJM6fP4+0tDR8/fXXWL58OZ588kmnnWPU6XQYPXo0fvvtN+zduxc9e/ZEWloaevfujYyMDLuMw9vbG3379kVGRgYSEhIAAPPmzcPIkSOt89SVnDhxArm5uWjcuLF1mMFggL+/P44fP37P95Vl/WrSpAm8vb1x+PDhEtVw8uRJ5ObmokOHDvftt6TtSuP2dN45TTdv3ix2dWBRURF0Ol2xc3OltWrVKiQlJWHTpk3w8vIqU193Xr0pIhgzZkyJ38vl65jlextD7w4GgwFbt25Fq1atMGPGDNSrVw/R0dHIy8tTujQAf634vr6+ePHFF5GQkICjR49i5syZTq/j2WefxTfffIMRI0bgypUr2LZtm936vn1By2effYbr169jxYoVGD58uN36L09ycnIAAFOmTLG5l+vs2bMOuYjiNp1OZ/2geVANFy5cAAD4+vret8+StiurLl264MCBA1izZg3y8vLw888/Y/Xq1ejWrVuZPhQTEhIwa9YsbN++3W730t1p/vz5NsHjSFy+98fQ+5tGjRph7dq1SE1NRUxMDBITEzF37lylyyqmfv36cHd3x9GjR+3e986dOzFv3jzr/xERETbfOm8bMGAAAPte5fbEE0+gefPm+OmnnzBs2DD07t0bPj4+duu/PLn9ATJv3jybvQIRwZ49exwyzsLCQly7dg2BgYElquH2VcP5+fn37bek7cpq2rRpaN++PQYNGgSz2Yzw8HBERkaW6L6ye4mNjcWXX36JrVu3ombNmnas1vm4fB+MoXeH1NRUHDt2DMBfK8s///lPPPXUU9ZhSkhPT0ffvn2LDU9JSUFRURFq165t93EeOHAAJpPJ+n9+fv5d58Htqywfe+wxu47/9t7eypUr8cYbb9i17/Lk9pV593vqhb1t27YNt27dwlNPPVWiGho3bgw3Nzfs2LHjvv2WtF1ZHT16FKdOncKVK1dgsVhw7tw5xMXFPdQXIxFBTEwMjhw5gtWrV6NSpUoOqNjWxYsXMXjwYIf1z+X7YAy9O6SmpmL48OE4fvw4CgoKcPDgQZw9exbNmzcHAFSpUgWpqak4c+YMsrOzy3R+rqRMJhM2b96MrVu3IisrCxaLBQcPHsQrr7wCk8mEN99809p2w4YNZbqs2mKx4PLly9i+fbtN6AFAWFgYkpKScP36dWRmZmLNmjV4++230aNHD7uHXmRkJKpVq4awsDDUq1fPrn2XJ3q9HoMHD0Z8fDzi4uKQlZWFoqIiXLhwwW4PHSgoKEBmZiYKCwvxyy+/YPTo0QgKCsKgQYNKVIOvry8iIiKwcuVKLF26FFlZWTh8+HCxe6ZK2q6sRo0ahcDAQLvcn3rs2DF88MEHWLx4MXQ6XbFHht15hKes25aIIC8vD8nJyTCbzWWu/TYu34dQqsthyrHSXp334YcfWq+GNJlMEh4eLmfOnJEWLVqIj4+PuLu7S82aNWXy5MlSWFgoIiK//PKLBAUFicFgkFatWsnEiRPFaDQKAKlTp47s2rVLZs2aJd7e3gJA/Pz85KuvvpKEhATruHx8fCQ+Pr5U09a9e3epW7euVKpUSTw9PSU4OFiio6OLXQ22fv168fLykvfff/+efa1ateqeV5fd+bdq1SrrezZv3ixRUVESHBwsnp6e4uHhIY8++qhMmzZNbt68WWwcWVlZ0qZNG6lSpYoAEDc3N6lfv77MmDHjnrVUq1ZNRo0aZX1twoQJ8uOPP1r/nzJlivj7+1v7Cw0NlV27dpV4HpbHqzdFRPLz8yUmJkYCAwNFq9WKr6+vREREyNGjR2X+/PllWr+WLVsm7dq1k+rVq4tWq5WqVatKnz595OzZsyWuQUQkOztbhgwZIlWrVpVKlSpJq1atZOrUqQJAAgIC5Ndffy1xu9jYWOtyNBqN0r17d1m4cKF1Oh955BE5deqULFq0SMxmswCQoKAg6yX4W7dulapVq9qsqzqdTho2bCjJycmlmvdHjhy57zYwZ84ca1t7bltTpkwREeHydfDyvdf2qBEp5w9KK6GkpCRERUWV++e+kTKcsX7cfl7jihUrHDYOtYuLi0NKSorNOeeCggK8/fbbiIuLQ0ZGhs2j8ahisefyvdf2qL1bYyKi8ubSpUsYPXp0sfNTHh4eCAwMhMVigcViYehVUM5avjynp4Djx4+X6Od3oqOjlS6VqNwwGAzQ6XRYunQpLl++DIvFgtTUVCxZsgRTp05FdHQ0UlNTuW1VUCVZvvY4H8o9PQWEhITwMCxRKXl7e2Pz5s2YPn06GjRogJycHFSqVAmNGjXCrFmzMHToUGi1Wm5bFVRJlq89MPSIqMJo3bo1vv/+e6XLIAdxxvLl4U0iIlINhh4REakGQ4+IiFSDoUdERKrB0CMiItVg6BERkWow9IiISDUYekREpBoMPSIiUg2GHhERqQZDj4iIVIOhR0REqsHQIyIi1XC5X1m4/Wu55By3bt1CTk4OvLy8lC7lvi5cuOCU8ezdu5froINkZ2fDaDTC3d1d6VKoAti7dy+aN29ebLjL7OnVrl0bvXr1UroM1Tl+/Di2bduGa9euKV3KfQUEBDh8/XjuuefuupFR2aWnp2Pbtm04ceKE0qVQBdG8eXM899xzxYZrhL+4SGWQn5+P6OhofP/991i9ejVeeOEFpUsiF7Nt2zb06NEDbdq0wYoVK2AwGJQuiSowl9nTI2V4enpixYoVCA8PR7du3bB69WqlSyIX8u2336JLly7o1q0bvvnmGwYelRlDj8pMq9Vi2bJl6N+/P6KiorBixQqlSyIX8OWXXyIiIgKDBg3Cl19+CZ1Op3RJ5AJc7kIWUoa7uzsWL14Ms9mMPn36IDs7G//4xz+ULosqqIULF2L06NEYP348Zs2apXQ55EIYemQ3Go0GH330Efz8/DBkyBBkZWVh7NixSpdFFczs2bMxceJEzJo1CxMmTFC6HHIxDD2yu5iYGGg0Grz55pu4dOkSv6lTiYgIxo8fj/nz5+Ozzz7D0KFDlS6JXBBDjxxiwoQJMJvNeP311wGAwUf3VVRUhGHDhuHzzz/HV199haioKKVLIhfF0COHGT58OMxmM1555RVkZmZi4cKFcHPjtVNkq6CgAP369cPatWuRlJSEnj17Kl0SuTCGHjlU37594eXlhcjISGRnZ+Pf//43tFqudvSX3NxchIeH44cffsB3332HDh06KF0SuTjenE5OsXXrVvTo0QPt27dHUlISPD09lS6JFHb9+nV069YNv/32G9avX49nn31W6ZJIBRh65DS7du1Ct27d0LJlSyQnJ/NGYxW7fPkyOnXqhMuXL2Pz5s1o3Lix0iWRSjD0yKkOHDiATp06ITQ0FGvXroXZbFa6JHKys2fPomPHjigsLMT333+P4OBgpUsiFeFVBeRUTZs2xc6dO3Hy5Em0b98eV69eVbokcqLjx4+jdevW8PDwwO7duxl45HQMPXK6hg0bYvfu3cjIyEDbtm2RmpqqdEnkBL/88gvatGkDf39/7NixAzVr1lS6JFIhhh4pom7duti2bRsKCwvRqlUr/PHHH0qXRA60a9cutG/fHo0bN8aWLVtQtWpVpUsilWLokWICAwOxc+dOmM1mPP/880hJSVG6JHKA9evX46WXXkLbtm2xfv36cv+Dw+TaGHqkKD8/P2zfvh21atVC69atcfjwYaVLIjtKSEhAz549ERERgeTkZOj1eqVLIpVj6JHiKleujO+//x5NmjTB888/j7179ypdEtnB559/jgEDBmDo0KFYvnw5H0pA5QJDj8oFk8mEdevWoU2bNujYsSO2bt2qdElUBp988gkGDRqEcePGYcGCBXz8HJUbXBOp3PD09ERSUhI6d+6Ml19+GRs3blS6JHoIs2fPxtixYzF37lw+aJzKHYYelSseHh6Ij49HdHQ0evTogZUrVypdEpWQiOCNN97A5MmTsXjxYrz55ptKl0RUDA+yU7nj7u6OJUuWwGw2Izo6GosXL8bgwYOVLovuo6ioCEOGDMHXX3+NhIQE9OrVS+mSiO6KoUflkkajwbx58+Dt7Y1XX30VWVlZGDNmjNJl0V3k5+ejT58+2Lx5M7799lu89NJLSpdEdE8MPSrXpk2bBoPBgLFjxyIzMxNTp05VuiS6w40bNxAWFoaff/4ZmzdvRosWLZQuiei+GHpU7sXExMBsNmPUqFHIzc3lxRHlREZGBrp06YI//vgD27dvx+OPP650SUQPxNCjCmHEiBEwm80YNGgQsrOzERsby8vgFXTp0iW8+OKLyMrKwq5du/DII48oXRJRiTD0qMLo168fKlWqhKioKGRlZWHZsmW84VkBZ86cQceOHaHVarFr1y7Url1b6ZKISoxflalC6dGjB7755hskJyejX79+sFgsSpekKseOHUOrVq1gNpuxc+dOBh5VOAw9qnA6d+6MjRs3YuPGjejZsyfy8vKULkkVfv75Z7Rt2xb16tXD1q1b4evrq3RJRKXG0KMKqU2bNtiyZQv27duHzp07Izs7W+mSXNr27dvRs7cMDgAAFjdJREFUoUMHPPPMM9i0aRO8vb2VLonooTD0qMJ6+umnsXPnTqSkpKB9+/ZIT09XuiSXtG7dOnTu3BldunTB6tWrYTAYlC6J6KEx9KhCCw0NxdatW3H58mW0bdsWFy9eVLokl/L1118jPDwcr7zyCr766ivodDqlSyIqE4YeVXiPPvoodu/ejYKCArRr1w7nz59XuiSXEBcXhwEDBmDEiBH49NNPeYsIuQSuxeQSAgMDsWvXLnh6eqJ169Y4efKk0iVVaLNnz8brr7+O8ePH4+OPP4ZGo1G6JCK7YOiRy7j9K+z+/v5o3bo1jhw5ctd2BQUFOHHihJOrKz9EBCkpKfd8bfz48Zg4cSLmz5/Pp9+Qy2HokUvx8fHBf/7zH4SGhqJt27bYt2+fzetFRUXo27cvBg0apEyB5UBycjJatmxZLPiKioowdOhQzJs3D8uWLeMDvsklaURElC6CyN5yc3MRHh6OvXv34rvvvkPLli0hIhg8eDC+/PJLFBUVYcOGDejUqZPSpTpVUVERQkJCcOrUKdSoUQN79+5F7dq1UVBQgAEDBmDNmjWIj49HWFiY0qUSOQRDj1xWQUEB+vbtiw0bNuCbb77Bpk2bMH/+fNy6dQvu7u5o1KgRDh06pKrzVf/+97/xj3/8AyICnU6HWrVqYdu2bRgxYgR2796Nb775Bi+88ILSZRI5DEOPXFphYSEGDx6MhIQEFBUV4e+r++rVq9GjRw+FqnMui8WC4OBg/Pnnn7h16xYAQKfTwWQywd3dHRs3bsTTTz+tcJVEjsVzeuTStFotnnzySRQWFhYLPHd3d8TExFgDwNX9v//3/5CammozvRaLBTdu3ECNGjXQsGFDBasjcg7u6ZFLi4uLw+uvv37P1zUaDRISEhAZGenEqpwvLy8PderUQVpa2l1f12q1aNOmDdavXw9PT08nV0fkPNzTI5e1fPlyjBo16r5tNBoNJk2ahKKiIidVpYxPPvnkvo9pKywsxI4dOzBgwADV7PmSOrlPmzZtmtJFENnb8ePH8fLLLz8wzEQEmZmZqFevnsv+8ndmZibCw8Nx8+bN+7bTaDQ4evQotFot2rZt66TqiJyLe3rkkkJCQnDmzBlMnToV3t7ecHNzu+dVmiKCSZMmoaCgwMlVOsfcuXORm5t7z9dvP0+zSZMmWL58OSZOnOis0oicjuf0yOXl5OTgq6++wgcffIDTp0/Dzc2t2B6gm5sbPvvsM7z22msKVekYV69eRWBg4P9v7+6DorruP45/7iILLGWRBISkiAppgmI0VWMUCc3YSUYnrW0CKiY+pB3bGPswJkZJgs04qUyaikI6o3HSOJnOtENASU216tgxSlLBNmkpNVWSqEFrCcIoQgJEEE7/6Li/H8EnHnYvcN+vmfvHnj33nO/eWfYz9+5Z7hXvORgcHKyOjg7NmjVLa9as0fTp022oEAgsQg+O0dnZqT/+8Y/auHGjDh48qODgYN+d1y3LUlxcnD755JMhtZDjySef1KZNm7q8TpfLpZCQEC1dulQrV65UQkKCzVUCgUPowZHKy8uVl5enHTt2KCgoSO3t7bIsSy+//LJ+8pOf2F1ev/j3v/+tpKQktbe3KygoSB0dHRozZoxWrVqlxYsXKzw83O4SgYAj9OB35eXlA/Z2P2fPntWePXu0f/9+tbW1KSIiQps2bRoSZ3tbtmzRgQMHJEljx47VnDlz9PWvf31A/geakSNHcnkVAUHowe/mzp2r7du3210GBrDMzExt27bN7jLgAKzeREBkZmbKGDPgt4sXL2r37t2219HX7S9/+Ytqa2ttr+NGtszMTLvfnnCQYXYXAAwkbrdbs2fPtruMPps6dardJQADEmd6AADHIPQAAI5B6AEAHIPQAwA4BqEHAHAMQg8A4BiEHgDAMQg9AIBjEHoAAMcg9AAAjkHoAQAcg9ADADgGoQcAcAxCDwNOXl6eRowYIcuytGXLFrvL6ZEvvvhCycnJWrNmTY/3LSkpUWJioizLkmVZiouL08KFC6+7X2VlpbKysjRmzBiFhIQoOjpaEydO1Lp163x9srKyfONeb9u1a1e3Wn72s59ds4aNGzfKsiy5XC4lJyfrnXfe6fHrBwKB0MOA8/TTT6usrMzuMnolJydHH374Ya/2zcjI0MmTJ5WUlKTIyEjV1tbqt7/97TX3OXLkiFJTUxUXF6cDBw6osbFRZWVlmjVrlg4ePNil7759+3ThwgW1t7fr008/lSTNmTNHbW1tam5uVl1dnX7wgx90q0WSXnvtNbW3t1+xho6ODv3qV7+SJM2cOVNVVVVKT0/v1TEA/I3Qw5DQ2tqq1NRUW2soKyvTBx98ENA58/LyNHz4cBUUFGj06NEKDQ3V7bffrp///OcKCwvz9bMsSzNmzFBkZKSGDRvWpT04OFgej0cxMTGaPHlytzkmT56s2tpa7dix44o1lJSU6Ktf/Wr/vzjADwg9DAlbt25VXV2dbfO3trZq1apVKigoCOi8586dU2Njo86fP9+l3e12a+fOnb7HhYWF8ng81x3v8ccf17e+9a0ubcuXL5ckvfLKK1fcZ+PGjVq5cmVPSwdsQehh0CgtLdXUqVPl8Xjk9Xp15513qqmpSStWrNDKlSt14sQJWZal2267TQUFBQoPD5fL5dLkyZMVGxur4OBghYeHa9KkSbr33ns1cuRIhYaGavjw4Vq9enWfasvJydGPfvQjxcTEXPH5vXv3yuv1Kjc3t0/zfNndd9+t5uZmzZw5U4cOHerXsS+bOXOmxo4dqwMHDnS7dHvo0CG1tLTogQce8MvcQH8j9DAoNDc3a86cOcrMzNT58+f18ccf6/bbb1dbW5sKCgr07W9/W0lJSTLG6Pjx41qxYoVWrVolY4xeeeUVffLJJ6qtrVV6eroqKir07LPPqqKiQufPn9eSJUu0fv16VVZW9qq2Q4cO6cSJE3rkkUeu2qejo0OS1NnZ2as5rmb16tWaMmWKKisrlZaWppSUFP3yl7/sdubXV8uWLZOkbguLNmzYoKeeeqpf5wL8idDDoFBdXa2mpialpKQoNDRUsbGxKikpUXR09HX3HTdunDwej26++WYtWLBAkpSQkKDo6Gh5PB7fCsmqqqoe19Xa2qoVK1Zo8+bN1+z34IMPqqmp6bqrIHsqLCxMZWVlevnll5WcnKyjR48qOztbY8eOVWlpab/Ns2TJEoWHh+s3v/mNWltbJUknT57Ue++9d82wBwYaQg+DQmJiokaMGKGFCxdq7dq1qq6u7tU4brdbknTp0iVfW3BwsCRddXXitTz33HP64Q9/aOtCjuDgYP30pz/VsWPHdPjwYX33u99VXV2d5s6dq4aGhn6ZIzIyUo888ogaGhr0xhtvSJLy8/O1fPly3zEFBgNCD4NCWFiY3n77baWlpSk3N1eJiYnKysrynXXY4c9//rOOHDmipUuX2lbDl91zzz36/e9/ryeeeEL19fU6cOBAv419eUHLli1bdOHCBW3bts132RMYLAg9DBopKSnauXOnampqlJ2draKiIuXl5dlWz9atW7V//365XC7fj7gvL2TJzc2VZVl6//33+3XOd955R/n5+b7HGRkZXc5aL1u0aJEkqaWlpd/mvuuuuzRt2jT99a9/1eOPP665c+cqKiqq38YHAoHQw6BQU1Ojo0ePSpJiYmL04osvatKkSb42O7z++usyxnTZ6uvrJf1vNacxRlOmTOnXOf/2t78pPDzc9/jixYtXPAaXV1lOmDChX+e/fLa3fft2Pfnkk/06NhAIhB4GhZqaGi1btkxVVVVqa2tTRUWFTp06pWnTpkmSbrrpJtXU1Ki6ulqfffZZr76f86c9e/b06ScL7e3tOnv2rA4ePNgl9CTpoYceUnFxsS5cuKDGxka99dZbeuaZZ/Sd73yn30Nv3rx5io6O1kMPPaTExMR+HRsICAP4WWZmpsnMzLzh/hs2bDCxsbFGkgkPDzcPP/ywqa6uNqmpqSYqKsoEBQWZW2+91eTk5JhLly4ZY4z5+9//bkaNGmXCwsJMWlqaefbZZ43H4zGSzOjRo827775rfvGLX5jIyEgjycTGxprf/e535o033vDNFRUVZQoLC/v0Wuvr640kk5OT06V99+7dJiIiwqxbt+6q+7755psmKSnJSLrm9uabb/r22bdvn5k/f75JSkoyISEhxu12mzvuuMOsXbvWfPHFF93maGpqMunp6eamm24ykozL5TK33Xabyc3NvWot0dHR5sc//rHvudWrV5uysjLf4zVr1pi4uDjfeOPGjTPvvvvuDR+znr4/gL6wjDEm8FELJ5k7d64kadu2bTZXgoGI9wcCicubAADHIPSA/6eqquqGbr+TlZVld6kAemHY9bsAzpGcnCyu+ANDF2d6AADHIPQAAI5B6AEAHIPQAwA4BqEHAHAMQg8A4BiEHgDAMQg9AIBjEHoAAMcg9AAAjkHoAQAcg9ADADgGoQcAcAxCDwDgGNxaCAFx5swZFRcX210GBqAzZ84oPj7e7jLgEIQeAuLw4cOaP3++3WVggMrMzLS7BDiEZbhjJhBQ8+bNkyTOfAEb8J0eAMAxCD0AgGMQegAAxyD0AACOQegBAByD0AMAOAahBwBwDEIPAOAYhB4AwDEIPQCAYxB6AADHIPQAAI5B6AEAHIPQAwA4BqEHAHAMQg8A4BiEHgDAMQg9AIBjEHoAAMcg9AAAjkHoAQAcg9ADADgGoQcAcAxCDwDgGIQeAMAxCD0AgGMQegAAxyD0AACOQegBAByD0AMAOAahBwBwDEIPAOAYhB4AwDGG2V0AMJSVlpbq8OHDXdqqqqokSS+99FKX9mnTpukb3/hGwGoDnMgyxhi7iwCGqj/96U964IEHFBwcLJfryhdWOjs71d7ern379un+++8PcIWAsxB6gB91dHQoNjZW586du2a/qKgo1dXVadgwLr4A/sR3eoAfBQUF6dFHH5Xb7b5qH7fbrUWLFhF4QAAQeoCfLViwQG1tbVd9vq2tTQsWLAhgRYBzcXkTCIBRo0bp9OnTV3wuPj5ep0+flmVZAa4KcB7O9IAAWLhwoYKDg7u1u91uLVmyhMADAoQzPSAAjh07pnHjxl3xuSNHjmj8+PEBrghwJkIPCJBx48bp2LFjXdqSk5O7tQHwHy5vAgGyePHiLpc4g4ODtWTJEhsrApyHMz0gQE6fPq3Ro0fr8p+cZVk6efKkRo8ebW9hgINwpgcESEJCgqZMmSKXyyXLsnT33XcTeECAEXpAAC1evFgul0tBQUFatGiR3eUAjsPlTSCA6uvrdcstt0iS/vOf/yg2NtbmigBnIfTgF/zuDH3BxxL8hX/2B79ZsWKFpk+fbncZA05paaksy1J6errdpQw45eXlKigosLsMDGGEHvxm+vTpmjdvnt1lDDizZs2SJHm9XpsrGZgIPfgToQcEGGEH2IfVmwAAxyD0AACOQegBAByD0AMAOAahBwBwDEIPAOAYhB4AwDEIPQCAYxB6AADHIPQAAI5B6AEAHIPQAwA4BqEHAHAMQg8D0tKlSxURESHLsvSPf/zD7nL6pLOzU/n5+UpNTe31GCUlJUpMTJRlWV02t9utESNG6L777tP69evV0NDQj5UDQw+hhwHptdde069//Wu7y+izjz/+WOnp6XrqqafU0tLS63EyMjJ08uRJJSUlKTIyUsYYdXZ2qq6uTsXFxRozZoyys7OVkpKi999/vx9fATC0EHqAn1RWVuqZZ57RE088obvuuqvfx7csS8OHD9d9992n119/XcXFxTp79qwefPBBNTY29vt8wFBA6GHAsizL7hL6ZOLEiSopKdGjjz6qkJAQv8+XmZmpxx57THV1ddqyZYvf5wMGI0IPA4IxRuvXr9cdd9yhkJAQRUZGatWqVd36dXR06Pnnn1dCQoLCwsI0YcIEFRUVSZI2b96s8PBweTwevfXWW5o9e7a8Xq/i4+NVWFjYZZzS0lJNnTpVHo9HXq9Xd955p5qamq47hz/s3btXXq9Xubm5fR7rsccekyTt2bPH1zYUjxnQawbwA0mmqKjohvvn5OQYy7LMhg0bTENDg2lpaTGbNm0ykkxFRYWv39NPP21CQkLM9u3bTUNDg3nuueeMy+Uy7733nm8cSWb//v2msbHR1NXVmXvvvdeEh4ebtrY2Y4wxn3/+ufF6veall14yra2tpra21jz88MOmvr7+hubojXvuucdMnDjxis/t2rXLREREmBdeeOG64yQlJZnIyMirPt/U1GQkmZEjR/raBtMxKyoqMnwswZ94d8EvehJ6LS0txuPxmPvvv79Le2FhYZfQa21tNR6Px2RlZXXZNyQkxCxfvtwY838f4K2trb4+l8Pz+PHjxhhjPvjgAyPJ7Nq1q1stNzJHb1wr9HrieqFnjDGWZZnhw4cbYwbfMSP04G9c3oTtjh8/rpaWFn3zm9+8Zr8PP/xQLS0tGj9+vK8tLCxMcXFxqqqquup+brdbktTe3i5JSkxM1IgRI7Rw4UKtXbtW1dXVfZ5joGhubpYxRl6vVxLHDPgyQg+2O3PmjCQpJibmmv2am5slSWvWrOnyW7VTp0716OcAYWFhevvtt5WWlqbc3FwlJiYqKytLra2t/TaHXT766CNJUnJysiSOGfBlhB5sFxoaKkm6ePHiNftdDsX8/HyZ/12a923l5eU9mjMlJUU7d+5UTU2NsrOzVVRUpLy8vH6dww579+6VJM2ePVsSxwz4MkIPths/frxcLpdKS0uv2W/kyJEKDQ3t839oqamp0dGjRyX9LxRefPFFTZo0SUePHu23OexQW1ur/Px8xcfH6/vf/74kjhnwZYQebBcTE6OMjAxt375dW7duVVNTk/75z3/q1Vdf7dIvNDRU3/ve91RYWKjNmzerqalJHR0dOnPmjD799NMbnq+mpkbLli1TVVWV2traVFFRoVOnTmnatGn9NkdP7Nmzp0c/WTDG6PPPP1dnZ6eMMaqvr1dRUZFmzJihoKAg7dixw/ed3lA9ZkCvBXjhDBxCPfzJwmeffWaWLl1qbr75ZvOVr3zFpKWlmeeff95IMvHx8aaystIYY8zFixdNdna2SUhIMMOGDTMxMTEmIyPD/Otf/zKbNm0yHo/HSDJf+9rXzIkTJ8yrr75qvF6vkWRGjRplPvroI1NdXW1SU1NNVFSUCQoKMrfeeqvJyckxly5duu4cPVFeXm5mzJhhbrnlFiPJSDJxcXEmNTXVlJaW+vrt3r3bREREmHXr1l11rD/84Q9mwoQJxuPxGLfbbVwul5HkW6k5depU88ILL5hz585123cwHTNWb8LfLGOMsStwMXRZlqWioiLNmzfP7lIwiBQXF2v+/PniYwn+wuVNAIBjEHrADaqqqup2a58rbVlZWXaXCuAqhtldADBYJCcnc9kNGOQ40wMAOAahBwBwDEIPAOAYhB4AwDEIPQCAYxB6AADHIPQAAI5B6AEAHIPQAwA4BqEHAHAMQg8A4BiEHgDAMQg9AIBjEHoAAMfgzunwC8uy7C4BgxgfS/AX7qcHvygqKrK7BADohjM9AIBj8J0eAMAxCD0AgGMQegAAxxgmaZvdRQAAEAj/BQA2IhsPkWSRAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KMy9GrVmfJZQ"
      },
      "source": [
        "# create tensorflow datasets for model which uses teacher forcing method\n",
        "train_dataset = create_dataset(X_train, y_train, teacher_forcing=True)\n",
        "valid_dataset = create_dataset(X_valid, y_valid, teacher_forcing=True)\n",
        "test_dataset = create_dataset(X_test, y_test, teacher_forcing=True)"
      ],
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ufgGA4C3eW6v",
        "outputId": "b70d6059-2c79-4b3c-f5d5-dd9486f54f6b"
      },
      "source": [
        "# set checkpoint path\n",
        "checkpoint_dir = os.path.join(checkpoint_2, 'model_2.h5')\n",
        "model_checkpoint = ModelCheckpoint(checkpoint_dir, monitor='val_loss', save_best_only=True)\n",
        "early_stopping = EarlyStopping(patience=7, monitor='val_loss', restore_best_weights=True)\n",
        "\n",
        "# train model\n",
        "history = model_2.fit(train_dataset, epochs=100, validation_data=valid_dataset,\n",
        "                    callbacks=[model_checkpoint, early_stopping])"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/100\n",
            "247/247 [==============================] - 16s 39ms/step - loss: 1.9835 - val_loss: 1.8046\n",
            "Epoch 2/100\n",
            "247/247 [==============================] - 7s 29ms/step - loss: 1.6641 - val_loss: 1.7124\n",
            "Epoch 3/100\n",
            "247/247 [==============================] - 7s 28ms/step - loss: 1.5269 - val_loss: 1.6390\n",
            "Epoch 4/100\n",
            "247/247 [==============================] - 7s 28ms/step - loss: 1.4080 - val_loss: 1.5688\n",
            "Epoch 5/100\n",
            "247/247 [==============================] - 7s 28ms/step - loss: 1.2939 - val_loss: 1.5239\n",
            "Epoch 6/100\n",
            "247/247 [==============================] - 7s 27ms/step - loss: 1.1952 - val_loss: 1.4861\n",
            "Epoch 7/100\n",
            "247/247 [==============================] - 7s 28ms/step - loss: 1.1046 - val_loss: 1.4641\n",
            "Epoch 8/100\n",
            "247/247 [==============================] - 7s 29ms/step - loss: 1.0242 - val_loss: 1.4445\n",
            "Epoch 9/100\n",
            "247/247 [==============================] - 7s 29ms/step - loss: 0.9533 - val_loss: 1.4354\n",
            "Epoch 10/100\n",
            "247/247 [==============================] - 7s 29ms/step - loss: 0.8892 - val_loss: 1.4259\n",
            "Epoch 11/100\n",
            "247/247 [==============================] - 7s 29ms/step - loss: 0.8270 - val_loss: 1.4183\n",
            "Epoch 12/100\n",
            "247/247 [==============================] - 7s 28ms/step - loss: 0.7696 - val_loss: 1.4185\n",
            "Epoch 13/100\n",
            "247/247 [==============================] - 7s 28ms/step - loss: 0.7145 - val_loss: 1.4205\n",
            "Epoch 14/100\n",
            "247/247 [==============================] - 7s 27ms/step - loss: 0.6641 - val_loss: 1.4210\n",
            "Epoch 15/100\n",
            "247/247 [==============================] - 7s 27ms/step - loss: 0.6177 - val_loss: 1.4259\n",
            "Epoch 16/100\n",
            "247/247 [==============================] - 7s 27ms/step - loss: 0.5759 - val_loss: 1.4318\n",
            "Epoch 17/100\n",
            "247/247 [==============================] - 7s 27ms/step - loss: 0.5362 - val_loss: 1.4369\n",
            "Epoch 18/100\n",
            "247/247 [==============================] - 7s 29ms/step - loss: 0.4995 - val_loss: 1.4395\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 404
        },
        "id": "DA5Bgd-WGl_S",
        "outputId": "c48f2beb-7fc4-4a1c-9534-68afced0007b"
      },
      "source": [
        "# plot loss\n",
        "plot_loss(history)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfQAAAGDCAYAAADd8eLzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3yV5f3/8dcnGwhhhkAGe0MYEpbKcIITB8OFgAO3trVWu37aWmu/trbaugBlqChQREVFHBVlCwHZKCIzzLDCSELW9fvjHDRgEhLIyZ3xfj4e55Gc+1znvj85rbzPdd3Xfd3mnENEREQqtiCvCxAREZGzp0AXERGpBBToIiIilYACXUREpBJQoIuIiFQCCnQREZFKQIEuUgxm9rGZjSjttl4ysy1mdnEA9vulmd3h//1mM/u0OG3P4DiNzeyomQWfaa0ilYkCXSot/z/2Jx55ZpaR7/nNJdmXc+4y59yk0m5bHpnZY2Y2t4Dt9c0sy8w6FndfzrnJzrlLS6muk76AOOe2OecinXO5pbH/U47lzKxlae9XJJAU6FJp+f+xj3TORQLbgKvybZt8op2ZhXhXZbn0JnCumTU7ZfsNwGrn3BoPahKR01CgS5VjZv3NLMXMHjWz3cAEM6tjZh+aWaqZHfT/Hp/vPfmHkUea2Xwz+4e/7WYzu+wM2zYzs7lmdsTMPjezF83szULqLk6NT5rZAv/+PjWz+vleH25mW81sv5n9vrDPxzmXAnwBDD/lpVuB109Xxyk1jzSz+fmeX2Jm35pZmpm9AFi+11qY2Rf++vaZ2WQzq+1/7Q2gMfCBf4TlN2bW1N+TDvG3iTWzmWZ2wMw2mtmd+fb9hJlNM7PX/Z/NWjNLKuwzKIyZ1fLvI9X/Wf7BzIL8r7U0s6/8f9s+M5vq325m9i8z22tmh81sdUlGOUSKS4EuVVVDoC7QBBiN77+FCf7njYEM4IUi3t8T+A6oDzwDvGZmdgZt3wKWAPWAJ/h5iOZXnBpvAkYBDYAw4NcAZtYeeNm//1j/8QoMYb9J+WsxszZAF3+9Jf2sTuyjPjAD+AO+z+IH4Lz8TYCn/fW1AxLwfSY454Zz8ijLMwUcYgqQ4n//YOCvZnZhvtev9repDcwsTs0F+A9QC2gO9MP3JWeU/7UngU+BOvg+2//4t18K9AVa+987FNh/BscWKZICXaqqPOBx59xx51yGc26/c+4d51y6c+4I8BS+f7ALs9U5N85//nYS0AiIKUlbM2sMdAf+n3Muyzk3H1/QFKiYNU5wzm1wzmUA0/CFMPgC7kPn3Fzn3HHgj/7PoDDv+ms81//8VuBj51zqGXxWJ1wOrHXOTXfOZQPPAbvz/X0bnXOf+f83SQX+Wcz9YmYJ+L4cPOqcy3TOrQBe9dd9wnzn3Cz//w5vAJ2Ls+98xwjGd9rht865I865LcCz/PTFJxvfl5xYfw3z822vCbQFzDm33jm3qyTHFikOBbpUVanOucwTT8ysupmN8Q+jHgbmArWt8BnU+YMo3f9rZAnbxgIH8m0D2F5YwcWscXe+39Pz1RSbf9/OuWMU0Uv01/Rf4Fb/aMLNwOslqKMgp9bg8j83sxgzm2JmO/z7fRNfT744TnyWR/Jt2wrE5Xt+6mcTYSWbP1EfCPXvt6Bj/AbfKMMS/5D+bQDOuS/wjQa8COw1s7FmFlWC44oUiwJdqqpTbzP4MNAG6Omci8I3RAr5zvEGwC6grplVz7ctoYj2Z1Pjrvz79h+z3mneMwnf8PAl+HqYH5xlHafWYJz89/4V3/8uif793nLKPou6NeROfJ9lzXzbGgM7TlNTSezjp174z47hnNvtnLvTORcL3AW8ZP6Z8s65fzvnugHt8Q29P1KKdYkACnSRE2riOxd8yMzqAo8H+oDOua1AMvCEmYWZWW/gqgDVOB240szON7Mw4M+c/r//ecAhYCwwxTmXdZZ1fAR0MLPr/D3jB/HNZTihJnAUSDOzOH4eenvwnbv+GefcdmAh8LSZRZhZJ+B2fL38MxXm31eEmUX4t00DnjKzmmbWBPjViWOY2ZB8kwMP4vsCkmdm3c2sp5mFAseATIo+3SFyRhToIj7PAdXw9cIWA7PL6Lg3A73xDX//BZgKHC+k7RnX6JxbC9yHb1LbLnyBk3Ka9zh8w+xN/D/Pqg7n3D5gCPA3fH9vK2BBviZ/As4B0vCF/4xTdvE08AczO2Rmvy7gEDcCTfH11t/FN0fi8+LUVoi1+L64nHiMAh7AF8qbgPn4Ps/x/vbdga/N7Ci+uRAPOec2AVHAOHyf+VZ8f/vfz6IukQKZ779ZESkP/Jc6feucC/gIgYhULuqhi3jIPxzbwsyCzGwgMAh4z+u6RKTi0QpZIt5qiG9ouR6+IfB7nHPfeFuSiFREGnIXERGpBDTkLiIiUgko0EVERCqBCncOvX79+q5p06ZelyEiIlJmli1bts85F11UmwoX6E2bNiU5OdnrMkRERMqMmW09XRsNuYuIiFQCCnQREZFKQIEuIiJSCVS4c+giIlKxZGdnk5KSQmZm5ukbV3ERERHEx8cTGhpa4vcq0EVEJKBSUlKoWbMmTZs2xXfXXCmIc479+/eTkpJCs2bNSvx+DbmLiEhAZWZmUq9ePYX5aZgZ9erVO+ORjIAFupklmNkcM1tnZmvN7KEC2piZ/dvMNprZKjM7J1D1iIiIdxTmxXM2n1Mge+g5wMPOufZAL+A+M2t/SpvL8N0TuRUwGng5gPWIiEgVFRkZ6XUJARewQHfO7XLOLff/fgRYD8Sd0mwQ8LrzWQzUNrNGgapJRESksiqTc+hm1hToCnx9yktxwPZ8z1P4eehjZqPNLNnMklNTUwNVpoiIVHLOOR555BE6duxIYmIiU6dOBWDXrl307duXLl260LFjR+bNm0dubi4jR478se2//vUvj6svWsBnuZtZJPAO8Avn3OEz2YdzbiwwFiApKUn3exURqaD+9MFa1u08oygoVPvYKB6/qkOx2s6YMYMVK1awcuVK9u3bR/fu3enbty9vvfUWAwYM4Pe//z25ubmkp6ezYsUKduzYwZo1awA4dOhQqdZd2gLaQzezUHxhPtk5N6OAJjuAhHzP4/3bykRmdi4frdpFdm5eWR1SREQ8NH/+fG688UaCg4OJiYmhX79+LF26lO7duzNhwgSeeOIJVq9eTc2aNWnevDmbNm3igQceYPbs2URFRXldfpEC1kM331S914D1zrl/FtJsJnC/mU0BegJpzrldgarpVAs27uO+t5bz6q1JXNw+pqwOKyJSZRW3J13W+vbty9y5c/noo48YOXIkv/rVr7j11ltZuXIln3zyCa+88grTpk1j/PjxXpdaqED20M8DhgMXmtkK/+NyM7vbzO72t5kFbAI2AuOAewNYz8/0bR1N/cgwpi9LKcvDioiIR/r06cPUqVPJzc0lNTWVuXPn0qNHD7Zu3UpMTAx33nknd9xxB8uXL2ffvn3k5eVx/fXX85e//IXly5d7XX6RAtZDd87NB4q8oM4554D7AlXD6YQGB3FNlzgmLdrCgWNZ1K0R5lUpIiJSBq699loWLVpE586dMTOeeeYZGjZsyKRJk/j73/9OaGgokZGRvP766+zYsYNRo0aRl+c7Lfv00097XH3RzJepFUdSUpIrzfuhr991mMuen8cTV7Vn5HklX2pPRESKtn79etq1a+d1GRVGQZ+XmS1zziUV9b4qv/Rru0ZRdIyL4p3lZTYXT0REpNRV+UAHGHxOPKt3pPHt7tK9lEJERKSsKNCBq7vEERpsvKPJcSIiUkEp0IG6NcK4sG0D3v1mp65JFxGRCkmB7je4WwL7jh5n7gYtLSsiIhWPAt2vf5to6tUI453lGnYXEZGKR4HuFxocxKAucXy+bi8Hj2V5XY6IiEiJKNDzGdwtnqzcPD5YtdPrUkRExENF3T99y5YtdOzYsQyrKR4Fej7tY6No3yhKS8GKiEiFE/Dbp1Y0g7vF8+cP17FhzxFax9T0uhwRkcrl48dg9+rS3WfDRLjsb0U2eeyxx0hISOC++3yrjT/xxBOEhIQwZ84cDh48SHZ2Nn/5y18YNGhQiQ6dmZnJPffcQ3JyMiEhIfzzn//kggsuYO3atYwaNYqsrCzy8vJ45513iI2NZejQoaSkpJCbm8sf//hHhg0bdsZ/9qnUQz/FoC6xhATpmnQRkcpk2LBhTJs27cfn06ZNY8SIEbz77rssX76cOXPm8PDDD1PS5dBffPFFzIzVq1fz9ttvM2LECDIzM3nllVd46KGHWLFiBcnJycTHxzN79mxiY2NZuXIla9asYeDAgaX6N6qHfop6keFc0LYBM77ZwSMD2hASrO88IiKl5jQ96UDp2rUre/fuZefOnaSmplKnTh0aNmzIL3/5S+bOnUtQUBA7duxgz549NGzYsNj7nT9/Pg888AAAbdu2pUmTJmzYsIHevXvz1FNPkZKSwnXXXUerVq1ITEzk4Ycf5tFHH+XKK6+kT58+pfo3Kq0KMLhbPKlHjjPv+31elyIiIqVkyJAhTJ8+nalTpzJs2DAmT55Mamoqy5YtY8WKFcTExJCZmVkqx7rpppuYOXMm1apV4/LLL+eLL76gdevWLF++nMTERP7whz/w5z//uVSOdYICvQAXtGlA3Rq6T7qISGUybNgwpkyZwvTp0xkyZAhpaWk0aNCA0NBQ5syZw9atW0u8zz59+jB58mQANmzYwLZt22jTpg2bNm2iefPmPPjggwwaNIhVq1axc+dOqlevzi233MIjjzxS6vdX15B7AcJCghjUJZbJi7eRlp5NreqhXpckIiJnqUOHDhw5coS4uDgaNWrEzTffzFVXXUViYiJJSUm0bdu2xPu89957ueeee0hMTCQkJISJEycSHh7OtGnTeOONNwgNDaVhw4b87ne/Y+nSpTzyyCMEBQURGhrKyy+/XKp/X5W/H3ph1uxI48r/zOfJazoyvFeTgB9PRKSy0v3QS0b3Qy9lHWKjaNuwpobdRUSkQtCQeyHMjMHd4vnLR+vZuPcILRvomnQRkapk9erVDB8+/KRt4eHhfP311x5VVDQFehGu6RrH3z7+lunLdvDYZSU/tyIiIhVXYmIiK1as8LqMYtOQexHqR4bTv000736TQm5exZprICJSnlS0+VpeOZvPSYF+GoO7xbPn8HHmfa/7pIuInImIiAj279+vUD8N5xz79+8nIiLijN6vIffTuLBtDHWqhzJ9WQr92zTwuhwRkQonPj6elJQUUlPVMTqdiIgI4uPjz+i9CvTT8F2THsdbS3RNuojImQgNDaVZs2Zel1Hpaci9GK4/J56snDw+XK37pIuISPmkQC+GjnFRtInRNekiIlJ+KdCL4cQ16d9sO8TGvUe9LkdERORnFOjFNKhrLMFBxjvL1UsXEZHyR4FeTA1qRtC/dTTvLt+ha9JFRKTcUaCXwPXd4tl9OJMFG3WfdBERKV8U6CVwUbsG1KoWqslxIiJS7gQs0M1svJntNbM1hbxey8w+MLOVZrbWzEYFqpbSEh4SzKAusXyydjdpGdlelyMiIvKjQPbQJwIDi3j9PmCdc64z0B941szCAljPz2WmwcwH4Nj+Yr9lcLd4jufk8dGqXQEsTEREpGQCFujOubnAgaKaADXNzIBIf9ucQNVToJRkWDkVxvT1/V4MiXG1aNUgUrPdRUSkXPHyHPoLQDtgJ7AaeMg5l1dQQzMbbWbJZpZcqmsBt7wIbv8EgoJg/EBYMg5Oc/OAE9ekL9t6kE2puiZdRETKBy8DfQCwAogFugAvmFlUQQ2dc2Odc0nOuaTo6OjSrSK2K4z+ClpcCLN+De/cAceLDupru8YRZKiXLiIi5YaXgT4KmOF8NgKbgbaeVFK9Ltw4BS78I6ydAa9eBKkbCm3eICqCfq2jmaFr0kVEpJzwMtC3ARcBmFkM0AbY5Fk1QUHQ99dwyww4lgrjLoA1MwptPrhbArvSMln0Q/En1ImIiARKIC9bextYBLQxsxQzu93M7jazu/1NngTONbPVwP+AR51z3q/Y0uICuGseNGgP00fB7N9C7s8vUbuoXQOiIkKYvmy7B0WKiIicLGD3Q3fO3Xia13cClwbq+GelVhyM/Ag++yMsfgl2LIMhEyEq9scmEaHBXN0llunLUjicmU1UhO6TLiIi3tFKcYUJCYPL/g8Gj4fda+CVPrDpy5OaDO6WQGZ2HrN0TbqIiHhMgX46Ha+H0XOgej1441qY+w/I811d1zm+Fi0bRGopWBER8ZwCvTii28CdX0CHa+GLJ2HKjZBxEDPj+nPiSd56kC37jnldpYiIVGEK9OIKj4TrX4PLnoGN/4Mx/WDXSl2TLiIi5YICvSTMoOddMOpjyMuBVy+h4Q/T6NMqmneWpZCna9JFRMQjCvQzkdAd7poLTXrDzAd4kpfZn3aYRZt0TbqIiHhDgX6matT3LULT9zc03jaD9yMe56tFi72uSkREqigF+tkICoYLfw83/ZfGwQe4f+OdZKx63+uqRESkClKgl4bWl7LpullscTFUm3ErfPY45JbtnWBFRKRqU6CXkg7tE3kk6hk+q3Y5LHgO3rgGju71uiwREakiFOilxMwYlNScOw/ewr6Ln4eUZN/qclsXeV2aiIhUAQr0UnRt1zjM4PWMc+GOzyGsOky83HeDl+NHvC5PREQqMQV6KWpUqxrnt6zvuya9QQcY/SV0GwmLX4YXesDa98DpWnURESl9CvRSNrhbPDsOZbB4836IqAVX/gtu/wxq1IP/joDJg+GAd7d9FxGRykmBXsoGdGhIzfCQk2/YktAd7vwSBjwN2xbDS73hq79DznHP6hQRkcpFgV7KIkKDubJzI2av2c2x4/kuXQsOgd73wv1LofUAmPMXePk82PSVd8WKiEiloUAPgMHd4knPymXW6gLukx4VC0Nfh5vfgbxseP1qeOdOXeImIiJnRYEeAOc0rkOz+jWKvk96q4vh3sXQ9zew9l34TxIsfRXycsuuUBERqTQU6AFgZgzuFs/Xmw+wbX964Q1Dq/mWjr13EcR2ho8ehlcvhp0ryq5YERGpFBToAXLimvRi3Se9fiu4dSZc9yqkbYdxF8DHj0Lm4cAXKiIilYICPUBia1fjvBb1eWd5CrnFuU+6GXQaAvcnQ9Jt8PUYeKE7rJmha9dFROS0FOgBdHPPxqQczOCvs9YX/03VasMVz8Kd/4OaMTB9FLx5Hez/IXCFiohIhadAD6DLEhsx6rymvDZ/MxMXbC7Zm+O6wZ1z4LJnYPtS37XrX/6frl0XEZECKdAD7A9XtOfS9jH86cN1fLp2d8neHBQMPe/yXbve9gr48q++YP9hTmCKFRGRCkuBHmDBQcbzN3SlU3xtHpzyDSu2Hyr5TqIawZAJcMsMwPluzTr9djiyp9TrFRGRikmBXgaqhQXz2ogkomuGc8ekpWw/UMSlbEVpeRHcswj6PQbrZ8ILSfD1WMjNOf17RUSkUlOgl5H6keFMHNWD7FzHiAlLOJSedWY7Co2AC37rW5Qmrht8/AiM7Qdb5pduwSIiUqEo0MtQi+hIxt2aRMqBDEa/sYzjOWexKly9FjD8Xd8ysplpMPEKmH4bpO0ovYJFRKTCUKCXsR7N6vKPoZ1ZsvkAv5m+irziXKNeGDNoPwjuW+Ibhv/2I98w/Nx/QHZm6RUtIiLlngLdA1d3juU3A9vw/oqdPPvZd2e/w7DqvmH4+5b4zrN/8SS81BO+naVFaUREqggFukfu6deCG3s05sU5P/D2km2ls9M6TWDYmzD8PQgOhyk3wuTBsO/70tm/iIiUWwELdDMbb2Z7zWxNEW36m9kKM1trZlXqxuBmxpODOtCvdTR/eG8NX35XirdPbXEB3LMABjwN25f4rl3/9I9aG15EpBILZA99IjCwsBfNrDbwEnC1c64DMCSAtZRLIcFBvHjzObSJqcl9k5ezdmda6e08OBR63wsPLIPOw2Dhv33n11dOgby80juOiIiUCwELdOfcXOBAEU1uAmY457b525diF7XiiAwPYcKo7kRVC+W2iUvZeSijlA/QAAa9CHd8AbXi4d27YPwA2PlN6R5HREQ85eU59NZAHTP70syWmdmthTU0s9FmlmxmyampqWVYYtmIiYpgwqjupB/P5baJSzmSmV36B4nvBrd/DoNegoObYewFMPNBOLav9I8lIiJlzstADwG6AVcAA4A/mlnrgho658Y655Kcc0nR0dFlWWOZadswipdv6cbGvUe5d/JysnMDMCweFARdb/YNw/e+D1ZMhv+c47tVq1abExGp0LwM9BTgE+fcMefcPmAu0NnDejx3fqv6PH1dIvO+38fv312NC9QlZxG1YMBTcM9CiD0HPv4NjOkDm+cG5ngiIhJwXgb6+8D5ZhZiZtWBnkAJbhxeOQ1JSuDBi1oxLTmFF77YGNiDRbfxrTY3bDJkHYVJV8G0EXBoe2CPKyIipS4kUDs2s7eB/kB9M0sBHgdCAZxzrzjn1pvZbGAVkAe86pwr9BK3quSXF7ci5UA6z362gfi61bi2a3zgDmYG7a70LUiz8D8w75+w4RPo8ys490Hf2vEiIlLuWcCGdQMkKSnJJScne11GwGXl5DFi/BKStx5g0m09OLdF/bI58KFtvmvW170HtZvARf8P2l0FIeFlc3wREfkZM1vmnEsqqo1WiiunwkKCeGV4N5rWq8Fdbyzj+z1HyubAtRvD0Ekw4gMIqwHv3A7PtoFZj/gudatgXwBFRKoK9dDLuZSD6Vz70kLCgoN4975zaVCzDIfA83Jh0xxY8Ras/xByj0OD9tDlJkgcCjVjyq4WEZEqrDg9dAV6BbA6JY2hYxbRskEkU+/qRfWwgE19KFzGIVg7wxfuKUvBgqHVJb5wbz1QQ/IiIgGkQK9E/rd+D3e+nsyFbRswZngSwUHmXTGpG2DlW75lZI/sgmp1IHGIL9wbdfFNtBMRkVKjQK9k3li8lT++t4ZbezfhT1d3wLwOTg3Ji4iUieIEugdjt3KmhvdqQsqBdMbM3URCnerc2be5twUFBUPLi32P/EPyn/4BPntcQ/IiImVIgV7BPDqwLSkHM3hq1nri6lTj8sRGXpfkU602JN3me+Qfkt8wW0PyIiJlQEPuFVBmdi43v/o1q3ek8fadPenWpK7XJRVMQ/IiIqVC59ArsQPHsrj+5YUcTM/irTt60T42yuuSilbYLPlOQ6HlJRBRzusXEfGQAr2S27Y/nRvGLiIjO5c37+hJh9haXpdUPKfOkg8Og+b9oe2V0OZyiKycd9QTETlTCvQqYNv+dG4ct5hjWTm8eXtPOsZVkFAH35D89iXw7Yew/gM4tBUsCBJ6+daXb3sl1GnidZUiIp5ToFcR2w+kc8PYxRw9nsPkOypYqJ/gHOxZ4zvX/u2Hvt8BGiZC26t8Ad+gvSbUiUiVpECvQipFqOd3YBN8+5Gv5759CeCgTjN/z/0qiO8OQboVgYhUDQr0KqbShfoJR/bAdx/5eu+b50JeNkTG+M63t7sSmvaFkDCvqxQRCRgFehW0/YDvnPrhjGwm39GLxPhKEuonZKbBhk/h2w/g+88h+xiE14LWl/rOube8GMIjva5SRKRUKdCrqJSDvp764Yxs3ryjJ53ia3tdUmBkZ8CmL3099+9mQcYBCImA5hf4eu6tL4Ma9byuUkTkrCnQq7CUg76e+qH0bN68vSedEyppqJ+QmwPbFvlnzH8Ih1N8M+ZjOkDDThDTERp29P2sXk4X4hERKYQCvYrbcSiDG8Yu4lB6Nm/c3pMulT3UT3AOdq3wTarbsQx2r4Fje396PSr+p3BvmOh71GmmSXYiUm4p0IUdhzK4cexiDh7L4o07qlCon+rIHtiz2hfue9bA7tWw73twub7XQ2tATHtfuJ8I+gbtdT5eRMoFBboAsPNQBjeOW8yBo1m8fnsPujau43VJ5UN2JqSu94X87tX+oF8Dx9P8DQzqNvf35v09+YYdISpO18OLSJlSoMuPdqVlcMNYX6hPur0H5yjUC+YcpG3PF/L+Xv3BzT+1iaj9U08+pj1Et4X6rX13nBMRCQAFupxkV5pv+H3f0Swm3daDbk0U6sV2/AjsWXtyT37vOshO/6lNzUYQ3cYX8Cd+1m+jmfYictYU6PIzu9MyuWHsIn+ody+/t16tCPJy4dA2SP0OUr89+Wf2sZ/aVa9/csif+BnZQEP3IlIsCnQp0O60TG4ct5i9hzN5/fYeCvXS5hykpeQL+BNh/12+8/P4hu4LCvqoWAW9iJxEgS6Fyh/qk27rQVJThXrAOQdHdv+8N5/6rW9RnBPCavrDvY2vF1+9HlSr6/tZPd/P8Fq61E6kilCgS5H2HM7kxrGL2XM4k4m39aC7Qt07x/ad0pv/1ndZ3bFUyMsp+D0W5A/6uvlC/9Tn9U7+IhBRC4KCy/ZvEykv8vJ8816yMyA3y3dfiNxs3++5/t/zTnmem+X7bzA3q2TtLn8WQiNKrXQFupzW3sOZ3DBuMbvTMpk4qgc9minUyxXn4PhhSD/ge2QcgPT9/uf7fY+MAz+9fuJ5blYhOzSoVidfT7+AR436+b4E1IfwmjoFIGUnLxeyjkLWMchK981HOfEzO+OUbem+dtnpxduekxGYmoNCIdj/CAqF4DC4b7HvC3QpUaBLsew97Bt+35WWyYSR3enZXLOyKzTnfP8g5g/4/IH/47Z8XwyO7fP1OAoSFJov6E98Eaif7wtA/i8E/jYh4WX7N4t3crL8AXwUjp8I4iP5fj/qu0rkx9+PFtw+65jveUlDNzgMQqtDWA3/z+q+haLCqhe+PaSa7w6NwWH5wjjs56EcnO+1oBD/tjAIDjn5vWXwhVeBLsW294hv+F2hXkU55/tH98eQ3/dT+B/bV/D2jIOF7y+spi/YI2r5rs+PqOV/1PY/Ctru/z20mkYESlNujn+YOT1fjzUjX4/X//Ok19N/Gpou8PWMn0K50NGgU1iQ7/8X4ZEQFukL2vBI37Yff8/32olHQSGdf1twSGA/v3JCgS4lsvdIJjeN+5odBzOYMKo7vRTqUpTcHF+op+8/OejT98Mx/8/MtHyPQ76f+a/dL0hwWMFBf+qXgDD/srw//huW79+y4mw76QQOSbYAACAASURBVN++Ara5XN/wb16O/5xpzsmP3Gz/6/leyz3x+oltuYW/1+X6D+v8xy3qZ14B2wp7r79t7nFf+BY28lIo84dmtZMD9McQreYP1xMhXCNfUJ/6uz+gwyN9d0LUF7UzpkCXEks9cpwbxy1mx8EMxo/sTu8WCnUpZTlZvnkBGYfyBf2hn4I/49DPvwT8uP1Q4ZMEy1JQiG+4NSjE10MMCsm3Ldg/bBty8iPY/9qJ9wUFA/ZTyJnle36GP/P/Hhx+SiBX8/d4/YFcWGAreMslTwPdzMYDVwJ7nXMdi2jXHVgE3OCcm366/SrQAy/1yHFuGreYlIMZvDYyiXNb1Pe6JBEf53zDvZmHfL3PU4PnpOdWjG35Xjt1mwXlC+GQk4NYgSdlzOtA7wscBV4vLNDNLBj4DMgExivQy4/UI8e5+dXFbDuQzpjhSfRrHe11SSIiVVZxAj1gq1I45+YCB07T7AHgHWDvadpJGYuuGc7bd/aief1I7pi0lNlrdntdkoiIFMGzZabMLA64Fni5GG1Hm1mymSWnpqYGvjgBoF5kOG+P7kXHuFrc99Zy3l+xw+uSRESkEF6uG/kc8KhzLu90DZ1zY51zSc65pOhoDf2WpVrVQnnj9p50b1qHX0xdwZQl27wuSURECuBloCcBU8xsCzAYeMnMrvGwHilEZHgIE0f1oF/raB6bsZrx8zef/k0iIlKmPAt051wz51xT51xTYDpwr3PuPa/qkaJFhAYzZng3BnZoyJ8/XMeLczZ6XZKIiOQTsEA3s7fxXY7WxsxSzOx2M7vbzO4O1DElsMJDgnnhpq5c2zWOv3/yHc/M/paKto6BiEhlFbA185xzN5ag7chA1SGlKyQ4iGeHdCYiNJiXvvyB9KxcHr+qPabrckVEPFU1FsGVUhUUZPz12o5UDwvmtfmbycjK5a/XJRIcpFAXEfGKAl3OiJnxhyvaUSMsmH9/sZGM7FyeHdqZ0GAv51mKiFRdCnQ5Y2bGry5tQ7WwEP5v9rdkZOfywk1dCQ8J9ro0EZEqR90pOWv39G/Bn67uwGfr9nDHpGQysnK9LklEpMpRoEupGHFuU54Z3IkFG/cxYvwSjmSW9JaNIiJyNhToUmqGJiXw/A1dWb7tILe8toRD6VlelyQiUmUo0KVUXdU5lpdv6cb6nYe5Yexi9h097nVJIiJVggJdSt0l7WMYP7I7W/enM3TMInanZXpdkohIpadAl4A4v1V9Xr+9B3sPH2fImIVsP5DudUkiIpWaAl0CpnvTurx1Z0+OZOYw5JVFbNx71OuSREQqLQW6BFSn+NpMGd2LnLw8ho1ZxPpdh70uSUSkUlKgS8C1bRjFtLt6ExYSxA1jF7Ni+yGvSxIRqXQU6FImmkdHMu2u3tSqFsotr37N15v2e12SiEilokCXMpNQtzrT7upNTFQ4IyYsYe6GVK9LEhGpNBToUqYa1opg2l29aV4/kjsmJTN7zS6vSxIRqRQU6FLm6kWG8/advegQF8U9k5fz0pcbcc55XZaISIWmQBdP1Koeytt39uLKTrE8M/s7fjF1BZnZuqmLiMiZKlagm1kNMwvy/97azK42s9DAliaVXURoMP++oQuPDGjD+yt2MmzMIvYc1qpyIiJnorg99LlAhJnFAZ8Cw4GJgSpKqg4z474LWjJmeDe+33uUq1+Yz6oUXdYmIlJSxQ10c86lA9cBLznnhgAdAleWVDUDOjTknXvOJSQoiCGvLGLmyp1elyQiUqEUO9DNrDdwM/CRf1twYEqSqqpdoyhm3n8eneNr8+Db3/CPT74jL0+T5UREiqO4gf4L4LfAu865tWbWHJgTuLKkqqoXGc6bd/Tkhu4JvDBnI3e/uYxjx3O8LktEpNyzkl4u5J8cF+mc82RR7qSkJJecnOzFoaUMOeeYuHALT364jtYxNRl3axIJdat7XZaIiCfMbJlzLqmoNsWd5f6WmUWZWQ1gDbDOzB4pjSJFCmJmjDqvGRNH9WDHoQwGvbiAJZsPeF2WiEi5Vdwh9/b+Hvk1wMdAM3wz3UUCqm/raN677zxqVwvl5lcXM2XJNq9LEhEpl4ob6KH+686vAWY657IBzVaSMtEiOpJ37z2PXs3r8diM1fzpg7Xk5OZ5XZaISLlS3EAfA2wBagBzzawJoBtbS5mpVT2UCSO7c9t5zZiwYAujJi4lLT3b67JERMqNEk+K+/GNZiHOuTKffqxJcTJt6XZ+/95qEupUZ9yIJFpER3pdkohIQJXmpLhaZvZPM0v2P57F11sXKXNDuyfw1p29SMvI5poXF+g2rCIiFH/IfTxwBBjqfxwGJgSqKJHT6d60Lu/ffx5xtasxcsISxs/frDu2iUiVVtxAb+Gce9w5t8n/+BPQPJCFiZxOfJ3qvHPPuVzcLoY/f7iOx95ZTVaOJsuJSNVU3EDPMLPzTzwxs/OAjKLeYGbjzWyvma0p5PWbzWyVma02s4Vm1rn4ZYv41AgP4ZVbuvHAhS2Zmrydm19dzL6jx70uS0SkzBU30O8GXjSzLWa2BXgBuOs075kIDCzi9c1AP+dcIvAkMLaYtYicJCjIePjSNvz7xq6sSklj0AsLWLdTF2GISNVSrEB3zq10znUGOgGdnHNdgQtP8565QKFLeznnFjrnDvqfLgbii1eySMGu7hzLf+/uTU5eHoNfWcjsNbu9LklEpMwUt4cOgHPucL413H9VinXcjm8FugKZ2egTM+xTUzWjWQrXKb42M+8/n1YxNbn7zWX867MN5OqObSJSBZQo0E9hpVGAmV2AL9AfLayNc26scy7JOZcUHR1dGoeVSiwmKoKpo3tx3TlxPP+/7xk5YYnOq4tIpXc2gX7W3R4z6wS8Cgxyzu0/2/2JnBARGsyzQzrzt+sS+XrzAa749zyWbtHNXUSk8ioy0M3siJkdLuBxBIg9mwObWWNgBjDcObfhbPYlUhAz44YejXn33nOpFhrMDWMXM+arH8jTELyIVEJnvPTraXds9jbQH6gP7AEeB0IBnHOvmNmrwPXAVv9bck63rB1o6Vc5M4czs3l0+io+XrObi9s14B9DOlO7epjXZYmIFEtxln4NWKAHigJdzpRzjkkLt/DUrPU0qBnBSzefQ+eE2l6XJSJyWqW2lrtIZWBmjDyvGdPu6g3A4FcWMmnhFi0ZKyKVggJdqpyujevw0YPn06dVNI/PXMv9b3/DkUzdilVEKjYFulRJtauH8eqtSTw6sC2z1+zmaq0uJyIVnAJdqqygIOOe/i14646eHDuew7UvLWDq0m0agheRCkmBLlVez+b1mPVQH5Ka1uHRd1bz6/+uIj0rx+uyRERKRIEuAtSPDOf123ry0EWtmPFNCte8uICNe496XZaISLEp0EX8goOMX17SmkmjerDvaBZXvzCf91fs8LosEZFiUaCLnKJv62hmPdiH9o2ieGjKCn7/7moys3O9LktEpEgKdJECNKwVwduje3FXv+ZM/nobg19ZyLb96V6XJSJSKAW6SCFCg4P47WXtGHdrEtv2p3PFf+bpHusiUm4p0EVO45L2MXz0YB+a1a/B3W8u4y8friM7N8/rskRETqJAFymGhLrV+e/dvRnRuwmvzt/MsDGL2Hkow+uyRER+pEAXKabwkGD+NKgjL9zUlQ17jnLFv+fx6VoNwYtI+aBAFymhKzvFMvP+82hUqxqj31jGr6auIC1da8GLiLcU6CJnoHl0JO/ddx4PXtSK91fu5NLnvmLOt3u9LktEqjAFusgZCgsJ4leXtOa9e8+jVrVQRk1cym+mr+Sw7twmIh5QoIucpcT4WnzwwPnc278F05elMPBfc5n3farXZYlIFaNAFykF4SHB/GZgW96551yqhQUz/LUl/O7d1Rw9rpu8iEjZUKCLlKKujevw0YN9GN23OW8v2cbA5+ay8Id9XpclIlWAAl2klEWEBvO7y9vx37t6ExocxE3jvubx99folqwiElAKdJEASWpal1kP9mHUeU2ZtGgrA5+bx5LNB7wuS0QqKQW6SABVCwvm8as6MGV0LxyOYWMX8eSH63T3NhEpdQp0kTLQq3k9Zj/Ul1t6NuG1+Zu5/Pl5LNt60OuyRKQSUaCLlJEa4SE8eU1HJt/Rk+M5eQx5ZSFPf7xevXURKRUKdJEydl7L+sz+RR+GdU9gzFebuPI/81m5/ZDXZYlIBadAF/FAzYhQnr6uE5Nu68HRzByue3kh//jkO47nqLcuImdGgS7ioX6to/nkl325tmscL8zZyKAXFrBmR5rXZYlIBaRAF/FYrWqh/GNIZ14bkcT+Y1lc8+ICnvt8A9m5eV6XJiIViAJdpJy4qF0Mn/2yL1d1juW5z7/nmhcXsCpF59ZFpHgU6CLlSO3qYfxrWBdeuaUbew4fZ9CLC/jtjFUcOJbldWkiUs4p0EXKoYEdG/LFr/tx+3nNmJacwgX/+JI3Fm0hN895XZqIlFMBC3QzG29me81sTSGvm5n928w2mtkqMzsnULWIVERREaH84cr2fPxQHzrERvHH99dy1X/mk7xFy8eKyM8Fsoc+ERhYxOuXAa38j9HAywGsRaTCah1Tk8l39OTFm87hYHoWg19ZxK+mrmDv4UyvSxORciRgge6cmwsU1ZUYBLzufBYDtc2sUaDqEanIzIwrOjXifw/34/4LWvLhql1c+OxXjJu7SbPhRQTw9hx6HLA93/MU/7afMbPRZpZsZsmpqallUpxIeVQ9LIRfD2jDp7/sS/emdXhq1noue34e87/XPddFqroKMSnOOTfWOZfknEuKjo72uhwRzzWtX4MJo3rw2ogksnLyuOW1r7l38jJ2HMrwujQR8UiIh8feASTkex7v3yYixXRRuxjOa1mfcXM38eKXG/ni273c178ld/ZtTkRosNfliUgZ8rKHPhO41T/bvReQ5pzb5WE9IhVSRGgwD1zUiv893J8L2zbg2c82cOm/5vK/9Xu8Lk1EylAgL1t7G1gEtDGzFDO73czuNrO7/U1mAZuAjcA44N5A1SJSFcTVrsZLN3dj8h09CQsJ4vZJydw2cSlb9h3zujQRKQPmXMVaqCIpKcklJyd7XYZIuZadm8ekhVt47vPvycrJ486+zbjvgpZUD/PyLJuInCkzW+acSyqqTYWYFCciJRMaHMQdfZrzxcP9uLJTI16c8wMXPfsVH63aRUX7Ei8ixaNAF6nEGkRF8M9hXZh+d2/qVA/jvreWc/OrX/P9niNelyYipUyBLlIFJDWtywcPnM+T13Rk7c7DXPb8PJ78cB1p6dlelyYipUSBLlJFBAcZw3s1Yc6v+zMkKYHxCzbT55kveOWrH8jMzvW6PBE5Swp0kSqmbo0wnr4ukVkP9iGpaV3+9vG39P/7l0xZso0cLSMrUmEp0EWqqHaNohg/sjtTR/eiUe0IHpuxmgHPzWX2mt2aOCdSASnQRaq4ns3rMeOecxkzvBsAd7+5jOteXsjiTfs9rkxESkKBLiKYGQM6NOSTX/Tl/65PZNehTG4Yu5hRE5awftdhr8sTkWLQwjIi8jOZ2blMXLiFl+Zs5MjxHK7tEscvL2lNQt3qXpcmUiUVZ2EZBbqIFCotPZuXvtrIxAVbcA5u7tWY+y9oSb3IcK9LE6lSFOgiUip2pWXw/OffMy15O9XDQhjdtzm3n9+MGuFaSlakLCjQRaRUbdx7hL9/8h2frN1D/chwHryoJTd0b0xYiKbjiASS1nIXkVLVskFNxgxP4p17zqV5dA3+3/trueRfXzFz5U7y8ipW50CkslGgi0iJdWtSh6mjezFhZHeqhQbz4NvfcPWL85n3farXpYlUWQp0ETkjZsYFbRvw0YN9+OfQzhw8ls3w15Zw86uLWZVyyOvyRKocnUMXkVJxPCeXNxdv44UvvudgejZXJDbiV5e2pkV0pNeliVR4mhQnImXucGY24+Zu4rX5m8nMzuX6c+J56OJWxNfRNewiZ0qBLiKe2Xf0OC/N+YE3F2/F4bipR2Puu7AlDWpGeF2aSIWjQBcRz+08lMF/vvieackphAYbI89txt39mlO7epjXpYlUGAp0ESk3tuw7xr8+38DMlTuJDAvhjj7Nub1PMyK1OI3IaSnQRaTc+Xb3Yf756QY+XbeHujXCuKdfC4b3bkJEaLDXpYmUWwp0ESm3Vmw/xLOffse87/cRExXOAxe2YmhSgladEymAAl1Eyr1FP+znH59+x7KtB0moW41fXNSaa7rGERxkXpcmUm5o6VcRKfd6t6jH9Lt7M2Fkd6IiQnn4vysZ+NxcPl69i4rW4RDxkgJdRDx3YtW5D+4/nxdvOoc857hn8nKuemE+X363V8EuUgwKdBEpN4KCjCs6NeKTX/TlH0M6cyg9m5ETljJ0zCK+3rTf6/JEyjWdQxeRcisrJ4+pS7fxny82svfIcfq2jubXl7amU3xtr0sTKVOaFCcilUJGVi6vL9rCy1/9wKH0bAZ0iOHhS9vQOqam16WJlAkFuohUKkcys3lt/mZenbeZY1k5DGjfkPsuaElifC2vSxMJKAW6iFRKB49lMX7BZiYu3MKRzBz6tY7mvgta0qNZXa9LEwkIzy9bM7OBZvadmW00s8cKeL2xmc0xs2/MbJWZXR7IekSkcqhTI4yHL23Dgscu5JEBbVizI42hYxYx9JVFfLUhVbPipUoKWA/dzIKBDcAlQAqwFLjRObcuX5uxwDfOuZfNrD0wyznXtKj9qocuIqfKyMrl7SXbGDt3E7sPZ5IYV4v7LmjJpe1jCNICNVIJeN1D7wFsdM5tcs5lAVOAQae0cUCU//dawM4A1iMilVS1sGBuO78ZX/2mP09fl0haRjZ3v7mMgc/P5b1vdpCTm+d1iSIBF8hAjwO253ue4t+W3xPALWaWAswCHghgPSJSyYWHBHNjj8Z88XA/nr+hCwC/mLqCC5/9ire+3sbxnFyPKxQJHK8XlrkRmOiciwcuB94ws5/VZGajzSzZzJJTU1PLvEgRqVhCgoMY1CWO2Q/1ZczwbtSuHsrv3l1Nv2e+5LX5m0nPyvG6RJFSF8hz6L2BJ5xzA/zPfwvgnHs6X5u1wEDn3Hb/801AL+fc3sL2q3PoIlJSzjnmfb+PF+ZsZMnmA9StEcbt5zdjeO8mREWEel2eyGl5fQ59KdDKzJqZWRhwAzDzlDbbgIsAzKwdEAGoCy4ipcrM6Ns6mml39ea/d/emU3wt/v7Jd5z39Bf8/ZNv2X/0uNclipy1gF6H7r8M7TkgGBjvnHvKzP4MJDvnZvpnto8DIvFNkPuNc+7TovapHrqIlIY1O9J4cc5GZq/dTYT/3Pvovs1pWCvC69JEfkYLy4iInMb3e47w8pc/8P7KnQSbcX23eO7p14LG9ap7XZrIjxToIiLFtG1/Oq/M/YHpySnkOsdVnRpxV78WtGsUdfo3iwSYAl1EpIR2p2Uybt4m3l6yjfSsXPq1juaufs3p3bweZlqkRryhQBcROUOH0rN4c/FWJi7cwr6jWXSOr8Vd/VowoENDgrX6nJQxBbqIyFnKzM7lneUpjJu7iS3702lSrzp39mnO4G7xRIQGe12eVBEKdBGRUpKb5/h07W5e+eoHVqakUT8yjJHnNuWWXk2oXT3M6/KkklOgi4iUMuccizcdYMzcH/jyu1SqhwVzQ/fG3N6nGXG1q3ldnlRSCnQRkQBav+swY+duYubKnRhwdedYRvdrTtuGmhkvpUuBLiJSBlIOpjN+/hamLPXNjO/fJpq7+7WgZ7O6mhkvpUKBLiJShk7MjJ+wYAv7j2XROaE2d/dtzqWaGS9nSYEuIuKBzOxcpi9LYdy8TWzdn06z+jW4s09zrjsnTjPj5Ywo0EVEPJSb5/jEPzN+VUoa9SPDGXVeU27p2YRa1XWXNyk+BbqISDngnGPRpv2M+WoTX21IpUZYMEO7JzC8VxOaR0d6XZ5UAAp0EZFy5sTM+A9X7SQ719G3dTQjejehf5sGOs8uhVKgi4iUU3uPZDJlyXYmf72VPYePk1C3GsN7NWFoUoIWqpGfUaCLiJRz2bl5fLp2D5MWbWHJ5gOEhwRxTZc4hvduQse4Wl6XJ+WEAl1EpAJZv+swry/aynvf7CAjO5ekJnW49dymDOzQkLCQIK/LEw8p0EVEKqC09Gz+u2w7byzeytb96UTXDOemHo25qWdjYqIivC5PPKBAFxGpwPLyHHO/T+X1RVuZ891egs0Y2LEhI85tSlKTOlqFrgopTqCHlFUxIiJSMkFBRv82DejfpgFb9h3jzcVbmZa8nQ9X7aJdoyhG9G7CoC5xVAvTYjWiHrqISIWSnpXD+yt2MmnhFr7dfYSoiBCGJiUwvHcTmtSr4XV5EiAachcRqaSccyzdcpBJi7bwyZrd5DpH/9bR3HpuU/q1iiZI17RXKhpyFxGppMyMHs3q0qNZXXanZfLWkm289fU2Rk1YStN61bmpZ2OuOyee+pHhXpcqZUQ9dBGRSiIrJ4/Za3fz+sItJG89SEiQcXG7GIZ1T6Bv62itRFeBqYcuIlKFhIUEcXXnWK7uHMvGvUeYunQ7M5bvYPba3TSqFcHgbvEMTUogoW51r0uVAFAPXUSkEsvKyeN/6/cwNXk7X21IxTk4r2U9hiYlMKBDQ93OtYLQpDgREfnRzkMZTF+WwtSl29lxKINa1UK5tmscw7on0K5RlNflSREU6CIi8jN5eY6FP+xnytJtfLp2D1m5eXSKr8Ww7glc1TmWqAjdq728UaCLiEiRDh7L4r0VO5i6dDvf7j5CRGgQVyTGMqx7At2bajW68kKBLiIixeKcY1VKGlOWbueDlTs5ejyH5vVrMLR7AtedE0eDmlpD3ksKdBERKbH0rBxmrd7N1KXbWLrlIMFBxkVtGzCsewL9WkcTEqw7v5U1BbqIiJyVjXuP8t/k7byzPIV9R7OIiQpncLd4rjsnnhbRkV6XV2V4HuhmNhB4HggGXnXO/a2ANkOBJwAHrHTO3VTUPhXoIiJlLzs3j/+t38u05O18+d1e8hx0jIvi6s6xXNkpltja1bwusVLzNNDNLBjYAFwCpABLgRudc+vytWkFTAMudM4dNLMGzrm9Re1XgS4i4q09hzP5cNUuZq7YwcqUNAB6NKvL1Z1juTyxEXVrhHlcYeXjdaD3Bp5wzg3wP/8tgHPu6XxtngE2OOdeLe5+FegiIuXHln3HmLlyJ++v2MEPqccICTL6tKrPoC5xXNI+hhrhWpC0NHi99GscsD3f8xSg5yltWgOY2QJ8w/JPOOdmB7AmEREpRU3r1+DBi1rxwIUtWbfrMDNX7uTDlbv4xdQVRIQGcXG7GK7uHEu/NtGEh2hVukDy+qtTCNAK6A/EA3PNLNE5dyh/IzMbDYwGaNy4cVnXKCIip2FmdIitRYfYWjw6oC3Lth1k5oqdfLR6Fx+u2kVURAiXdWzE1V1i6dW8nm4UEwCBDPQdQEK+5/H+bfmlAF8757KBzWa2AV/AL83fyDk3FhgLviH3gFUsIiJnLSjI6N60Lt2b1uX/XdWeBRv3MXPFTj5ctZOpyduJrhnOlZ0acXXnWLok1NbiNaUkkOfQQ/BNirsIX5AvBW5yzq3N12YgvolyI8ysPvAN0MU5t7+w/eocuohIxZSZncv/1u9l5sodzPk2lazcPBrXrc7VnWMZ1CWWVjE1vS6x3CoPl61dDjyH7/z4eOfcU2b2ZyDZOTfTfF/LngUGArnAU865KUXtU4EuIlLxpWVk88na3XywcicLNu4jz0HbhjW5ukssV3WK1S1eT+F5oAeCAl1EpHLZeySTWat28f7KnXyzzTeFqktCba5IbMRliQ2Jr6NwV6CLiEiFsm1/Oh+s2sms1btYu/MwAJ0TanNFYkMuT2xUZcNdgS4iIhXWln3HmLVmF7NW72LNjpPD/bKOjarUsLwCXUREKoWt+4/x0epTwj2+FpcnNuLyxMof7gp0ERGpdLbuP8as1buZtXoXq3f4lp7t5A/3KyppuCvQRUSkUtu2P/3HYflVKZU33BXoIiJSZWw/kM4s/7D8iZvGJMb9FO6N61XccFegi4hIlVRQuHeMi/ox3JvUq+FxhSWjQBcRkSpv+4F0Pl6zi49W72bldt917m1ianJJ+xgubh9Dp7haBJXzteUV6CIiIvmkHExn9prdfLZuD0u3HCDPQYOa4VzcPoZL2sXQu0U9IkLL313hFOgiIiKFOHgsiznf7eWzdXv4akMq6Vm5VA8Lpl/raC5uF8OFbRtQp0aY12UCCnQREZFiyczOZdGm/Xy2bg+fr9vD3iPHCQ4ykprU4ZL2MVzSPsbT8+4KdBERkRLKy3Os3pHmC/f1e/h29xEAWsdEcnE7X7h3jq9dpufdFegiIiJnadv+dD5b7+u5L9lygNw8R3TNcC5u14BL2sdwbov6AT/vrkAXEREpRYfSfefdP1+3ly+/28sx/3n3Pq3qc0n7hlzYtgF1A3DeXYEuIiISIMdzcln0w34+X7+Hz9ftZffhTIIMkprU5YWbutIgKqLUjlWcQA8ptaOJiIhUIeEhwfRv04D+bRrw5CDfeffP1+3h680HqBcZXub1KNBFRETOkpnRKb42neJre1ZDkGdHFhERkVKjQBeR/9/e3cfIVZVxHP/+0hZtgJQtNbVStFYbE4mKm4YgQULE1NIY6lukhMQKJAYUA3/40oSEEOM/YDSmSDSgaDVEGxW0MaCtxaiJtojNtrSCdGlqpNm+ibYSDUJ9/OOeba7Te7e73dm5M+f+PsnNnDn3zOx5cu6dZ+7LzjGzDDihm5mZZcAJ3czMLANO6GZmZhlwQjczM8uAE7qZmVkGnNDNzMwy4IRuZmaWASd0MzOzDDihm5mZZcAJ3czMLANO6GZmZhkYuPnQJR0B/tLFt1wAHO3i+/U7x5u3NsXbpljB8eZsMrG+ISJeM1GDgUvo3SbpydNNGp8Tx5u3NsXbpljB8easW7H6lLuZmVkGnNDNzMwy4IQO3LNqUwAABlJJREFU9zfdgR5zvHlrU7xtihUcb866Emvrr6GbmZnlwEfoZmZmGWhNQpe0UtKfJY1KWlex/lWSNqb12yUt6X0vu0PShZJ+JelPkvZIuq2izZWSjkkaScudTfS1WyTtl/RUiuXJivWStD6N7y5Jw030c7okvaU0ZiOSjku6vaPNQI+tpAclHZa0u1Q3X9IWSXvT41DNa9emNnslre1dr89cTbxfkvRM2lYfkXRezWsn3O77UU28d0k6UNpmV9W8dsLP8X5TE+vGUpz7JY3UvHbqYxsR2S/ALOA5YClwFrATeGtHm08C30jlNcDGpvs9jXgXAcOpfC7wbEW8VwI/a7qvXYx5P7BggvWrgMcAAZcC25vucxdingUcpPj/1GzGFrgCGAZ2l+ruAdal8jrg7orXzQf2pcehVB5qOp4zjHcFMDuV766KN62bcLvvx6Um3ruAz5zmdaf9HO+3pSrWjvVfBu7s1ti25Qj9EmA0IvZFxH+AHwCrO9qsBjak8o+AqySph33smogYi4gdqfxP4GnggmZ71bjVwHejsA04T9Kipjs1TVcBz0VEN39oqXER8RvghY7q8v65AfhAxUvfB2yJiBci4u/AFmDljHW0S6rijYjNEfFKeroNWNzzjs2QmvGdjMl8jveViWJN+eWjwPe79ffaktAvAP5aev48pya4k23SjnQMOL8nvZtB6dLBO4HtFavfJWmnpMckXdTTjnVfAJsl/VHSJyrWT2YbGDRrqP8wyGlsARZGxFgqHwQWVrTJcYwBbqQ4u1TldNv9ILk1XWJ4sOaSSm7j+27gUETsrVk/5bFtS0JvJUnnAD8Gbo+I4x2rd1Ccqn0HcC/wk173r8suj4hh4GrgU5KuaLpDM0nSWcA1wA8rVuc2tv8nivORrfj3HEl3AK8AD9U0yWW7/zrwJuBiYIziVHTurmPio/Mpj21bEvoB4MLS88WprrKNpNnAPOBvPendDJA0hyKZPxQRD3euj4jjEfFiKj8KzJG0oMfd7JqIOJAeDwOPUJyeK5vMNjBIrgZ2RMShzhW5jW1yaPwSSXo8XNEmqzGW9HHg/cD16UvMKSax3Q+EiDgUESci4r/AA1THkc34phzzIWBjXZszGdu2JPQ/AMskvTEd2awBNnW02QSM3xX7EeDxup2o36VrM98Cno6Ir9S0ee34PQKSLqHYFgbyC4yksyWdO16muKFod0ezTcDH0t3ulwLHSqdwB1Htt/ucxrakvH+uBX5a0eYXwApJQ+mU7YpUN3AkrQQ+B1wTEf+qaTOZ7X4gdNzP8kGq45jM5/igeC/wTEQ8X7XyjMe26bsAe7VQ3OX8LMVdknekui9Q7DAAr6Y4fTkKPAEsbbrP04j1copTkruAkbSsAm4Gbk5tbgX2UNwpug24rOl+TyPepSmOnSmm8fEtxyvgvjT+TwHLm+73NOI9myJBzyvVZTO2FF9UxoCXKa6T3kRxP8tWYC/wS2B+arsc+GbptTemfXgUuKHpWKYR7yjF9eLx/Xf8P3BeBzyaypXbfb8vNfF+L+2XuyiS9KLOeNPzUz7H+3mpijXVf2d8fy21nfbY+pfizMzMMtCWU+5mZmZZc0I3MzPLgBO6mZlZBpzQzczMMuCEbmZmlgEndLOWkXSiY8a2rs1aJWlJeWYpM+ud2U13wMx67t8RcXHTnTCz7vIRupkBJ+dfvifNwfyEpDen+iWSHk8TZ2yV9PpUvzDN1b0zLZelt5ol6QFJeyRtljS3saDMWsQJ3ax95naccr+2tO5YRLwN+Brw1VR3L7AhIt5OMUnI+lS/Hvh1FJPADFP8ohXAMuC+iLgI+Afw4RmOx8zAvxRn1jaSXoyIcyrq9wPviYh9aXKfgxFxvqSjFD/F+XKqH4uIBZKOAIsj4qXSeyyhmJN8WXr+eWBORHxx5iMzazcfoZtZWdSUp+KlUvkEvlfHrCec0M2s7NrS4+9T+XcUM1sBXA/8NpW3ArcASJolaV6vOmlmp/I3Z7P2mStppPT85xEx/q9rQ5J2URxlX5fqPg18W9JngSPADan+NuB+STdRHInfQjGzlJk1wNfQzQw4eQ19eUQcbbovZjZ1PuVuZmaWAR+hm5mZZcBH6GZmZhlwQjczM8uAE7qZmVkGnNDNzMwy4IRuZmaWASd0MzOzDPwPMT0oimq1lMEAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 576x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "edqCuf_MfJGg",
        "outputId": "6e74b3d6-5a98-464b-f0b0-ecbe5c0e5099"
      },
      "source": [
        "# evaluate model on a test set\n",
        "print(\"Test set loss: {:.4f}\".format(model_2.evaluate(test_dataset, verbose=0)))"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test set loss: 1.4082\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cf9uz8bwfJEK"
      },
      "source": [
        "def prepare_data(sent):\n",
        "    sent = preprocess_english(sent)\n",
        "    sent_tokens = english_tokenizer.texts_to_sequences([sent])\n",
        "    sent_padded = pad_sequences(sent_tokens, maxlen=english_maxlen, padding='post')\n",
        "    return sent_padded\n",
        "    \n",
        "# write a function which returns the translated text in nepali\n",
        "def translate(eng_sent, model):\n",
        "    \n",
        "    # prepare y_pred\n",
        "    Y_pred = tf.fill(dims=(1, 1), value=1)\n",
        "    # prepare encoder input\n",
        "    X_encoder = prepare_data(eng_sent)\n",
        "    \n",
        "    index = 0\n",
        "    next_ = 0\n",
        "    end_token_id = nepali_tokenizer.texts_to_sequences(['end_'])[0][0]\n",
        "    # loop until max target length or until end token is predicted\n",
        "    while index < nepali_maxlen and next_ != end_token_id:\n",
        "        # calculate the padding size\n",
        "        padsize = nepali_maxlen - Y_pred.shape[1]\n",
        "        # create input for decoder\n",
        "        X_decoder = tf.pad(Y_pred, [[0, 0], [0, padsize]])\n",
        "        # get the probabilities of next prediction\n",
        "        Y_proba = model.predict([X_encoder, X_decoder])[:, index:index+1]\n",
        "        # get the next predicted token\n",
        "        Y_proba_next = tf.argmax(Y_proba, axis=-1, output_type=tf.int32)\n",
        "        # concatenate with Y_pred to predict next word\n",
        "        Y_pred = tf.concat([Y_pred, Y_proba_next], axis=1)\n",
        "        # assign current predicted token to next\n",
        "        next_ = Y_proba_next.numpy()[0][0]\n",
        "        # increment the index value\n",
        "        index += 1\n",
        "    # translate the sentences and return all except start and end tokens\n",
        "    translated_sent = nepali_tokenizer.sequences_to_texts(Y_pred.numpy())\n",
        "    return \" \".join(translated_sent[0].split()[1:-1])"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "3SW6qrPNunAg",
        "outputId": "4827aa4d-7c58-406f-9e48-97b51eec7c49"
      },
      "source": [
        "# save predictions as a dataframe\n",
        "save_path = os.path.join(prediction_dir, 'model_2_predictions.csv')\n",
        "model_2_predictions = predict_and_save_as_dataframe(model=model_2, save_path=save_path,\n",
        "                                                    translate_fn=translate)\n",
        "model_2_predictions.sample(5)"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>enlgish</th>\n",
              "      <th>nepali</th>\n",
              "      <th>predicted</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>tom is guilty</td>\n",
              "      <td>टम दोषी छ</td>\n",
              "      <td>टम एक खी छ</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>266</th>\n",
              "      <td>this is life</td>\n",
              "      <td>यो जिवन हो</td>\n",
              "      <td>यो राम्रो छ</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57</th>\n",
              "      <td>did tom go</td>\n",
              "      <td>टोम गए</td>\n",
              "      <td>टम छोड्नुहोस्</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>hold your breath</td>\n",
              "      <td>सास रोक</td>\n",
              "      <td>एक मिनेट दिनुहोस्</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>908</th>\n",
              "      <td>he fell backward</td>\n",
              "      <td>ऊ पछिल्तिर खसे</td>\n",
              "      <td>ऊ टम मन पर्दैन</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              enlgish          nepali          predicted\n",
              "12      tom is guilty       टम दोषी छ         टम एक खी छ\n",
              "266      this is life      यो जिवन हो        यो राम्रो छ\n",
              "57         did tom go          टोम गए      टम छोड्नुहोस्\n",
              "33   hold your breath         सास रोक  एक मिनेट दिनुहोस्\n",
              "908  he fell backward  ऊ पछिल्तिर खसे     ऊ टम मन पर्दैन"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d5I0-Uugj_qg"
      },
      "source": [
        "## Seq-to-Seq Model with Pretrained-embedding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ap30nAb3LqAm"
      },
      "source": [
        "# download glove embedding\n",
        "!wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1Yf1XvZ3pgvNzXqnQZLHxjTunNqJIPtjJ' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\\1\\n/p')&id=1Yf1XvZ3pgvNzXqnQZLHxjTunNqJIPtjJ\" -O glove_embedding_50k.txt && rm -rf /tmp/cookies.txt\n",
        "clear_output()\n"
      ],
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B3WzDNm3M8t2",
        "outputId": "76cf02b8-c8b1-4d6a-cda5-607341b4c999"
      },
      "source": [
        "# load glove word embedding file\n",
        "glove_emb = open('glove_embedding_50k.txt')\n",
        "# initialize a dictionary to store embeddings\n",
        "glove_embedding_dict = {}\n",
        "# loop over all the embeddings in glove_emb\n",
        "for line in glove_emb:\n",
        "    # get the word\n",
        "    word = line.split()[0]\n",
        "    # get the embedding for the word and store in a dict\n",
        "    glove_embedding_dict[word] = np.array(line.split()[1:])\n",
        "\n",
        "# close the file\n",
        "glove_emb.close()\n",
        "    \n",
        "# verify\n",
        "glove_embedding_dict['the']"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['0.418', '0.24968', '-0.41242', '0.1217', '0.34527', '-0.044457',\n",
              "       '-0.49688', '-0.17862', '-0.00066023', '-0.6566', '0.27843',\n",
              "       '-0.14767', '-0.55677', '0.14658', '-0.0095095', '0.011658',\n",
              "       '0.10204', '-0.12792', '-0.8443', '-0.12181', '-0.016801',\n",
              "       '-0.33279', '-0.1552', '-0.23131', '-0.19181', '-1.8823',\n",
              "       '-0.76746', '0.099051', '-0.42125', '-0.19526', '4.0071',\n",
              "       '-0.18594', '-0.52287', '-0.31681', '0.00059213', '0.0074449',\n",
              "       '0.17778', '-0.15897', '0.012041', '-0.054223', '-0.29871',\n",
              "       '-0.15749', '-0.34758', '-0.045637', '-0.44251', '0.18785',\n",
              "       '0.0027849', '-0.18411', '-0.11514', '-0.78581'], dtype='<U11')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EthPfOQkNGYh"
      },
      "source": [
        "# Now we have to extract the embedding for the words that are in the vocabulary\n",
        "\n",
        "# create a helper function for creating embedding matrix\n",
        "def create_embedding_matrix(glove_emb, vocab_size):\n",
        "    vocab_words = list(english_tokenizer.word_index.keys())\n",
        "    # initialize a matrix with shape (vocab_size, 50)\n",
        "    matrix = np.zeros((vocab_size+1, 50))\n",
        "    # loop over the vocab words\n",
        "    for word in vocab_words:\n",
        "        if word in glove_emb:\n",
        "            matrix[english_tokenizer.texts_to_sequences([word])[0][0]] = glove_emb[word]\n",
        "    # return the updated matrix\n",
        "    return matrix"
      ],
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-LzbhmkPNNp-",
        "outputId": "791940c6-93fc-4d97-fb55-bca4a7175518"
      },
      "source": [
        "# create embedding_matrix\n",
        "embedding_matrix = create_embedding_matrix(glove_embedding_dict, english_vocabsize)\n",
        "embedding_matrix.shape"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(2826, 50)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hQ4-LKkGmjkV",
        "outputId": "f8735ab1-6502-4fc2-9178-51699530d41c"
      },
      "source": [
        "# define embedding size\n",
        "src_embed = 50\n",
        "target_embed = 128\n",
        "units = 256\n",
        "\n",
        "# define inputs\n",
        "encoder_inputs = Input(shape=[None], dtype=tf.int32)\n",
        "decoder_inputs = Input(shape=[None], dtype=tf.int32)\n",
        "\n",
        "# encoder embeddings\n",
        "encoder_embeddings = Embedding(english_vocabsize+1, src_embed, mask_zero=True)(encoder_inputs)\n",
        "# define encoder lstm layers\n",
        "encoder_lstm_layer1 = LSTM(units, return_sequences=True, dropout=0.4)\n",
        "encoder_lstm_layer2 = LSTM(units, return_state=True, dropout=0.4)\n",
        "# get encoder states\n",
        "layer1_out = encoder_lstm_layer1(encoder_embeddings)\n",
        "_, en_state_h, en_state_c = encoder_lstm_layer2(layer1_out)\n",
        "encoder_states = [en_state_h, en_state_c]\n",
        "\n",
        "# Decoder embeddings\n",
        "decoder_embeddings = Embedding(nepali_vocabsize+1, target_embed, mask_zero=True)(decoder_inputs)\n",
        "decoder_lstm_layer = LSTM(units, return_sequences=True)\n",
        "decoder_lstm_output = decoder_lstm_layer(decoder_embeddings, initial_state=encoder_states)\n",
        "# get decoder final output\n",
        "decoder_output = Dense(nepali_vocabsize+1, activation=\"softmax\")(decoder_lstm_output)\n",
        "\n",
        "# Create a model\n",
        "model_emb = Model(inputs=[encoder_inputs, decoder_inputs], outputs=[decoder_output])\n",
        "# Initialize embedding layer\n",
        "model_emb.layers[1].set_weights([embedding_matrix])\n",
        "# also do not train this layer\n",
        "model_emb.layers[1].trainable = False\n",
        "# display model summary\n",
        "model_emb.summary()\n"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_4 (InputLayer)            [(None, None)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_3 (Embedding)         (None, None, 50)     141300      input_4[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "input_5 (InputLayer)            [(None, None)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "lstm_5 (LSTM)                   (None, None, 256)    314368      embedding_3[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "embedding_4 (Embedding)         (None, None, 128)    557312      input_5[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lstm_6 (LSTM)                   [(None, 256), (None, 525312      lstm_5[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "lstm_7 (LSTM)                   (None, None, 256)    394240      embedding_4[0][0]                \n",
            "                                                                 lstm_6[0][1]                     \n",
            "                                                                 lstm_6[0][2]                     \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, None, 4354)   1118978     lstm_7[0][0]                     \n",
            "==================================================================================================\n",
            "Total params: 3,051,510\n",
            "Trainable params: 2,910,210\n",
            "Non-trainable params: 141,300\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PiO_QcWSnCh2"
      },
      "source": [
        "## Train the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w_60epejm6tj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ae73553e-422b-4b2d-91e9-33fd4597ce2e"
      },
      "source": [
        "# checkpoint model\n",
        "save_path = os.path.join(checkpoint_3, \"model_3.h5\")\n",
        "checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(save_path, save_best_only=True)\n",
        "early_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True)\n",
        "# define optimizer\n",
        "optimizer = tf.keras.optimizers.Adam()\n",
        "# compile model\n",
        "model_emb.compile(loss='sparse_categorical_crossentropy', optimizer=optimizer)\n",
        "\n",
        "# fit the model\n",
        "history = model_emb.fit(train_dataset, epochs=200, batch_size=64, validation_data=valid_dataset, callbacks=[checkpoint_cb, early_stopping_cb])"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/200\n",
            "247/247 [==============================] - 12s 25ms/step - loss: 1.9755 - val_loss: 1.8263\n",
            "Epoch 2/200\n",
            "247/247 [==============================] - 4s 15ms/step - loss: 1.6792 - val_loss: 1.6991\n",
            "Epoch 3/200\n",
            "247/247 [==============================] - 4s 16ms/step - loss: 1.5090 - val_loss: 1.5976\n",
            "Epoch 4/200\n",
            "247/247 [==============================] - 4s 17ms/step - loss: 1.3853 - val_loss: 1.5424\n",
            "Epoch 5/200\n",
            "247/247 [==============================] - 4s 15ms/step - loss: 1.2829 - val_loss: 1.4887\n",
            "Epoch 6/200\n",
            "247/247 [==============================] - 4s 16ms/step - loss: 1.1824 - val_loss: 1.4437\n",
            "Epoch 7/200\n",
            "247/247 [==============================] - 4s 16ms/step - loss: 1.0941 - val_loss: 1.4170\n",
            "Epoch 8/200\n",
            "247/247 [==============================] - 4s 17ms/step - loss: 1.0152 - val_loss: 1.3969\n",
            "Epoch 9/200\n",
            "247/247 [==============================] - 4s 17ms/step - loss: 0.9446 - val_loss: 1.3761\n",
            "Epoch 10/200\n",
            "247/247 [==============================] - 4s 18ms/step - loss: 0.8793 - val_loss: 1.3630\n",
            "Epoch 11/200\n",
            "247/247 [==============================] - 4s 17ms/step - loss: 0.8196 - val_loss: 1.3548\n",
            "Epoch 12/200\n",
            "247/247 [==============================] - 4s 18ms/step - loss: 0.7646 - val_loss: 1.3473\n",
            "Epoch 13/200\n",
            "247/247 [==============================] - 4s 17ms/step - loss: 0.7148 - val_loss: 1.3460\n",
            "Epoch 14/200\n",
            "247/247 [==============================] - 4s 17ms/step - loss: 0.6699 - val_loss: 1.3444\n",
            "Epoch 15/200\n",
            "247/247 [==============================] - 4s 16ms/step - loss: 0.6272 - val_loss: 1.3442\n",
            "Epoch 16/200\n",
            "247/247 [==============================] - 4s 16ms/step - loss: 0.5874 - val_loss: 1.3467\n",
            "Epoch 17/200\n",
            "247/247 [==============================] - 4s 15ms/step - loss: 0.5505 - val_loss: 1.3508\n",
            "Epoch 18/200\n",
            "247/247 [==============================] - 4s 16ms/step - loss: 0.5204 - val_loss: 1.3509\n",
            "Epoch 19/200\n",
            "247/247 [==============================] - 4s 17ms/step - loss: 0.4891 - val_loss: 1.3611\n",
            "Epoch 20/200\n",
            "247/247 [==============================] - 4s 15ms/step - loss: 0.4617 - val_loss: 1.3589\n",
            "Epoch 21/200\n",
            "247/247 [==============================] - 4s 16ms/step - loss: 0.4362 - val_loss: 1.3605\n",
            "Epoch 22/200\n",
            "247/247 [==============================] - 4s 16ms/step - loss: 0.4132 - val_loss: 1.3657\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aew_hcpuptxS"
      },
      "source": [
        "### Plot Loss"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 404
        },
        "id": "Cx4sgortndI-",
        "outputId": "1f358d48-a95b-45eb-f4cf-f4e8e11b034b"
      },
      "source": [
        "# plot loss\n",
        "plot_loss(history)"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfEAAAGDCAYAAAA72Cm3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hUVf7H8fd3UgghoYcSIPQqVUKRakOxIGBDRbGzKqvuutb9ubu6q+uurq5iL2vBtbF2RawgCCIQeu8toSX0lpByfn/cQQKEECCTm0k+r+eZZ2buPXPnm1HmM/fcc8815xwiIiISfgJ+FyAiIiInRiEuIiISphTiIiIiYUohLiIiEqYU4iIiImFKIS4iIhKmFOIiRWBmY83s2uJu6yczW21mZ4dguz+a2U3Bx0PN7NuitD2B90kys91mFnGitYqEO4W4lFnBL/gDtzwz25fv+dDj2ZZz7jzn3FvF3bY0MrP7zWxiActrmtl+M2tb1G05595xzp1TTHUd8qPDObfWORfnnMstju0f9l7OzJoV93ZFiptCXMqs4Bd8nHMuDlgLDMi37J0D7cws0r8qS6X/Aj3MrPFhy68A5jnn5vtQk4gUQCEu5Y6ZnW5mqWZ2n5ltBN4ws2pm9qWZpZvZtuDj+vlek7+L+Dozm2Rm/wq2XWVm551g28ZmNtHMdpnZ92b2vJn99yh1F6XGv5nZ5OD2vjWzmvnWX2Nma8xsi5n939E+H+dcKjAOuOawVcOAUceq47CarzOzSfme9zOzxWa2w8yeAyzfuqZmNi5YX4aZvWNmVYPr3gaSgC+CPSn3mlmj4B5zZLBNopl9bmZbzWy5md2cb9sPmdloMxsV/GwWmFny0T6DozGzKsFtpAc/ywfNLBBc18zMJgT/tgwz+yC43Mzs32a22cx2mtm84+nNECmMQlzKqzpAdaAhMBzv38IbwedJwD7guUJe3w1YAtQEHgf+Y2Z2Am3fBaYBNYCHODI48ytKjVcB1wO1gGjgbgAzawO8GNx+YvD9CgzeoLfy12JmLYGOwXqP97M6sI2awMfAg3ifxQqgZ/4mwGPB+loDDfA+E5xz13Bob8rjBbzF+0Bq8PWXAn83szPzrb8o2KYq8HlRai7As0AVoAnQF++HzfXBdX8DvgWq4X22zwaXnwP0AVoEX3s5sOUE3lvkCApxKa/ygL8457Kcc/ucc1uccx855/Y653YBj+J9SR/NGufcq8HjsW8BdYHax9PWzJKALsCfnXP7nXOT8MKlQEWs8Q3n3FLn3D5gNF7wghdqXzrnJjrnsoA/BT+Do/kkWGOP4PNhwFjnXPoJfFYHnA8scM596JzLBp4GNub7+5Y7574L/jdJB54q4nYxswZ4Pwjuc85lOudmA68F6z5gknPuq+B/h7eBDkXZdr73iMA7pPCAc26Xc2418CQHf+xk4/2wSQzWMCnf8nigFWDOuUXOuQ3H894iR6MQl/Iq3TmXeeCJmcWa2cvBLtKdwESgqh195HP+8NkbfBh3nG0Tga35lgGsO1rBRaxxY77He/PVlJh/2865PRSyNxis6X/AsGCvwVBg1HHUUZDDa3D5n5tZbTN738zSgtv9L94ee1Ec+Cx35Vu2BqiX7/nhn02MHd94iJpAVHC7Bb3HvXi9CdOC3fU3ADjnxuHt9T8PbDazV8ys8nG8r8hRKcSlvDr88n1/AFoC3ZxzlfG6PyHfMdsQ2ABUN7PYfMsaFNL+ZGrckH/bwfescYzXvIXX9dsPb0/yi5Os4/AajEP/3r/j/XdpF9zu1Ydts7BLLq7H+yzj8y1LAtKOUdPxyODg3vYR7+Gc2+icu9k5lwj8BnjBgiPcnXMjnXOdgTZ43er3FGNdUo4pxEU88XjHdrebWXXgL6F+Q+fcGiAFeMjMos3sNGBAiGr8ELjQzHqZWTTwV4797/8nYDvwCvC+c27/SdYxBjjFzC4O7gHfgTc24YB4YDeww8zqcWTQbcI7Fn0E59w64GfgMTOLMbP2wI14e/MnKjq4rRgziwkuGw08ambxZtYQuOvAe5jZZfkG+G3D+9GRZ2ZdzKybmUUBe4BMCj+UIVJkCnERz9NARby9rV+Ar0vofYcCp+F1bT8CfABkHaXtCdfonFsAjMAbmLYBL2RSj/Eah9eF3jB4f1J1OOcygMuAf+D9vc2ByfmaPAycCuzAC/yPD9vEY8CDZrbdzO4u4C2uBBrh7ZV/gjfm4fui1HYUC/B+rBy4XQ/cjhfEK4FJeJ/n68H2XYCpZrYbb2zDnc65lUBl4FW8z3wN3t/+xEnUJfIr8/6dikhpEDwtabFzLuQ9ASIS/rQnLuKjYFdrUzMLmFl/YCDwqd91iUh40ExVIv6qg9dtXAOve/tW59wsf0sSkXCh7nQREZEwpe50ERGRMKUQFxERCVNhd0y8Zs2arlGjRn6XISIiUmJmzJiR4ZxLOHx52IV4o0aNSElJ8bsMERGREmNmawparu50ERGRMKUQFxERCVMKcRERkTAVdsfERUQkvGRnZ5OamkpmZuaxG5dzMTEx1K9fn6ioqCK1D1mIm1kDvIsm1Ma7ms8rzrlnDmtjwDPA+XjX973OOTczVDWJiEjJS01NJT4+nkaNGuF97UtBnHNs2bKF1NRUGjduXKTXhLI7PQf4g3OuDdAdGGFmbQ5rcx7elYyaA8OBF0NYj4iI+CAzM5MaNWoowI/BzKhRo8Zx9ViELMSdcxsO7FU753YBi4B6hzUbCIxynl+AqmZWN1Q1iYiIPxTgRXO8n1OJDGwzs0ZAJ2DqYavqAevyPU/lyKAXERE5KXFxcX6XEBIhD3EziwM+An7nnNt5gtsYbmYpZpaSnp5evAWKiIiEqZCGuJlF4QX4O865jwtokgY0yPe8fnDZIZxzrzjnkp1zyQkJR8w6JyIiUiTOOe655x7atm1Lu3bt+OCDDwDYsGEDffr0oWPHjrRt25affvqJ3Nxcrrvuul/b/vvf//a5+iOFcnS6Af8BFjnnnjpKs8+B35rZ+0A3YIdzbkOoahIREX89/MUCFq4/oU7Zo2qTWJm/DDilSG0//vhjZs+ezZw5c8jIyKBLly706dOHd999l3PPPZf/+7//Izc3l7179zJ79mzS0tKYP38+ANu3by/WuotDKM8T7wlcA8wzs9nBZX8EkgCccy8BX+GdXrYc7xSz60NYzxH27c9l3OLN9GtTm+hIzXsjIlLWTZo0iSuvvJKIiAhq165N3759mT59Ol26dOGGG24gOzubQYMG0bFjR5o0acLKlSu5/fbbueCCCzjnnHP8Lv8IIQtx59wkoNBhds45B4wIVQ3HMmVlBiPencnr1yVzZqvafpUhIlJuFHWPuaT16dOHiRMnMmbMGK677jruuusuhg0bxpw5c/jmm2946aWXGD16NK+//rrfpR6iXO9+9mqWQJWKUXwxRz34IiLlQe/evfnggw/Izc0lPT2diRMn0rVrV9asWUPt2rW5+eabuemmm5g5cyYZGRnk5eVxySWX8MgjjzBzZumbi6xcT7saHRngvLZ1+GLOejKzc4mJivC7JBERCaHBgwczZcoUOnTogJnx+OOPU6dOHd566y2eeOIJoqKiiIuLY9SoUaSlpXH99deTl5cHwGOPPeZz9Ucyr0c7fCQnJ7vivJ745OUZDH1tKi8OPZXz2mmeGRGR4rZo0SJat27tdxlho6DPy8xmOOeSD29brrvTAbo3qUHNuAp8MXe936WIiIgcl3If4hEB44J2dfhh0WZ2Z+X4XY6IiEiRlfsQBxjQIZGsnDy+X7jJ71JERESKTCEOnJpUjcQqMXwxR13qIiISPhTiQCBgXNghkYnL0tm+d7/f5YiIiBSJQjxoQPtEsnMd3yzY6HcpIiIiRaIQD2pbrzKNasRq4hcREQkbCvEgM2NAh0R+XpFB+q4sv8sREREfFXb98dWrV9O2bdsSrOboFOL5DOiQSJ6DsfO1Ny4iIqVfuZ529XAtasfTsnY8X8xZz7DTGvldjohI2TP2ftg4r3i3WacdnPePQpvcf//9NGjQgBEjvGtuPfTQQ0RGRjJ+/Hi2bdtGdnY2jzzyCAMHDjyut87MzOTWW28lJSWFyMhInnrqKc444wwWLFjA9ddfz/79+8nLy+Ojjz4iMTGRyy+/nNTUVHJzc/nTn/7EkCFDTvjPBoX4EQZ0qMu/vl3K+u37SKxa0e9yRESkGAwZMoTf/e53v4b46NGj+eabb7jjjjuoXLkyGRkZdO/enYsuugizQi/AeYjnn38eM2PevHksXryYc845h6VLl/LSSy9x5513MnToUPbv309ubi5fffUViYmJjBkzBoAdO3ac9N+lED/Mhe0T+de3SxkzdwM392nidzkiImXLMfaYQ6VTp05s3ryZ9evXk56eTrVq1ahTpw6///3vmThxIoFAgLS0NDZt2kSdOnWKvN1JkyZx++23A9CqVSsaNmzI0qVLOe2003j00UdJTU3l4osvpnnz5rRr144//OEP3HfffVx44YX07t37pP8uHRM/TKOalWhfv4rmUhcRKWMuu+wyPvzwQz744AOGDBnCO++8Q3p6OjNmzGD27NnUrl2bzMzMYnmvq666is8//5yKFSty/vnnM27cOFq0aMHMmTNp164dDz74IH/9619P+n0U4gUY0D6Ruak7WJ2xx+9SRESkmAwZMoT333+fDz/8kMsuu4wdO3ZQq1YtoqKiGD9+PGvWrDnubfbu3Zt33nkHgKVLl7J27VpatmzJypUradKkCXfccQcDBw5k7ty5rF+/ntjYWK6++mruueeeYrk+uUK8ABe09y5J+qX2xkVEyoxTTjmFXbt2Ua9ePerWrcvQoUNJSUmhXbt2jBo1ilatWh33Nm+77Tby8vJo164dQ4YM4c0336RChQqMHj2atm3b0rFjR+bPn8+wYcOYN28eXbt2pWPHjjz88MM8+OCDJ/03lfvriR/NpS/+zK7MHL75fZ+Qv5eISFmm64kfH11PvBgM6JDIkk27WLppl9+liIiIFEghfhTntatDwOBLXdlMRKRcmjdvHh07djzk1q1bN7/LOoROMTuKWvExnNa0Bl/M3cDv+7U4rvMGRUQk/LVr147Zs2f7XUahtCdeiAHtE1mVsYcF63f6XYqISFgLt/FXfjnez0khXoj+besQGTC+UJe6iMgJi4mJYcuWLQryY3DOsWXLFmJiYor8GnWnF6JqbDR9WiTw5dwN3Ne/FYGAutRFRI5X/fr1SU1NJT093e9SSr2YmBjq169f5PYK8WMY0KEu4xZvZta6bXRuWN3vckREwk5UVBSNGzf2u4wySd3pe7cWuvrs1rWpEBngizm6PKmIiJQu5TvEV4yDp9vBmp+P2iQ+JoozW9Xiy7kbyM3T8RwRESk9yneIJ54KcbVh9DDYkXbUZgM6JJKxO4upK7eUYHEiIiKFC1mIm9nrZrbZzOYfZX0VM/vCzOaY2QIzuz5UtRxVxapwxbuQvQ8+uBqyC756zRkta1EpOkJXNhMRkVIllHvibwL9C1k/AljonOsAnA48aWbRIaynYLVaweCXYf1MGPMHKOAUiIrREfRrU5ux8zeyPyevxEsUEREpSMhC3Dk3EShs1JgD4s2bCi0u2DYnVPUUqvWF0OdemP1fmP5agU0GdEhk+95sJi/PKOHiRERECubnMfHngNbAemAecKdzzr/d3NMfgBb94ev7YfXkI1b3bp5A5ZhITfwiIiKlhp8hfi4wG0gEOgLPmVnlghqa2XAzSzGzlJBNFhAIwMWvQLVG8L9rYUfqIaujIwOc17Yu3y7cRGZ2bmhqEBEROQ5+hvj1wMfOsxxYBRR4RXbn3CvOuWTnXHJCQkLoKoqpEhzolgkfXHPEQLcBHRLZnZXDj0s2h64GERGRIvIzxNcCZwGYWW2gJbDSx3o8CS3h4gMD3e46ZKBb9ybVqRkXrYlfRESkVAjlKWbvAVOAlmaWamY3mtktZnZLsMnfgB5mNg/4AbjPOVc6Ro21ugD63gez34Fpr/66ODIiwPnt6vLD4k3szvJnDJ6IiMgBIZs73Tl35THWrwfOCdX7n7S+98OGufDNA1D7FGjUE/C61EdNWcMPizYxsGM9n4sUEZHyrHzP2FaYQMDrVq/WODijmzfQrXNSNepUjtEodRER8Z1CvDAHBrrlZP060C0QMC5sX5cJS9PZsTfb7wpFRKQcU4gfS0IL79SzfAPdBnRIJDvX8c2CjX5XJyIi5ZhCvChane8dIw8OdGtfvwpJ1WM1l7qIiPhKIV5Ufe+DFufBNw9ga35mQIe6/LxiCxm7s/yuTEREyimFeFEdNtBtcFPIzXOMna8udRER8YdC/HjkG+jWdNwtnJIQpVHqIiLiG4X48QoOdLP1s3ii4ltMX72FDTv2+V2ViIiUQwrxExEc6NZm85cMC3zLmLmahlVEREqeQvxE9b0PWp7Pn6PeZmXKt35XIyIi5ZBC/EQFAjD4ZXbGJnHX9kdJW73U74pERKScUYifjJjKZF0yigpkE/XhMMjWsXERESk5CvGTVKdpB56rei+1di+CL39/yKVLRUREQkkhXgzqdh3Mv7MvgTnvweRn/C5HRETKCYV4MTi/fV2eyxvM4pr94Pu/wM/P+l2SiIiUAwrxYlArPoZuTRK4fd8tuDaD4NsHYfJIv8sSEZEyTiFeTAZ0SGTZliwWnPYUnDIYvvuTutZFRCSkFOLFpP8pdYgMGB/M2AAXvxYM8j/DpKf9Lk1ERMoohXgxqVYpmiFdGvDutLUsTt8bDPKLvWPkCnIREQkBhXgxuufcllSOieTPny7ABSLg4leh7SXBIP+33+WJiEgZoxAvRlVjo7m3fyumrd7Kp7PTICISBr8CbS+F7x+Cn57yu0QRESlDFOLFbEhyAzrUr8Lfv1rMrszsYJC/DO0ugx8ehp+e9LtEEREpIxTixSwQMP46sC0Zu7N4+vtl3sKISBj0UjDI/woT/+VvkSIiUiYoxEOgQ4OqXNEliTd/Xs2Sjbu8hfn3yMf9DSY+4W+RIiIS9hTiIXLvuS2Jj4nkz5/Nxx2YTz0QEQzyy2HcIzBBQS4iIidOIR4i1SpFc8+5LZm6aiufz1l/cEUgAga/BO2HwPhHYMLj/hUpIiJhTSEeQld0SaJdvSo8OmaRN8jtgEAEDHoR2l8B4x+FH//pX5EiIhK2FOIhFBEw/jaoLem7sxj5w7JDVwYiYNAL0OFK+PHvCnIRETluCvEQ69igKkOSG/DG5NUs27Tr0JWBCBj4PHS4Khjk//CnSBERCUshC3Eze93MNpvZ/ELanG5ms81sgZlNCFUtfru3fysqVYjkz58tODjI7YBABAx8Lhjkj8H4x/wpUkREwk4o98TfBPofbaWZVQVeAC5yzp0CXBbCWnxVvVI0d5/bkikrt/DF3A1HNjgQ5B2HwoR/KMhFRKRIQhbizrmJwNZCmlwFfOycWxtsvzlUtZQGV3VNom29yjw6ZiG7s3KObBCIgIuehY5XB4P873D4XruIiEg+fh4TbwFUM7MfzWyGmQ3zsZaQiwjO5LZpZxbPHj7I7YBDgvyf8O2DkL2vZAsVEZGw4WeIRwKdgQuAc4E/mVmLghqa2XAzSzGzlPT09JKssVidmlSNy5Pr859Jq1i+eVfBjQIBL8g7Xw9TnoORp8LMtyEvt2SLFRGRUs/PEE8FvnHO7XHOZQATgQ4FNXTOveKcS3bOJSckJJRokcXtvv6tiI2O4C+fFzDI7YBAAAY8DdeNgcp14fPfwos9YMlYdbGLiMiv/Azxz4BeZhZpZrFAN2CRj/WUiBpxFbj73JZMXr6Fr+ZtLLxxo15w0w9w2VuQmw3vXQFvnA/rppVMsSIiUqqF8hSz94ApQEszSzWzG83sFjO7BcA5twj4GpgLTANec84d9XS0smRot4ackliZv325kD0FDXLLzwxOGQQjpsIFT8KW5fCffvDB1ZBxlGPrIiJSLthRu3RLqeTkZJeSkuJ3GSdtxpptXPLiz9zStyn3n9eq6C/M2g1TnoefR3qD3k4dBqffD/F1QlesiIj4ysxmOOeSD1+uGdt80rlhNS7tXJ//TFrJivTdRX9hhTg4/T64YxYk3wCz3oaRnbyromXuDF3BIiJS6ijEfXT/ea2IiYrgocIGuR1NXC244F8wYhq0ONe7PvnIjvDLS5CzPzQFi4hIqaIQ91HNuAr8oV8LflqWwdfzjzHI7WhqNIXL3oSbx0GtNvD1ffB8F5j3IeTlFWu9IiJSuijEfXZ194a0rusNctu7/xiD3ApTrzNc+wUM/RCiKsFHN8KrZ8DKH4utVhERKV0U4j6LjAjwt4GnsH5HJs+NW35yGzOD5v3glp9g0EuwdwuMGghvD4YNc4unYBERKTUU4qVAcqPqXHxqPV79aSUrj2eQ29EEIqDjlfDbFDjnEUibCS/3gQ9vhC0rTn77IiJSKijES4kHzmtNTGQED32x8PgHuR1NVAz0uB3unAO9fgeLx8DzXeHL38POAq6mJiIiYUUhXkokxFfg9/1aMHFpOt8s2FS8G69YFc5+CO6cDZ2vg5mjvNPSvvsz7C3sQnMiIlKaKcRLkWGnNaRVnXj+9uVC9u0PwQVP4ut4s779djq0HgCTR8IzHb3T07KKoRtfRERKlEK8FImMCPDXgW1J276P58ef5CC3wlRvApe8CrdOhoY9vIliRnaEqS9DTlbo3ldERIqVQryU6dq4OoM71eOViStZlbEntG9W+xS46n244Vuo2RLG3gvPJcPs93TpUxGRMKAQL4UeOK8V0ZEB/vzZfPLySmBu+6RucN2XcPVHULEafHoLvNgTFn2pS5+KiJRiCvFSqFblGB44vxU/Lcvgye+WlMybmkGzs+HmH70Z4PKy4YOh8NrZsHJCydQgIiLHRSFeSl3VNYkruzbg+fEr+Gx2Wsm9cSAApwyG26bCgJGwawOMusibNCZtRsnVISIix6QQL6XMjIcvakvXRtW598O5zE3dXrIFRERC52vh9plwzqPejG+vngkfXAPpS0u2FhERKZBCvBSLjgzw4tWnUjOuAsNHzWDzzsySLyIqBnr81pswpu/9sGIcvNDNm/1Ne+YiIr5SiJdyNeIq8Nq1yezMzGb42zPIzPZp1HhMZTjjAS/MTxsBS7/x9sz/cy4s/Eyj2UVEfKAQDwOt61bmqcs7Mnvddv748bzim5b1RFSq6c3HftdCOPcx75j56GHeeeZTnofMnf7VJiJSzijEw0T/tnW4q18LPp6VxisTV/pdjrdnftptcMcsuPxtqFwPvvkjPNUGvv4jbFvjd4UiImWeQjyM3H5mMy5oV5d/fL2YcYuLeX71ExWIgDYXwQ1fw83joWV/mPayt2f+wTWw9heday4iEiLma9fsCUhOTnYpKSl+l+GbfftzufSln1mzZS+fjuhBs1rxfpd0pB1pMO0VmPEmZG6HxFO94+htBkJElN/ViYiEHTOb4ZxLPny59sTDTMXoCF4dlkxMVAQ3vpXC9r37/S7pSFXqQb+HvePm5/8LMnfARzfCMx1g0tOwb5vfFYqIlAkK8TCUWLUiL1/TmQ3bMxnx7kyyc/P8Lqlg0ZWg683w2xS48gOo0RS+/ws8dQqMuRu2rPC7QhGRsKYQD1OdG1bj0cFtmbx8C4+OWeR3OYULBLxj5dd+AbdM8rrVZ74Fz3aGd6+Apd/C/r1+VykiEnYi/S5ATtxlyQ1YsnEXr01aRcs68VzZNcnvko6tTjsY/CKc/RBMfw1S/gNLx0JENNTvCk36QuO+UO9UHT8XETkGDWwLc7l5jhvenM7k5Rm8c1M3ujWp4XdJxyc7E9ZMhpU/wqoJ3vSuOIiOh0Y9vUBv0hdqtfEu0iIiUg4dbWCbQrwM2LEvm8EvTGb73mw+G9GTBtVj/S7pxO3dCqt/8kJ95QTYGjxuXikBGveBJqd7wV6toY9FioiULIV4GbcyfTeDnp9MYtWKfHRrDypVKCNHSnakemG+aoIX7LuD58dXa3RwL71xX28mORGRMkohXg5MXJrOdW9M4+zWtXnp6s4EAmWs+9k5SF8SDPQJ3h57VnCa19rtgoHexzsvPS7B31pFRIpRiYe4mb0OXAhsds61LaRdF2AKcIVz7sNjbVchXrj/TFrF375cyB1nNuOuc1r6XU5o5ebAhtkHj6evnQq5Wd66yvWgbgeo2zF43wEq1/W1XBGRE3W0EA9ln+ubwHPAqEKKigD+CXwbwjrKlRt6NmLJxp2MHLecFnXiubB9ot8lhU5EJNRP9m597obsfd7lUdfPhg1zvIBfMhYI/lCNq31osCd29MJeA+ZEJEyFLMSdcxPNrNExmt0OfAR0CVUd5Y2Z8bdBbVmZvoe7/zeHRjUq0bZeFb/LKhlRFaFRL+92QNZu2DT/0GBf/j244AQ5sTWO3GOv1kjBLiJhwbfRT2ZWDxgMnIFCvFhViIzgxas7M/C5Sdw8KoXPftuTWvExfpfljwpxkNTdux2wfy9sXgjrZwWDfQ78PBLycrz1MVUOBntiR+8Yu4JdREohP4cwPw3c55zLs2N8OZrZcGA4QFJSGExoUgokxFfg1WuTufTFKdzy9gzeG96dCpERfpdVOkTHHuyGPyAnKxjs+fbYp74EucG56WOqQmInbxKaxE7eTV3xIuKzkI5OD3anf1nQwDYzWwUc+AasCewFhjvnPi1smxrYdnzGztvAre/M5JJT6/Ovy9pzrB9Mkk/Ofkhf5O2xp8307jcvPLjHXqnWwUA/EO5xtfytWUTKJD8GthXKOdf4wGMzexMv7AsNcDl+57Wry51nNeeZH5ZRMz6a+/u3UpAXVWT0wePkna/zlmVnBo+x5wv25d8dPMZeud7BYD9wi63u258gImVbyELczN4DTgdqmlkq8BcgCsA591Ko3leO9Luzm7NlTxYvT1iJYdzXv6WC/ERFxRzZFZ+1GzbO9QL9QLgv/vLg+mqNvDCvfQpUbeTNNle1obfXrv8OInISQjk6/crjaHtdqOoQb8T6Xy9qS56DlyaswAzuPVdBXmwqxEHDHt7tgH3bvWPr64N766kzYMEnh74usiJUTToY6offV6xasn+HiISdMjI3pxxLIGA8MrAtzsGLP64gYHD3OQrykKlY1bfZlpcAACAASURBVJtBrknfg8v274Hta73btjWwfQ1sW+3dr50KWTsO3UZMlcPCvZEX+lUbevfRYTxHvki4ycuD7D1ez9v+3ZC1y/s3vX93cNmufOt2Q7+HS+RKjArxciQQMB4d1BZwPD9+BYbxh3NaKMhLSnQlqNXauxVk37Z84Z7vPn0JLPsOcjIPbR9VCSrV8M51j63p3VcK3v/6+MDyGlChindtd5FQysuFPRmwa4N3rYNdG2DXRu+2bxsEIrxLDwcivfuIqMMeR3n3EVH52kUV/BrnvIGmh9xyweUe+rygNoc/z96XL5APhHS+5/v38OvEUccSVcmbgKoExsMoxMsZL8jb4Rw8N345ZnBXPwV5qVCxmndL7HjkOue8L8Rta7w9+R1rYc8W2LsF9mbAnnQv7PdmQPbegrdvEUcG/YHnFat7XzgVq0NstYPPK1TWcXvxFBbOuzbC7gP3m70QPVxsTe//qbxcyMuG3Hy3vHz3JcUC3g+BQKT3byOygndoLDreu6+UANUbQ3QcVIgP3scV8ryS9zg6rkR/LCvEy6FAwPj7YC/Inx23HDPjrn4t/C5LCmMG8XW8W1K3wtvu35sv3PMF/d4t3pfw3uCyTQu85fu2FfK+Ed4Pi18D/rCgL2hddKXgl2OE9/pAhH4IFIVz3rwE2Xu9/4bZ+7zHv972BZcfWLfn6MsOnC1xYLuHvtGR73u09c55e6HHCuf4uhBf2xu8GRf8/zS+7sH/ZyvV8s72KMpnkJcTDPf9wcf7gwGf7/GBwD/w/9eBMP71cf5lBTy3iDLTK6UQL6cCAeOxi9vhcIz8YRkG/F5BXjZEx3q3qg2K1j43BzJ3wL6t3vXcj7jfdvDx9rXehDj7th7ZvV8YCxz6hXvgS9QiDgv8QL71h31BW1G+oA//Uo88dPsW/OI2Ayzfjws7uCz/eg7cFdA2L/dg2OS/5ew/ctmBdjlZ+V6T73HOfi+A84dv0T5Y70dTVEWIig3eYry/+fB2hzw9/EdVIesrVj8snPMFdFHDuajMDnalozEfRaEQL8cCAeMfF7cnz8EzPyzDDH53toK83ImI9I6ZV6pxfK/bv9cL833bDg39/XuCxyRzvVA6cPzxkGX5j13mHvY4p4Dl+Y5d5mRB3p6jH9t0BSw78Ng5wB28Lw6BqIPHaSMrHHwcUeHgsdzICl7QxlQJro8+8jVRsV6bI0I5+Dj6sOdRsd5r1ctRrinEy7lAwPjnJe1xDp7+fhkBM+44q7nfZUk4OLDHX6W+35WcPOfydSsX9NgdbHfgsQW88FWIio8U4kJEwHj80vY4HE99txQDbleQS3lipjCWsKQQF8AL8icu7QAOnvxuKWbw2zMV5CIipZlCXH4VETCeuKwDDvjXt0sxM0ac0czvskRE5CgU4nKIiIDxr8s64JzjiW+WYAa3na4gFxEpjRTicoSIgPHk5R1xwONfL8Ewbj29qd9liYjIYRTiUqCIgPHkZR1wDv759WICBr/pqyAXESlNFOJyVJERAZ663DtG/tjYxZjB8D4KchGR0kIhLoWKjAjw78u9Y+R//2oxhnFznyZ+lyUiIijEpQgiIwI8PaQjzsGjXy3CDG7qrSAXEfGbQlyKJDIiwNNXdMTheGTMIkBBLiLiN4W4FFlURIBnrugEzOKRMYvIzXMa7CYi4iOFuByXA0EesNk8NnYxuc7pPHIREZ8oxOW4RQWPkUcEjMe/XkJurtNc6yIiPlCIywnxTj/rSIQZT363lFzndBlTEZESphCXE3ZgrvVAwHj6+2Xk5Tl+368FpqtBiYiUCIW4nJSIgPH4Je2JMGPkuOXkOsfd57RUkIuIlACFuJy0QMB47OJ2BALG8+NXkJsH9/VXkIuIhJpCXIpFIGA8OqgtEQF4acIKcvPy+OP5rRXkIiIhVKQQN7NKwD7nXJ6ZtQBaAWOdc9khrU7CSiBg/G1gWyLMePWnVeTmwZ8uVJCLiIRKUffEJwK9zawa8C0wHRgCDA1VYRKezIyHLjqFQMB4ffIq8pzjLwPaKMhFREKgqCFuzrm9ZnYj8IJz7nEzmx3KwiR8mRl/vrANkQFvjzwnL4+/XtSWQEBBLiJSnIoc4mZ2Gt6e943BZRGhKUnKAjPjj+e3JhAwXp6wktw8eHSQglxEpDgVNcR/BzwAfOKcW2BmTYDxhb3AzF4HLgQ2O+faFrB+KHAfYMAu4Fbn3JzjKV5KNzPj/v6tiDDjhR9XkJfnfh3FLiIiJ69IIe6cmwBMADCzAJDhnLvjGC97E3gOGHWU9auAvs65bWZ2HvAK0K0o9Uj4MDPuObclkYGD55H/85L2RCjIRUROWqAojczsXTOrHBylPh9YaGb3FPYa59xEYGsh6392zm0LPv0FqF/EmiXMmBl3ndOS353dnA9npHLP/+aQm+f8LktEJOwVKcSBNs65ncAgYCzQGLimGOu4MbhdKcN+d3YL/tCvBR/PSuOu0bPJyc3zuyQRkbBW1GPiUWYWhRfizznnss2sWHalzOwMvBDvVUib4cBwgKSkpOJ4W/HJ7Wc1JxAwnvhmCbl5jqeHdCQyoqi/JUVEJL+ihvjLwGpgDjDRzBoCO0/2zc2sPfAacJ5zbsvR2jnnXsE7Zk5ycrL6YcPciDOaERkwHhu7mDzneOaKTkQpyEVEjltRB7aNBEbmW7QmuAd9wswsCfgYuMY5t/RktiXh5zd9mxIRMB4Zs4js3Jk8e2UnYqJ01qKIyPEo6sC2Kmb2lJmlBG9PApWO8Zr3gClASzNLNbMbzewWM7sl2OTPQA3gBTObbWYpJ/OHSPi5qXcTHr7oFL5ftIkrX/2FjN1ZfpckIhJWzLlj906b2Ud4o9LfCi66BujgnLs4hLUVKDk52aWkKO/Lkq/nb+DO92dTu3IMb1zfhaYJcX6XJCJSqpjZDOdc8uHLi3ogsqlz7i/OuZXB28NAk+ItUcqr/m3r8v7w7uzJyuHiF35m2qqjnpkoIiL5FDXE95nZr6PHzawnsC80JUl51CmpGp/c1pMacdFc/dpUPpud5ndJIiKlXlFD/BbgeTNbbWar8WZi+03IqpJyKalGLB/f2oOOSVW58/3ZPD9+OUU53CMiUl4VKcSdc3Occx2A9kB751wn4MyQViblUtXYaN6+sSsDOybyxDdLeODjeWRrUhgRkQId18m5zrmdwZnbAO4KQT0iVIiM4OkhHbn9zGa8P30dN7w5nV2Z2X6XJSJS6pzMDBu6goWEjJnxh3Na8vgl7ZmyYguXvTSFDTs0DENEJL+TCXEdrJSQu7xLA964vgtp2/Yx6PnJLFi/w++SRERKjUJD3Mx2mdnOAm67gMQSqlHKud7NE/jfracRYcblL01h/JLNfpckIlIqFBrizrl451zlAm7xzrmizrsuctJa1anMJyN60qhmJW56K4V3pq7xuyQREd/pqhMSNmpXjmH0b06jT/Oa/N8n83ls7CLydF1yESnHFOISVipViOTVYckM7ZbEyxNWcvt7s8jMzvW7LBERX6hLXMJOZESARwa1pWGNWP7+1WI27szk1WHJVK8U7XdpIiIlSnviEpbMjOF9mvLC0FOZn7aDi1+YzKqMPX6XJSJSohTiEtbOb1eXd2/uzs7MHC5+YTIpq3XxFBEpPxTiEvY6N6zGJ7f1oGpsNFe9NpXR09f5XZKISIlQiEuZ0LBGJT6+tQddG1Xn3o/mct+HczXgTUTKPIW4lBnVKkXz1g1d+e0ZzfggZR2XvvQz67bu9bssEZGQUYhLmRIRMO4+tyX/uTaZtVv2csHInxi3eJPfZYmIhIRCXMqks1rX5svbe1O/Wiw3vJnCU98uIVcTw4hIGaMQlzIrqUYsH9/Wg8uT6zNy3HKue2MaW/fs97ssEZFioxCXMi0mKoLHL+3APy9px9RVW7lw5E/MWrvN77JERIqFQlzKhSFdkvj41h4EAsblL0/h7SmrcU7d6yIS3hTiUm60rVeFL2/vRa9mNfnTZwu4a/Qc9u7P8bssEZETphCXcqVqbDT/ubYLf+jXgk9npzH4+Z9Zmb7b77JERE6IQlzKnUDAuP2s5rx1fVc278rkoucm8/X8DX6XJSJy3BTiUm71aZHAl3f0pmmtOG7570we+2oRObl5fpclIlJkCnEp1+pVrcjo33Tn6u5JvDxxJVe9NpXNuzL9LktEpEgU4lLuVYiM4JFB7Xjq8g7MTd3OBSMnMW2VroYmIqWfQlwk6OJT6/PpiJ5Uio7gyld/4bWfVuo0NBEp1UIW4mb2upltNrP5R1lvZjbSzJab2VwzOzVUtYgUVas6lfn89l6c3boWj4xZxM2jUkjfleV3WSIiBQrlnvibQP9C1p8HNA/ehgMvhrAWkSKrHBPFS1d35k8XtmHisgz6Pz2Rbxds9LssEZEjhCzEnXMTgcIOLA4ERjnPL0BVM6sbqnpEjoeZcWOvxnx5ey9qV45h+NszuPfDOezO0uQwIlJ6+HlMvB6wLt/z1OAykVKjRe14Ph3Rk9tOb8qHM1I575mJGvQmIqVGWAxsM7PhZpZiZinp6el+lyPlTHRkgHv7t2L0b07DMIa8MoXHxi4iKyfX79JEpJzzM8TTgAb5ntcPLjuCc+4V51yycy45ISGhRIoTOVxyo+qMvbM3V3RpwMsTVjLwucks3rjT77JEpBzzM8Q/B4YFR6l3B3Y45zT3pZRqlSpE8tjF7fnPtclk7M7iomcn88rEFeTm6VQ0ESl5oTzF7D1gCtDSzFLN7EYzu8XMbgk2+QpYCSwHXgVuC1UtIsXtrNa1+eZ3fTi9ZQJ//2oxV736C6nb9vpdloiUMxZuk1kkJye7lJQUv8sQAcA5x4czUnn4i4UA/GVAGy7tXB8z87kyESlLzGyGcy758OVhMbBNpLQyMy5LbsDYO3vTJrEy93w4l1v+O4MtuzVBjIiEnkJcpBg0qB7Lezd354/nt2L84nTOffonfli0ye+yRKSMU4iLFJOIgDG8T1M++21PasZFc+NbKTzw8Tz2aIIYEQkRhbhIMWtdtzKf/bYnv+nbhPenr+X8kT8xY40miBGR4qcQFwmBCpERPHBea96/uTu5eY7LXprCP79eTGa2JogRkeKjEBcJoW5NajD2zt5c2rk+L/64gvOf+YmpK7f4XZaIlBEKcZEQi4+J4vFLO/D2jV3Zn5vHkFd+4Y+fzGNnZrbfpYlImFOIi5SQ3s0T+Pb3fbipV2Pen7aWfk9N0CVOReSkKMRFSlBsdCQPXtiGT27rSbXYaIa/PYPb3pnB5l2ZfpcmImFIIS7igw4NqvLF7b2459yWfL9oM2c/OYHR09cRbjMoioi/FOIiPomKCDDijGaMvbM3repW5t6P5jL0tamsztjjd2kiEiYU4iI+a5oQx/s3d+fvg9sxL3UH5z49kZcnrCAnN8/v0kSklFOIi5QCgYBxVbckvrurL31aJPDY2MUMfH4y89N2+F2aiJRiCnGRUqROlRheuaYzLww9lU07sxj4/GT+MVaTxIhIwRTiIqWMmXF+u7r8cFdfLj21Pi9NWEH/pyfy84oMv0sTkVJGIS5SSlWJjeKfl7bn3Zu64YCrXp3K/R/NZcdeTRIjIh6FuEgp16NZTb6+sw+/6duE/81I5ex/T2DsvA06HU1EFOIi4aBitHdBlc9G9KRWfAVufWcm174xneWbd/tdmoj4SCEuEkba1qvCZyN68qcL2zBr7Tb6Pz2RR8csZJfmYRcplxTiImEmMiLAjb0aM/7u07m0c31em7SKM/41gf+lrCMvT13sIuWJQlwkTNWMq8A/LmnPZyN6klS9Ivd8OJfBL/7M7HXb/S5NREqIQlwkzLWvX5UPb+nBU5d3YMP2fQx6fjL3/G+OLqoiUg4oxEXKgEDAuPjU+oy7+3Ru6duUT2encea/JvDqxJXsz9H0rSJllUJcpAyJqxDJ/ee14tvf96Vr4+o8+tUi+j8zkQlL0/0uTURCQCEuUgY1rlmJ16/rwuvXJeMcXPv6NG56K4U1W3SFNJGyRCEuUoad2ao2X/+uN/ef14opKzLo99REnvhmMXuycvwuTUSKgUJcpIyrEBnBLX2bMu7u07mwfV2eH7+Cs56cwGez0zTrm0iYU4iLlBO1K8fw1JCOfHRrDxLiK3Dn+7O5/OUpLFivy52KhCuFuEg507lhNT4b0ZN/XtKOlel7GPDsJO77cC4bduzzuzQROU4hDXEz629mS8xsuZndX8D6JDMbb2azzGyumZ0fynpExBMIGEO6JDHu7tO5oWdjPpmVxulP/Mg/v17Mjn2awlUkXFiojomZWQSwFOgHpALTgSudcwvztXkFmOWce9HM2gBfOecaFbbd5ORkl5KSEpKaRcqrdVv38tR3S/l0dhpVKkbx2zOacXX3hsRERfhdmogAZjbDOZd8+PJQ7ol3BZY751Y65/YD7wMDD2vjgMrBx1WA9SGsR0SOokH1WP49pCNf3t6L9vWr8siYRZz15AQ+nplKruZjFym1Qhni9YB1+Z6nBpfl9xBwtZmlAl8Bt4ewHhE5hlMSqzDqhq68c1M3qleK5q7Rc7hg5E/8uGSzRrKLlEJ+D2y7EnjTOVcfOB9428yOqMnMhptZipmlpKdr5imRUOvZrCafjejJs1d2Yu/+XK57YzpXvTqVObq4ikipEsoQTwMa5HteP7gsvxuB0QDOuSlADFDz8A05515xziU755ITEhJCVK6I5BcIGAM6JPL9XX15+KJTWLppFwOfn8yId2eyOkMzv4mUBqEM8elAczNrbGbRwBXA54e1WQucBWBmrfFCXLvaIqVIdGSAa3s04sd7TueOs5ozfvFmzn5qAn/+bD7pu7L8Lk+kXAvZ6HSA4CljTwMRwOvOuUfN7K9AinPu8+CI9FeBOLxBbvc6574tbJsanS7ir827Mhn5wzLem7aOmMgAN/dpwk29mxBXIdLv0kTKrKONTg9piIeCQlykdFiZvpt/fbuEr+ZtpGZcNHee1ZwruiYRFeH3UBuRssePU8xEpAxrkhDHC0M788ltPWiaEMefPltAv6cm8Pmc9TotTaSEKMRF5KR0SqrG+8O788Z1XagQGcEd783inH9P4JNZqeTk5vldnkiZpu50ESk2eXmOsfM38uy4ZSzeuIuGNWIZcXozBp9aT93sIidBx8RFpMTk5Tm+W7SJZ8ctY37aTupVrcitpzflsuT6VIjUVK4ix0shLiIlzjnHj0vSGTluGbPWbqdO5Rhu6duEK7omaV52keOgEBcR3zjnmLx8CyPHLWPaqq3UjKvAb/o0YWj3JGKjdWqayLEoxEWkVPhl5RaeHbeMycu3UL1SNDf2asyw0xoSHxPld2kipZZCXERKlRlrtvHsuGX8uCSdKhWjuKFnY67r2YgqFRXmIodTiItIqTQ3dTsjf1jO94s2EV8hkmt7NOLGXo2pVina79JESg2FuIiUagvX7+S58csYO38jsVERXH1aQ27u3YSacRX8Lk3EdwpxEQkLSzft4rlxy/ly7nqiIgJcfGp9buzViGa14v0uTcQ3CnERCSsr03fzysSVfDwrjf05efRtkcANvRrTp3lNzMzv8kRKlEJcRMLSlt1ZvDt1LaN+WUP6riya14rjhl6NGdypns41l3JDIS4iYS0rJ5cxczfwn0mrWLB+J9VioxjarSHDTmtIrcoxfpcnElIKcREpE5xzTF21ldcnreK7RZuIDBgD2idyQ6/GtK1Xxe/yRELiaCGuqZJEJKyYGd2b1KB7kxqs2bKHNyav5n8p6/h4VhpdG1fnxl6NObt1bSICOm4uZZ/2xEUk7O3MzGb09HW8MXk1adv3kVQ9lut6NOLyLg2Iq6B9FQl/6k4XkTIvJzePbxdu4vVJq0hZs434CpEM6dKAa3s0okH1WL/LEzlhCnERKVdmr9vO65NW8dW8DeQ5x7mn1OG6Ho3o2ri6TlGTsKMQF5FyacOOfYyasoZ3p65lx75sWtWJ55rTGjKoYz0qqatdwoRCXETKtX37c/l8ThqjpqxhwfqdxFeI5NLk+lzTvSFNEuL8Lk+kUApxERG8U9Rmrt3OqCmr+WreBrJzHb2b12TYaY04s1UtjWqXUkkhLiJymPRdWbw/bS3vTF3Lxp2Z1KtakaHdk7iiSxLVdRU1KUUU4iIiR5GTm8f3izbx1s9rmLJyC9GRAS5sX5drT2tEhwZV/S5PRCEuIlIUSzft4u0pa/h4Zip79ufSoX4VrjmtERe2r6u52sU3CnERkeOwKzObT2al8dbPq1mRvodqsVEM6ZLE0G5JOudcSpxCXETkBDjnmLJiC29NWc13CzcBcGar2lzdPYnezRM0EE5KhOZOFxE5AWZGj2Y16dGsJuu37+PdqWt5b9pavl+0iXpVK3JZcn0uT25AYtWKfpcq5ZD2xEVEjlNWTi7fL9zM+9PX8tOyDMygb4sErujSgDNb1SY6MuB3iVLG+NKdbmb9gWeACOA159w/CmhzOfAQ4IA5zrmrCtumQlxESpN1W/fyv5R1jE5JZePOTGrGRXPJqfW5vEsDmmoSGSkmJR7iZhYBLAX6AanAdOBK59zCfG2aA6OBM51z28yslnNuc2HbVYiLSGmUm+eYuDSd96at5YfFm8nNc3RtVJ0rujbgvLZ1qRitke1y4vwI8dOAh5xz5wafPwDgnHssX5vHgaXOudeKul2FuIiUdpt3ZfLRjDQ+mL6W1Vv2Eh8TyaCO9RjSpQFt61XxuzwJQ34MbKsHrMv3PBXodlibFgBmNhmvy/0h59zXIaxJRCTkasXHcOvpTbmlbxOmrtrK+9PW8kHKOt7+ZQ1t61Xmii5JXNQxkcoxUX6XKmHO79HpkUBz4HSgPjDRzNo557bnb2Rmw4HhAElJSSVdo4jICTEzujepQfcmNXh4bzafzk7jvWlrefDT+TwyZiEXtEvkiq4NSG5YTZdHlRMSyhBPAxrke14/uCy/VGCqcy4bWGVmS/FCfXr+Rs65V4BXwOtOD1nFIiIhUiU2imt7NGLYaQ2Zl7aD96at4/PZaXw0M5UmCZW4uFM9BnWqR/1qmkhGii6Ux8Qj8Qa2nYUX3tOBq5xzC/K16Y832O1aM6sJzAI6Oue2HG27OiYuImXFnqwcxszbwIcpqUxbvRWAro2rc3GnepzXri5VKqq7XTx+nWJ2PvA03vHu151zj5rZX4EU59zn5vUfPQn0B3KBR51z7xe2TYW4iJRF67bu5dNZaXwyK42VGXuIjgxwdutaDO5Un74tEnTueTmnaVdFRMKAc465qTv4ZFYan89Zz9Y9+6kWG8WADokM6lSPTg2q6vh5OaQQFxEJM9m5eUxcms4ns9L4buEmsnLyaFyzEoM61mNwp3ok1dDx8/JCIS4iEsZ2Zmbz9byNfDwrlV9WesfPkxtWY1CnelzYvi5VY6N9rlBCSSEuIlJGpG3fx2ez0/hkZhrLNu8mOiLAGa0SGNypPme0SqBCpGaHK2sU4iIiZYxzjgXrd/LJrDQ+m72ejN1ZVKkYxVmtanFm61r0bp6gEe5lhEJcRKQMy8nNY9LyDD6fvZ5xSzazfW82kQEjuVE1zmpVmzNa1aJpQiUNigtTCnERkXIiN88xe902fli0mXGLN7N44y4AGtaI5YyWtTirdS26Nq6ubvcwohAXESmn0rbvY9zizYxbtImfV2whKyePStER9Gpek7Na1eb0VgnUio/xu0wphEJcRETYtz+Xn1dkeKG+eDMbdmQC0L5+lV/30tsmViEQULd7aaIQFxGRQzjnWLRhF+OXbOaHRZuYtW47zkFCfAXOaJnAma1q0at5AnEV/L5WlijERUSkUFt2ZzFhaTo/LN7MxKXp7MrMIToiQI9mNTi7dW36talN7crqdveDQlxERIosOzePlNXb+H7RJr5buIm1W/cC0KF+Ffq1qU2/NnVoUTtOo91LiEJcREROiHOOZZt3893CTXy7cBNz1m0HIKl67K976F0aVSMyQhdpCRWFuIiIFItNOzP5YdFmvlu4kckrtrA/J4+qsVGc2bIWZ7epTZ8WOo5e3BTiIiJS7PZk5TBxaTrfLdrEuMXeJDMHjqP3a1Obs1vrOHpxUIiLiEhI5eTmkbJmG98tLPg4+tltatOydryOo58AhbiIiJSYox1HrxVfgV7NatKreU16NatJLe2lF4lCXEREfLN5Zybjl2zmp2UZ/LxiC1v37AegZe14L9Cb16Rb4+rERutYekEU4iIiUirk5TkWbtjJpOUZTFqWwbTVW9mfk0d0RIBTG1ald/MEejWrSdt6VYjQzHGAQlxEREqpzOxcpq/eyqRlGfy0LIOFG3YCUKViFD2a1qBX85r0bpZAUo1Ynyv1j0JcRETCQsbuLCYH99InLc/4dX73pOqxwUCvSY+mNakSW36ula4QFxGRsOOcY0X6HiYv9/bSf1m5hd1ZOQQMTkmsQvcm1enepAZdGlenckzZDXWFuIiIhL3s3DzmrNvOpOVeoM9cu539OXkEDNrWq0L3JjXo3qQ6yY3KVqgrxEVEpMzJzM5l1trt/LJyC7+s3MKstdvZn1v2Ql0hLiIiZV5ZDXWFuIiIlDuFhXq7X0O9Bp0bVSvVoa4QF/n/9u4/tq6yjuP4+9O71tVRuo3NpqwbRazJhpZZJyFIzJRoJiRMQyKgf6AxwRAwMyFE4h9q/JGoiajIYjIU2R8oISjS6KLg/JmouLWOwdYY59KWzbJ1odvEMNiPr3+c03Hp7GSh554+PZ9XcnPPee7d7bfn5Olnzzn3nMfMKm+6UJfgrW9qo++iRbwzf3Rf8MZZc4tYh7iZmdkUx46fZHB0goHhCQZGJxgcmeDosRMALF7QQt+KV0K9t6ud+c21UuqcLsR9fzszM6us+c01rrwku+4csrvJ/XP8BQZGJrLH6AS/HjoAwLwmcemF579qtN7Z3lpm+R6Jm5mZnc3z/3mZv41OnA72p/Yd5tjxUwBc2D7/VaG+svN8mmtNM15DHkEw6AAABgVJREFUKSNxSeuA7wA14PsR8bVp3nc98AjwrohwQpuZ2ayxeEELV6/s4OqVHUB2rfrQ2FEGRyYYGD3M4MgEP985BsD85iYu61rIvR/tY2nbGwqvrbAQl1QDNgLvB/YB2yT1R8TuKe9rAzYATxZVi5mZ2UxprjXR27WQ3q6FfPzdWdvYkRcZHDnMwMgEu/51hMULWhpSS5Ej8cuBPRGxF0DSQ8B6YPeU930Z+DpwZ4G1mJmZFaazvZVre1u5trezoT935g/cv2IZ8Gzd+r687TRJfcDyiPjF2T5I0i2StkvaPj4+PvOVmpmZJajIED8rSU3A3cAd/++9EbEpItZExJqlS5cWX5yZmVkCigzx/cDyuvWuvG1SG/A24HeShoErgH5JZ3z7zszMzM5UZIhvA3okXSypBbgR6J98MSKORMSSiOiOiG7gL8B1/na6mZnZa1NYiEfECeB24FfAEPBwROyS9CVJ1xX1c83MzKqi0OvEI2ILsGVK2+enee/aImsxMzOba0r7YpuZmZm9Pg5xMzOzRDnEzczMEuUQNzMzS5RD3MzMLFEOcTMzs0Q5xM3MzBKliCi7hnMiaRwYmcGPXAIcmsHPs3PnfVA+74NyefuXb7bvg4si4ozJQ5IL8ZkmaXtE+H7tJfI+KJ/3Qbm8/cuX6j7w4XQzM7NEOcTNzMwS5RCHTWUXYN4Hs4D3Qbm8/cuX5D6o/DlxMzOzVHkkbmZmlqhKh7ikdZL+LmmPpLvKrqeKJA1LelrSDknby66nCiTdL+mgpGfq2hZLekLSP/LnRWXWOJdNs/2/KGl/3g92SLqmzBrnMknLJf1W0m5JuyRtyNuT7AOVDXFJNWAj8EFgFXCTpFXlVlVZ742I1Sle3pGoB4B1U9ruArZGRA+wNV+3YjzAmdsf4Ft5P1gdEVsaXFOVnADuiIhVwBXAbfnf/iT7QGVDHLgc2BMReyPiZeAhYH3JNZkVLiL+ADw/pXk9sDlf3gx8qKFFVcg0298aJCLGImIwX/43MAQsI9E+UOUQXwY8W7e+L2+zxgrgcUkDkm4pu5gK64iIsXz5OaCjzGIq6nZJO/PD7Ukcyk2dpG7gHcCTJNoHqhziNjtcFRF9ZKc1bpP0nrILqrrILlnxZSuN9T3gEmA1MAZ8s9xy5j5J5wE/AT4TEUfrX0upD1Q5xPcDy+vWu/I2a6CI2J8/HwQeJTvNYY13QFInQP58sOR6KiUiDkTEyYg4BdyH+0GhJDWTBfiDEfHTvDnJPlDlEN8G9Ei6WFILcCPQX3JNlSJpgaS2yWXgA8AzZ/9XVpB+4OZ8+WbgsRJrqZzJ8Mh9GPeDwkgS8ANgKCLurnspyT5Q6Zu95JdxfBuoAfdHxFdLLqlSJL2ZbPQNMA/4kfdB8ST9GFhLNmvTAeALwM+Ah4EVZLMEfiQi/OWrAkyz/deSHUoPYBj4VN35WZtBkq4C/gg8DZzKmz9Hdl48uT5Q6RA3MzNLWZUPp5uZmSXNIW5mZpYoh7iZmVmiHOJmZmaJcoibmZklyiFuVjGSTtbNlrVjJmfwk9RdPzuXmRVrXtkFmFnDvRgRq8suwsxeP4/EzQw4Pbf7N/L53f8q6S15e7ek3+STc2yVtCJv75D0qKSn8seV+UfVJN2Xz9X8uKTW0n4psznOIW5WPa1TDqffUPfakYh4O3Av2d0MAb4LbI6IXuBB4J68/R7g9xFxGdAH7Mrbe4CNEXEpcBi4vuDfx6yyfMc2s4qR9EJEnPc/2oeB90XE3nyCiOci4gJJh4DOiDiet49FxBJJ40BXRLxU9xndwBMR0ZOvfxZojoivFP+bmVWPR+JmVi+mWT4XL9Utn8TfvTErjEPczOrdUPf853z5T2Sz/AF8jGzyCICtwK0AkmqS2htVpJll/D9ks+pplbSjbv2XETF5mdkiSTvJRtM35W2fBn4o6U5gHPhE3r4B2CTpk2Qj7lsBz7xl1kA+J25mwOlz4msi4lDZtZjZa+PD6WZmZonySNzMzCxRHombmZklyiFuZmaWKIe4mZlZohziZmZmiXKIm5mZJcohbmZmlqj/AkdFc495n66qAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 576x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XPgdrQT3p9bc"
      },
      "source": [
        "## Evaluate on a Test set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hA8WIoijpzVW",
        "outputId": "6fcd23ae-0698-4506-ba92-64ef3e250ea9"
      },
      "source": [
        "loss = model_emb.evaluate(test_dataset, verbose=0)\n",
        "print(\"{:25s}\".format(\"Performance on Test Set\"))\n",
        "print(\"Loss: {:.4f}\".format(loss))"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Performance on Test Set  \n",
            "Loss: 1.3428\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "QG7Q-fbD9SXx",
        "outputId": "d4a882ec-45cf-4f2f-dbe9-c9403dcdf1ff"
      },
      "source": [
        "# save predictions as a dataframe\n",
        "save_path = os.path.join(prediction_dir, 'model_3_predictions.csv')\n",
        "model_3_predictions = predict_and_save_as_dataframe(model=model_emb, save_path=save_path,\n",
        "                                                    translate_fn=translate)\n",
        "model_3_predictions.sample(5)"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>enlgish</th>\n",
              "      <th>nepali</th>\n",
              "      <th>predicted</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>287</th>\n",
              "      <td>sit down tom</td>\n",
              "      <td>टम राख्नुहोस्</td>\n",
              "      <td>यहाँ टम छोड्नुहोस्</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56</th>\n",
              "      <td>here they come</td>\n",
              "      <td>यहाँ तिनीहरू आउँछन्</td>\n",
              "      <td>यहाँ यहाँ जाऊ</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>319</th>\n",
              "      <td>look at that</td>\n",
              "      <td>त्यो हेर</td>\n",
              "      <td>यहाँ यहाँ यहाँ रहनुहोस्</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>424</th>\n",
              "      <td>tom hung up on me</td>\n",
              "      <td>टम मलाई झुण्ड्याइयो</td>\n",
              "      <td>टम एक नाक उडायो</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>539</th>\n",
              "      <td>do you watch tv</td>\n",
              "      <td>के तपाई टिभी हेर्नुहुन्छ</td>\n",
              "      <td>के तपाई उसलाई खेल्न सक्नुहुन्छ</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "               enlgish  ...                       predicted\n",
              "287       sit down tom  ...              यहाँ टम छोड्नुहोस्\n",
              "56      here they come  ...                   यहाँ यहाँ जाऊ\n",
              "319       look at that  ...         यहाँ यहाँ यहाँ रहनुहोस्\n",
              "424  tom hung up on me  ...                 टम एक नाक उडायो\n",
              "539    do you watch tv  ...  के तपाई उसलाई खेल्न सक्नुहुन्छ\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5pXs8Gn_LXgc"
      },
      "source": [
        "## Attention-Based Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w6gI6BjM_BQ_"
      },
      "source": [
        "### Encoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8FVDEF_2_Jye"
      },
      "source": [
        "embedding_dim = 256\n",
        "units = 256\n",
        "BATCH_SIZE = 32\n",
        "steps_per_epoch = X_train.shape[0] // BATCH_SIZE\n",
        "val_steps_per_epoch = X_valid.shape[0] // BATCH_SIZE"
      ],
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y460EMGN-_4e"
      },
      "source": [
        "class Encoder(keras.Model):\n",
        "    def __init__(self, vocab_size, embedding_dim, enc_units,\n",
        "                batch_size):\n",
        "        super(Encoder, self).__init__()\n",
        "        self.batch_size = batch_size\n",
        "        self.enc_units = enc_units\n",
        "        self.embedding = Embedding(vocab_size+1, embedding_dim)\n",
        "        self.gru = GRU(self.enc_units,\n",
        "                       return_sequences=True,\n",
        "                       return_state=True,\n",
        "                       recurrent_initializer=\"glorot_uniform\")\n",
        "    \n",
        "    def call(self, x, hidden):\n",
        "        x = self.embedding(x)\n",
        "        output, state = self.gru(x, initial_state=hidden)\n",
        "        return output, state\n",
        "    \n",
        "    def initialize_hidden_state(self):\n",
        "        return tf.zeros((self.batch_size, self.enc_units))\n",
        "    \n",
        "encoder = Encoder(english_vocabsize, embedding_dim, units, BATCH_SIZE)\n",
        "# encoder.layers"
      ],
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P_TfLXgC_bgw"
      },
      "source": [
        "### Attention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OOLgUXMr_DTn"
      },
      "source": [
        "class BahdanauAttention(tf.keras.layers.Layer):\n",
        "    def __init__(self, units):\n",
        "        super(BahdanauAttention, self).__init__()\n",
        "        self.W1 = keras.layers.Dense(units)\n",
        "        self.W2 = keras.layers.Dense(units)\n",
        "        self.V = keras.layers.Dense(1)\n",
        "        \n",
        "    def call(self, query, values):\n",
        "        hidden_with_time_axis = tf.expand_dims(query, 1)\n",
        "        score = self.V(tf.nn.tanh(\n",
        "            self.W1(values) + self.W2(hidden_with_time_axis)))\n",
        "        attention_weights = tf.nn.softmax(score, axis=1)\n",
        "        context_vector = attention_weights * values\n",
        "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
        "        return context_vector, attention_weights"
      ],
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5kPiHSU__gMs"
      },
      "source": [
        "### Decoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M6mzN5-k_dRD"
      },
      "source": [
        "class Decoder(keras.Model):\n",
        "    def __init__(self, vocab_size, embedding_dim, dec_units,\n",
        "                 batch_size):\n",
        "        super(Decoder, self).__init__()\n",
        "        self.batch_size = batch_size\n",
        "        self.dec_units = dec_units\n",
        "        self.embedding = Embedding(vocab_size+1, embedding_dim)\n",
        "        self.gru = GRU(dec_units,\n",
        "                       return_sequences=True,\n",
        "                       return_state=True,\n",
        "                       recurrent_initializer=\"glorot_uniform\")\n",
        "        self.fc = keras.layers.Dense(vocab_size+1)\n",
        "        self.attention = BahdanauAttention(self.dec_units)\n",
        "        \n",
        "    def call(self, x, hidden, enc_output):\n",
        "        context_vector, attention_weights = self.attention(hidden,\n",
        "                                                           enc_output)\n",
        "        x = self.embedding(x)\n",
        "        x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
        "        output, state = self.gru(x)\n",
        "        output = tf.reshape(output, (-1, output.shape[2]))\n",
        "        x = self.fc(output)\n",
        "        return x, state, attention_weights\n",
        "    \n",
        "decoder = Decoder(nepali_vocabsize, embedding_dim, units, BATCH_SIZE)\n"
      ],
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fc2S1Bw4_rIh"
      },
      "source": [
        "### Define Optimizer and the Loss Function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZFigZYY2_hKl"
      },
      "source": [
        "optimizer = keras.optimizers.Adam()\n",
        "loss_object = keras.losses.SparseCategoricalCrossentropy(from_logits=True,\n",
        "                                                         reduction='none')\n",
        "\n",
        "def loss_function(y_true, y_pred):\n",
        "    # mask = tf.math.logical_not(tf.math.equal(true, 0))\n",
        "    loss_ = loss_object(y_true, y_pred)\n",
        "    mask = tf.cast(y_true != 0, tf.float32)\n",
        "    loss_ *= mask\n",
        "    return tf.reduce_mean(loss_)"
      ],
      "execution_count": 70,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D6lZzczD_yVm"
      },
      "source": [
        "# Create checkpoints\n",
        "checkpoint_prefix = os.path.join(checkpoint_4, \"ckpt\")\n",
        "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
        "                                 encoder=encoder,\n",
        "                                 decoder=decoder)"
      ],
      "execution_count": 71,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RtYAEFn8AGlo"
      },
      "source": [
        "### Training the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mn_tPAv_CK3Q"
      },
      "source": [
        "# create a dataset comaptible with Attention Model\n",
        "train_dataset = create_dataset(X_train, y_train)\n",
        "valid_dataset = create_dataset(X_valid, y_valid)\n",
        "test_dataset = create_dataset(X_test, y_test)"
      ],
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-Nlsf6TjtTwG",
        "outputId": "ad0de3a0-fa9b-471e-f871-17646f611364"
      },
      "source": [
        "for row in train_dataset:\n",
        "  print(english_tokenizer.sequences_to_texts([row[0][0, :].numpy()]))\n",
        "  # print(nepali_tokenizer.sequences_to_texts([row[0][1][0, :].numpy()]))\n",
        "  print(nepali_tokenizer.sequences_to_texts([row[1][0, :].numpy()]))\n",
        "  break"
      ],
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['start_ i like puzzles end_ UNK UNK UNK']\n",
            "['start_ मलाई पजलहरू मन पर्छ end_ UNK UNK UNK UNK UNK']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6xCHf0BHAFmm"
      },
      "source": [
        "@tf.function\n",
        "def train_step(inp, targ, enc_hidden, test=False):\n",
        "    loss = 0\n",
        "    val_loss = 0\n",
        "    with tf.GradientTape() as tape:\n",
        "        enc_output, enc_hidden = encoder(inp, enc_hidden)\n",
        "        dec_hidden = enc_hidden\n",
        "        dec_input = tf.expand_dims([english_tokenizer.word_index['start_']]*BATCH_SIZE, 1)\n",
        "        \n",
        "        if test:\n",
        "          # teacher forcing method\n",
        "          for i in range(1, targ.shape[1]):\n",
        "              predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
        "              val_loss += loss_function(targ[:, i], predictions)\n",
        "              predicted_tokens = tf.argmax(predictions, axis=-1)\n",
        "              dec_input = tf.expand_dims(predicted_tokens, 1)\n",
        "              return val_loss / targ.shape[0]\n",
        "\n",
        "\n",
        "        # teacher forcing method\n",
        "        for i in range(1, targ.shape[1]):\n",
        "            predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
        "            loss += loss_function(targ[:, i], predictions)\n",
        "            dec_input = tf.expand_dims(targ[:, i], 1)\n",
        "            \n",
        "        batch_loss = (loss / int(targ.shape[0]))\n",
        "        variables = encoder.trainable_variables + decoder.trainable_variables\n",
        "        gradients = tape.gradient(loss, variables)\n",
        "        optimizer.apply_gradients(zip(gradients, variables))\n",
        "        return batch_loss\n"
      ],
      "execution_count": 74,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xV83gOc1ALE3",
        "outputId": "d80a4fd6-69ec-46ed-fa03-b94947d3bba3"
      },
      "source": [
        "# train for 30 epochs\n",
        "EPOCHS = 30\n",
        "training_loss, validation_loss = [], []\n",
        "# initialize previous validation loss to a large value\n",
        "prev_val_loss = np.inf\n",
        "\n",
        "# loop over number of epochs\n",
        "for epoch in range(EPOCHS):\n",
        "    start = time.time()\n",
        "    # initialize encoder's hidden state\n",
        "    enc_hidden = encoder.initialize_hidden_state()\n",
        "    # initialize total training loss\n",
        "    total_loss = 0\n",
        "    # initialize total validation loss\n",
        "    total_val_loss = 0\n",
        "\n",
        "    # loop over all training batches\n",
        "    for (batch, (inp, targ)) in enumerate(train_dataset.take(steps_per_epoch)):\n",
        "        # calculate batch loss\n",
        "        batch_loss = train_step(inp, targ, enc_hidden)\n",
        "        # update total loss\n",
        "        total_loss += batch_loss\n",
        "        # print loss after every 100 batches\n",
        "        if batch % 100 == 0:\n",
        "            print(\"Epoch {} Batch {} Loss {:.4f}\".format(epoch+1,\n",
        "                                                         batch,\n",
        "                                                         batch_loss.numpy()))\n",
        "    \n",
        "    # loop over all validation batches\n",
        "    for (batch, (inp, targ)) in enumerate(valid_dataset.take(val_steps_per_epoch)):\n",
        "      val_loss = train_step(inp, targ, enc_hidden, test=True)\n",
        "      total_val_loss += val_loss\n",
        "\n",
        "    # save model every five epochs\n",
        "    if (epoch+1 % 5 == 0):\n",
        "      checkpoint.save(file_prefix=checkpoint_prefix)\n",
        "\n",
        "    # calculate current validation loss\n",
        "    current_val_loss = total_val_loss / val_steps_per_epoch\n",
        "    \n",
        "    # save model if validation_loss improves\n",
        "    if current_val_loss < prev_val_loss:\n",
        "      print(\"[INFO] Validation loss improved from {} to {}, saving model weights...\".format(prev_val_loss, \n",
        "                                                                   current_val_loss))\n",
        "      checkpoint.save(file_prefix=checkpoint_prefix)\n",
        "      # update previous validation loss\n",
        "      prev_val_loss = current_val_loss\n",
        "    \n",
        "    # Display training status\n",
        "    print(\"Epoch {} [Training Loss] : {:.4f}\".format(epoch+1,\n",
        "                                        total_loss / steps_per_epoch))\n",
        "    print(\"Epoch {} [Validation Loss] : {:.4f}\".format(epoch+1,\n",
        "                                        total_val_loss / val_steps_per_epoch))\n",
        "    print(\"Time taken for 1 epoch {} sec\\n\".format(time.time() - start))\n",
        "\n",
        "    # store training loss\n",
        "    training_loss.append(total_loss / steps_per_epoch)\n",
        "    # store validation loss\n",
        "    validation_loss.append(total_val_loss / val_steps_per_epoch)"
      ],
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1 Batch 0 Loss 1.1620\n",
            "Epoch 1 Batch 100 Loss 0.6708\n",
            "Epoch 1 Batch 200 Loss 0.6125\n",
            "[INFO] Validation loss improved from inf to 0.12866294384002686, saving model weights...\n",
            "Epoch 1 [Training Loss] : 0.7015\n",
            "Epoch 1 [Validation Loss] : 0.1287\n",
            "Time taken for 1 epoch 20.743913650512695 sec\n",
            "\n",
            "Epoch 2 Batch 0 Loss 0.6049\n",
            "Epoch 2 Batch 100 Loss 0.5623\n",
            "Epoch 2 Batch 200 Loss 0.5479\n",
            "[INFO] Validation loss improved from 0.12866294384002686 to 0.10828609019517899, saving model weights...\n",
            "Epoch 2 [Training Loss] : 0.5713\n",
            "Epoch 2 [Validation Loss] : 0.1083\n",
            "Time taken for 1 epoch 8.622982263565063 sec\n",
            "\n",
            "Epoch 3 Batch 0 Loss 0.5320\n",
            "Epoch 3 Batch 100 Loss 0.5049\n",
            "Epoch 3 Batch 200 Loss 0.4967\n",
            "[INFO] Validation loss improved from 0.10828609019517899 to 0.08974621444940567, saving model weights...\n",
            "Epoch 3 [Training Loss] : 0.5123\n",
            "Epoch 3 [Validation Loss] : 0.0897\n",
            "Time taken for 1 epoch 9.110194444656372 sec\n",
            "\n",
            "Epoch 4 Batch 0 Loss 0.4649\n",
            "Epoch 4 Batch 100 Loss 0.4500\n",
            "Epoch 4 Batch 200 Loss 0.4507\n",
            "[INFO] Validation loss improved from 0.08974621444940567 to 0.08065206557512283, saving model weights...\n",
            "Epoch 4 [Training Loss] : 0.4514\n",
            "Epoch 4 [Validation Loss] : 0.0807\n",
            "Time taken for 1 epoch 9.498546123504639 sec\n",
            "\n",
            "Epoch 5 Batch 0 Loss 0.4043\n",
            "Epoch 5 Batch 100 Loss 0.4053\n",
            "Epoch 5 Batch 200 Loss 0.4100\n",
            "[INFO] Validation loss improved from 0.08065206557512283 to 0.07570390403270721, saving model weights...\n",
            "Epoch 5 [Training Loss] : 0.3978\n",
            "Epoch 5 [Validation Loss] : 0.0757\n",
            "Time taken for 1 epoch 9.814576864242554 sec\n",
            "\n",
            "Epoch 6 Batch 0 Loss 0.3451\n",
            "Epoch 6 Batch 100 Loss 0.3565\n",
            "Epoch 6 Batch 200 Loss 0.3693\n",
            "[INFO] Validation loss improved from 0.07570390403270721 to 0.07279948145151138, saving model weights...\n",
            "Epoch 6 [Training Loss] : 0.3494\n",
            "Epoch 6 [Validation Loss] : 0.0728\n",
            "Time taken for 1 epoch 9.449053525924683 sec\n",
            "\n",
            "Epoch 7 Batch 0 Loss 0.2982\n",
            "Epoch 7 Batch 100 Loss 0.3118\n",
            "Epoch 7 Batch 200 Loss 0.3317\n",
            "[INFO] Validation loss improved from 0.07279948145151138 to 0.07107286900281906, saving model weights...\n",
            "Epoch 7 [Training Loss] : 0.3047\n",
            "Epoch 7 [Validation Loss] : 0.0711\n",
            "Time taken for 1 epoch 8.899318933486938 sec\n",
            "\n",
            "Epoch 8 Batch 0 Loss 0.2592\n",
            "Epoch 8 Batch 100 Loss 0.2731\n",
            "Epoch 8 Batch 200 Loss 0.2920\n",
            "[INFO] Validation loss improved from 0.07107286900281906 to 0.07065702229738235, saving model weights...\n",
            "Epoch 8 [Training Loss] : 0.2654\n",
            "Epoch 8 [Validation Loss] : 0.0707\n",
            "Time taken for 1 epoch 8.799949169158936 sec\n",
            "\n",
            "Epoch 9 Batch 0 Loss 0.2206\n",
            "Epoch 9 Batch 100 Loss 0.2421\n",
            "Epoch 9 Batch 200 Loss 0.2514\n",
            "[INFO] Validation loss improved from 0.07065702229738235 to 0.07034726440906525, saving model weights...\n",
            "Epoch 9 [Training Loss] : 0.2300\n",
            "Epoch 9 [Validation Loss] : 0.0703\n",
            "Time taken for 1 epoch 9.074395179748535 sec\n",
            "\n",
            "Epoch 10 Batch 0 Loss 0.1890\n",
            "Epoch 10 Batch 100 Loss 0.2056\n",
            "Epoch 10 Batch 200 Loss 0.2294\n",
            "[INFO] Validation loss improved from 0.07034726440906525 to 0.06926889717578888, saving model weights...\n",
            "Epoch 10 [Training Loss] : 0.1982\n",
            "Epoch 10 [Validation Loss] : 0.0693\n",
            "Time taken for 1 epoch 8.654568195343018 sec\n",
            "\n",
            "Epoch 11 Batch 0 Loss 0.1631\n",
            "Epoch 11 Batch 100 Loss 0.1698\n",
            "Epoch 11 Batch 200 Loss 0.1941\n",
            "Epoch 11 [Training Loss] : 0.1682\n",
            "Epoch 11 [Validation Loss] : 0.0694\n",
            "Time taken for 1 epoch 9.683337688446045 sec\n",
            "\n",
            "Epoch 12 Batch 0 Loss 0.1397\n",
            "Epoch 12 Batch 100 Loss 0.1460\n",
            "Epoch 12 Batch 200 Loss 0.1702\n",
            "Epoch 12 [Training Loss] : 0.1416\n",
            "Epoch 12 [Validation Loss] : 0.0705\n",
            "Time taken for 1 epoch 9.226902484893799 sec\n",
            "\n",
            "Epoch 13 Batch 0 Loss 0.1170\n",
            "Epoch 13 Batch 100 Loss 0.1233\n",
            "Epoch 13 Batch 200 Loss 0.1398\n",
            "Epoch 13 [Training Loss] : 0.1185\n",
            "Epoch 13 [Validation Loss] : 0.0696\n",
            "Time taken for 1 epoch 8.975070714950562 sec\n",
            "\n",
            "Epoch 14 Batch 0 Loss 0.0949\n",
            "Epoch 14 Batch 100 Loss 0.1013\n",
            "Epoch 14 Batch 200 Loss 0.1194\n",
            "Epoch 14 [Training Loss] : 0.0998\n",
            "Epoch 14 [Validation Loss] : 0.0696\n",
            "Time taken for 1 epoch 8.457971334457397 sec\n",
            "\n",
            "Epoch 15 Batch 0 Loss 0.0817\n",
            "Epoch 15 Batch 100 Loss 0.0880\n",
            "Epoch 15 Batch 200 Loss 0.0977\n",
            "Epoch 15 [Training Loss] : 0.0836\n",
            "Epoch 15 [Validation Loss] : 0.0704\n",
            "Time taken for 1 epoch 8.554429292678833 sec\n",
            "\n",
            "Epoch 16 Batch 0 Loss 0.0695\n",
            "Epoch 16 Batch 100 Loss 0.0736\n",
            "Epoch 16 Batch 200 Loss 0.0816\n",
            "Epoch 16 [Training Loss] : 0.0698\n",
            "Epoch 16 [Validation Loss] : 0.0704\n",
            "Time taken for 1 epoch 8.597604990005493 sec\n",
            "\n",
            "Epoch 17 Batch 0 Loss 0.0571\n",
            "Epoch 17 Batch 100 Loss 0.0596\n",
            "Epoch 17 Batch 200 Loss 0.0763\n",
            "Epoch 17 [Training Loss] : 0.0585\n",
            "Epoch 17 [Validation Loss] : 0.0715\n",
            "Time taken for 1 epoch 8.450416088104248 sec\n",
            "\n",
            "Epoch 18 Batch 0 Loss 0.0484\n",
            "Epoch 18 Batch 100 Loss 0.0498\n",
            "Epoch 18 Batch 200 Loss 0.0598\n",
            "Epoch 18 [Training Loss] : 0.0499\n",
            "Epoch 18 [Validation Loss] : 0.0721\n",
            "Time taken for 1 epoch 8.904979705810547 sec\n",
            "\n",
            "Epoch 19 Batch 0 Loss 0.0418\n",
            "Epoch 19 Batch 100 Loss 0.0409\n",
            "Epoch 19 Batch 200 Loss 0.0491\n",
            "Epoch 19 [Training Loss] : 0.0422\n",
            "Epoch 19 [Validation Loss] : 0.0727\n",
            "Time taken for 1 epoch 9.23619556427002 sec\n",
            "\n",
            "Epoch 20 Batch 0 Loss 0.0410\n",
            "Epoch 20 Batch 100 Loss 0.0398\n",
            "Epoch 20 Batch 200 Loss 0.0424\n",
            "Epoch 20 [Training Loss] : 0.0364\n",
            "Epoch 20 [Validation Loss] : 0.0735\n",
            "Time taken for 1 epoch 9.218698978424072 sec\n",
            "\n",
            "Epoch 21 Batch 0 Loss 0.0309\n",
            "Epoch 21 Batch 100 Loss 0.0289\n",
            "Epoch 21 Batch 200 Loss 0.0336\n",
            "Epoch 21 [Training Loss] : 0.0304\n",
            "Epoch 21 [Validation Loss] : 0.0740\n",
            "Time taken for 1 epoch 8.720405578613281 sec\n",
            "\n",
            "Epoch 22 Batch 0 Loss 0.0299\n",
            "Epoch 22 Batch 100 Loss 0.0257\n",
            "Epoch 22 Batch 200 Loss 0.0264\n",
            "Epoch 22 [Training Loss] : 0.0258\n",
            "Epoch 22 [Validation Loss] : 0.0746\n",
            "Time taken for 1 epoch 9.119285821914673 sec\n",
            "\n",
            "Epoch 23 Batch 0 Loss 0.0229\n",
            "Epoch 23 Batch 100 Loss 0.0226\n",
            "Epoch 23 Batch 200 Loss 0.0232\n",
            "Epoch 23 [Training Loss] : 0.0218\n",
            "Epoch 23 [Validation Loss] : 0.0750\n",
            "Time taken for 1 epoch 9.249422788619995 sec\n",
            "\n",
            "Epoch 24 Batch 0 Loss 0.0195\n",
            "Epoch 24 Batch 100 Loss 0.0178\n",
            "Epoch 24 Batch 200 Loss 0.0203\n",
            "Epoch 24 [Training Loss] : 0.0191\n",
            "Epoch 24 [Validation Loss] : 0.0757\n",
            "Time taken for 1 epoch 9.235056400299072 sec\n",
            "\n",
            "Epoch 25 Batch 0 Loss 0.0144\n",
            "Epoch 25 Batch 100 Loss 0.0143\n",
            "Epoch 25 Batch 200 Loss 0.0213\n",
            "Epoch 25 [Training Loss] : 0.0174\n",
            "Epoch 25 [Validation Loss] : 0.0789\n",
            "Time taken for 1 epoch 8.678094863891602 sec\n",
            "\n",
            "Epoch 26 Batch 0 Loss 0.0163\n",
            "Epoch 26 Batch 100 Loss 0.0121\n",
            "Epoch 26 Batch 200 Loss 0.0160\n",
            "Epoch 26 [Training Loss] : 0.0163\n",
            "Epoch 26 [Validation Loss] : 0.0778\n",
            "Time taken for 1 epoch 8.71278691291809 sec\n",
            "\n",
            "Epoch 27 Batch 0 Loss 0.0116\n",
            "Epoch 27 Batch 100 Loss 0.0099\n",
            "Epoch 27 Batch 200 Loss 0.0144\n",
            "Epoch 27 [Training Loss] : 0.0142\n",
            "Epoch 27 [Validation Loss] : 0.0807\n",
            "Time taken for 1 epoch 8.623615026473999 sec\n",
            "\n",
            "Epoch 28 Batch 0 Loss 0.0103\n",
            "Epoch 28 Batch 100 Loss 0.0139\n",
            "Epoch 28 Batch 200 Loss 0.0149\n",
            "Epoch 28 [Training Loss] : 0.0136\n",
            "Epoch 28 [Validation Loss] : 0.0809\n",
            "Time taken for 1 epoch 8.394057750701904 sec\n",
            "\n",
            "Epoch 29 Batch 0 Loss 0.0114\n",
            "Epoch 29 Batch 100 Loss 0.0103\n",
            "Epoch 29 Batch 200 Loss 0.0123\n",
            "Epoch 29 [Training Loss] : 0.0134\n",
            "Epoch 29 [Validation Loss] : 0.0809\n",
            "Time taken for 1 epoch 8.672465801239014 sec\n",
            "\n",
            "Epoch 30 Batch 0 Loss 0.0084\n",
            "Epoch 30 Batch 100 Loss 0.0122\n",
            "Epoch 30 Batch 200 Loss 0.0134\n",
            "Epoch 30 [Training Loss] : 0.0123\n",
            "Epoch 30 [Validation Loss] : 0.0818\n",
            "Time taken for 1 epoch 9.275505542755127 sec\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "16B7osDWQ-uw",
        "outputId": "9ad71ce1-22f5-4dbe-86b3-6eb936875469"
      },
      "source": [
        "# Calculate test loss\n",
        "total_test_loss = 0\n",
        "for (batch, (inp, targ)) in enumerate(test_dataset.take(val_steps_per_epoch)):\n",
        "      test_loss = train_step(inp, targ, enc_hidden, test=True)\n",
        "      total_test_loss += test_loss\n",
        "\n",
        "print(\"Test Loss: {:.4f}\".format((total_test_loss/val_steps_per_epoch).numpy()))"
      ],
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test Loss: 0.0839\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "vsUFmidtydqC",
        "outputId": "d2491949-b2a9-4e97-8ab5-9ad6a21e0742"
      },
      "source": [
        "# plot training and validation loss\n",
        "plt.plot(training_loss, label='Training Loss')\n",
        "plt.plot(validation_loss, label='Validation Loss')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD4CAYAAAATpHZ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwU9f3H8ddnN/dBLsKZQIIkIGeAAAoqUI+CWqiKCtUq2opaFcVatf31hxalVetPW1vv+6qIqIgVxaMqCB6Em3DJEUo4k0BCDnLs7vf3xy5hCQlZwiaT3Xyej8c+dmd2duYzLHnnm+/MfEeMMSillAoONqsLUEop5T8a6kopFUQ01JVSKohoqCulVBDRUFdKqSASYtWG27dvb9LS0qzavFJKBaTly5cXGmOSG3rfslBPS0sjJyfHqs0rpVRAEpEdJ3pfu1+UUiqIaKgrpVQQ0VBXSqkgYlmfulKqZdTU1JCfn09lZaXVpaiTEBERQUpKCqGhoSf1OQ11pYJcfn4+sbGxpKWlISJWl6N8YIyhqKiI/Px80tPTT+qz2v2iVJCrrKwkKSlJAz2AiAhJSUlN+uvKp1AXkbEisklEtojIvfW8/7iIrPI8NotI8UlXopRqNhrogaep31mjoS4iduBJYBzQB5gsIn28lzHGTDfGZBljsoB/AO81qRofLN9xkIc/2dhcq1dKqYDmS0t9GLDFGLPNGFMNzAYmnGD5ycBb/iiuPrm7S3j6q63kFZY31yaUUn5UVFREVlYWWVlZdOrUia5du9ZOV1dXn/CzOTk5TJs2rdFtjBgxwi+1fvXVV1x88cV+WZdVfDlQ2hXY6TWdDwyvb0ER6Q6kA/9p4P2pwFSAbt26nVShR4zKdF8d+/XmAtLaRzdpHUqplpOUlMSqVasAuP/++4mJieGuu+6qfd/hcBASUn8UZWdnk52d3eg2li5d6p9ig4C/D5ROAuYaY5z1vWmMec4Yk22MyU5ObnDoghPqnhRNWlIUX28uOJU6lVIWmjJlCjfddBPDhw/n7rvv5ocffuDMM89k0KBBjBgxgk2bNgHHtpzvv/9+rr/+ekaPHk2PHj144oknatcXExNTu/zo0aOZOHEivXv35qqrruLI3d0WLFhA7969GTJkCNOmTTupFvlbb71F//796devH/fccw8ATqeTKVOm0K9fP/r378/jjz8OwBNPPEGfPn0YMGAAkyZNOvV/rJPkS0t9F5DqNZ3imVefScAtp1pUY0ZlJjMnJ5/KGicRofbm3pxSQeNPH+ayfvchv66zT5d23Pezvif9ufz8fJYuXYrdbufQoUMsXryYkJAQPv/8c/7whz/w7rvvHveZjRs38uWXX1JaWkqvXr24+eabjzuPe+XKleTm5tKlSxdGjhzJkiVLyM7O5sYbb2TRokWkp6czefJkn+vcvXs399xzD8uXLychIYELLriAefPmkZqayq5du1i3bh0AxcXu80Meeughtm/fTnh4eO28luRLS30ZkCEi6SIShju459ddSER6AwnAt/4t8XijeiVzuMZJTt7B5t6UUqqZXH755djt7kZZSUkJl19+Of369WP69Onk5ubW+5mLLrqI8PBw2rdvT4cOHdi3b99xywwbNoyUlBRsNhtZWVnk5eWxceNGevToUXvO98mE+rJlyxg9ejTJycmEhIRw1VVXsWjRInr06MG2bdu47bbb+OSTT2jXrh0AAwYM4KqrruKNN95osFupOTW6RWOMQ0RuBRYCduAlY0yuiMwEcowxRwJ+EjDbtMCdrM/okUSY3cbXm/dzVkb75t6cUkGjKS3q5hIdffSY2P/+7/8yZswY3n//ffLy8hg9enS9nwkPD699bbfbcTgcTVrGHxISEli9ejULFy7kmWeeYc6cObz00kt89NFHLFq0iA8//JBZs2axdu3aFg13n/rUjTELjDGZxpjTjDGzPPNmeAU6xpj7jTHHncPeHKLCQhiWnqj96koFiZKSErp27QrAK6+84vf19+rVi23btpGXlwfA22+/7fNnhw0bxtdff01hYSFOp5O33nqLUaNGUVhYiMvl4rLLLuPBBx9kxYoVuFwudu7cyZgxY3j44YcpKSmhrKzM7/tzIgE7TMCozGRmLdjA7uLDdImPtLocpdQpuPvuu7n22mt58MEHueiii/y+/sjISJ566inGjh1LdHQ0Q4cObXDZL774gpSUlNrpd955h4ceeogxY8ZgjOGiiy5iwoQJrF69muuuuw6XywXAX/7yF5xOJ1dffTUlJSUYY5g2bRrx8fF+358TkRboLalXdna2OZWbZGzeV8oFjy/ioUv7M2lY006PVKot2LBhA6effrrVZViurKyMmJgYjDHccsstZGRkMH36dKvLOqH6vjsRWW6MafA8z4Ad+yWjQwyd4yK0C0Yp5ZPnn3+erKws+vbtS0lJCTfeeKPVJTWLgO1+ERFGZSbz0Zo91DhdhNoD9veTUqoFTJ8+vdW3zP0hoJNwVGYypVUOVu3U8cOUUgoCPNRH9GyP3SZ8vUm7YJRSCgI81OMiQxncLV771ZVSyiOgQx3cXTBrd5VQWFZldSlKKWW5IAj1DgAs/lFb60q1RmPGjGHhwoXHzPvb3/7GzTff3OBnRo8ezZFTni+88MJ6x1C5//77efTRR0+47Xnz5rF+/fra6RkzZvD555+fTPn1as1D9AZ8qPft0o6k6DDtV1eqlZo8eTKzZ88+Zt7s2bN9Hn9lwYIFTb6Ap26oz5w5k/POO69J6woUAR/qNptwTmYyi34sxOWy5kIqpVTDJk6cyEcffVR7Q4y8vDx2797N2Wefzc0330x2djZ9+/blvvvuq/fzaWlpFBYWAjBr1iwyMzM566yzaofnBfc56EOHDmXgwIFcdtllVFRUsHTpUubPn8/vfvc7srKy2Lp1K1OmTGHu3LmA+8rRQYMG0b9/f66//nqqqqpqt3ffffcxePBg+vfvz8aNvt9prTUM0Ruw56l7G5WZzPsrd7FudwkDUlr2klylAsrH98Letf5dZ6f+MO6hBt9OTExk2LBhfPzxx0yYMIHZs2dzxRVXICLMmjWLxMREnE4n5557LmvWrGHAgAH1rmf58uXMnj2bVatW4XA4GDx4MEOGDAHg0ksv5YYbbgDgj3/8Iy+++CK33XYb48eP5+KLL2bixInHrKuyspIpU6bwxRdfkJmZyTXXXMPTTz/NHXfcAUD79u1ZsWIFTz31FI8++igvvPBCo/8MrWWI3oBvqQOcndEeEbQLRqlWyrsLxrvrZc6cOQwePJhBgwaRm5t7TFdJXYsXL+aSSy4hKiqKdu3aMX78+Nr31q1bx9lnn03//v158803Gxy694hNmzaRnp5OZmYmANdeey2LFi2qff/SSy8FYMiQIbWDgDWmtQzRGxQt9aSYcPp3jePrzQXcdm6G1eUo1XqdoEXdnCZMmMD06dNZsWIFFRUVDBkyhO3bt/Poo4+ybNkyEhISmDJlCpWVlU1a/5QpU5g3bx4DBw7klVde4auvvjqleo8M3+uPoXtbeojeoGipg7sLZsV/D1JSUWN1KUqpOmJiYhgzZgzXX399bSv90KFDREdHExcXx759+/j4449PuI5zzjmHefPmcfjwYUpLS/nwww9r3ystLaVz587U1NTw5ptv1s6PjY2ltLT0uHX16tWLvLw8tmzZAsDrr7/OqFGjTmkfW8sQvUHRUgd3qP/jP1tYsrWQC/t3trocpVQdkydP5pJLLqnthhk4cCCDBg2id+/epKamMnLkyBN+fvDgwVx55ZUMHDiQDh06HDN87gMPPMDw4cNJTk5m+PDhtUE+adIkbrjhBp544onaA6QAERERvPzyy1x++eU4HA6GDh3KTTfddFL701qH6A3YoXfrcjhdDHrgMy7s15mHJ9Z/oEWptkiH3g1cbWro3bpC7DbOzmjP15sLsOoXlVJKWS1oQh3cXTB7D1WyeV/L3j5KKaVaC59CXUTGisgmEdkiIvXeh1RErhCR9SKSKyL/8m+ZvjknMxmArzfvt2LzSrVa+tdr4Gnqd9ZoqIuIHXgSGAf0ASaLSJ86y2QAvwdGGmP6Anc0qZpT1Dkukl4dY3XURqW8REREUFRUpMEeQIwxFBUVERERcdKf9eXsl2HAFmPMNgARmQ1MALyvErgBeNIYc9BTkGVN5VG9knllSR7lVQ6iw4Pm5B6lmiwlJYX8/HwKCrSxE0giIiKOObvGV76kXldgp9d0PjC8zjKZACKyBLAD9xtjPqm7IhGZCkwF6NateW4WPSozmecWbeO7bUWce3rHZtmGUoEkNDSU9PR0q8tQLcRfB0pDgAxgNDAZeF5Ejjvp0hjznDEm2xiTnZyc7KdNHys7LYHIULt2wSil2iRfQn0XkOo1neKZ5y0fmG+MqTHGbAc24w75FhceYmfEaUka6kqpNsmXUF8GZIhIuoiEAZOA+XWWmYe7lY6ItMfdHbPNj3WelFG9ktlRVEFeYblVJSillCUaDXVjjAO4FVgIbADmGGNyRWSmiBwZJm0hUCQi64Evgd8ZY4qaq+jGjPKc2rhI74aklGpjfDo9xBizAFhQZ94Mr9cGuNPzsFz3pGjSkqL4elMB15yZZnU5SinVYoLqilJvozKTWbq1iCqH0+pSlFKqxQRvqPdK5nCNk5y8g1aXopRSLSZoQ/2MHkmEhdiYuzzf6lKUUqrFBG2oR4WFcMPZ6by/chffbrXsmK1SSrWooA11gNt+kkG3xCj+Z95a7VtXSrUJQR3qEaF2Zk7oy7aCcp772rLT5pVSqsUEdagDjO7VgYsGdOYfX27Ri5GUUkEv6EMdYMbFfQi32/jfD9bp8KNKqaDWJkK9Y7sI7vppLxb/WMi/1+yxuhyllGo2bSLUAa4+ozsDUuKY+e/1lByusbocpZRqFm0m1O02YdbP+1NUVsWjCzdZXY5SSjWLNhPqAP1T4rjmzDTe+H4Hq3YWW12OUkr5XZsKdYDfXpBJh9hw/vDeWhxOl9XlKKWUX7W5UI+NCOW+n/Vl/Z5DvPrtDqvLUUopv2pzoQ4wrl8nxvRK5rFPN7Gn5LDV5SillN+0yVAXEWZO6IfTGP40f73V5SillN+0yVAHSE2MYtq5GXySu5cvNuyzuhyllPKLNhvqAL8+qwcZHWKY8UEuFdUOq8tRSqlT1qZDPSzExqxL+rOr+DB//+JHq8tRSqlT1qZDHWBYeiJXZKfwwuLtbNpbanU5Sil1SnwKdREZKyKbRGSLiNxbz/tTRKRARFZ5Hr/2f6nN595xpxMTHsIMHfBLKRXgGg11EbEDTwLjgD7AZBHpU8+ibxtjsjyPF/xcZ7NKjA7j7rG9+H77Aeav3m11OUop1WS+tNSHAVuMMduMMdXAbGBC85bV8iYN7caAlDge/GgDpZU64JdSKjD5EupdgZ1e0/meeXVdJiJrRGSuiKTWtyIRmSoiOSKSU1BQ0IRym4/dJjwwoR+FZVX8/XM9aKqUCkz+OlD6IZBmjBkAfAa8Wt9CxpjnjDHZxpjs5ORkP23afwamxjNpaDdeXpqnB02VUgHJl1DfBXi3vFM882oZY4qMMVWeyReAIf4pr+Xd/dNexEaE6F2SlFIByZdQXwZkiEi6iIQBk4D53guISGevyfHABv+V2LISosO4Z2xvfth+gA9W6UFTpVRgaTTUjTEO4FZgIe6wnmOMyRWRmSIy3rPYNBHJFZHVwDRgSnMV3BKuzE5lYGo8sxZs4JAeNFVKBRCxqoshOzvb5OTkWLJtX6zJL2bCk0u4bkQ6M35W3xmcSinV8kRkuTEmu6H32/wVpQ0ZkBLPL4Z149Vv89iw55DV5SillE801E/gdz/tRbsIvdJUKRU4NNRPID4qjHvH9WZZ3kHeX7mr8Q8opZTFNNQbcfmQVLJS4/nzgg2UHNaDpkqp1k1DvRE2m/Dgz/tRVF7N459ttrocpZQ6IQ11H/TrGsfVw7vz2rd5rN+tB02VUq2XhrqP7rqgF/FRYcz4YB0ulx40VUq1ThrqPoqLCuXecb3J2XGQucvzrS5HKaXqpaF+EiYOTmFYWiIP/Hs9u4sPW12OUkodR0P9JNhswl8vH4DTGO6eu0bPXVdKtToa6iepe1I0f7jwdL7ZUsgb3+2wuhyllDqGhnoTXDW8G+dkJvPnBRvJKyy3uhyllKqlod4EIsIjlw0g1C789p3VOPVsGKVUK6Gh3kSd4iKYOaEfy3cc5PnF26wuRymlAA31UzIhqwtj+3bisU83s3GvXpSklLKehvopEBFmXdKPdpEh3Pn2aqodLqtLUkq1cRrqpygpJpw/X9Kf9XsO8Y///Gh1OUqpNk5D3Q8u6NuJywan8NRXW1m1s9jqcpRSbZiGup/M+FkfOsSG89s5q6iscVpdjlKqjdJQ95O4yFD+OnEgWwvKeeSTTVaXo5Rqo3wKdREZKyKbRGSLiNx7guUuExEjIg3eFDWYnZXRnmvO7M5LS7bz7dYiq8tRSrVBjYa6iNiBJ4FxQB9gsoj0qWe5WOB24Ht/FxlI7h3Xm7SkKH43dzVlVQ6ry1FKtTG+tNSHAVuMMduMMdXAbGBCPcs9ADwMVPqxvoATFRbC/10xkN3Fh3nw3+utLkcp1cb4EupdgZ1e0/meebVEZDCQaoz56EQrEpGpIpIjIjkFBQUnXWygGNI9kRtHncbsZTv595rdVpejlGpDTvlAqYjYgMeA3za2rDHmOWNMtjEmOzk5+VQ33ardeX4mg7rFc++7a9mug34ppVqIL6G+C0j1mk7xzDsiFugHfCUiecAZwPy2erD0iFC7jX/+YjAhduGWN1foaY5KqRbhS6gvAzJEJF1EwoBJwPwjbxpjSowx7Y0xacaYNOA7YLwxJqdZKg4gXeMjeeyKgazfc4g/faj960qp5tdoqBtjHMCtwEJgAzDHGJMrIjNFZHxzFxjoftK7IzeNOo23fvgvH6za1fgHlFLqFIT4spAxZgGwoM68GQ0sO/rUywoud12QyfIdB/j9e2vp2yWOnh1irC5JKRWk9IrSFhBit/GPyYOJCLVzy5srOFyt/etKqeahod5COsVF8Lcrs9i8v5QZH6yzuhylVJDSUG9B52Qmc+uYnryzPJ+5y/OtLkcpFYQ01FvYHedlcmaPJP44by2b9pZaXY5SKshoqLcwu034++QsYsJD+c2byynX8WGUUn6koW6BDrERPDEpi+2F5fxx3jqMMVaXpJQKEhrqFhnRsz13nJfJ+yt38faynY1/QCmlfKChbqFbxvTk7Iz23Dc/l/W7D1ldjlIqCGioW8huEx6/Mov4qFCmvp7DgfJqq0tSSgU4DXWLtY8J57lfZrO/tIrfvLmcGqfL6pKUUgFMQ70VGJgaz8OX9ee7bQeYqQN/KaVOgU9jv6jmd8mgFDbuKeXZRdvo3TmWq4Z3t7okpVQA0pZ6K3L32N6M7pXMfR/k8v02vXG1Uurkaai3Inab8MTkQXRLiuLmN1ew80CF1SUppQKMhnor0y4ilBeuyabG6eKG13L0ilOl1EnRUG+FeiTH8M9fDGbzvlLuemc1LpdecaqU8o2Geis1KjOZ3487nY/X7eUf/9lidTlKqQChZ7+0Yr8+O50New/x+Oeb6dUphrH9OltdklKqldOWeismIvz5kv5kpcZz55zVbNijQwkopU7Mp1AXkbEisklEtojIvfW8f5OIrBWRVSLyjYj08X+pbVNEqJ1nfzmEmPAQbnhNhxJQSp1Yo6EuInbgSWAc0AeYXE9o/8sY098YkwU8Ajzm90rbsI7tInjuGvdQAje9sZwqh97jVClVP19a6sOALcaYbcaYamA2MMF7AWOMd79ANKCna/hZVmo8f504gB+2H2DaWytx6BgxSql6+BLqXQHvAb/zPfOOISK3iMhW3C31afWtSESmikiOiOQUFBQ0pd42bUJWV2Zc3IeFufu49721eqqjUuo4fjtQaox50hhzGnAP8McGlnnOGJNtjMlOTk7216bblOvPSuf2czOYuzyfWQs26F2TlFLH8OWUxl1Aqtd0imdeQ2YDT59KUerE7jgvg5LDNbz4zXbiI0O57dwMq0tSSrUSvoT6MiBDRNJxh/kk4BfeC4hIhjHmR8/kRcCPqGYjIsy4uA+HKmv4v8820y4ylGtHpFldllKqFWg01I0xDhG5FVgI2IGXjDG5IjITyDHGzAduFZHzgBrgIHBtcxatwGYTHrlsAIcOO7hvfi5xkaH8fNBxhzqUUm2MWNUnm52dbXJycizZdjCprHFy3cvL+CHvAM9ePYTz+nS0uiSlVDMSkeXGmOyG3tcrSgNcRKid56/Npl+XdvzmXyv4dquOw65UW6ahHgRiwkN4+bphdE+M4obXcliTX2x1SUopi2ioB4nE6DBe/9Vw4qNCufalH9iyv9TqkpRSFtBQDyKd4iJ441fDsdtsXP3CD3rnJKXaIA31IJPWPprXfzWMimoHk577jh1F5VaXpJRqQRrqQej0zu341w1nUFHt4Ipnv2XL/jKrS1JKtRAN9SDVr2scs6eeidNlmPTct2zaq33sSrUFGupBrFenWGZPPRO7TZj03Les21VidUlKqWamoR7kenaIYc6NZxIVFsLk579j5X8PWl2SUqoZaai3Ad2Tonn7xjNIjA7j6he+54ftB6wuSSnVTDTU24iUhCjennomneIiuPalH1iypdDqkpRSzUBDvQ3pFBfB7Kln0j0piuteWcaXm/ZbXZJSys801NuY5Nhw3rrhDDI7xjD1tRw+zd1rdUlKKT/SUG+DEqLDePPXZ9C3Sxy/eXMFH67ebXVJSik/0VBvo+IiQ3nj18MZ3C2BabNX8vKS7VaXpJTyAw31NiwmPIRXrx/GBX068qcP1/OnD3Nx6s2slQpoGuptXGSYnaeuGsL1I9N5eUkev3lzOYernVaXpZRqIg11hd0mzPhZH+77WR8+Xb+Pyc9/R2FZldVlKaWaQENd1bpuZDrPXD2EjXsPcclTS9haoAOBKRVoNNTVMX7atxOzp57J4Wonlz61VK8+VSrA+BTqIjJWRDaJyBYRubee9+8UkfUiskZEvhCR7v4vVbWUrNR43rt5JEkx7mEF5uspj0oFjEZDXUTswJPAOKAPMFlE+tRZbCWQbYwZAMwFHvF3oapldUuK4r2bR5DVLZ5pb63kqa+2YIyeGaNUa+dLS30YsMUYs80YUw3MBiZ4L2CM+dIYc+Tead8BKf4tU1khPiqM1381jAlZXXjkk0384f21OJwuq8tSSp1AiA/LdAV2ek3nA8NPsPyvgI/re0NEpgJTAbp16+ZjicpK4SF2Hr8ii5SESJ78cis7Dxzmn78YRHxUmNWlKaXq4dcDpSJyNZAN/LW+940xzxljso0x2cnJyf7ctGpGNpvwu5/25pGJA/hh+wEmPLmEH/fpnZSUao18CfVdQKrXdIpn3jFE5Dzgf4Dxxhg9yTkIXZGdyltTh1Ne5eSSp5byxYZ9VpeklKrDl1BfBmSISLqIhAGTgPneC4jIIOBZ3IGu47kGsSHdE5l/60jS20fz69dyePqrrXoAValWpNFQN8Y4gFuBhcAGYI4xJldEZorIeM9ifwVigHdEZJWIzG9gdSoIdImPZM6NZ3LxgC48/MlG7nh7FZU1OrSAUq2BWNXKys7ONjk5OZZsW/mHMYanvtrKo59uon/XOJ795RA6x0VaXZZSQU1Elhtjsht6X68oVU0mItwypifP/zKbrfvLGP/PJazQG1srZSkNdXXKzuvTkfdvGUlUmJ1Jz37H3OX5VpekVJuloa78IrNjLB/cMpKh6Qnc9c5qHvj3emr0QiWlWpyGuvKb+KgwXr1uGFNGpPHiN9uZ+My37Cgqt7ospdoUDXXlVyF2G/eP78vTVw1me0EZFz3xDR+sOu6yBqVUM9FQV81iXP/OfHzHOZzeOZbbZ6/it3NWU1blsLospYKehrpqNl3jI3nrhjO4/dwM3l+Zz8/+8Q1r80usLkupoKahrppViN3G9PMzeeuGM6iscXLp00t4ftE2XHqDa6WahYa6ahHDeyTx8e1n85PeHZi1YAPXvbKMglIdIkgpf9NQVy0mPiqMZ64ewoM/78d324oY9/fFLNpcYHVZSgUVDXXVokSEq8/ozvxbzyIxOpRrXvqBGR+so7SyxurSlAoKGurKEr06xTL/1rOYMiKN17/bwfmPLWJh7l6ry1Iq4GmoK8tEhNq5f3xf3rt5BPFRodz4+nJufD2HvSWVVpemVMAKvFA/XAy7V1pdhfKjQd0S+PC2s7hnbG++2lTAeY99zWvf5uHUM2SUOmmBF+rfPgnPjYb3pkLxzkYXV4Eh1G7j5tGn8en0cxjULZ4ZH+Qy8ZmlbNx7yOrSlAoogRfqI26Fs6ZD7jz4ZzZ8/ieo1B/8YNE9KZrXrh/G41cOZEdRBRc/8Q1/XbhRb8KhlI8C9yYZxf+FLx6AtXMgqj2M+T0MngL2EL/VqKx1oLyaPy/YwNzl+aQlRfHAz/txdobesFy1bY3dJCNwQ/2IXSvg0z/CjiXQPhPOfwAyfwoip75u1Sos3VLIH95fS15RBSN7JnHn+b0Y0j3B6rKUskTwhzqAMbBpAXw2A4q2QPo5cMEs6DzAP+tXlquscfLGdzt45uutFJZVM7pXMneen8mAlHirS1OqRbWNUD/CWQM5L8NXf4HDB2HgZPjJ/0Bcin+3oyxTUe3g1aU7eHbRVooraji/T0emn5dJny7trC5NqRbhl1AXkbHA3wE78IIx5qE6758D/A0YAEwyxsxtbJ3NeuPpw8XwzWPw3dPu6cHXwtl3QrsuzbM91eJKK2t4eUkezy/eRmmlg4v6d+aO8zLI6BhrdWlKNatTDnURsQObgfOBfGAZMNkYs95rmTSgHXAXMN/yUD+ieCcsfhRWvgFih+zr3GfOxHZq3u2qFlNSUcML32zjpW+2U1HjZMLALtx+Xibp7aOtLk2pZuGPUD8TuN8Y81PP9O8BjDF/qWfZV4B/t5pQP+JgHix6FFb9C+yhMPTXMPJ2iOnQMttXze5AeTXPLtrKq0vzqHEaLuzfmV+dlU5Wqva5q+DSWKj7cp56V8D7Kp98z7ymFDNVRHJEJKegoAVH50tIgwn/hNtyoO+l8N1T8PHnq6MAAA6OSURBVPeB7gOr5UUtV4dqNonRYfx+3OksvvsnXD8yja827ufnTy5h4tNL+XjtHr06VbUZvrTUJwJjjTG/9kz/EhhujLm1nmVfoTW21Osq3AKLHoG170BIJAy/EUbcBlGJ1tSj/K60soY5Ofm8snQ7Ow8cJiUhkikj0rhyaCqxEaFWl6dUk2n3y4kUbIavH4Z177q7ZbpmQ9pZkDYSUoZBWJS19alT5nQZPlu/lxe/2c6yvIPEhIdwRXYq141MIzVRv18VePwR6iG4D5SeC+zCfaD0F8aY3HqWfYVACvUj9m+AVW9C3hLYswqMC2yh0HWIJ+TPgtRhEKYH3wLZmvxiXvxmOx+t2YPLGC7o04nrRqYxLD0R0YvVVIDw1ymNF+I+ZdEOvGSMmSUiM4EcY8x8ERkKvA8kAJXAXmNM3xOts1WFurfKQ7Dze8hb7A753SvBOMEW4g757iOhUz9ISHf31WuXTcDZU3KY177dwb++/y8lh2tIbx/NxCEpXDKoK13iI60uT/mTy+k+xfnwAagoOvqoKnM33jDuixcx7una197PdTSUmcYJjipwVh99buj1iGlw+sVN2qW2dfFRc6gqhf9+Dzu+gbxv3MMSGK/BpSLijgZ8ouc5Id39ul1XsNmtqlw1oqLawUdr9jB3eT7fbz+ACJzVsz0Th6Tw076diAjV767ZOB3uoC0vgLL9UF4I5fs90wXu53LP/OpyCAl3P+zhR1+HRIA9zP0cEu5+XVV6bHgfPgi0YMbZwzw1hnlehx2t2x56tM4zboZe45q0CQ11f6uugIPb4cB296mS3q+L/wsur9uyiQ0iEyEqyf2ITjr6Oqr9sfPC20FYjLuLJzQKbIE3gGYg21FUzrsrdvHu8nx2FR8mNiKE8QO7MHFIClmp8c3bPWM8rUSXw+vhdF8h7T3vSGvPWePV8qs+fr7L4W54uFyeZ6f72bg8r11H5x23jRr3ey6H+/+yy+EOYO/1uOqsq+57zhr3Z52eddQ33VDQ2kIgOvnoI6aD+2eitgVc6d5nR2U901Xun6OoxGN/7mofiUefw2I8DS7xjBMl7p/X2tee6dr366pnnjS0rH9pqLcklxNK8o+GfUm+u7VQXggVR/7887w2jQwlGxrt/s8cFn007MOiITTS/Z9R7J5nm+e1zWue1/OR/5y1y3o/PO/b7Me3eo5rFXmmRY7/ExWOn+dyuX/I6v0hrPL8kHqexX5sK8YeerRlU9vaCXP/wNduizrbrjPtcnoFisMrWGrqBJjj2K/QwM6DFazbVcLmfaXUOA1JMWH079KO3skRtAv1/IntvV+101VHw6U2GL1C+kTTlhD3v7Et1P1va/N8D7aQo48j07X/h+r+H5Pj5x1Z55HPNjQdlQjR7SG6gyfE20Nkgg7G14jGQl3HqfUnmx0SursfjGp4OZcLqkrc58jX9vGVQnWZ+0/N6vI6r8uhuhQqi6F0T50Wk/NoiB3TYnJ5WlGuo60074eqlw3o7nlcZPPMqAK2ux/VhOKyh2MPiyQkLAIJifD6RRjh7o6zh7mHgPYOR5u9zrQnKI8J0brL1Jmu/VM+7NhfeMf8EjwSwl6/9Gt/wdcNZg3PYKShbgWbzd0iiUwAelpTg3fouxxHW811W9G1rU9Pq9SYY/9EPfJnaN15NrtXy9/zXF/L3x4GGK+DSTV1uhO859d4BZH3duuZPqalGeoJ2dBjQ/TI6/r+lK4jr6icTzcW8dmG/eTsOIgph67xkZx/Wkcu6NORoemJhNq1y0xZT7tflDpJhWVV/GfDfj5dv5fFPxZS5XDRLiKEn/TuwAV9O3FOZjIx4dpeUs1D+9SVakYV1Q4W/1jIp7n7+M/GfRysqMFuE/p1aUd2WiJD0xIY0j2R5Nhwq0tVQUJDXakW4nC6yNlxkMU/FpCTd5BVO4upcriPX6QlRdWGfHZaIj3aR+sFT6pJ9ECpUi0kxG7jjB5JnNEjCYBqh4t1u0vIyTvAsryD/GfjfuYuzwcgKTqMId0TGNQtgYGpcfTvGqdj0ii/0Ja6Ui3EGMO2wvLakM/JO0BeUQXgPr57WnIMA1PiGZgax8CUeHp3jiU8RC+AUsfS7helWrGD5dWs2VXCmp3FrM4vZtXOEgrLqgAIs9s4vXMsA1Pj6dc1jp4dYujZIYZ22qJv0zTUlQogxhh2l1SyZmcxq/KLWb2zmLX5JZRXH71YrUNseG3A9+wQw2nJ7ucOseHaT98GaJ+6UgFEROgaH0nX+EjG9e8MuIcP/u+BCrbsLzv6KCjjvRW7KKs6ejVqbEQIpyXHkNEhhoyOMWR0iKVnhxi6xkdis2nYtxXaUlcqQBlj2F9adWzYewK/oLSqdrnIUDs9O7jDvqcn7DM6xJCaGIVdwz7gaEtdqSAlInRsF0HHdhGM7Nn+mPeKK6rZsr+MH/eX8eO+Mn7cX8q324p4b+Wu2mVC7UKX+EhSEiJJiY8iNTGSlIQoUhIiSU2MIjkmXFv4AUhDXakgFB8VRnZaItlpx473X1pZUxv22wrK2VV8mPyDFXyxcX/tAdojwuw2uia4Q79LXCSd4iLoHBfheXZPt4sI0X78VkZDXak2JDYilEHd3OfH13W42lkb8jsPup/zDx4m/0AFm/aWUlBWddz9IaLC7EfDvl0kHduFkxAVRnxUKAlRYSREhxIfFUZCVBhxkaHa3dMCNNSVUgBEhtlrz6ipT43Txf7SKvaWHGZPSSV7Syq9ng/z7dZC9pdW4XDVf5xOBNpFhJIY7Q79mPAQosLsRIeFEBXueQ4LITrcTuSR+WHu1+EhdsJCbISH2I59ttsJD7URZrdpV5GHhrpSyiehdlvtmTkNMcZQVuXgYHkNByuqOVhRTXGF53V5NQc9r4sraiitdLDvUCUV1U4qqp2UVzlqh1VoWn1CdHgIsREhxISHEhsRQruIEGIj3L9AYj2vYyPcvyxC7DZCbILdJl7PNuw2IdTunrbbBJune8kmUnsfDEGw1d4Twz0/xCaeXzhHfwGF2KTFu6c01JVSfiMinuAMpVtS1El/3uF0UVHj5LAn5I8EfrXDRbXTSVWNi2qni6oaF1VOF9UOF1UO9/uVNS4qqh2UVjoorXT/0thdXElpVSllle75Df0V0VxEILxO0IeF2LjjvEzGD+zSLNv0KdRFZCzwd9w3nn7BGPNQnffDgdeAIUARcKUxJs+/pSqlgl2I3UY7u61Zrpo1xlBZ46K0qoaKKidOY3C6DDVOF06XweFyTzucBofLhcPz2hiDwXPXQe/XGFyeeQAOp/H8wnHW/uKpdrqochz95VPlcE8nRDXfVcGNhrqI2IEngfOBfGCZiMw3xqz3WuxXwEFjTE8RmQQ8DFzZHAUrpVRTiAiRnj56Yq2upvn4cquWYcAWY8w2Y0w1MBuYUGeZCcCrntdzgXNFz3NSSqkW50uodwV2ek3ne+bVu4wxxgGUAEl1VyQiU0UkR0RyCgoKmlaxUkqpBrXoTRWNMc8ZY7KNMdnJycktuWmllGoTfAn1XUCq13SKZ169y4hICBCH+4CpUkqpFuRLqC8DMkQkXUTCgEnA/DrLzAeu9byeCPzHWDVSmFJKtWGNnv1ijHGIyK3AQtynNL5kjMkVkZlAjjFmPvAi8LqIbAEO4A5+pZRSLcyn89SNMQuABXXmzfB6XQlc7t/SlFJKnawWPVCqlFKqeVl2kwwRKQB2NPHj7YFCP5bTGgTbPgXb/kDw7VOw7Q8E3z7Vtz/djTENnj5oWaifChHJOdGdPwJRsO1TsO0PBN8+Bdv+QPDtU1P2R7tflFIqiGioK6VUEAnUUH/O6gKaQbDtU7DtDwTfPgXb/kDw7dNJ709A9qkrpZSqX6C21JVSStVDQ10ppYJIwIW6iIwVkU0iskVE7rW6nlMlInkislZEVolIjtX1NIWIvCQi+0Vknde8RBH5TER+9Dwff/v6VqqB/blfRHZ5vqdVInKhlTWeLBFJFZEvRWS9iOSKyO2e+QH5PZ1gfwL2exKRCBH5QURWe/bpT5756SLyvSfz3vaMwdXwegKpT91zF6bNeN2FCZhc5y5MAUVE8oBsY0zAXjAhIucAZcBrxph+nnmPAAeMMQ95fvkmGGPusbJOXzWwP/cDZcaYR62sralEpDPQ2RizQkRigeXAz4EpBOD3dIL9uYIA/Z48NxaKNsaUiUgo8A1wO3An8J4xZraIPAOsNsY83dB6Aq2l7stdmFQLM8Yswj2Qmzfvu2G9ivsHLiA0sD8BzRizxxizwvO6FNiA++Y2Afk9nWB/ApZxK/NMhnoeBvgJ7jvKgQ/fUaCFui93YQo0BvhURJaLyFSri/GjjsaYPZ7Xe4GOVhbjJ7eKyBpP90xAdFPUR0TSgEHA9wTB91RnfyCAvycRsYvIKmA/8BmwFSj23FEOfMi8QAv1YHSWMWYwMA64xfOnf1DxjK0fOP189XsaOA3IAvYA/2dtOU0jIjHAu8AdxphD3u8F4vdUz/4E9PdkjHEaY7Jw34xoGND7ZNcRaKHuy12YAooxZpfneT/wPu4vMhjs8/R7Hun/3G9xPafEGLPP8wPnAp4nAL8nTz/tu8Cbxpj3PLMD9nuqb3+C4XsCMMYUA18CZwLxnjvKgQ+ZF2ih7stdmAKGiER7DvIgItHABcC6E38qYHjfDeta4AMLazllR4LP4xIC7HvyHIR7EdhgjHnM662A/J4a2p9A/p5EJFlE4j2vI3GfELIBd7hP9CzW6HcUUGe/AHhOUfobR+/CNMvikppMRHrgbp2D+4Yl/wrE/RGRt4DRuIcJ3QfcB8wD5gDdcA+xfIUxJiAOPjawP6Nx/0lvgDzgRq++6FZPRM4CFgNrAZdn9h9w90MH3Pd0gv2ZTIB+TyIyAPeBUDvuBvccY8xMT07MBhKBlcDVxpiqBtcTaKGulFKqYYHW/aKUUuoENNSVUiqIaKgrpVQQ0VBXSqkgoqGulFJBRENdKaWCiIa6UkoFkf8HzLxSQuFStJIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zK7Dq9UbANn-"
      },
      "source": [
        "def evaluate(sentence):\n",
        "    attention_plot = np.zeros((nepali_maxlen, english_maxlen))\n",
        "    inputs = [english_tokenizer.texts_to_sequences([i])[0][0] for i in sentence.split()]\n",
        "    inputs = pad_sequences([inputs], maxlen=english_maxlen, padding=\"post\")\n",
        "    inputs = tf.convert_to_tensor(inputs)\n",
        "    result = ''\n",
        "    hidden = [tf.zeros((1, units))]\n",
        "    enc_output, enc_hidden = encoder(inputs, hidden)\n",
        "    dec_hidden = enc_hidden\n",
        "    dec_input = tf.expand_dims([nepali_tokenizer.word_index['start_']], 0)\n",
        "    for i in range(nepali_maxlen):\n",
        "        predictions, dec_hidden, attention_weights = decoder(dec_input, dec_hidden,\n",
        "                                                              enc_output)\n",
        "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
        "        if nepali_tokenizer.index_word[predicted_id] == 'end_':\n",
        "            return result, sentence\n",
        "        result += nepali_tokenizer.index_word[predicted_id] + ' '\n",
        "        dec_input = tf.expand_dims([predicted_id], 0)\n",
        "    return result, sentence"
      ],
      "execution_count": 79,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tDNZArcrAPtU"
      },
      "source": [
        "def translate(sentence):\n",
        "    result, sentence = evaluate(sentence)\n",
        "    # print(\"Input: {}\".format(sentence))\n",
        "    # print(\"Translated {}\".format(result))\n",
        "    return result"
      ],
      "execution_count": 80,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C3gHPDGAE9wJ",
        "outputId": "c29bf647-a376-4e81-b718-2963afa75c40"
      },
      "source": [
        "# restoring the latest checkpoint in checkpoint_dir\n",
        "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.training.tracking.util.InitializationOnlyStatus at 0x7f446a125390>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qf61jr_pENJs"
      },
      "source": [
        "# list of sentences to translate\n",
        "english_test = list(X_test.values)\n",
        "# original translation\n",
        "nepali_test = list(y_test.values)\n",
        "\n",
        "# initialize a list to store model translation\n",
        "translations = []\n",
        "\n",
        "# loop over all english sentences\n",
        "for sent in english_test:\n",
        "  translation = translate(sent)\n",
        "  translations.append(translation)\n",
        "\n",
        "# create a dataframe\n",
        "prediction_df = pd.DataFrame()\n",
        "prediction_df['english'] = english_test\n",
        "prediction_df['nepali'] = nepali_test\n",
        "prediction_df['predicted'] = translations\n",
        "\n",
        "# remove start and end tokens\n",
        "prediction_df['english'] = prediction_df.english.str.strip('start_')\n",
        "prediction_df['english'] = prediction_df.english.str.strip('end_')\n",
        "\n",
        "prediction_df['nepali'] = prediction_df.nepali.str.strip('start_')\n",
        "prediction_df['nepali'] = prediction_df.nepali.str.strip('end_')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "bNyrq49P7uI2",
        "outputId": "34aba3f1-9bbd-4f9a-dfdc-568169e00e9f"
      },
      "source": [
        "prediction_df.sample(5)"
      ],
      "execution_count": 87,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>english</th>\n",
              "      <th>nepali</th>\n",
              "      <th>predicted</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>937</th>\n",
              "      <td>i am lefthanded</td>\n",
              "      <td>म बाँचेको छु</td>\n",
              "      <td>म ठट्टामा छु</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>377</th>\n",
              "      <td>this book is mine</td>\n",
              "      <td>यो पुस्तक मेरो हो</td>\n",
              "      <td>यो पुस्तक मेरो हो</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>754</th>\n",
              "      <td>i can swim</td>\n",
              "      <td>म पौडी खेल्न सक्छु</td>\n",
              "      <td>म पौडिन सक्छु</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>540</th>\n",
              "      <td>i have been shot</td>\n",
              "      <td>मलाई गोली लागेको छ</td>\n",
              "      <td>मा शट</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>640</th>\n",
              "      <td>i miss tom</td>\n",
              "      <td>म टम सम्झन्छु</td>\n",
              "      <td>म टम घृणा गर्छु</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 english                nepali           predicted\n",
              "937     i am lefthanded          म बाँचेको छु        म ठट्टामा छु \n",
              "377   this book is mine     यो पुस्तक मेरो हो   यो पुस्तक मेरो हो \n",
              "754          i can swim    म पौडी खेल्न सक्छु       म पौडिन सक्छु \n",
              "540    i have been shot    मलाई गोली लागेको छ               मा शट \n",
              "640          i miss tom         म टम सम्झन्छु     म टम घृणा गर्छु "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 87
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TVol5HhRJ7OF"
      },
      "source": [
        "# save predictions to a disk\n",
        "save_path = os.path.join(prediction_dir, 'model_4_predictions.csv')\n",
        "prediction_df.to_csv(save_path, index=False)"
      ],
      "execution_count": 105,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bCS6Coa473XS"
      },
      "source": [
        "# Results\n",
        "### Evaluated using BLEU Score\n",
        "\n",
        "**Bilingual Evaluation Understudy Score**\n",
        "\n",
        "The Bilingual Evaluation Understudy Score, or BLEU for short, is a metric for evaluating a generated sentence to a reference sentence.\n",
        "\n",
        "A perfect match results in a score of 1.0, whereas a perfect mismatch results in a score of 0.0.\n",
        "\n",
        "The score was developed for evaluating the predictions made by automatic machine translation systems. It is not perfect, but does offer 5 compelling benefits:\n",
        "\n",
        "- It is quick and inexpensive to calculate.\n",
        "- It is easy to understand.\n",
        "- It is language independent.\n",
        "- It correlates highly with human evaluation.\n",
        "- It has been widely adopted.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e92jG-Ko6L3Q"
      },
      "source": [
        "# load the predicitons\n",
        "model_1_predictions = pd.read_csv(prediction_dir + '/model_1_predictions.csv')\n",
        "model_2_predictions = pd.read_csv(prediction_dir + '/model_2_predictions.csv')\n",
        "model_3_predictions = pd.read_csv(prediction_dir + '/model_3_predictions.csv')\n",
        "model_4_predictions = pd.read_csv(prediction_dir + '/model_4_predictions.csv')"
      ],
      "execution_count": 106,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M98Trz5o8v_w"
      },
      "source": [
        "# calculate blue_score\n",
        "def calculate_score(pred_df):\n",
        "    # list to store reference and translation\n",
        "    actual, predicted = [], []\n",
        "    for row in pred_df.itertuples():\n",
        "        org = row.nepali.split()\n",
        "        pred = row.predicted.split()\n",
        "        actual.append([org])\n",
        "        predicted.append(pred)\n",
        "    print('BLUE-1: {:.4f}'.format(corpus_bleu(actual, predicted, weights=[1.0, 0, 0, 0], smoothing_function=SmoothingFunction().method1)))\n",
        "    print('BLUE-2: {:.4f}'.format(corpus_bleu(actual, predicted, weights=[0.5, 0.5, 0, 0], smoothing_function=SmoothingFunction().method1)))\n",
        "    print('tBLUE-3: {:.4f}'.format(corpus_bleu(actual, predicted, weights=[0.3, 0.3, 0.3, 0], smoothing_function=SmoothingFunction().method1)))\n",
        "    print('BLUE-4: {:.4f}'.format(corpus_bleu(actual, predicted, weights=[0.25, 0.25, 0.25, 0.25], smoothing_function=SmoothingFunction().method1)))"
      ],
      "execution_count": 135,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ps1qY-MA80HZ",
        "outputId": "a307f4a2-81bb-46fd-bda2-ac02392da473"
      },
      "source": [
        "# calculate bleu score for all predictions\n",
        "helper = [\"Model-1 Scores\", \"Model-2 Scores\", \"Model-3 Scores\", \"Model-4 Scores\"]\n",
        "model_preds = [model_1_predictions, model_2_predictions, model_3_predictions,\n",
        "               model_4_predictions]\n",
        "for i,predictions in enumerate(model_preds):\n",
        "    print('-----' +helper[i]+ ' -----')\n",
        "    calculate_score(predictions)\n",
        "    print(\" \")"
      ],
      "execution_count": 136,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-----Model-1 Scores -----\n",
            "BLUE-1: 0.1265\n",
            "BLUE-2: 0.0409\n",
            "tBLUE-3: 0.0070\n",
            "BLUE-4: 0.0014\n",
            " \n",
            "-----Model-2 Scores -----\n",
            "BLUE-1: 0.2377\n",
            "BLUE-2: 0.0895\n",
            "tBLUE-3: 0.0460\n",
            "BLUE-4: 0.0159\n",
            " \n",
            "-----Model-3 Scores -----\n",
            "BLUE-1: 0.2467\n",
            "BLUE-2: 0.1076\n",
            "tBLUE-3: 0.0625\n",
            "BLUE-4: 0.0204\n",
            " \n",
            "-----Model-4 Scores -----\n",
            "BLUE-1: 0.4859\n",
            "BLUE-2: 0.3555\n",
            "tBLUE-3: 0.3068\n",
            "BLUE-4: 0.1872\n",
            " \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WuU_7krz829s"
      },
      "source": [
        "# Check out some good and bad predictions\n",
        "\n",
        "# calculate sentence bleu\n",
        "def calculate_sentence_bleu(prediction_df):\n",
        "    scores = []\n",
        "    for row in prediction_df.itertuples():\n",
        "        org = row.nepali.split()\n",
        "        pred = row.predicted.split()\n",
        "        score = sentence_bleu([org], pred, smoothing_function=SmoothingFunction().method1)\n",
        "        scores.append(score)\n",
        "    return scores"
      ],
      "execution_count": 126,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jR7SL5QQ-YBx"
      },
      "source": [
        "# calculate sentence_bleu score for each translation\n",
        "model_4_predictions['sentence_bleu'] = calculate_sentence_bleu(model_4_predictions)\n",
        "# sort using sentence_bleu score\n",
        "sorted_ = model_4_predictions.sort_values(by='sentence_bleu', ascending=False)"
      ],
      "execution_count": 127,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "id": "E3Zfb8mN-qb7",
        "outputId": "82064aa6-60cc-4e8a-a89a-8acb1a19589b"
      },
      "source": [
        "# Some good predictions\n",
        "sorted_.head(10)"
      ],
      "execution_count": 128,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>english</th>\n",
              "      <th>nepali</th>\n",
              "      <th>predicted</th>\n",
              "      <th>sentence_bleu</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>494</th>\n",
              "      <td>it is my book</td>\n",
              "      <td>यो मेरो किताब हो</td>\n",
              "      <td>यो मेरो किताब हो</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>654</th>\n",
              "      <td>i can see tom</td>\n",
              "      <td>म टम देख्न सक्छु</td>\n",
              "      <td>म टम देख्न सक्छु</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86</th>\n",
              "      <td>let me see that</td>\n",
              "      <td>मलाई त्यो हेर्न दिनुहोस्</td>\n",
              "      <td>मलाई त्यो हेर्न दिनुहोस्</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>256</th>\n",
              "      <td>he is my teacher</td>\n",
              "      <td>उहाँ मेरो शिक्षक हुनुहुन्छ</td>\n",
              "      <td>उहाँ मेरो शिक्षक हुनुहुन्छ</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>888</th>\n",
              "      <td>i love snow</td>\n",
              "      <td>मलाई हिउँ मन पर्छ</td>\n",
              "      <td>मलाई हिउँ मन पर्छ</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>894</th>\n",
              "      <td>it is a doll</td>\n",
              "      <td>यो एक पुतली हो</td>\n",
              "      <td>यो एक पुतली हो</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>608</th>\n",
              "      <td>it is so easy</td>\n",
              "      <td>यो एकदम सजिलो छ</td>\n",
              "      <td>यो एकदम सजिलो छ</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>906</th>\n",
              "      <td>my house is here</td>\n",
              "      <td>मेरो घर यहाँ छ</td>\n",
              "      <td>मेरो घर यहाँ छ</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>282</th>\n",
              "      <td>that is my dog</td>\n",
              "      <td>त्यो मेरो कुकुर हो</td>\n",
              "      <td>त्यो मेरो कुकुर हो</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>584</th>\n",
              "      <td>i feel very good</td>\n",
              "      <td>म धेरै राम्रो महसुस गर्छु</td>\n",
              "      <td>म धेरै राम्रो महसुस गर्छु</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                english  ... sentence_bleu\n",
              "494      it is my book   ...           1.0\n",
              "654      i can see tom   ...           1.0\n",
              "86     let me see that   ...           1.0\n",
              "256   he is my teacher   ...           1.0\n",
              "888        i love snow   ...           1.0\n",
              "894       it is a doll   ...           1.0\n",
              "608      it is so easy   ...           1.0\n",
              "906   my house is here   ...           1.0\n",
              "282     that is my dog   ...           1.0\n",
              "584   i feel very good   ...           1.0\n",
              "\n",
              "[10 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 128
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "id": "h4SfKMRe-tUV",
        "outputId": "f154552b-a9cd-4730-d985-55d0eb41069c"
      },
      "source": [
        "# Some bad predictions\n",
        "sorted_.tail(10)"
      ],
      "execution_count": 129,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>english</th>\n",
              "      <th>nepali</th>\n",
              "      <th>predicted</th>\n",
              "      <th>sentence_bleu</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>217</th>\n",
              "      <td>aim higher</td>\n",
              "      <td>उच्च लक्ष्य</td>\n",
              "      <td>लीड समात्नुहोस्</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>697</th>\n",
              "      <td>do not lose heart</td>\n",
              "      <td>नबिर्सनुहोस्</td>\n",
              "      <td>नडराऊ</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>685</th>\n",
              "      <td>i feel refreshed</td>\n",
              "      <td>मलाई स्फूर्ति पाएको छ</td>\n",
              "      <td>म शक्तिहीन महसुस गर्छु</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>221</th>\n",
              "      <td>silence is golden</td>\n",
              "      <td>मौन सुनौलो हो</td>\n",
              "      <td>रसिया बच्चा</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>695</th>\n",
              "      <td>i do not use it</td>\n",
              "      <td>म यसलाई प्रयोग गर्दिन</td>\n",
              "      <td>मलाई यो पाउँदिन</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>694</th>\n",
              "      <td>ask yourself why</td>\n",
              "      <td>आफैलाई किन सोध्नुहोस्</td>\n",
              "      <td>भन्नुहोस् भन्नुहोस्</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>222</th>\n",
              "      <td>you smell bad</td>\n",
              "      <td>तपाईं खराब गन्ध</td>\n",
              "      <td>तिमि नराम्रो छौ</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>223</th>\n",
              "      <td>tom needs to pay</td>\n",
              "      <td>टमले तिर्नु पर्छ</td>\n",
              "      <td>टम दु खी छ</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>224</th>\n",
              "      <td>she rode a camel</td>\n",
              "      <td>उनी ऊँट चढिन्</td>\n",
              "      <td>उनले एउटा नाक उडायो</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>she woke up</td>\n",
              "      <td>उनी उठे</td>\n",
              "      <td>उनले समात्दै</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 english  ... sentence_bleu\n",
              "217          aim higher   ...           0.0\n",
              "697   do not lose heart   ...           0.0\n",
              "685    i feel refreshed   ...           0.0\n",
              "221   silence is golden   ...           0.0\n",
              "695     i do not use it   ...           0.0\n",
              "694    ask yourself why   ...           0.0\n",
              "222       you smell bad   ...           0.0\n",
              "223    tom needs to pay   ...           0.0\n",
              "224    she rode a camel   ...           0.0\n",
              "0           she woke up   ...           0.0\n",
              "\n",
              "[10 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 129
        }
      ]
    }
  ]
}