RNN_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gX6rKfD8CyNH"
   },
   "source": [
    "## Load processed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1021,
     "status": "ok",
     "timestamp": 1596873868610,
     "user": {
      "displayName": "Richard Sun",
      "photoUrl": "",
      "userId": "07723018862498440951"
     },
     "user_tz": -480
    },
    "id": "1-AwwneWTY-F",
    "outputId": "1410042b-546c-4fe3-8cbd-2a7e904e7de1"
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5379,
     "status": "ok",
     "timestamp": 1597469728944,
     "user": {
      "displayName": "Richard Sun",
      "photoUrl": "",
      "userId": "07723018862498440951"
     },
     "user_tz": -480
    },
    "id": "ejldGJAGCyNJ",
    "outputId": "0cd2e0db-307a-4b92-b274-81b902190009"
   },
   "outputs": [],
   "source": [
    "import yaml\n",
    "import numpy as np\n",
    "import torch\n",
    "from collections import Counter\n",
    "from torch import nn\n",
    "from torchtext import data\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "import torch.nn.functional as F\n",
    "%matplotlib inline\n",
    "import copy\n",
    "import argparse\n",
    "from helper import data_generator\n",
    "import sys; sys.argv=['']; del sys\n",
    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "file = 'args/sst_attn_all.yaml'\n",
    "# file = 'args/IMDB_rnn.yaml'\n",
    "# file ='args/mr_rnn.yaml'\n",
    "with open(file) as f:\n",
    "    args = yaml.load(f, Loader=yaml.Loader)\n",
    "    parser = argparse.ArgumentParser(description='attention')\n",
    "    config = parser.parse_args()\n",
    "    for k, v in args['common'].items():\n",
    "        setattr(config, k, v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(config.dic_path, 'rb') as f:\n",
    "    vocab, word2id, id2word = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# config.batch_size = 512#for mr\n",
    "# config.batch_size = 256#for imdb\n",
    "config.batch_size = 1024#for sst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dg = data_generator(config, config.train_path)#training data\n",
    "dev_dg = data_generator(config, config.dev_path, False)#dev data\n",
    "test_dg = data_generator(config, config.test_path, False)#test data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2065,
     "status": "ok",
     "timestamp": 1597469731628,
     "user": {
      "displayName": "Richard Sun",
      "photoUrl": "",
      "userId": "07723018862498440951"
     },
     "user_tz": -480
    },
    "id": "DsDOEdpkCyNc"
   },
   "outputs": [],
   "source": [
    "from torch.nn import utils as nn_utils\n",
    "import torch\n",
    "from torch import optim\n",
    "import pickle\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.init as init\n",
    "from self_defined_gru_no_combination import GRU as MVM_G\n",
    "from self_defined_lstm_no_combination import LSTM as MVM_L\n",
    "from self_defined_rnn_no_combination import RNN as MVM_E\n",
    "from self_defined_gru_linear import GRU as MVMA_G\n",
    "from self_defined_lstm_linear import LSTM as MVMA_L\n",
    "from self_defined_rnn_linear import RNN as MVMA_E\n",
    "\n",
    "\n",
    "# from dynamic_lstm import dynamicLSTM\n",
    "# from dynamic_gru import dynamicGRU\n",
    "# from dynamic_rnn import dynamicRNN\n",
    "from torch.nn import TransformerEncoder, TransformerEncoderLayer  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "class PositionalEncoding(nn.Module):\n",
    "\n",
    "    def __init__(self, d_model, dropout=0.1, max_len=5000, trainable=False):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "        if trainable:\n",
    "            pe = nn.Parameter(torch.randn(max_len, d_model).uniform_(-0.1, 0.1))\n",
    "        else:\n",
    "            pe = torch.zeros(max_len, d_model)\n",
    "            position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
    "            div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
    "            pe[:, 0::2] = torch.sin(position * div_term)\n",
    "            pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        self.pe = pe.unsqueeze(0).transpose(0, 1).to(device)\n",
    "        #self.register_buffer('pe', pe)\n",
    "        #print(pe.shape)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x + self.pe[:x.size(0), :]\n",
    "        #print(self.pe[:x.size(0), :].shape)\n",
    "        return self.dropout(x)\n",
    "def length_to_mask(length, max_len=None, dtype=None):\n",
    "    \"\"\"length: B.\n",
    "    return B x max_len.\n",
    "    If max_len is None, then max of length will be used.\n",
    "    \"\"\"\n",
    "    assert len(length.shape) == 1, 'Length shape should be 1 dimensional.'\n",
    "    max_len = max_len or length.max().item()\n",
    "    mask = torch.arange(max_len, device=length.device,\n",
    "                        dtype=length.dtype).expand(len(length), max_len) < length.unsqueeze(1)\n",
    "    if dtype is not None:\n",
    "        mask = torch.as_tensor(mask, dtype=dtype, device=length.device)\n",
    "    return mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleClassifier(nn.Module):\n",
    "    def __init__(self, vocab_size, embed_dim, hidden_dim, label_dim=1, cell_type='MVMA_G'):\n",
    "        super(SimpleClassifier, self).__init__()\n",
    "        \n",
    "        self.embeddings = nn.Embedding(vocab_size, embed_dim)\n",
    "        #self.embeddings.weight.data.uniform_(-0.05, 0.05)\n",
    "        self.embeddings.weight.data.uniform_(-0.1, 0.1)\n",
    "        self.dropout = nn.Dropout(0.5)\n",
    "        encoder_layer = TransformerEncoderLayer(d_model=embed_dim, nhead=5, \n",
    "                                                dim_feedforward=hidden_dim, dropout=0.1)\n",
    "        \n",
    "\n",
    "        self.affine = nn.Linear(embed_dim, hidden_dim)\n",
    "        self.softmax = nn.Softmax(dim=-1)\n",
    "        self.scale = scale\n",
    "        \n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "        if cell_type == 'MVMA_E':\n",
    "            self.encoder = MVMA_E(embed_dim, hidden_dim, \n",
    "                          bidirectional=False)\n",
    "        elif cell_type == 'MVMA_G':\n",
    "            self.encoder = MVMA_G(embed_dim, hidden_dim, \n",
    "                          bidirectional=False)\n",
    "        elif cell_type == 'MVMA_L':\n",
    "            self.encoder = MVMA_L(embed_dim, hidden_dim, \n",
    "                          bidirectional=False)\n",
    "        elif cell_type == 'transformer':\n",
    "            self.pos_encoder = PositionalEncoding(embed_dim, trainable=False)\n",
    "            self.encoder = TransformerEncoder(encoder_layer, 2)\n",
    "#         self.rnn = RNN(embed_dim, hidden_dim, \n",
    "#                       bidirectional=False)\n",
    "        self.cell_type = cell_type\n",
    "        self.linear = nn.Linear(embed_dim, hidden_dim)\n",
    "        self.label_dim = label_dim\n",
    "\n",
    "        self.decoder = nn.Linear(hidden_dim, label_dim, bias=False)\n",
    "\n",
    "    # batch_size * sent_l * dim\n",
    "    def forward(self, seq_ids, seq_lengths=None):\n",
    "        '''\n",
    "        Args:\n",
    "            seq_ids: word indexes, batch_size, max_len, Long Tensor\n",
    "            seq_lengths: lengths of sentences, batch_size, Long Tensor\n",
    "        attention:\n",
    "            score = v h\n",
    "            att = softmax(score)\n",
    "        '''\n",
    "        \n",
    "        seq_embs = self.embeddings(seq_ids)\n",
    "        seq_embs = self.dropout(seq_embs)\n",
    "        #print(seq_embs.shape)\n",
    "        batch_size, max_len, hidden_dim = seq_embs.size()\n",
    "        # batch * max_len * hidden_states\n",
    "        #hidden_vecs = self.affine(seq_embs)\n",
    "        if self.cell_type != 'transformer':\n",
    "            hidden_vecs, final_vec = self.encoder(seq_embs, seq_lengths)\n",
    "        else:\n",
    "            seq_embs = self.pos_encoder(seq_embs.transpose(0, 1))\n",
    "            src_key_mask = ~length_to_mask(seq_lengths)\n",
    "            hidden_vecs = self.encoder(seq_embs, src_key_padding_mask=src_key_mask.to(device))\n",
    "            hidden_vecs = hidden_vecs.transpose(0, 1)\n",
    "            final_vec = hidden_vecs.mean(1)\n",
    "            \n",
    "        #final_vec = self.linear(seq_embs)\n",
    "        final_vec = self.dropout(final_vec)\n",
    "        senti_scores = self.decoder(final_vec)\n",
    "        #multi class\n",
    "        if self.label_dim == 1:\n",
    "            probs = self.sigmoid(senti_scores)\n",
    "        else:\n",
    "            probs = self.softmax(senti_scores)\n",
    "            logits = torch.log(probs + 0.000000000001)\n",
    "            return logits, senti_scores\n",
    "        return probs, senti_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1072,
     "status": "ok",
     "timestamp": 1597469734506,
     "user": {
      "displayName": "Richard Sun",
      "photoUrl": "",
      "userId": "07723018862498440951"
     },
     "user_tz": -480
    },
    "id": "9fTKpWOECyNf"
   },
   "outputs": [],
   "source": [
    "# class SimpleAttnClassifier(nn.Module):\n",
    "#     def __init__(self, vocab_size, embed_dim, hidden_dim, label_dim=1, cell_type='elman'):\n",
    "#         super(SimpleAttnClassifier, self).__init__()\n",
    "        \n",
    "#         self.embeddings = nn.Embedding(vocab_size, embed_dim)\n",
    "#         #self.embeddings.weight.data.uniform_(-0.05, 0.05)\n",
    "#         self.dropout = nn.Dropout(0.5)\n",
    "\n",
    "#         self.affine = nn.Linear(embed_dim, hidden_dim)\n",
    "#         encoder_layer = TransformerEncoderLayer(d_model=embed_dim, nhead=2, \n",
    "#                                                 dim_feedforward=hidden_dim, dropout=0.1)\n",
    "#         self.softmax = nn.Softmax(dim=-1)\n",
    "#         self.scale = 10#np.sqrt(hidden_dim)\n",
    "        \n",
    "#         self.sigmoid = nn.Sigmoid()\n",
    "#         if cell_type == 'elman':\n",
    "#             self.rnn = RNN(embed_dim, hidden_dim//2, \n",
    "#                           bidirectional=True)\n",
    "#         elif cell_type == 'gru':\n",
    "#             self.rnn = GRU(embed_dim, hidden_dim//2, \n",
    "#                           bidirectional=True)\n",
    "#         elif cell_type == 'lstm':\n",
    "#             self.rnn = LSTM(embed_dim, hidden_dim//2, \n",
    "#                           bidirectional=True)\n",
    "#         elif cell_type == 'transformer':\n",
    "#             self.pos_encoder = PositionalEncoding(embed_dim, trainable=False)\n",
    "#             self.encoder = TransformerEncoder(encoder_layer, 1)\n",
    "#         self.cell_type = cell_type\n",
    "#         self.linear = nn.Linear(embed_dim, hidden_dim)\n",
    "#         self.label_dim = label_dim\n",
    "#         self.V = nn.Parameter(torch.randn(hidden_dim, 1))\n",
    "#         self.decoder = nn.Linear(hidden_dim, label_dim, bias=False)\n",
    "\n",
    "#     # batch_size * sent_l * dim\n",
    "#     def forward(self, seq_ids, seq_lengths=None):\n",
    "#         '''\n",
    "#         Args:\n",
    "#             seq_ids: word indexes, batch_size, max_len, Long Tensor\n",
    "#             seq_lengths: lengths of sentences, batch_size, Long Tensor\n",
    "#         attention:\n",
    "#             score = v h\n",
    "#             att = softmax(score)\n",
    "#         '''\n",
    "        \n",
    "#         seq_embs = self.embeddings(seq_ids)\n",
    "#         seq_embs = self.dropout(seq_embs)\n",
    "#         #print(seq_embs.shape)\n",
    "#         batch_size, max_len, hidden_dim = seq_embs.size()\n",
    "#         # batch * max_len * hidden_states\n",
    "#         if self.cell_type != 'transformer':\n",
    "#             hidden_vecs, final_vec = self.rnn(seq_embs, seq_lengths)\n",
    "#         else:\n",
    "#             seq_embs = self.pos_encoder(seq_embs.transpose(0, 1))\n",
    "#             src_key_mask = ~length_to_mask(seq_lengths)\n",
    "#             hidden_vecs = self.encoder(seq_embs, src_key_padding_mask=src_key_mask.to(device))\n",
    "#             hidden_vecs = hidden_vecs.transpose(0, 1)\n",
    "        \n",
    "#         inter_out = hidden_vecs\n",
    "#         #batch * max_len\n",
    "#         scores = torch.matmul(inter_out, self.V).squeeze(-1)\n",
    "\n",
    "#         #scores = torch.matmul(inter_out, self.decoder.weight.transpose(0, 1)).squeeze(-1)\n",
    "#         scores = scores/self.scale#we found this is very important to handle variances of scores\n",
    "        \n",
    "#         #Mask the padding values\n",
    "#         mask = torch.zeros(batch_size, max_len)\n",
    "#         for i in range(batch_size):\n",
    "#             mask[i, seq_lengths[i]:] = 1\n",
    "#         scores = scores.masked_fill(mask.to(device).bool(), -np.inf)\n",
    "#         #print(scores)\n",
    "#         #Softmax, batch_size*1*max_len\n",
    "#         attn = self.softmax(scores).unsqueeze(1)\n",
    "#         #weighted sum, batch_size*hidden_dim\n",
    "#         final_vec = torch.bmm(attn, hidden_vecs).squeeze(1)\n",
    "        \n",
    "#         #final_vec = self.linear(seq_embs)\n",
    "#         final_vec = self.dropout(final_vec)\n",
    "#         senti_scores = self.decoder(final_vec)\n",
    "#         #multi class\n",
    "#         if self.label_dim == 1:\n",
    "#             probs = self.sigmoid(senti_scores)\n",
    "#         else:\n",
    "#             probs = self.softmax(senti_scores)\n",
    "#             logits = torch.log(probs + 0.000000000001)\n",
    "#             return logits, senti_scores\n",
    "#         return probs, senti_scores\n",
    "    \n",
    "#     def load_vector(self, path, trainable=True):\n",
    "#         '''\n",
    "#         Load pre-savedd word embeddings\n",
    "#         '''\n",
    "#         with open(path, 'rb') as f:\n",
    "#             vectors = pickle.load(f)\n",
    "#             print(\"Loaded from {} with shape {}\".format(path, vectors.shape))\n",
    "#             #self.word_embed.weight = nn.Parameter(torch.FloatTensor(vectors))\n",
    "#             np_data = torch.from_numpy(vectors)#/4\n",
    "#             self.embeddings.weight.data.copy_(np_data)\n",
    "#             self.embeddings.weight.requires_grad = trainable\n",
    "#             print('embeddings loaded')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1032,
     "status": "ok",
     "timestamp": 1597469738705,
     "user": {
      "displayName": "Richard Sun",
      "photoUrl": "",
      "userId": "07723018862498440951"
     },
     "user_tz": -480
    },
    "id": "4mu6fXJWCyNk"
   },
   "outputs": [],
   "source": [
    "##Evaluation classification\n",
    "def evaluate_cls(dg, model, binary=True):\n",
    "    #Make prediction\n",
    "    model.eval()\n",
    "    count = 0\n",
    "    #record the gold and the prediction\n",
    "    gold_labels = []\n",
    "    pred_labels = []\n",
    "    dg.reset_samples()\n",
    "    while dg.index<dg.data_len:\n",
    "        sent_ids, label_list, sent_lens = next(dg.get_ids_samples())\n",
    "        outputs, _ =  model(sent_ids.to(device), sent_lens.to(device))\n",
    "        if binary:\n",
    "            preds = (outputs>0.5).squeeze()\n",
    "            gold_labels += list(label_list.cpu().numpy())\n",
    "            pred_labels += list(preds.cpu().numpy())\n",
    "            num = (preds.cpu() == label_list.bool()).sum().cpu().item()\n",
    "        else:\n",
    "            preds = outputs.argmax(1)\n",
    "            num = (label_list==preds.cpu()).sum().item()\n",
    "\n",
    "        count += num\n",
    "\n",
    "    accuracy = count*1.0/dg.data_len\n",
    "    print('Evaluation accuracy:', accuracy)\n",
    "    return accuracy\n",
    "\n",
    "\n",
    "#regression\n",
    "def evaluate_reg(dg, model):\n",
    "    #Make prediction\n",
    "    model.eval()\n",
    "    count = 0\n",
    "    #record the gold and the prediction\n",
    "    error = 0\n",
    "    dg.reset_samples()\n",
    "    while dg.index<dg.data_len:\n",
    "        sent_ids, label_list, sent_lens = next(dg.get_ids_samples())\n",
    "        probs, scores = model(sent_ids.to(device), sent_lens)#.to(device))\n",
    "        #label_list.apply_(scale_value)\n",
    "        loss = loss_func(scores.squeeze(), label_list.float().to(device))\n",
    "        num = len(sent_lens)\n",
    "        error += loss.item() * num\n",
    "        count += num\n",
    "\n",
    "    mse = error/count\n",
    "    return mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 494
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 55364,
     "status": "error",
     "timestamp": 1597469809620,
     "user": {
      "displayName": "Richard Sun",
      "photoUrl": "",
      "userId": "07723018862498440951"
     },
     "user_tz": -480
    },
    "id": "STPSEMhcCyNo",
    "outputId": "cf30a75b-52c9-4c61-d28e-03c636fd7d95"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MVMA-L\n",
      "Epoch: 0\n",
      "####################\n",
      "Sample Loss:1.610\n",
      "Sample Loss:1.024\n",
      "Sample Loss:0.927\n",
      "Sample Loss:0.878\n",
      "Sample Loss:0.834\n",
      "Evaluation accuracy: 0.5149082568807339\n",
      "Epoch: 1\n",
      "####################\n",
      "Sample Loss:0.803\n",
      "Sample Loss:0.711\n",
      "Sample Loss:0.660\n",
      "Sample Loss:0.600\n",
      "Sample Loss:0.588\n",
      "Evaluation accuracy: 0.7637614678899083\n",
      "Epoch: 2\n",
      "####################\n",
      "Sample Loss:0.498\n",
      "Sample Loss:0.476\n",
      "Sample Loss:0.482\n",
      "Sample Loss:0.450\n",
      "Sample Loss:0.454\n",
      "Evaluation accuracy: 0.8119266055045872\n",
      "Epoch: 3\n",
      "####################\n",
      "Sample Loss:0.408\n",
      "Sample Loss:0.408\n",
      "Sample Loss:0.342\n",
      "Sample Loss:0.342\n",
      "Sample Loss:0.312\n",
      "Evaluation accuracy: 0.823394495412844\n",
      "Epoch: 4\n",
      "####################\n",
      "Sample Loss:0.328\n",
      "Sample Loss:0.339\n",
      "Sample Loss:0.324\n",
      "Sample Loss:0.335\n",
      "Sample Loss:0.318\n",
      "Evaluation accuracy: 0.823394495412844\n",
      "Epoch: 5\n",
      "####################\n",
      "Sample Loss:0.322\n",
      "Sample Loss:0.294\n",
      "Sample Loss:0.294\n",
      "Sample Loss:0.298\n",
      "Sample Loss:0.274\n",
      "Evaluation accuracy: 0.819954128440367\n",
      "Epoch: 6\n",
      "####################\n",
      "Sample Loss:0.261\n",
      "Sample Loss:0.271\n",
      "Sample Loss:0.285\n",
      "Sample Loss:0.263\n",
      "Sample Loss:0.281\n",
      "Evaluation accuracy: 0.8222477064220184\n",
      "Epoch: 7\n",
      "####################\n",
      "Sample Loss:0.226\n",
      "Sample Loss:0.276\n",
      "Sample Loss:0.286\n",
      "Sample Loss:0.242\n",
      "Sample Loss:0.261\n",
      "Evaluation accuracy: 0.8153669724770642\n",
      "Epoch: 8\n",
      "####################\n",
      "Sample Loss:0.271\n",
      "Sample Loss:0.249\n",
      "Sample Loss:0.261\n",
      "Sample Loss:0.251\n",
      "Sample Loss:0.245\n",
      "Evaluation accuracy: 0.823394495412844\n",
      "Epoch: 9\n",
      "####################\n",
      "Sample Loss:0.230\n",
      "Sample Loss:0.285\n",
      "Sample Loss:0.271\n",
      "Sample Loss:0.245\n",
      "Sample Loss:0.242\n",
      "Evaluation accuracy: 0.8325688073394495\n",
      "Epoch: 10\n",
      "####################\n",
      "Sample Loss:0.212\n",
      "Sample Loss:0.270\n",
      "Sample Loss:0.250\n",
      "Sample Loss:0.260\n",
      "Sample Loss:0.229\n",
      "Evaluation accuracy: 0.823394495412844\n",
      "Epoch: 11\n",
      "####################\n",
      "Sample Loss:0.234\n",
      "Sample Loss:0.245\n",
      "Sample Loss:0.248\n",
      "Sample Loss:0.247\n",
      "Sample Loss:0.233\n",
      "Evaluation accuracy: 0.8256880733944955\n",
      "Epoch: 12\n",
      "####################\n",
      "Sample Loss:0.235\n",
      "Sample Loss:0.305\n",
      "Sample Loss:0.253\n",
      "Sample Loss:0.261\n",
      "Sample Loss:0.257\n",
      "Evaluation accuracy: 0.8314220183486238\n",
      "Epoch: 13\n",
      "####################\n",
      "Sample Loss:0.233\n",
      "Sample Loss:0.268\n",
      "Sample Loss:0.248\n",
      "Sample Loss:0.247\n",
      "Sample Loss:0.240\n",
      "Evaluation accuracy: 0.8268348623853211\n",
      "Epoch: 14\n",
      "####################\n",
      "Sample Loss:0.222\n",
      "Sample Loss:0.243\n",
      "Sample Loss:0.237\n",
      "Sample Loss:0.245\n",
      "Sample Loss:0.231\n",
      "Evaluation accuracy: 0.8211009174311926\n",
      "Epoch: 15\n",
      "####################\n",
      "Sample Loss:0.205\n",
      "Sample Loss:0.251\n",
      "Sample Loss:0.214\n",
      "Sample Loss:0.220\n",
      "Sample Loss:0.259\n",
      "Evaluation accuracy: 0.8245412844036697\n",
      "Epoch: 16\n",
      "####################\n",
      "Sample Loss:0.221\n",
      "Sample Loss:0.268\n",
      "Sample Loss:0.253\n",
      "Sample Loss:0.259\n",
      "Sample Loss:0.230\n",
      "Evaluation accuracy: 0.8348623853211009\n",
      "Epoch: 17\n",
      "####################\n",
      "Sample Loss:0.206\n",
      "Sample Loss:0.237\n",
      "Sample Loss:0.251\n",
      "Sample Loss:0.246\n",
      "Sample Loss:0.204\n",
      "Evaluation accuracy: 0.8348623853211009\n",
      "Epoch: 18\n",
      "####################\n",
      "Sample Loss:0.204\n",
      "Sample Loss:0.259\n",
      "Sample Loss:0.256\n",
      "Sample Loss:0.225\n",
      "Sample Loss:0.221\n",
      "Evaluation accuracy: 0.8360091743119266\n",
      "Epoch: 19\n",
      "####################\n",
      "Sample Loss:0.184\n",
      "Sample Loss:0.220\n",
      "Sample Loss:0.216\n",
      "Sample Loss:0.211\n",
      "Sample Loss:0.231\n",
      "Evaluation accuracy: 0.8360091743119266\n",
      "Epoch: 20\n",
      "####################\n",
      "Sample Loss:0.214\n",
      "Sample Loss:0.230\n",
      "Sample Loss:0.202\n",
      "Sample Loss:0.223\n",
      "Sample Loss:0.186\n",
      "Evaluation accuracy: 0.8360091743119266\n",
      "Epoch: 21\n",
      "####################\n",
      "Sample Loss:0.189\n",
      "Sample Loss:0.222\n",
      "Sample Loss:0.225\n",
      "Sample Loss:0.232\n",
      "Sample Loss:0.204\n",
      "Evaluation accuracy: 0.8348623853211009\n",
      "Epoch: 22\n",
      "####################\n",
      "Sample Loss:0.183\n",
      "Sample Loss:0.230\n",
      "Sample Loss:0.221\n",
      "Sample Loss:0.192\n",
      "Sample Loss:0.195\n",
      "Evaluation accuracy: 0.8428899082568807\n",
      "Epoch: 23\n",
      "####################\n",
      "Sample Loss:0.194\n",
      "Sample Loss:0.201\n",
      "Sample Loss:0.215\n",
      "Sample Loss:0.216\n",
      "Sample Loss:0.196\n",
      "Evaluation accuracy: 0.8440366972477065\n",
      "Epoch: 24\n",
      "####################\n",
      "Sample Loss:0.195\n",
      "Sample Loss:0.239\n",
      "Sample Loss:0.198\n",
      "Sample Loss:0.185\n",
      "Sample Loss:0.208\n",
      "Evaluation accuracy: 0.8428899082568807\n",
      "Epoch: 25\n",
      "####################\n",
      "Sample Loss:0.153\n",
      "Sample Loss:0.226\n",
      "Sample Loss:0.205\n",
      "Sample Loss:0.194\n",
      "Sample Loss:0.193\n",
      "Evaluation accuracy: 0.8520642201834863\n",
      "Epoch: 26\n",
      "####################\n",
      "Sample Loss:0.155\n"
     ]
    }
   ],
   "source": [
    "# torch.manual_seed(555)\n",
    "# np.random.seed(555)\n",
    "hidden_dim = 300\n",
    "label_dim = 5\n",
    "\n",
    "loss_func = nn.BCELoss()\n",
    "if label_dim > 1:\n",
    "    loss_func = nn.NLLLoss()\n",
    "\n",
    "valid_accs = []\n",
    "test_accs = []\n",
    "for scale in range(5):\n",
    "    model = SimpleClassifier(config.vocab_size, 300, hidden_dim, label_dim, 'MVMA_L')\n",
    "    #model.load_vector(config.emb_path, trainable=True)\n",
    "    \n",
    "    model = model.to(device)\n",
    "    ##################################\n",
    "    ####Weight decay can influence the result, if the value is too large, the model will not converge after iterations\n",
    "    #optimizer = optim.SGD(model.parameters(), lr=0.5, weight_decay=0.000000)\n",
    "    #optimizer = optim.Adagrad(model.parameters(), lr=0.5, weight_decay=0.00000, lr_decay=0.001)\n",
    "    #optimizer = optim.Adagrad(model.parameters(), lr=0.01, weight_decay=0.000001, lr_decay=0.001)\n",
    "\n",
    "    \n",
    "    loop_num = int(train_dg.data_len/config.batch_size)+1\n",
    "    best_model = None\n",
    "    best_acc = -1\n",
    "    train_acc = []\n",
    "    valid_acc = []\n",
    "    \n",
    "    for i in range(30):\n",
    "        print('Epoch:', i)\n",
    "        print('#'*20)\n",
    "        \n",
    "        ##For gru\n",
    "        #optimizer = optim.Adagrad(model.parameters(), lr=0.01, weight_decay=0.00001, lr_decay=0.001)\n",
    "        #for transformer\n",
    "        optimizer = optim.Adagrad(model.parameters(), lr=0.002, weight_decay=0.00001, lr_decay=0.001)\n",
    "\n",
    "        total_loss = 0\n",
    "        model.train()\n",
    "        #shuffle the training set given the random seed\n",
    "        train_dg.shuffle_data()\n",
    "        #sequential sampling, use all the dataset\n",
    "        train_dg.reset_samples()\n",
    "        #model.embeddings.required_grad = config.update_emb\n",
    "        for j in range(loop_num):\n",
    "            model.zero_grad()\n",
    "            # generate dataset\n",
    "            sent_ids,  label_list, sent_lens = next(train_dg.get_sequential_ids_samples())\n",
    "            logits, scores = model(sent_ids.to(device), sent_lens.to(device))\n",
    "            #label_list.apply_(scale_value)\n",
    "            \n",
    "            #print(logits.item())\n",
    "            if label_dim == 1:\n",
    "                loss = loss_func(logits.squeeze(), label_list.to(device).float())\n",
    "            else:\n",
    "                loss = loss_func(logits.squeeze(), label_list.to(device))#.float())#\n",
    "\n",
    "            loss.backward()\n",
    "            nn.utils.clip_grad_norm_(model.parameters(),5)\n",
    "            optimizer.step()\n",
    "            if j%20== 0:\n",
    "                l = loss.cpu().item()\n",
    "                print('Sample Loss:{:.3f}'.format(l))\n",
    "\n",
    "        dev_acc = evaluate_cls(dev_dg, model, binary=label_dim==1)\n",
    "        valid_acc.append(dev_acc)\n",
    "        if best_acc < dev_acc:\n",
    "            best_acc = dev_acc\n",
    "            best_model = copy.deepcopy(model)\n",
    "\n",
    "    print('Best dev:', best_acc)\n",
    "    test_acc = evaluate_cls(test_dg, best_model, binary=label_dim==1)\n",
    "    #Test performance       \n",
    "    print('Scale', scale, 'Test ACC:', test_acc)\n",
    "    valid_accs.append(best_acc)\n",
    "    test_accs.append(test_acc)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8543577981651376\n",
      "0.8543577981651376\n",
      "0.8520642201834863\n"
     ]
    }
   ],
   "source": [
    "for item in valid_accs:\n",
    "    print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8583196046128501\n",
      "0.8506315211422295\n",
      "0.8489840746842394\n"
     ]
    }
   ],
   "source": [
    "test_accs\n",
    "for item in test_accs:\n",
    "    print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2163,
     "status": "ok",
     "timestamp": 1596872327132,
     "user": {
      "displayName": "Richard Sun",
      "photoUrl": "",
      "userId": "07723018862498440951"
     },
     "user_tz": -480
    },
    "id": "OdxaGl4sCyOI",
    "outputId": "797ad37b-57aa-4dec-dc59-e0cbb62a224a"
   },
   "outputs": [],
   "source": [
    "#print('Best dev:', best_acc)\n",
    "scale = 10\n",
    "test_acc = evaluate_cls(test_dg, best_model)\n",
    "#Test performance       \n",
    "print('Scale', scale, 'Test ACC:', test_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "imdb valid 89.0   test 0.8804\n",
    "mr valid 0.796 test 0.767"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# best_model = torch.load(\"checkpoints/sst_spectral_norm_gru_20210927.pt\")\n",
    "#best_model = torch.load(\"checkpoints/sst_spectral_norm_elman_20210927.pt\")\n",
    "# best_model = torch.load(\"checkpoints/sst_spectral_norm_lstm_20210927.pt\")\n",
    "\n",
    "#torch.save(best_model, \"checkpoints/imdb_spectral_norm_gru_20210927.pt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Examine results from self-defined RNNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from all_ngram_feature import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "88"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word2id['bad']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.3225, -0.0042,  0.4190], device='cuda:0')"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent = [29, 88]\n",
    "gru_phrase_polarity_multi(sent, best_model, hidden_dim=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 2.8089, -0.5653, -0.1237]], device='cuda:0', grad_fn=<MmBackward>)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs = torch.LongTensor([sent]).to(device)\n",
    "length = torch.LongTensor([2])\n",
    "probs, scores = best_model(inputs, length.to(device))\n",
    "#_, scores = get_higher_degree_ngram(inputs, best_model, degree=3, hidden_dim=300)\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Elman RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "amazing\n",
      "real 1.7740238904953003\n",
      "first-degree approx 1.7679108381271362\n",
      "seond degree approx 1.7679108381271362\n",
      "very amazing\n",
      "real 3.4417803287506104\n",
      "first-degree approx 1.865437626838684\n",
      "seond degree approx 1.6167563199996948\n",
      "not amazing\n",
      "real 0.12541809678077698\n",
      "first-degree approx -1.709215760231018\n",
      "seond degree approx -1.9143320322036743\n",
      "not not amazing\n",
      "real 0.1223953366279602\n",
      "first-degree approx -0.0673142522573471\n",
      "seond degree approx 0.03873423859477043\n",
      "awful\n",
      "real -1.7218923568725586\n",
      "first-degree approx -1.6802411079406738\n",
      "seond degree approx -1.6802411079406738\n",
      "very awful\n",
      "real -3.481248617172241\n",
      "first-degree approx -2.0158090591430664\n",
      "seond degree approx -1.4931972026824951\n",
      "not awful\n",
      "real 0.10105571150779724\n",
      "first-degree approx 1.882822036743164\n",
      "seond degree approx 2.3339004516601562\n",
      "not not awful\n",
      "real 0.0360625796020031\n",
      "first-degree approx -0.017644550651311874\n",
      "seond degree approx -0.25441500544548035\n"
     ]
    }
   ],
   "source": [
    "best_model.eval()\n",
    "elman_first_degree_scores = []\n",
    "elman_second_degree_scores = []\n",
    "awful = word2id['awful']\n",
    "amazing = word2id['amazing']\n",
    "not_ = word2id['not']\n",
    "very = word2id['very']\n",
    "sents = [[amazing], [very, amazing], [not_, amazing], \n",
    "         [not_, not_, amazing], [awful], [very, awful], [not_, awful], [not_, not_, awful]]\n",
    "labels = ['amazing', 'very amazing', 'not amazing', \n",
    "          'not not amazing', 'awful', 'very awful', 'not awful', 'not not awful']\n",
    "lengths = [1, 2, 2, 3, 1, 2, 2, 3]\n",
    "\n",
    "for i, sent in enumerate(sents):\n",
    "    print(' '.join([id2word[w] for w in sent]))\n",
    "    inputs = torch.LongTensor([sent]).to(device)\n",
    "    length = torch.LongTensor([lengths[i]])\n",
    "    probs, scores = best_model(inputs, length.to(device))\n",
    "    print('real', scores.item())\n",
    "    scores = rnn_phrase_polarity_new(sent, best_model, 300)\n",
    "    print('first-degree approx', scores)\n",
    "    elman_first_degree_scores.append(scores)\n",
    "    phi, scores = get_higher_degree_ngram(inputs, best_model, degree=2, hidden_dim=300)\n",
    "    elman_second_degree_scores.append(scores[:,-1][0].item())\n",
    "    print('seond degree approx', scores[:,-1][0].item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f7da406f390>"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD+CAYAAAAnIY4eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU1f3/8dchRhIIAgVBZEclwZAQQgiLTAiIShFQkVVU0CKbaEWtiqWsla8W+LFUKbIoiGBJUVwQWsASAYFCgAFkF4zsO0Qia+D+/hgYiZNAwkxyM5P38/G4j8nMuXPuZyZ3Pjk5c+/nGsuyEBER/1XE7gBERMQ7SuQiIn5OiVxExM8pkYuI+DklchERP6dELiLi57xO5MaYEGPMamPMBmPMZmPMUF8EJiIiOWO8PY7cGGOA4pZlpRtjgoHlwB8ty1rliwBFROT6bvG2A8v1lyD9yt3gK4vOMhIRySc+mSM3xgQZY5zAEWCRZVn/80W/IiJyY16PyAEsy7oExBhjSgFzjTG1Lcv6/tp1jDE9gZ4AxYsXrxcREeGLTYv4te3bXbfh4fbGIf5h7dq1xyzLuv23j3s9R+7RoTGDgV8syxqV3TpxcXFWSkqKT7cr4o8SE123ycl2RiH+whiz1rKsuN8+7oujVm6/MhLHGBMKtAC2eduviIjkjC+mVioA040xQbj+MCRZljXPB/2KiEgO+OKolY1AXR/EIiIiN8EnX3aK/7p48SL79u3j3LlzdodSKI0c6brdutXeOKRgCQkJoVKlSgQHB+dofSXyQm7fvn2UKFGCatWq4Tq3S0TsZFkWx48fZ9++fVSvXj1Hz1GtlULu3LlzlClTRkncJhcvuhaRq4wxlClTJlf/JSuRi5K4jXbvdi0i18rtZ1KJXAqNxo0b2x2C+NiyZcuIjIwkJiaGs2fPZrteYmIigXzuiubIJZNqb3zt0/5S337Yp/15Y8WKFXaH4Bfe6/1fn/b3/MTmPu3vWjNnzuTVV1/lmWeeybNt+AONyMV2jz76KPXq1SMyMpJJkyYBEBYWxuuvv069evVo0aIFq1evJjExkRo1avDll18CkJqaisPhIDY2ltjYWHeiHjRoEDExMcTExFCxYkX3hzwsLAyA5ORkEhMTad++PREREXTt2pWrZzjPnz+fiIgImjRpwosvvkjr1q3z++0otH67HyQlJfHyyy8DMG7cOGrUqAHArl27aNKkCVOmTCEpKYlhw4bRtWtXkpOTM/2++vXrx7Rp0+x4KflOiVxs98EHH7B27VpSUlIYP348x48f55dffiExMZG1a9dSokQJBg4cyKJFi5g7dy6DBg0CoFy5cixatIh169Yxe/ZsXnzxRQCGDRuG0+nk22+/pUyZMvTr189jm+vXr2fs2LFs2bKF3bt3891333Hu3Dl69erFggULWL58OUePHs3X96Gw++1+cN9997Fs2TLANYVSpkwZ9u/fz/Lly3E4HPTo0YO2bdsycuRIZs6caXP09tLUithu/PjxzJ07F4C9e/eyc+dObr31Vlq2bAlAVFQURYsWJTg4mKioKFJTUwHXMfD9+vXD6XQSFBTEjh073H1alkXXrl3p378/9erV89hmfHw8lSpVAiAmJobU1FTCwsKoUaOG+5CvLl26uP9DyCu3e5Q/Krx+ux/s3buX9PR0Tp8+zd69e3niiSdYunQpy5Yto127djZHW7AokYutkpOTWbx4MStXrqRYsWIkJiZy7tw5goOD3d/cFylShKJFi7p/zsjIAGDMmDGUL1+eDRs2cPnyZUJCQtz9DhkyhEqVKmU7d3q1P4CgoCAyMjLwdQG5nPjd7/J9kwVSdvtBo0aN+PDDDwkPD8fhcPDBBx+wcuVKRo8e7dHHLbfcwuXLl933C9NJbppaEVulpaVRunRpihUrxrZt21i1KucXlkpLS6NChQoUKVKEGTNmcOnSJQDmzZvHokWLGD9+fK5iiYiIYPfu3e4R/+zZs3P1/Jtx4YJrKeyy2w8SEhIYNWoUCQkJ1K1blyVLllC0aFFKlizp0UfVqlXZsmUL58+fJy0tjW+++Sa/X4ZtlMjFVi1btiQjI4Po6Gj+8pe/0LBhwxw/t2/fvkyfPp2GDRuyY8cOihcvDsDo0aM5cOAA8fHxxMTEuOfUbyQ0NJQJEybQsmVLmjRpQvny5bNMGL7044+upbDLbj9wOBzs3buXhIQEgoKCqFy5Mk2aNMmyj8qVK9OxY0eio6Pp2rUrdesWnhJQPq9HnhOqR15wbN26lVq1atkdRoGRnp5OWFgYlmXx/PPPc88999C/f/88254uLCHZyeqzmWf1yEUCyeTJk4mJiSEyMpK0tDR69epld0giN6QvO0Wu0b9//zwdgYvkBY3IRUT8nEbkIjYqX97uCCQQKJGL2KhUKbsjkECgqRURG50751pEvKERuYiPtWrVilmzZlEqB8Ptn35y3RaGww8v7Dud79u8tVIJr/vYtm0bnTt3xhjDnDlzuOuuu7Jcr3v37rRu3Zr27dt7vc3cUiKXzIb4+ASYIWm+7e8almVhWRZFihSsfyznz59vdwheGd3JtxUfX5k9z6f9XSs/9oHPP/+cRx55hKFDh+bZNrxVsD4BUui8/vrrTJgwwX1/yJAh7joaI0eOpH79+kRHRzN48GDAVbq2Vq1a9O3bl9jYWIYPH57pcMHJkye7S59eq0+fPsTFxREZGenuC6BatWq8+eabNGrUiLi4ONatW8dDDz3EXXfdxcSJEwHXSUL3338/sbGxREVF8cUXXwAwceJEd7nc6tWr06xZM3efx44dc8f63HPPERkZyYMPPui++MGaNWuIjo6mU6dG/O1vf6J27dq+fFv9ypsjBjFx+mT3/eH/bwRj3v87AKMnjqPxw02p90Ajho1+C4DUvT8R3SyOF97sT4PfOxgx7h1eHfKG+/lTZ03jT0MHeGwnq31g9erV7gJcX3zxBaGhoVy4cIFz585Ro0YN5s+fz9ixY5kyZQrNmjUjNTU10+9q1KhRDBkyxOfvSW4pkYutOnfunKmmSVJSEh06dGDhwoXs3LmT1atX43Q6Wbt2LUuXLgVg+/btPP3006xfv55XX32VL7/8kotXLnz54YcfZlko66233iIlJYWNGzfy7bffsnHjRndb5cqVWblyJQ6Hg+7duzNnzhxWrVrlPrU/JCSEuXPnsm7dOpYsWcIrr7yCZVn07t0bp9PJmjVrqFSpUpZ/QHbu3Mnzzz/P5s2bKVWqFJ9++ikAzzzzDBMnTmT27JUEBQX57g31Qx3bPs6crz5z358zby6Pt36URd9+ww8/7uK7ecms+c93rNvkZNmq7wDYsWsnT7bvwup/L6d/rxeZt2i+ex/4KOljunV80mM7We0DsbGxrF+/HnCVyq1duzZr1qzhf//7Hw0aNKBVq1b07t2b/v37s2TJknx4N26OplbEVnXr1uXIkSMcOHCAo0ePUrp0aapUqcL48eNZuHChu15Geno6O3fupEqVKlStWtVdi6N48eI0b96cefPmUatWLS5evEhUVJTHdpKSkpg0aRIZGRkcPHiQLVu2EB0dDUDbtm0BV7nc9PR0SpQoQYkSJQgJCeHUqVMUL16cN998k6VLl1KkSBH279/P4cOHueOOOwD44x//SPPmzWnTpo3HdqtXr05MTAwA9erVIzU1lVOnTnH69GkaN27M9u3QuvUTrFiRd9MPBV1M7TocOX6UA4cOcuzEMUqXLEWVipV574OJfLP0v8S3dNVWSf8lnR9Sd1G5YiWqVKpCg9h4AIoXK06z+5oyf/G/Cb8nnIsXL1K7VqTHdrLbB+6++262bt3K6tWrefnll1m6dCmXLl3C4XDk6/vgDSVysV379u2ZM2cOhw4donPnzoBr7nPAgAEep8inpqa6i2Nd1aNHD0aMGEFERESWo/Eff/yRUaNGsWbNGkqXLk337t0zlTi9tkTuteVtr5bMnTlzJkePHmXt2rUEBwdTrVo19/OnTZvGTz/9xLvvvpvla/ttudyzZ89mKpdboQIcP56jtymgtWv1CJ/N/5zDR47Qoe3jgGsf+NPzL/Pck89mWjd1708UDy2W6bFnujzNO38fTfjdNXk6i9H49fYBh8PBggULCA4OpkWLFnTv3p1Lly4xatQoj34KaqlcTa2I7Tp37sw///lP5syZ4/7G/6GHHuKDDz4gPT0dgP3793PkyJEsn9+gQQP27t3LrFmz6NKli0f7zz//TPHixSlZsiSHDx9mwYIFuYovLS2NcuXKERwczJIlS/jpyqEma9euZdSoUXz88ce5+rKtdOnSlChRglWrVnHbbTBv3j9zFU8g6ti2Pf/68lM+m/857Vo9CsADTe9n+uwZpP9yZR84eIAjx7K+alN83frsO7if2Z//i06PeB41cr19ICEhgbFjx9KoUSNuv/12jh8/zrZt24iM9BzVly9fniNHjnD8+HHOnz/PvHkF4z8pjcjFdpGRkZw+fZqKFStSoUIFAB588EG2bt1Ko0aNANf1Nj/++ONs55M7duyI0+mkdOnSHm116tShbt26REZGUqNGDe67775cxde1a1fatGlDXFwcMTExREREAPDuu+9y4sQJ95eccXFxTJkyJUd9Tp06leeee47Q0OI4HIl5Xi63oLs3vBan09OpeMedVCjvmrJ6oOn9bPthOwmPtAAgrHhxPhw3Odt9oH3rx9iweSOlS+VuH2jQoAGHDx8mISEBgOjoaMqVK+e+sMm1goODGTRoEA0aNKB69erufcFuXpexNcZUBj4C7gAuA5Msyxp3veeojG3BEShlbFu3bk3//v25//777Q4lR66Wy92+HSZNepuMjIOMG3fdj43fy+vjyB/t3oEXezxP8yaJ7sd8cRy5XfK7jG0G8IplWbWAhsDzxph7fdCvyA2dOnWKmjVrEhoa6jdJHODrr78mJiaGNm1qs3btMgYOHGh3SH7rVNopIhPqEhoSmimJFyZeT61YlnUQOHjl59PGmK1ARWCLt32L3EipUqUyXXTZX3Tq1IlOnTq5LyyhizDfvFIlS7F56Xq7w7CVT7/sNMZUA+oC//NlvyIikj2fJXJjTBjwKfCSZVk/Z9He0xiTYoxJOXo062+eRUQk93ySyI0xwbiS+EzLsj7Lah3LsiZZlhVnWVbc7fo/UgSAihVdi4g3vJ4jN65jdKYCWy3L+n/ehyRSeISF2R2BBAJfjMjvA54CmhtjnFeWVj7oVyRL06ZN48CBA3aHwaBBg1i8eLFXfaSnuxbJnY+SZnLg0EGf9nn+/HlatGhBTExMpvo/vzVt2jT69evn0217yxdHrSwHPI+cF78UNd2zTok3NnXb5NP+wPVBql27NnfeeafP+86NYcOGed3H/v2u24JUj3zfG8t82l+lt31fs2TGv2YSGV6LO++o4LM+169fz8WLF3E6nT7rM7/oFH2x1fVKvTqdTho2bEh0dDSPPfYYJ0+eZM6cOaSkpNC1a1diYmLc6141efJk6tevT506dXj88cc5c+YM4Cr636dPH5o1a0aNGjX49ttvefbZZ6lVqxbdu3d3Pz+rUqcpKSnucrVRUVHuM/6uVkoEV+nawYMHu0vdbtu2DYCjR4/ywAMPEBsbS69evahatSrHjh3L0/fU31wtS9vntReIuT+eVk884v69bti8EUfb5tR7oBEdejzByVMn+ezrz1m7cT3dXuxB/Yfu89gHps6aRuOHmxL3YGP3PnDp0iVq1KiBZVmcOnWKIkWKuKtpOhwOVq9ezZNPPonT6SQmJoZdu3a5yxGDax9ITEzM1/clN5TIxXbZlXp9+umneeedd9i4cSNRUVEMHTqU9u3bExcXx8yZM3E6nYSGhmbqq127dqxZs4YNGzZQq1Ytpk6d6m47efIk//3vfxkzZgxt2rShf//+bN68mU2bNrlHYVmVOo2Li8PpdOJ0OmnZsiWvvvpqlq+jbNmyrFu3jj59+rgLLg0dOpTmzZuzbt06HnvsMfbs2ZMXb6Hf++HHXfTu9hzOb1ZTqmQp5i5w1Xx/9qVevDVgGGsXraR2xL38dezbtHv4UepF12X6+Cms+c93HvvAo79vw4qvvyVl4Qr3PhAUFETNmjXZsmULy5cvp169eixbtozz58+zb98+4uPjmTJlCg6HA6fTme1VgAoqJXKxXValXtPS0jh16hRNmzYFoFu3bu4R1PV8//33OBwOoqKimDlzJps3b3a3tWnTBmMMUVFRlC9fnqioKIoUKUJkZCSpqamAq9RpbGwsdevWZfPmzWzZ8ut5bUlJSaxbt4633347y21fvUDB1dcAsHz5cndFx5YtW2ZZC0agWuWq1Il0lRWOjYrhp717SPs5jbSf00ho5Cpj+2T7J1j+vxU37Gvztq00b/cQsS0aZtoHHA4HS5cuZenSpQwYMIDly5ezZs0a6tevn3cvLJ+oaNYV1d74Ot+3mfr2w/m+zYIoq1KvN6t79+58/vnn1KlTh2nTppGcnOyxnezK1V6v1OnmzZsZPHgwS5cuzbZo09U+g4KCyMjIAMDbWkaFRdFbr/19BJFx6eb3gR6v9GHOlFlE3xvFrMWfuvcBh8PBxIkTOXDgAMOGDWPkyJEkJye7i2X91rUlawtKudrsaEQuBVLJkiUpXbo0y5a5vnibMWOGe3ReokQJTp/OugDT6dOnqVChAhcvXmTmzJm52mZ2pU7T0tLo3LkzH330Ebk9B6JJkyYkJSUBsHDhQk6ePJmpvXJl1yKeSt5WklIlS7lH4bM++yeOBq6qhWFhYZz+JevDfdLTT3NHuTs89oEGDRqwYsUKihQpQkhICDExMbz//vvZXkCiWrVqrF27FsA93VdQaUQuBdb06dPp3bs3Z86coUaNGnz44YeAa9Tdu3dvQkNDWblyZaY50uHDh9OgQQOqVq1KVFRUtgk/K9mVOv3888/56aefeO6559zr5vTIhsGDB9OlSxdmz55N06ZNqVChAiVK/FqRr1ix6zxZmDpmIv0GvMSZs2epXqUak0e7ru/6dIeu9BvwEqEhoSz9fHGmfWDwqwNp0rY5VStWJjouxr0PFC1alMqVK7uvLuVwOPjkk0+yvKIUuH53f/jDHxgxYgQNGjTI41fqHa/L2N6MgljGtrBOrQRKGduC6vz58wQFBXHLLbewcuVK+vTpk+mPwM9XilncdptNAeajvC5jm5XCUsZWI3KRPLRnzx46duzI5cuXufXWW5k8eXKm9oNXzmkpDIlc8o4SuUgeuueee9xXaRfJK0rkBdWQfLr010NJcODKN/J31s2fbYqIT+moFRERP6dELiLi5zS1ImKjqlXtjkACgUbk4nd8WcbW6XQyf/58n/R1M0JCXEtKSgovvviibXH4G1+Wsc2rfWD8+PHUqlWLrl27Xne9MB8UpdeIXDLZGuHbY8prbdvq0/7At2VsnU4nKSkptGplTwn9U6dct3FxccTFeRwebIshQ4YU6P7At2Vs82ofmDBhAgsWLKB69eo+7TcrGpGLrXxdxjYxMZHXX3+d+Ph4atas6T7F/9y5czzzzDNERUVRt25dlixZwoULFxg0aBCzZ8/O8mICqampOBwOYmNjiY2NZcUK16niycnJNG3alI4dO1KzZk3eeOMNZs6cSXx8PFFRUezatQuAr776igYNGlC3bl1atGjB4cOHAWjVqpW7LG7lyiWZOHE6ycnJtG7dGnAlvmeffZbExERq1KjB+PHj3TENHz6ciIgIHnjgAbp06eKusujPfF3G9oEOrXhzxCDua52YZ/tA3759+fLLLwF47LHHePbZZwGYOnUqAwcOpHfv3uzevZu2bdsyZswYhgwZkul3Vbt2bXdhNV9QIhfb+bKMLUBGRgarV69m7NixDB06FID33nsPgE2bNvHJJ5/QrVs3Ll++zLBhw+jUqRNOp5NOnTpl6qdcuXIsWrSIdevWMXv27ExTHxs2bGDcuHFs2rSJGTNmsGPHDlavXk2PHj34+9//DrjqrKxatYr169fTuXNn/va3vwEwf/58nE4nU6dO5c47q9KixaMer2Hbtm385z//YfXq1QwdOpSLFy+SkpLCp59+yvr16/nss88oaGdHe8OXZWzBtQ98Ny85z/aBhIQE9x+I/fv3u6tkLl++3F2c684772TJkiX0798/b960a2hqRWyX0zK2HTp0yFF/2ZWTfeGFFwCIiIigatWq7Nix47r9XLx4kX79+uF0OgkKCsq0fv369alQwfVv/V133cWDDz4IQFRUFEuWLAFg3759dOrUiYMHD3LhwoVM/2IfO3aMp556ilGjkihRoqTH5d4efvhhihYtStGiRSlXrhyHDx9m+fLlPPLII+7E1aZNmxy9H/4gp2Vsn+jTLUf9Pfr7tkDe7QMOh4OxY8eyZcsW7r33Xk6ePMnBgwdZuXJlpv+g8osSudjOl2Vsr+3P23KyY8aMoXz58mzYsIHLly8TEhKSZczXlsW9WhIX4IUXXuDll1+mbdu2JCcnu+eKL126ROfOnRk0aBA1a9a+7mu49nUEcklcX5axdfV3K5B3+0DFihU5efIk//73v0lISODEiRMkJSURFhaWqSjaVdeWxAXfl8XV1IoUSDdbxjY7CQkJ7pKmO3bsYM+ePYSHh1+3r7S0NCpUqECRIkWYMWMGly5dytU209LSqFixIuCq5HjVG2+8QXR0tPuCEznVpEkTvvrqK86dO0d6ejpff53/hd7y082Wsc2Or/eBRo0aMXbsWBISEnA4HIwaNeq6JXHXrVsHwLp16/jxxx9zFfuNKJFLgTV9+nT+9Kc/ER0djdPpZNCgQcCvZWyz+rIzO3379uXSpUtERUXRqVMnpk2bRtGiRWnWrBlbtmzJ8ouuvn37Mn36dBo2bMiOHTsoXrx4ruIfMmQIHTp0wOFwULZsWffjo0aNYuHChcTExNCxYwzff/9ljvqrX78+bdu2pU6dOrRr1464uDhKlsynUg42mTpmIgPeGki9BxqxYfMm/vzS68CvZWyz+rIzO77eBxwOBxkZGdx9993ExsZy4sSJbBP5448/zokTJ4iJieEf//gHNWvWvMl3JGsqY3tFgStjm0+1VrY+lEStquVcd1RrpcBLT08nLCyMM2fOkJCQwKRJk4iNjbU7rBxRGdvcURlbET9x4oTr9ne/y9n6PXv2ZMuWLZw7d45u3br5TRKXvKVELmKjo0ddtzlN5LNmzcq7YMRvaY5cRMTPKZEXelZAH9Ym4o9y+5lUIi/kQtJ2c/yXwD5GWcSfWJbF8ePHM523cCOaIy/kKq17h328ztGSNeBn3xe4kus7dMh1e825IgEr46RvT4LJiVtO5zwZFiQhISFUqlQpx+v7JJEbYz4AWgNHLMvK+lQ1KZCCL5yi+qoBrjtD0uwNphC6/XbX7TWHmQesfW8sy/dtVnq7cBxS66uplWlASx/1JVJolC1bOJK45C2fJHLLspYCJ3zRl0hhMm2aaxHxRr592WmM6WmMSTHGpBy9evCsSCGnRC6+kG+J3LKsSZZlxVmWFXf71YlBERHxmg4/FBHxc0rkIiJ+zieJ3BjzCbASCDfG7DPG/MEX/YqIyI355Dhyy7K6+KIfkcJm/ny7I5BAoDM7RWxUrJjdEUgg0By5iI0mTHAtIt5QIhexUVKSaxHxhhK5iIifUyIXEfFzSuQiIn5OiVxExM/p8EMRGyUn2x2BBAKNyEVE/JwSuYiNRo1yLSLeUCIXsdG8ea5FxBtK5CIifk6JXETEzymRi4j4OR1+KGKj0FC7I5BAoEQuYqMFC+yOQAKBplZERPycErmIjYYPdy0i3lAiF7HRN9+4FhFvKJGLiPg5JXIRET+nRC4i4ud0+KGIjcqUsTsCCQRK5CI2+vRTuyOQQKCpFRERP6dELmKjAQNci4g3NLUiYqOVK+2OQAKBT0bkxpiWxpjtxpgfjDFv+KJPERHJGa8TuTEmCHgP+D1wL9DFGHOvt/2KiEjO+GJqJR74wbKs3QDGmH8CjwBbfNC3h8REz8c6doS+feHMGWjVyrO9e3fXcuwYtG/v2d6nj+s24+cQjs2L8Wi/LX43xe4+wsXjxTn+nyiP9pKNdxJa7TgXDt/GiW88/4aVSthOSKWTnNtXmlNLw399Latct2PHQkwMLF4Mf/3rlcbUX6//9X7rlwgv+wNfbW/J6JX9PPqf8VgvKpfcz+zv2/GPlGc92ud0fJqyxU4wzfkE05xPeLTP79qBYsFnmTABkpI8mt1Xeh81yvOyZKGhv1bwGz7c83TzMmV+PTJjwADPqYRKleDjj10/v/QSOJ2Z22vWhEmTXD/37Ak7dvzadmb1aiJCtjGg3P8B8NqBdziccUem59cJdfLy7WMA+OP+cZy6VCpTe8Niq+hT9h+u/ve9z/nLIZnam4Yl8+zvPgSg257pFIuPz9Tu7b535AiUKwd798JTT3m2v/IKtGkD27dDr16e7QMHQosWrvftpZc820eMgMaNYcUKePNNz/Ys971rvP8+hIfDV1/B6NGe7TNmQOXKMHs2/OMfnu1z5kDZsjBtGrw/y/Oz81GHzYQGX2b6ugrM21bWo/1fT2zy7FQ8+CKRVwT2XnN/H9DgtysZY3oCPQGqVKnig836VurbD7s+TFn8+XmlW5lfP0w7PdsH9ijz64fpR8/2EX0b//ph2pfDgKo5fv35hbUQDnwFHMxi3Ze3QGVgNnAsi/bXfoSywDTgVBbtfz4ExYAJ2YcTNT2KYynd+PlQ00yPF7n1PFHTXX8Jj6zvRfqhzL/6W9JPETX9ZQAObfojZw7VydTuvHCYqOmub/sObnmNs4ci3G3176iffUBAsfh4fhcTT62xTwNQ8kk4/Zv3t2yjeGr9X08ASjwOF45nbr/9/nhq/eVFAMJ+D0FnM7eXbx1PrVdfc20v8brhsH/HSY/Hvpl+kF9WHSL9bDD7d9T2aL9sGc5cPMf0AbvYv8NzEPD1e3vY8/VxDp8qxv4d4R7tUDrbePZu2cQnf5nOyrLbSD0Wwd4t3TzW+ei1SXxT+kd2HK7D3i2dPdo/eOldyt22n80H4tm7/TGP9kl9R1Oq2DGcexzs3dWKyvd6JuuritYo5fFYxeH3UawYlJ4ARbMYRFR62+H5oA8NGTIkT1RNmjYAAAg5SURBVPvPr20ay7K868CYDsBDlmX1uHL/KSDesqwXsntOXFyclZKS4tV2JX9FTc/+A5pXNnXzr9HYe73/m+/bfH5i82zbRndqnY+RuLwy27+uJO1vidwYs9ayrLjfPu6LLzv34RoPXlUJOOCDfkVEJAd8kcjXAPcYY6obY24FOgNf+qBfkYA357t7mPPdPXaHIX7O6zlyy7IyjDH9gP8AQcAHlmVt9joykUJg3/Ewu0OQAOCTE4Isy5oPzPdFXyIikjs6RV9ExM8pkYuI+DnVWhGxUbmSZ+wOQQKAErmIjZ5out3uECQAKJGLSKFlxwlBeUFz5CI2mvVtOLO+zerUe5Gc04hcxEZH0orZHYIEAI3IRUT8nBK5iIifUyIXEfFzmiMXsVGlMul2hyABQIlcxEbt78viSiUiuaSpFRERP6dELmKj6d/cy/QsrvMqkhuaWhGx0clfitodggQAjchFRPycErmIiJ9TIhcR8XOaIxexUfXyaXaHIAFAiVzERo802G13CBIANLUiIuLnlMhFbDR5YW0mL6xtdxji5zS1ImKjX84F2x2CBACNyEVE/JwSuYiIn1MiFxHxc5ojF7FReMWTdocgAcCrEbkxpoMxZrMx5rIxJs5XQYkUFr+vl8rv66XaHYb4OW+nVr4H2gFLfRCLiIjcBK+mVizL2gpgjPFNNCKFzIT5dQDo22qDzZGIP9McuYiNLmToeAPx3g0TuTFmMXBHFk1/tizri5xuyBjTE+gJUKVKlRwHKCIi13fDRG5ZVgtfbMiyrEnAJIC4uDjLF32KiIiOIxcR8XtezZEbYx4D/g7cDnxtjHFalvWQTyITKQRqVz1mdwgSALw9amUuMNdHsYgUOi3q7LU7BAkAmloREfFzSuQiNhr7ZV3GflnX7jDEzymRi4j4OSVyERE/p0QuIuLnlMhFRPycaq2I2Cj2riN2hyABQIlcxEYJkfvtDkECgKZWRGx04WIRLlzUx1C8oz1IxEYTFtRhwoI6dochfk6JXETEzymRi4j4OX3ZKRKgXpk9z+4QJJ9oRC4i4uc0IhexUcPwg3aHIAFAiVzERg3DD9kdggQATa2I2Cj9bDDpZ4PtDkP8nBK5iI2mLKrNlEW17Q5D/JwSuYiIn9McuYiPPD+xea6f869tN/9ckas0IhcR8XNK5CIifk5TKyI26tPH7ggkECiRi9ioUye7I5BAoKkVERvt3etaRLyhEbmIjZ56ynWbnGxrGOLnNCIXEfFzXiVyY8xIY8w2Y8xGY8xcY0wpXwUmIiI54+3UyiJggGVZGcaYd4ABwOvehyUFzaZum+wOQUSy4dWI3LKshZZlZVy5uwqo5H1IIiKSG778svNZYLYP+xMJeK+8YncEEghumMiNMYuBO7Jo+rNlWV9cWefPQAYw8zr99AR6AlSpUuWmghUJNG3a2B2BBIIbJnLLslpcr90Y0w1oDdxvWZZ1nX4mAZMA4uLisl1PpDDZvt11Gx5ubxzi37yaWjHGtMT15WZTy7LO+CYkkcKjVy/XrY4jF294exz5u0AJYJExxmmMmeiDmEREJBe8GpFblnW3rwIREZGbozM7RUT8nBK5iIifU9EsERsNHGh3BBIIlMhFbNTiugf3iuSMplZEbOR0uhYRb2hELmKjl15y3eo4cvGGRuQiIn5OiVxExM8pkYuI+DklchERP6cvO0VsNGKE3RFIIFAiF7FR48Z2RyCBQFMrIjZascK1iHhDI3IRG735putWx5GLNzQiFxHxc0rkIiJ+TolcRMTPKZGLiPg5fdkpYqOxY+2OQAKBErmIjWJi7I5AAoGmVkRstHixaxHxhkbkIjb6619dt7pSkHhDI3IRET+nRC4i4ueUyEVE/JwSuYiIn9OXnSI2ev99uyOQQKBELmKj8HC7I5BA4NXUijFmuDFmozHGaYxZaIy501eBiRQGX33lWkS84e0c+UjLsqIty4oB5gGDfBCTSKExerRrEfGGV4ncsqyfr7lbHLC8C0dERHLL6zlyY8xbwNNAGtDsOuv1BHpeuZtujNnu7bZFAoUxdkcgfqJqVg8ay7r+INoYsxi4I4umP1uW9cU16w0AQizLGuxNlCIikjs3TOQ57siYqsDXlmXV9kmHIiKSI94etXLPNXfbAtu8C0dERHLLqxG5MeZTIBy4DPwE9LYsa7+PYhMRkRzw2dSKiIjYQ7VWRET8nBK5FArGmM7GmP8ZY34xxhy58nNf4zLNGHPBGJNujDlhjFlkjIm45rlDjDEfZ9GnZYy5O39fiYgnJXIJeMaYV4BxwEhch9KWB3oD9wG3Xlntb5ZlhQEVgf3AVBtCFbkpKpolAc0YUxIYBjxtWdan1zStB7peWcf9oGVZZ40xScC/8jNOEW9oRC6BrhFQFPjiRisCGGOKA12AH/IyKBFfUiKXQFcWOGZZVsbVB4wxK4wxp4wxZ40xCVceftUYcwo4DTQBnrIhVpGbokQuge44UNYY455GtCyrsWVZpa60Xf0MjLryWDXgLK7zI67KAIKv7dQYc/X+xTyKWyTHlMgl0K0EzgOP5GRly7L2AH8ExhljQq88vAdXgr9WdeASri9GRWylRC4BzbKsU8BQYIIxpr0xJswYU8QYE4Or9HJWz1kEHODXap3/BsKNMU8ZY4KNMb8DRgBzrp2yEbGLErkEPMuy/ga8DLwGHAEOA+8DrwMrsnnaSOA1Y0xRy7KOAK2AXlee/z2uss198jh0kRzRKfoiIn5OI3IRET+nRC4i4ueUyEVE/JwSuYiIn1MiFxHxc0rkIiJ+TolcRMTPKZGLiPg5JXIRET/3/wHqdzavQi/wAgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# for i in range(len(elman_first_degree_scores)):\n",
    "#     plt.bar(i, elman_first_degree_scores[i], label=labels[i])\n",
    "for i in range(len(elman_second_degree_scores)):\n",
    "    plt.bar(i, elman_second_degree_scores[i], label=labels[i])\n",
    "plt.plot([-1,6], [0, 0], 'b', linestyle='--')\n",
    "plt.plot([4,4], [-6.5, 6.5], 'b', linestyle='--')\n",
    "ax = plt.gca()\n",
    "ax.set_title('GRU', y=-0.1)\n",
    "ax.axes.xaxis.set_visible(False)\n",
    "plt.ylim(-3, 3)\n",
    "plt.legend(ncol=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "amazing\n",
      "real 9.08558177947998\n",
      "approx 9.999212265014648\n",
      "very amazing\n",
      "real 8.987044334411621\n",
      "approx 0.08840620517730713\n",
      "not amazing\n",
      "real -0.06087500602006912\n",
      "approx -7.169424057006836\n",
      "not not amazing\n",
      "real -0.09008053690195084\n",
      "approx 0.04791463911533356\n",
      "awful\n",
      "real -6.310919284820557\n",
      "approx -8.080079078674316\n",
      "very awful\n",
      "real -4.444940567016602\n",
      "approx 1.8684308528900146\n",
      "not awful\n",
      "real -5.530052661895752\n",
      "approx 1.4990994930267334\n",
      "not not awful\n",
      "real -5.441023826599121\n",
      "approx 0.4996188282966614\n"
     ]
    }
   ],
   "source": [
    "best_model.eval()\n",
    "gru_first_degree_scores = []\n",
    "awful = word2id['awful']\n",
    "amazing = word2id['amazing']\n",
    "not_ = word2id['not']\n",
    "very = word2id['very']\n",
    "sents = [[amazing], [very, amazing], [not_, amazing], \n",
    "         [not_, not_, amazing], [awful], [very, awful], [not_, awful], [not_, not_, awful]]\n",
    "labels = ['amazing', 'very amazing', 'not amazing', \n",
    "          'not not amazing', 'awful', 'very awful', 'not awful', 'not not awful']\n",
    "lengths = [1, 2, 2, 3, 1, 2, 2, 3]\n",
    "for i, sent in enumerate(sents):\n",
    "    print(' '.join([id2word[w] for w in sent]))\n",
    "    inputs = torch.LongTensor([sent]).to(device)\n",
    "    length = torch.LongTensor([lengths[i]])\n",
    "    probs, scores = best_model(inputs, length.to(device))\n",
    "    print('real', scores.item())\n",
    "    scores = gru_phrase_polarity_new(sent, best_model, 300)\n",
    "    print('approx', scores)\n",
    "    gru_first_degree_scores.append(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fd3c0a3c350>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD+CAYAAAAnIY4eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de3wU1f3/8dcBYwIEgYIgEm5RIRgSNiEEUDZGRaUIWJWrVEGr3EQraqtYy0VaazX+RGoREZSIqOSLVRGhBS0IFGgIYQG5F4wQ7rdEIgQIzO+PhTUhCSTsJsMk7+fjMY/N7pk989nN7CeTs2c+YyzLQkREnKuK3QGIiIh/lMhFRBxOiVxExOGUyEVEHE6JXETE4ZTIRUQczu9EbowJMcakGmPWGGPWG2PGBiIwEREpGePvPHJjjAFqWJaVY4wJApYCv7Usa0UgAhQRkQu7wt8OLO9fgpyzd4POLjrLSESknARkjNwYU9UY4wH2Awssy/pvIPoVEZGL8/uIHMCyrNOAyxhTG/jMGNPasqzv8q9jjBkEDAKoUaNG24iIiEBsOmDW7cou921GNapV7tuUy8vmzd7bli3tjUOcYdWqVQcty7r6/Mf9HiMv1KExo4GfLMtKKm6duLg4Ky0tLaDb9Vez578q921mvHJ3uW9TLi+Jid7bRYvsjEKcwhizyrKsuPMfD8SslavPHoljjKkGdAY2+duviIiUTCCGVhoCycaYqnj/MKRYljUnAP2KiEgJBGLWylogJgCxiIjIJQjIl53iXKdOnSIzM5Pc3Fy7Q6mUXnvNe7txo71xyOUlJCSEsLAwgoKCSrS+Enkll5mZSc2aNWnWrBnec7tExE6WZXHo0CEyMzNp3rx5iZ6jWiuVXG5uLnXr1lUSt8mpU95F5BxjDHXr1i3Vf8lK5KIkbqPt272LSH6l/UwqkUulcdNNN9kdggTYkiVLiIyMxOVycfz48WLXS0xM5HI7dyWQNEYuBQT6xKjL6aSnZcuW2R2CI/x9yL8D2t/jk24LaH/5zZgxg2effZaHH364zLbhBDoiF9v96le/om3btkRGRjJ58mQAQkNDee6552jbti2dO3cmNTWVxMREwsPDmT17NgAZGRm43W5iY2OJjY31JepRo0bhcrlwuVw0atTI9yEPDQ0FYNGiRSQmJtKzZ08iIiLo378/585wnjt3LhEREXTq1Iknn3ySbt26lffbUWmdvx+kpKTw9NNPA/Dmm28SHh4OwLZt2+jUqRNTpkwhJSWFl156if79+7No0aICv6/hw4czbdo0O15KuVMiF9u99957rFq1irS0NCZMmMChQ4f46aefSExMZNWqVdSsWZMXX3yRBQsW8NlnnzFq1CgA6tevz4IFC0hPT2fmzJk8+eSTALz00kt4PB6+/fZb6taty/Dhwwttc/Xq1YwfP54NGzawfft2/vOf/5Cbm8vgwYOZN28eS5cu5cCBA+X6PlR25+8HN998M0uWLAG8Qyh169Zl165dLF26FLfbzaOPPkqPHj147bXXmDFjhs3R20tDK2K7CRMm8NlnnwGwc+dOtm7dypVXXkmXLl0AiIqKIjg4mKCgIKKiosjIyAC8c+CHDx+Ox+OhatWqbNmyxdenZVn079+fESNG0LZt20LbjI+PJywsDACXy0VGRgahoaGEh4f7pnz169fP9x9CWbm6UPmjyuv8/WDnzp3k5ORw9OhRdu7cyQMPPMDixYtZsmQJ9913n83RXl6UyMVWixYt4uuvv2b58uVUr16dxMREcnNzCQoK8n1zX6VKFYKDg30/5+XlAfDGG2/QoEED1qxZw5kzZwgJCfH1O2bMGMLCwoodOz3XH0DVqlXJy8sj0AXkSuIXvyj3TV6WitsPOnbsyPvvv0/Lli1xu9289957LF++nNdff71QH1dccQVnzpzx3a9MJ7lpaEVslZ2dTZ06dahevTqbNm1ixYqSX1gqOzubhg0bUqVKFaZPn87p06cBmDNnDgsWLGDChAmliiUiIoLt27f7jvhnzpxZqudfipMnvUtlV9x+kJCQQFJSEgkJCcTExLBw4UKCg4OpVatwCeimTZuyYcMGTpw4QXZ2Nt988015vwzbKJGLrbp06UJeXh7R0dH88Y9/pEOHDiV+7rBhw0hOTqZDhw5s2bKFGjVqAPD666+ze/du4uPjcblcvjH1i6lWrRoTJ06kS5cudOrUiQYNGhSZMALp+++9S2VX3H7gdrvZuXMnCQkJVK1alcaNG9OpU6ci+2jcuDG9e/cmOjqa/v37ExNTeUpABbweeUmoHrnX5TA1b+PGjbRq1cruMC4bOTk5hIaGYlkWjz/+ODfccAMjRowos+3pwhJSnKI+m2VWj1ykInn33XdxuVxERkaSnZ3N4MGD7Q5J5KL0ZadIPiNGjCjTI3CRsqAjchERh9MRuYiNGjSwOwKpCJTIRWxUu7bdEUhFoKEVERvl5noXEX8okYsEWNeuXcnKyirRuj/84F3k8rVp0yZcLhcxMTFs27at2PUGDhzIrFmzyjGyn2loRQoaE+ATYMZkB7a/fCzLwrIsqlS5vI5H5s6da3cIfnm9T2ArPj4zc05A+8uvPPaBzz//nHvuuYexY8eW2Tb8dXl9AqTSee6555g4caLv/pgxY3x1NF577TXatWtHdHQ0o0ePBryla1u1asWwYcOIjY1l3LhxBaYLvvvuu77Sp/kNHTqUuLg4IiMjfX0BNGvWjBdeeIGOHTsSFxdHeno6d911F9dddx2TJk0CvCcJ3X777cTGxhIVFcUXX3wBwKRJk3zlcps3b86tt97q6/PgwYO+WB977DEiIyO58847fRc/WLlyJdHR0fTp05FXX/0drVu3DuTb6ih27gOpqam+AlxffPEF1apV4+TJk+Tm5hIeHs7cuXMZP348U6ZM4dZbbyUjI6PA7yopKYkxY8YE/D0pLSVysVXfvn0L1DRJSUmhV69ezJ8/n61bt5KamorH42HVqlUsXrwYgM2bN/PQQw+xevVqnn32WWbPns2psxe+fP/994sslPXnP/+ZtLQ01q5dy7fffsvatWt9bY0bN2b58uW43W7fv8crVqzwndofEhLCZ599Rnp6OgsXLuSZZ57BsiyGDBmCx+Nh5cqVhIWFFZk8tm7dyuOPP8769eupXbs2n376KQAPP/wwkyZNYubM5VStWjVwb6gD2bkPxMbGsnr1asBbKrd169asXLmS//73v7Rv356uXbsyZMgQRowYwcKFC8vh3bg0GloRW8XExLB//352797NgQMHqFOnDk2aNGHChAnMnz/fVy8jJyeHrVu30qRJE5o2beqrxVGjRg1uu+025syZQ6tWrTh16hRRUVGFtpOSksLkyZPJy8tjz549bNiwgejoaAB69OgBeMvl5uTkULNmTWrWrElISAhZWVnUqFGDF154gcWLF1OlShV27drFvn37uOaaawD47W9/y2233Ub37t0Lbbd58+a4XC4A2rZtS0ZGBllZWRw9epSbbrqJzZuhW7cHWLas7IYfLnd27wPXX389GzduJDU1laeffprFixdz+vRp3G53ub4P/lAiF9v17NmTWbNmsXfvXvr27Qt4xz5HjhxZ6BT5jIwMX3Gscx599FFefvllIiIiijwS+/7770lKSmLlypXUqVOHgQMHFihxmr9Ebv7ytudK5s6YMYMDBw6watUqgoKCaNasme/506ZN44cffuCtt94q8rWdXy73+PHjBcrlNmwIhw6V6G2q0OzcB9xuN/PmzSMoKIjOnTszcOBATp8+TVJSUqF+LtdSuRpaEdv17duXTz75hFmzZtGzZ08A7rrrLt577z1ycnIA2LVrF/v37y/y+e3bt2fnzp189NFH9OvXr1D7jz/+SI0aNahVqxb79u1j3rx5pYovOzub+vXrExQUxMKFC/nh7DSTVatWkZSUxIcffliqL9vq1KlDzZo1WbFiBVddBXPmfFKqeCoiO/eBhIQExo8fT8eOHbn66qs5dOgQmzZtIjIyslA/DRo0YP/+/Rw6dIgTJ04wZ87l8Z+UjsjFdpGRkRw9epRGjRrRsGFDAO688042btxIx44dAe/1Nj/88MNix5N79+6Nx+OhTp06hdratGlDTEwMkZGRhIeHc/PNN5cqvv79+9O9e3fi4uJwuVxEREQA8NZbb3H48GHfl5xxcXFMmTKlRH1OnTqVxx57jGrVauB2J5Z5udzLnZ37QPv27dm3bx8JCQkAREdHU79+fd+FTfILCgpi1KhRtG/fnubNm/v2Bbv5XcbWGNMY+AC4BjgDTLYs680LPUdlbL1UxjZwunXrxogRI7j99tvtDqVEzpXL3bwZJk9+hby8Pbz55gU/NnIRTtsHLqa8y9jmAc9YltUK6AA8boy5MQD9ilxUVlYWLVq0oFq1ao76AH/11Ve4XC66d2/NqlVLePHFF+0OybGcug8Ekt9DK5Zl7QH2nP35qDFmI9AI2OBv3yIXU7t27QIXXXaKPn360KdPH9+FJXQR5kvn1H0gkAL6ZacxphkQA/w3kP2KiEjxApbIjTGhwKfAU5Zl/VhE+yBjTJoxJu3AgQOB2qyISKUXkERujAnCm8RnWJb1j6LWsSxrsmVZcZZlxV2t/yNFAGjUyLuI+MPvMXLjnaMzFdhoWdb/8z8kkcojNNTuCKQiCMQR+c3Ag8BtxhjP2aVrAPoVKdK0adPYvXu33WEwatQovv76a7/6yMnxLlI6ZbEPnDhxgs6dO+NyuQrUfilq28OHDw/otv0ViFkrS4HCM+fFkaKSC9eo8Me6AesC2h94P0itW7fm2muvDXjfpfHSSy/53ceuXd7bli397ipgMp9fEtD+wl4JfM2SstgHVq9ezalTp/B4PAHrs7zoFH2x1YVKvXo8Hjp06EB0dDT33nsvR44cYdasWaSlpdG/f39cLpdv3XPeffdd2rVrR5s2bbj//vs5duwY4C36P3ToUG699VbCw8P59ttveeSRR2jVqhUDBw70Pb+oUqdpaWm+crVRUVG+M/7yX0igWbNmjB492lfqdtOmTQAcOHCAO+64g9jYWAYPHkzTpk05ePBgmb6n5+zdtvWSlvJWHvvA6dOnCQ8Px7IssrKyqFKliq+SotvtJjU1lV//+td4PB5cLhfbtm3zlSMG7z6QmJhYru9LaSiRi+2KK/X60EMP8de//pW1a9cSFRXF2LFj6dmzJ3FxccyYMQOPx0O1atUK9HXfffexcuVK1qxZQ6tWrZg6daqv7ciRI/z73//mjTfeoHv37owYMYL169ezbt0631FYUaVO4+Li8Hg8eDweunTpwrPPPlvk66hXrx7p6ekMHTrUV3Bp7Nix3HbbbaSnp3PvvfeyY8eOsngLHa+s94GqVavSokULNmzYwNKlS2nbti1LlizhxIkTZGZmEh8fz5QpU3C73Xg8Hq677jo73oZLpkQutiuq1Gt2djZZWVnccsstAAwYMMB3BHUh3333HW63m6ioKGbMmMH69et9bd27d8cYQ1RUFA0aNCAqKooqVaoQGRlJRkYG4C11GhsbS0xMDOvXr2fDhp/Pa0tJSSE9PZ1XXnmlyG2fu0DBudcAsHTpUl81vy5duhRZB0TKZx9wu90sXryYxYsXM3LkSJYuXcrKlStp165d2b2wcqJELrY7v9RrXl7eJfc1cOBA3nrrLdatW8fo0aNLVa72XKnTb775hrVr13L33Xf7nr9+/XpGjx7NJ598UmzRpnN95n8N/tYyqizKYx9wu90sWbKE1NRU33VVFy1a5CuWdb78JWsvl3K1xVEil8tSrVq1qFOnDkuWeL94mz59uu/IrGbNmhw9erTI5x09epSGDRty6tQpZsyYUaptFlfqNDs7m759+/LBBx9Q2nMgOnXqREpKCgDz58/nyJEjBdobN/YuUlig94H27duzbNkyqlSpQkhICC6Xi3feeafYC0g0a9aMVatWAfiGei5XKmMrl63k5GSGDBnCsWPHCA8P5/333we8R1xDhgyhWrVqLF++vMAY6bhx42jfvj1NmzYlKiqq2A97UYordfr555/zww8/8Nhjj/nWLenMhtGjR9OvXz9mzpzJLbfcQsOGDalZs6avvXr1EodXKQVyHwgODqZx48a+Kwu53W4+/vjjIq8mBN7f3W9+8xtefvll2rdvX8av1D9+l7G9FCpj66UythXfiRMnqFq1KldccQXLly9n6NChBf4I/Hi2mMVVVwV+23bMQLnmuhvKfZsVVWnK2OqIXKQM7dixg969e3PmzBmuvPJK3n333QLte/Z4b8sikUvloUQuUoZuuOEG31XaRcqKvuwUEXE4JXIREYdTIhcRcTiNkYvYqGlTuyOQikCJXBxn2rRp3HnnnQGpfOfxeNi9ezddu9pTeTkkxHublpbGBx98wIQJE2yJozyczCz5nP6L+SBlBp0TbuPaaxpecL0rw2pesB3Kbh+YMGECb7/9NrGxsRc8OS00NJQcP2sZK5FLARsjAjunvNWmjQHtDwJbwtTj8ZCWlmZbIs/K8t7GxcURF1doerAtJk0v3RmxFzNmzJiA9gcw/f9mENmy1UUTeUmU1T4wceJE5s2bR/PmzQPab1E0Ri62CnQJ08TERJ577jni4+Np0aKF7/Tu3NxcHn74YaKiooiJiWHhwoWcPHmSUaNGMXPmzCIvJpCRkYHb7SY2NpbY2FiWLVsGwKJFi7jlllvo3bs3LVq04Pnnn2fGjBnEx8cTFRXFtm3bAPjyyy9p3749MTExdO7cmX379gHQtWtXX1ncxo1rMWlSMosWLaJbt26AN/E98sgjJCYmEh4eXuAofdy4cURERHDHHXfQr18/X5VFJ8vY+QPRt8Yx9PdP4Lo9nq4P3OP7va5ZvxZ3j9toe0dHej36AEeyjvCPrz5n1drVDHjyUdrddXOhfeCOXl154eVR3Nwtscz2gWHDhjF79mwA7r33Xh555BEApk6dyosvvsiQIUPYvn07PXr04I033mDMmDEFfletW7f2FVYLBCVysV0gS5gC5OXlkZqayvjx4xk7diwAf//73wFYt24dH3/8MQMGDODMmTO89NJL9OnTB4/HQ58+fQr0U79+fRYsWEB6ejozZ87kySef9LWtWbOGN998k3Xr1jF9+nS2bNlCamoqjz76KH/7298Ab52VFStWsHr1avr27curr74KwNy5c/F4PEydOpVrr21K586/KvQaNm3axL/+9S9SU1MZO3Ysp06dIi0tjU8//ZTVq1fzj3/8g8vt7Gh//O/7bQwZ8Bieb1KpXas2n837AoBHnhrMn0e+xKoFy2kdcSN/Gv8K9939K9pGx5A8YQor//WfYveB/8xZVGb7QEJCgu8PxK5du3xVMpcuXYrb7WbSpElce+21LFy4kBEjRpTNm5aPhlbEdiUtYdqrV68S9VdcOdknnngCgIiICJo2bcqWLVsu2M+pU6cYPnw4Ho+HqlWrFli/Xbt2NGzo/bf+uuuu48477wQgKiqKhQsXApCZmUmfPn3Ys2cPJ0+eLPAv9sGDB3nwwQdJSkqhZs1ahS73dvfddxMcHExwcDD169dn3759LF26lHvuuceXuLp3716i98MJmjVuSpvIaABio1z8sHMH2T9mk/1jNgkdOwHw654P8MDQASXq71e/7AGU3T7gdrsZP348GzZs4MYbb+TIkSPs2bOH5cuX2/I9hxK52O78Eqbn/6t8qf35W072jTfeoEGDBqxZs4YzZ84Qcu6byfNizl8W91xJXIAnnniCp59+mh49erBo0SLfWPHp06fp27cvo0aNokWL1hd8DflfR0UuiRt8Zf73syp5p/3cB668Eii7faBRo0YcOXKEf/7znyQkJHD48GFSUlIIDQ0tUBTtnPwlcSHwZXE1tCKXpUstYVqchIQE38yBLVu2sGPHDlq2bHnBvrKzs2nYsCFVqlRh+vTpnD59ulTbzM7OplGjRoC3it85zz//PNHR0b4LTpRUp06d+PLLL8nNzSUnJ4evvir/Qm/lqdZVtahdqzZL/+sdl/7oH5/gbu+tSBkaGsrRn0o30yPQ+0DHjh0ZP348CQkJuN1ukpKSLlgSNz09HYD09HS+//77UsV+MUrkctlKTk7md7/7HdHR0Xg8HkaNGgX8XMK0qC87izNs2DBOnz5NVFQUffr0Ydq0aQQHB3PrrbeyYcOGIr/oGjZsGMnJyXTo0IEtW7ZQo0aNUsU/ZswYevXqhdvtpl69er7Hk5KSmD9/Pi6Xi969XXz33ewS9deuXTt69OhBmzZtuO+++4iLi6NWrVqlislppr4xiZF/fpG2d3Rkzfp1/OGp5wB4qFd/ho98qsgvO4sT6H3A7XaTl5fH9ddfT2xsLIcPHy42kd9///0cPnwYl8vF22+/TYsWLS7xHSmaytiepTK24gQ5OTmEhoZy7NgxEhISmDx5MrGxsUWue7mVsQ3kPPKSKsk88suVytiKOMThw97bX/yiZOsPGjSIDRs2kJuby4ABA4pN4lK5KJGL2OjAAe9tSRP5Rx99VHbBiGNpjFxExOGUyKVCT2sTcaLSfiaVyCu5kJAQDh06pGQucpmwLItDhw4VOG/hYjRGXsmFhYWRmZnJgXODtVKu9u713uY7VyRgfjywP/CdXsSRk3nFtuUdCexJMCVxxdGSJ8PLSUhICGFhYSVePyCJ3BjzHtAN2G9ZVtGnqsllKSgoqFyqs0nRrr7ae5tvmnnAvD7md4Hv9CKemTmn2LbM55eUYyReYa/ElPs27RCooZVpQJcA9SVSadSrVzZJXCqXgCRyy7IWA4cD0ZdIZTJtmncR8Ue5fdlpjBlkjEkzxqRpPFbES4lcAqHcErllWZMty4qzLCvu6nMDgyIi4jdNPxQRcTglchERhwtIIjfGfAwsB1oaYzKNMb8JRL8iInJxAZlHbllWv0D0I1LZzJ1rdwRSEejMThEbVa9udwRSEWiMXMRGEyd6FxF/KJGL2CglxbuI+EOJXETE4ZTIRUQcTolcRMThlMhFRBxO0w9FbLRokd0RSEWgI3IREYdTIhexUVKSdxHxhxK5iI3mzPEuIv5QIhcRcTh92SkildaYMWMqxDZ1RC4i4nA6IhexUbVqdkcgFYESuYiN5s2zOwKpCDS0IiLicErkIjYaN867iPhDiVzERt98411E/KFELiLicErkIiIOp0QuIuJwmn4oYqO6de2OQCoCJXIRG336qd0RSEWgoRUREYdTIhex0ciR3kXEHxpaEbHR8uV2RyAVQUASuTGmC/AmUBWYYlnWK4Hot1IbU8uGbWaX/zZFxG9+D60YY6oCfwd+CdwI9DPG3OhvvyIiUjKBOCKPB/5nWdZ2AGPMJ8A9wIYA9F1IYmLhx3r3hmHD4Ngx6Nq1cPvAgd7l4EHo2bNw+9Ch3tu8H0M4OMdVqP2q+O1Uv34/pw7V4NC/ogq117ppK9WaHeLkvqs4/E3hv2G1EzYTEnaE3Mw6ZC1u+fNrWeG9HT8eXC74+mv405/ONmb8fP2vd7o9Rct6/+PLzV14ffnwQv1Pv3cwjWvtYuZ39/F22iOF2mf1foh61Q8zzfMA0zwPFGqf278X1YOOM3EipKQUavZd6T0pqfBlyapV+7mC37hxhU83r1v355kZI0cWHkoIC4MPP/T+/NRT4PEUbG/RAiZP9v48aBBs2fJz27HUVCJCNjGy/l8A+P3uv7Iv75oCz29TzcPTV78BwG93vUnW6doF2jtUX8HQem97+898hxNnQgq03xK6iEd+8T4AA3YkUz0+vkC7v/ve/v1Qvz7s3AkPPli4/ZlnoHt32LwZBg8u3P7ii9C5s/d9e+qpgm07N/yFrlHJNKu3iYyDEcxdN6DQ8+9xTaZRne/Zsq8NX2/oW6i9Z9u3qH/VLtbvjufbzfcWan+g/evUrn4Qzw43y7Z15cvEgu2zZkG9ejBtGrzzUeHPzge91lMt6AzJ6Q2Zs6leofb/e2Bd4RcthQQikTcCdua7nwm0P38lY8wgYBBAkyZNArDZwMp45W7vh6mIPz/PDKj784dpa+H2Fx+t+/OH6fvC7S8Pu4mbboJly+CFzBIG1Mz9889PrIKWwJfAniLWfXoDNAZmAgeLaP/991APmAZkFdH+h71QHZhYfDhRyVEcTBvAj3tvKfB4lStPEJXs/Uu4f/VgcvYW/NVfkZNFVPLTAOxd91uO7W1ToN1zch9Ryd5v+/Zs+D3H90b42tpd0674gIDq8fH8whVPq/EPAVDr13D0vPe3Xsd4Wv1lEAA174eThwq2X317PK3++CQAob+EqscLtjfoFk+rZ3/v3V7iBcNh15YjhR77JnkPP63YS87xIHZtaV2o/YxlOHYql+SR29i1pfBBwFd/38GOrw6xL6s6u7a0LNQOdYqNp/GNUfQbl+Tb99a9UHidh179m+8gYvOfCrc/Mv4dWraEL7+E7a8Xbh80cRqNG8PMmbDz7WJDASA4vHahxxqNu5nq1aHORAgu4iAi7BV34QcDyI4rBJUFY1mWfx0Y0wu4y7KsR8/efxCItyzrieKeExcXZ6Wlpfm1XSlfUcmFj6bK2roBzjoa+/uQf5f7Nh+fdFu5b1PsY4xZZVlW3PmPB2L6YSbe48FzwoDdAehXRERKIBCJfCVwgzGmuTHmSqAvMDsA/YpUeLP+cwOz/nOD3WGIw/k9Rm5ZVp4xZjjwL7zTD9+zLGu935GJVAKZh0LtDkEqgIDMI7csay4wNxB9iYhI6egUfRERh1MiFxFxONVaEbFR/VrH7A5BKgAlchEbPXDLZrtDkApAQysiIg6nRC5io4++bclH3xZ16r1IyWloRcRG+7Or2x2CVAA6IhcRcTglchERh1MiFxFxOI2Ri9gorG6O3SFIBaBELmKjnjcXcaUSkVLS0IqIiMMpkYvYKPmbG0ku4jqvIqWhoRURGx35KdjuEKQC0BG5iIjDKZGLiDicErmIiMNpjFzERs0bZNsdglQASuQiNrqn/Xa7Q5AKQEMrIiIOp0QuYqN357fm3fmt7Q5DHE5DKyI2+ik3yO4QpALQEbmIiMMpkYuIOJwSuYiIw2mMXMRGLRsdsTsEqQD8OiI3xvQyxqw3xpwxxsQFKiiRyuKXbTP4ZdsMu8MQh/N3aOU74D5gcZwkBGQAAAZZSURBVABiERGRS+DX0IplWRsBjDGBiUakkpk4tw0Aw7qusTkScTKNkYvY6GSe5huI/y6ayI0xXwPXFNH0B8uyvijphowxg4BBAE2aNClxgCIicmEXTeSWZXUOxIYsy5oMTAaIi4uzAtGniIhoHrmIiOP5NUZujLkX+BtwNfCVMcZjWdZdAYlMpBJo3fSg3SFIBeDvrJXPgM8CFItIpdO5zU67Q5AKQEMrIiIOp0QuYqPxs2MYPzvG7jDE4ZTIRUQcTolcRMThlMhFRBxOiVxExOFUa0XERrHX7bc7BKkAlMhFbJQQucvuEKQC0NCKiI1OnqrCyVP6GIp/tAeJ2GjivDZMnNfG7jDE4ZTIRUQcTolcRMThlMhFRBxOiVxExOE0/VDERh1a7rE7BKkAlMhFbNSh5V67Q5AKQEMrIjbKOR5EzvEgu8MQh1MiF7HRlAWtmbKgtd1hiMMpkYuIOJwSuYiIwymRi4g4nBK5iIjDafqhiI3cN6qMrfhPiVzERm2v14UlxH8aWhGx0ZGcYI7kBNsdhjicErmIjZL/fSPJ/77R7jDE4ZTIRUQczq9Ebox5zRizyRiz1hjzmTGmdqACExGRkvH3iHwB0NqyrGhgCzDS/5BERKQ0/ErklmXNtywr7+zdFUCY/yGJiEhpBHL64SPAzAD2J1Lh3R69w+4QpAK4aCI3xnwNXFNE0x8sy/ri7Dp/APKAGRfoZxAwCKBJkyaXFKxIRRPV7JDdIUgFcNFEbllW5wu1G2MGAN2A2y3Lsi7Qz2RgMkBcXFyx64lUJvuyqgPQoPYxmyMRJ/N31koX4Dmgh2VZ2hNFSunjxS35eHFLu8MQh/N31spbQE1ggTHGY4yZFICYRESkFPz6stOyrOsDFYiIiFwandkpIuJwSuQiIg6nMrYiNuoSm2F3CFIBKJGL2Cgi7IjdIUgFoKEVERtlHgwl82Co3WGIwymRi9ho1rIbmLXsBrvDEIdTIhcRcTglchERh1MiFxFxOCVyERGH0/RDERv1iN9mdwhSASiRi9go/Jof7Q5BKgANrYjYaPveq9i+9yq7wxCHUyIXsdHs1OuYnXqd3WGIwymRi4g4nBK5iIjDKZGLiDicErmIiMNp+qGIjXretNXuEKQCUCIXsVFYvRy7Q5AKQEMrIjbalFmHTZl17A5DHE5H5CI2+md6M0BXChL/6IhcRMThlMhFRBxOiVxExOGUyEVEHE5fdorYqF/CZrtDkApAiVzERg1qH7M7BKkA/BpaMcaMM8asNcZ4jDHzjTHXBiowkcpgXUZd1mXUtTsMcTh/x8hfsywr2rIsFzAHGBWAmEQqjW/WNuGbtU3sDkMczq9EbllW/utU1QAs/8IREZHSMpblX+41xvwZeAjIBm61LOtAMesNAgadvdsS0Lc8IiKl09SyrKvPf/CiidwY8zVwTRFNf7As64t8640EQizLGu1vpCIiUnJ+H5H7OjKmKfCVZVmtA9KhiIiUiL+zVm7Id7cHsMm/cEREpLT8OiI3xnyKd7z7DPADMMSyrF0Bik1EREogYEMrIiJiD9VaERFxOCVyqRSMMX2NMf81xvxkjNl/9udhxmuaMeakMSbHGHPYGLPAGBOR77ljjDEfFtGnZYy5vnxfiUhhSuRS4RljngHeBF7DO5W2ATAEuBm48uxqr1qWFQo0AnYBU20IVeSSqGiWVGjGmFrAS8BDlmV9mq9pNdD/7Dq+By3LOm6MSQH+rzzjFPGHjsilousIBANfXGxFAGNMDaAf8L+yDEokkJTIpaKrBxy0LCvv3APGmGXGmCxjzHFjTMLZh581xmQBR4FOwIM2xCpySZTIpaI7BNQzxviGES3LusmyrNpn2859BpLOPtYMOI73/Ihz8oCg/J0aY87dP1VGcYuUmBK5VHTLgRPAPSVZ2bKsHcBvgTeNMdXOPrwDb4LPrzlwGu8XoyK2UiKXCs2yrCxgLDDRGNPTGBNqjKlijHHhLb1c1HMWALv5uVrnP4GWxpgHjTFBxphfAC8Ds/IP2YjYRYlcKjzLsl4FngZ+D+wH9gHvAM8By4p52mvA740xwZZl7Qe6AoPPPv87vGWbh5Zx6CIlolP0RUQcTkfkIiIOp0QuIuJwSuQiIg6nRC4i4nBK5CIiDqdELiLicErkIiIOp0QuIuJwSuQiIg73/wE6OX5wS7DZ1QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(len(gru_first_degree_scores)):\n",
    "    plt.bar(i, gru_first_degree_scores[i], label=labels[i])\n",
    "plt.plot([-1,6], [0, 0], 'b', linestyle='--')\n",
    "plt.plot([4,4], [-6.5, 6.5], 'b', linestyle='--')\n",
    "ax = plt.gca()\n",
    "ax.set_title('GRU', y=-0.1)\n",
    "ax.axes.xaxis.set_visible(False)\n",
    "plt.ylim(-3, 3)\n",
    "plt.legend(ncol=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "amazing\n",
      "real 0.9530318379402161\n",
      "approx 0.9486384987831116\n",
      "very amazing\n",
      "real 3.1096980571746826\n",
      "approx 2.8145484924316406\n",
      "not amazing\n",
      "real -0.467763751745224\n",
      "approx -1.6512925624847412\n",
      "not not amazing\n",
      "real 0.3846995234489441\n",
      "approx 1.2782796621322632\n",
      "awful\n",
      "real -0.9194007515907288\n",
      "approx -0.9077060222625732\n",
      "very awful\n",
      "real -3.0138728618621826\n",
      "approx -2.71238112449646\n",
      "not awful\n",
      "real 0.7023808360099792\n",
      "approx 1.819990634918213\n",
      "not not awful\n",
      "real -0.37374943494796753\n",
      "approx -1.3033583164215088\n"
     ]
    }
   ],
   "source": [
    "best_model.eval()\n",
    "awful = word2id['awful']\n",
    "amazing = word2id['amazing']\n",
    "not_ = word2id['not']\n",
    "very = word2id['very']\n",
    "lstm_first_degree_scores = []\n",
    "sents = [[amazing], [very, amazing], [not_, amazing], \n",
    "         [not_, not_, amazing], [awful], [very, awful], [not_, awful], [not_, not_, awful]]\n",
    "labels = ['amazing', 'very amazing', 'not amazing', \n",
    "          'not not amazing', 'awful', 'very awful', 'not awful', 'not not awful']\n",
    "lengths = [1, 2, 2, 3, 1, 2, 2, 3]\n",
    "for i, sent in enumerate(sents):\n",
    "    print(' '.join([id2word[w] for w in sent]))\n",
    "    inputs = torch.LongTensor([sent]).to(device)\n",
    "    length = torch.LongTensor([lengths[i]])\n",
    "    probs, scores = model(inputs, length.to(device))\n",
    "    print('real', scores.item())\n",
    "    scores = lstm_phrase_polarity_new(sent, model, 300)\n",
    "    print('approx', scores)\n",
    "    lstm_first_degree_scores.append(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f7d85e87a50>"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD+CAYAAAAnIY4eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU1fnH8c9JjGEVkE1kz09ZDIEQw6YmBEVRBFREQFCgVNlEKy5FLGVtqa2hIlpF3IiICkKtgFAByypYCBBBwlYgmrDvECFI4Pz+GIiBSSBhhtzcyff9et3XZObOnPvM5MyTk3Pvfa6x1iIiIu4V5HQAIiLiGyVyERGXUyIXEXE5JXIREZdTIhcRcTklchERl/M5kRtjihljVhpjvjfGbDDGjPRHYCIikjfG1+PIjTEGKGmtTTfGhADLgN9Za7/zR4AiInJp1/jagPX8JUg/dzfk3KKzjERECohf5siNMcHGmCRgHzDfWvtff7QrIiKX5/OIHMBaewaINMaUBb4wxjSw1v6Q/TnGmD5AH4CSJUveWq9ePX9sWsTVNm/23Nat62wc4g6rV68+YK2tePHjPs+RezVozHDgZ2ttfG7PiY6OtomJiX7drogbxcV5bhctcjIKcQtjzGprbfTFj/vjqJWK50biGGOKA62BTb62KyIieeOPqZUqQIIxJhjPH4Zp1trZfmhXRETywB9HrawDGvshFhERuQJ+2dnpD6dPnyYtLY2MjAynQxEpMK++6rnduNHZOHJTrFgxqlWrRkhIiNOhyCUUmkSelpZG6dKlqVWrFp5zjIq4XWsLfps36h8r+ZW1loMHD5KWlkbt2rWdDkcuodDUWsnIyKB8+fJK4lKknD7tWQojYwzly5fXf8kuUGgSOaAkLkXO9u2epbDSd9IdClUiL0puu+02p0MQP1q6dCnh4eFERkZy8uTJXJ8XFxeHzqEQfys0c+QXq/XSV35tL+WV+/3anq+WL1/udAiF3j/6/cev7T014U6/tpfdlClTeOGFF/jNb35z1bYhkhuNyC/y4IMPcuuttxIeHs7EiRMBKFWqFIMHD+bWW2+ldevWrFy5kri4OMLCwpg5cyYAKSkpxMTEEBUVRVRUVFaiHjZsGJGRkURGRlK1atWsL3qpUqUAWLRoEXFxcXTq1Il69erRvXt3zp9tO+ebZdSL7cgdD/bmmT/+jXY9ninoj6NIurgPTJs2jeeeew6A119/nbCwMAC2bdvGHXfcwXvvvce0adMYNWoU3bt3Z9GiRbRr1y6rvYEDBzJp0iQn3ooUEUrkF/nggw9YvXo1iYmJjB8/noMHD/Lzzz8TFxfH6tWrKV26NEOHDmX+/Pl88cUXDBs2DIBKlSoxf/581qxZw9SpU3nmGU/SHTVqFElJSSxevJjy5cszcOBAr22uXbuWcePGkZyczPbt2/n222/JyDhF38F/Zu7Hb7DsXx+w/+DhAv0cirKL+8Dtt9/O0qVLAc8USvny5dm5cyfLli0jJiaGJ554gg4dOvDqq68yZcoUh6OXoqjQTq04Zfz48XzxxRcApKamsnXrVq699lruvfdeACIiIggNDSUkJISIiAhSUlIAz3HwAwcOJCkpieDgYLZs2ZLVprWW7t27M2jQIG699VavbTZt2pRq1aoBEBkZSUpKCqVOXENYzarUrlEVgEcfvJeJH8+4mm9dzrm4D6SmppKens7x48dJTU2lW7duLFmyhKVLl9KxY0eftlXRq/yRSP4pkWezaNEiFixYwIoVKyhRogRxcXFkZGQQEhKStfc+KCiI0NDQrJ8zMzMBeO2116hcuTLff/89Z8+epVixYlntjhgxgmrVquU6f3q+PYDg4GAyMzOxBF+ttymXkFsfaNGiBR9++CF169YlJiaGDz74gBUrVjB27FivNq655hrOnj2bdf9Sh+9df/1VeRtSxGhqJZujR49Srlw5SpQowaZNm/juu7xf5Ojo0aNUqVKFoKAgJk+ezJkzZwCYPXs28+fPZ/z48fmKpd7/1WL7jztJSd0FwNSZ8/L1erkyufWB2NhY4uPjiY2NpXHjxixcuJDQ0FDKlCnj1UbNmjVJTk7m1KlTHD16lG+++SbX7f3yi2cR8YVG5Nnce++9TJgwgYYNG1K3bl2aN2+e59cOGDCAhx9+mM8//5xWrVpRsmRJAMaOHcuuXbto2rQpAB06dGDUqFGXba948WK8NeYl7u0+kArXl6VpZPiVvSnJl9z6QExMDKmpqcTGxhIcHEz16tXJraZ+9erV6dy5Mw0bNuTmm2+mcePcz5jdscNzq3rk4gu/1yPPi5zqkW/cuJH69esXeCyF1q61pP98glIlS2Ct5amXX+Hm2tUZ1Oexq7dNnaJf4NxwYQl9NwuP3OqRa0ReiL075Z8kfD6bX06fpnGDevR9/GGnQxKRQkiJvBAb1OexqzsCF5GAoJ2dIiIupxG5iIMqV3Y6AgkESuQiDipb1ukIJBBoakXEQRkZnkXEF0rkAazt409z5Ohxp8OQS/jxR8+SF5s2bSIyMpLGjRuzbdu2XJ/Xq1cvpk+f7qcIxQ0K79TKCO8z5nxr76h/28vGWou1lqCgwvV3cc7kN5wOwSdju7S7/JPy4fmps/3aXnYF0Qf+9a9/8cADDzBy5Mirtg1xp8KVeRw0ePBg3nrrraz7I0aMyKqj8eqrr9KkSRMaNmzI8OHDAU/Z2vr16zNgwACioqIYPXo0gwYNynr9u+++m1X6NLv+/fsTHR1NeHh4VlsAtWrV4uWXX6ZFixZER0ezZv1G2nQbwP/d1oEJH3lGV+k/n+Cuzn2JatONiLs68+XXiwCY8NF0Iu/uSuTdXandvB2tOvXxtNnsfg4cOkxK6i7qt+zIky+OJrxVJ+55dAAnT3r+n1+VtIGGrTvTon1PXnzxRRo0aODHT9VdnOgD48d72lq5cmVWAa4vv/yS4sWL88svv5CRkUFYWBhz5sxh3LhxvPfee7Rq1YqUlJQLflfx8fGMGDHC75+JuIMS+Tldu3Zl6tSpWfenTZvGI488wrx589i6dSsrV64kKSmJ1atXs2TJEgA2b95Mjx49WLt2LS+88AIzZ87k9LkLMH744Yc5Fsn685//TGJiIuvWrWPx4sWsW7cua1316tVZsWIFMTEx9Bo0nOkTX+W7WQkMi38bgGKh1/LF+2NZ8/UnLPz8HZ4f9XestfTr0Ymk+Z+xas5kqlWpxHN9unttd+uOVJ7q2ZkNC6dT9rrSzJjjqf/xm+dGMOEvf2DFrASCg4t2oS4n+sCqVYvZvHkdUVFRrF3rueD20qVLadCgAatWreK///0vzZo1o23btvTr149BgwaxcOHCAvg0xE0K79RKAWvcuDH79u1j165d7N+/n3LlylGjRg3Gjx/PvHnzsuplpKens3XrVmrUqEHNmjWzanGULFmSO++8k9mzZ1O/fn1Onz5NRESE13amTZvGxIkTyczMZPfu3SQnJ9OwYUPAU4cFPKVy0/f9SOlSJSldqiTFQkM5cvQ4JUsU4+VX3mTJf9cQZILYuWc/e/cf5IZKFQD43bB47ry9Ce3vaem13drVbySygec88Fsb1icldTdHjh7nePrP3NakEQDdunVj9uyrN/1Q2DnRB9LSdvO//yVzzTUNuemmm9i4cSMrV67kueeeY8mSJZw5c4aYmJgC/RzEfZTIs+nUqRPTp09nz549dO3aFfDMfQ4ZMoS+ffte8NyUlJSswljnPfHEE4wZM4Z69erlOBLbsWMH8fHxrFq1inLlytGrV68LSpxmL48bem1I1uNBQYbMM5lM+edc9h88zOq5UwgJCaFWs/vJOOUpnTdp6kx+TNvNm38enON7Cw29Nuvn4OAgTmZk4kSdncKuoPtA9+69KF7c0wdiYmKYO3cuISEhtG7dml69enHmzBni4+O92slPqVwJfJpayaZr16589tlnTJ8+nU6dOgHQpk0bPvjgA9LT0wHYuXMn+/bty/H1zZo1IzU1lU8++YRHH33Ua/2xY8coWbIkZcqUYe/evcydOzdf8R09nk6lCtcTEhLCwm9X8WPabgBWr0sm/p3JfPzGn/K1s61c2esoXaok3632TO989tln+YonEBV0H1iwYC7Fi3vWxcbGMm7cOFq0aEHFihU5ePAgmzZtIjzcu/Jl5cqV2bdvHwcPHuTUqVNF+j8p0Yj8AuHh4Rw/fpyqVatSpUoVAO655x42btxIixYtAM+1Nj/++ONc55M7d+5MUlIS5cqV81rXqFEjGjduTHh4OGFhYdx+++35iq97x/to3/NZou/rTmR4XerdVAuANz+cyqEjR2n1iGcnZ3SjW3gvflie2nw/fhhP/n40JYsXJ+6e+3Osr12UFHQfaN78dk6d8qxr1qwZe/fuJTY2FoCGDRtSqVKlrIuaZBcSEsKwYcNo1qwZtWvXzrWkrhQNPpexNcZUBz4CbgDOAhOtta9f6jWBXMa2Xbt2DBo0iLvuusu3hnat9U9Al3G+VC7AKx99ze7du3n99Uv++uQy8tMHVMZW8iO3Mrb+mFrJBJ631tYHmgNPGWNu8UO7rnLkyBHq1KlD8eLFfU/iBeirBUuJvLsrDe58hKVLlzJ06FCnQ3Itt/YBcT+fp1astbuB3ed+Pm6M2QhUBZJ9bdtNypYte8EFl92iywNt6PJAG88dXVjCJ27tA+J+ft3ZaYypBTQG/uvPdkVEJHd+S+TGmFLADOBZa+2xHNb3McYkGmMS9+/f76/NiogUeX5J5MaYEDxJfIq19p85PcdaO9FaG22tja5YsaI/NivielWrehYRX/g8R248x0a9D2y01v7d95BEio5SpZyOQAKBP0bktwOPA3caY5LOLW390G6hN2nSJHbt2uV0GAx79W0WLNFuCSf42gfS0z1LdqdOnaJ169ZERkZeUPslp20PHDjwirctgcMfR60sA7zPWPBRRIJ3jQpfrO+53q/tgeeL1KBBA2688Ua/t50fo17s7+j2r5a0l5b6tb1qr/i/ZomvfWDnTs9t9uPI165dy+nTp0lKSvJDhFIU6BT9c86XJH3yyScJDw/nnnvu4eTJkwAkJSXRvHlzGjZsyEMPPcThw4eZPn06iYmJdO/encjIyKznnvfuu+/SpEkTGjVqxMMPP8yJEycAT9H//v3706pVK8LCwli8eDG9e/emfv369OrVK+v1/V8aQ/R93Qlv1Ynh56ofJn6fnFWuNuKuzpiqUZ42nx3O9NkLAE/p2uHxb2eVut30vx0A7D94mLu79ieqTTf6/v5P1GzalgOHDl/Vz9RtCqIPnDlzhrCwMKy1HDlyhPr1g1i1ylNJMSYmhpUrV/LYY4+RlJREZGQk27Zto1atWhw4cACAxMRE4uLiCvRzkcJPp+hns3XrVj799FPeffddOnfuzIwZM3jsscfo0aMHb7zxBi1btmTYsGGMHDmScePG8eabbxIfH090tNeJVnTs2JEnn3wSgKFDh/L+++/z9NNPA3D48GH+85//MHPmTNq3b8+3337Le++9R5MmTbK+wH8e/BTXlyvDmTNnuKtLP9YlbyG60S0kzffUQ3lx9GvcG9cix/dR4fpyrPn6E96aNI34CZN5L34YI//+Dnfe3oQhT/fm3wu/ZeKUHPdJF3kF0Qfq1KlDcnIyO3bsIDz8VlavXsojjzQjLS2Npk2b8t577xEfHx9w9VP8/R9WXlyN/8IKI43Is6lduzaRkZEA3HrrraSkpHD06FGOHDlCy5ae0rA9e/bMqkV9KT/88AMxMTFEREQwZcoUNmzYkLWuffv2GGOIiIigcuXKREREEBQURHh4OCkpKQBMmzWfqDbdaNzmUTZs3kby1h1Zr582cx5r1m/ilZefyXHbHe+70/MeGtYnJdUzf7tsZRJdz534c2+r2ylX9rp8fjpFQ0H0gZiYGJYsWcKSJUvo02cIq1cvY9WqVTRp0uTqvTEJaErk2ZwvIwsQHBxMZmbmFbfVq1cv3nzzTdavX8/w4cNzL1ebbZtBQUFkZmZ6Sp2+8xHfTJ3AugXTuP+uGDIyPJWVNmzexvCxE/js7VdyLdoUGhry63s4cwZAJWvzqCD6QExMDEuXLmXlypW0bNmWY8eOsGjRoqxiWRfLXrJW5WolJ0rkl1GmTBnKlSvH0qWefwsnT56cNTIrXbo0x4/nfHHj48ePU6VKFU6fPs2UKVPytc1jx45RsnhxylxXir37DzJ34bcAHD12nK4DhvDR66OpWN67st6l3NE0kmmz5gMwb/EKDh/xOmdLcuHvPtCsWTOWL19OUFAQN91UjCZNInnnnXdyvYBErVq1WL16NQAzZszw51uTAKE58jxISEigX79+nDhxgrCwMD788EPAM+Lq168fxYsXZ8WKFRQ/X1gaGD16NM2aNaNmzZpERETk+mXPSaNGjWjcoB7hrToRVqMat5+7gs+//r2IH9N28+SLo7Oee37O/HKGP9eXRwcMYerMebRsHkWVyhUofdFFESR3/uwDoaGhVK9enebNm1OiBLRqFcPnn3+a49WEAIYPH85vf/tbxowZQ7Nmza7+mxXX8bmM7ZUI5DK2fuPnMranTv1CcHAQ11xzDSsSv6f/kL94/xFQ0awCd+zcP0bXFeJdFv76bmpnp+9yK2OrEXkR8dPOPXTuN5izZ89y7bUhvPvqH50OSYDdnos8FepELoWfEnkRcXNYDdbO+9TpMETkKtDOThERl1MiFxFxOSVyERGX0xy5iINq1nQ6AgkEGpH7wJ9lbJOSkpgzZ45f2vJFYmIizzyT86n/4s3XPlCsmGeBq9cHxo8fT/369enevfsln1dKxdFdq9COyDfW8+8x5fU3bfRre+DfMrZJSUkkJibStq2zpdyjo6NzLADlhBEjRhTq9sD3PnDkiOe2bNmr1wfeeust5s6dS+3atf3arhQeGpGf4+8SpnFxcQwePJimTZtSp06drNO7MzIy+M1vfkNERASNGzdm4cKF/PLLLwwbNoypU6fmeDGBlNRdxDzUm6g23Yhq043lq74HYNHyRFo+/ASd+w6mzh0P8tKY8Uz55xya3v84EXd1ZltKKgCz5i2mWbseNL7nUVp36cfe/QcBaPv401llccvUiyUhIYFFixbRrl07wJP4evfuTVxcHGFhYYwfPz4rptGjR1OvXj3uvvtuHn30UeLj46/Cb6VgOdEHbrutMV9+mYc+kJJCTEwMUVFRREVFsXz5cgAGDBjAzJkzAXjooYfo3bs3AO+//z5Dhw6lX79+bN++nQ4dOvDaa68xYsSIC35XDRo0yCrUJu6lRJ7N1q1beeqpp9iwYQNly5bNqmvRo0cP/vrXv7Ju3ToiIiIYOXIknTp1Ijo6milTppCUlHTBqdnnZWZmsnLlSsaNG8fIkSMB+Mc//gHA+vXr+fTTT+nZsydnz55l1KhRdOnShaSkJLp06XJBO5UqlGP+p2+z5utPmPr2Kzwz7G9Z675P3sLro15g/TfTmDzjK7Zs/5GVX03miUcf5I0PPGdu3tG0Md/NSmDtvE/p+kAb/vZWAgBzJr9B0vzPeH/sMGpWu4EHH3zQ6z1s2rSJr7/+mpUrVzJy5EhOnz5NYmIiM2bMYO3atfzzn//k4rN03ayg+8DYsZ8yZEge+kClSsyfP581a9YwderUrOmv2NjYrD8QO3fuJDk5GYBly5YRExPDhAkTuPHGG1m4cCGDBg26Oh+aOK7QTq04Ia8lTB955JE8tdexY8cL2gLPF+x8XfJ69epRs2ZNtmzZcsl2Tp/OZOAf/kpS8haCg4LYsv2nrHVNGoVTpbLnYtb/V7Ma97T01CiPqHcTC5d7Emza7r106f8Su/cd4JdfTlO7xq/TAAcOHebxZ/7ItAl/pUyZMl7bvv/++wkNDSU0NJRKlSqxd+9eli1bxgMPPJCVuNq3b5+nz8MNCroPhIXV48Yb89IHTjNw4ECSkpIIDg7Oen5MTAzjxo0jOTmZW265hcOHD7N7925WrFhxwX9QEtiUyLO5uITpxf8qX2l72cuhXkltm9fenULliuX5fv5nnD17lmJhv15QIvTakKyfg4KCCL322qyfz2/z6T/+jef6PEaHe1qyaHkiI/7+DgBnzpyha/8hDBvUhwb1brrke8j+PgK5JG6h7QOvvUblypX5/vvvPX3g3B7SqlWrcvjwYf79738TGxvLoUOHmDZtGqVKlaJ06dJe7WQviQsqixsoNLVyGVdawjQ3sbGxWSVNt2zZwk8//UTdunUv2dbRY+lUqVSBoKAgJs/4ijPnaozn1dFj6VS9wTNqT/h8VtbjL40ZT8P6N2ddcCKv7rjjDmbNmkVGRgbp6el89dVX+Xq921zNPrBjxxZ2785DHzh6lCpVqnj6wOTJF/SBFi1aMG7cOGJjY4mJiSE+Pv6SJXHXrFkDwJo1a9ixY0eOzxN3USLPg4SEBF588UUaNmxIUlISw4YNA34tYZrTjq7cDBgwgDNnzhAREUGXLl2YNGkSoaGhtGrViuTk5Bx3dA3o2ZmE6bNo3q4HW7b/RMkS3nOxlzLi+b480ncwMQ/1psL1ZbMej58wmXlLvsva4Xl+p9nlNGnShA4dOtCoUSM6duxIdHR0jtMygeRq9YGXXurChx/moQ8MGEBCQgLNmzdny5YtlMxWgjgmJobMzExuuukmoqKiOHToUK6J/OGHH+bQoUNERkby9ttvU6dOnSv8RKQwURnbwsrPZWzzJB9lbNPT0ylVqhQnTpwgNjaWiRMnEhUVdRWD83byhx8KdHsAxRs0KPBtOk1lbAsPlbEVv+rTpw/JyclkZGTQs2fPAk/igeLQIc/t9dc7G4e4mxK5XJFPPvnE6RACwv79nlslcvGF5shFRFyuUCXyQD6sTcSN9J10h0KTyIsVK8bBgwfVcUQKCWstBw8ezDpmXQqvQjNHXq1aNdLS0th/ftKwqDuyr+C3edT/hcWuptN79xb4NkOCg/3a3p49ntts5+gUKsWKFaNatWpOhyGX4ZdEboz5AGgH7LPWXtHxWSEhIarOlt2I5g5s82jBb9MHGx/qWODb9HcVzYqe87SoUMGvzUoR46+plUnAvX5qS6TIqFBBSVx855dEbq1dAhzyR1siRcmkSZ5FxBcFtrPTGNPHGJNojEnUPLiIhxK5+EOBJXJr7URrbbS1Nrri+YlBERHxWaE5asVptV4q+Ap+Ka/cX+DbFJHAU2iOIxcRkSvjl0RujPkUWAHUNcakGWN+6492RUTk8vwytWKtfdQf7YgUNXPmOB2BBALNkYs4qEQJpyOQQKA5chEHvfWWZxHxhRK5iIOmTfMsIr5QIhcRcTklchERl1MiFxFxOSVyERGX0+GHIg5atMjpCCQQaEQuIuJySuQiDoqP9ywivlAiF3HQ7NmeRcQXSuQiIi6nRC4i4nJK5CIiLqfDD0UcVLy40xFIIFAiF3HQ3LlORyCBQFMrIiIup0Qu4qDRoz2LiC+UyEUc9M03nkXEF0rkIiIup0QuIuJySuQiIi6nww9FHFS+vNMRSCBQIhdx0IwZTkcggUBTKyIiLqdELuKgIUM8i4gvNLUi4qAVK5yOQAKBX0bkxph7jTGbjTH/M8a85I82RUQkb3xO5MaYYOAfwH3ALcCjxphbfG1XRETyxh9TK02B/1lrtwMYYz4DHgCS/dC2l7g478c6d4YBA+DECWjb1nt9r16e5cAB6NTJe33//p7bzGPFODA70mv9dU23U+KmfZw+WJKDX0d4rS9z21aK1zrIL3uv49A33n/DysZupli1w2SklePIkrq/vpfvPLfjxkFkJCxYAH/607mVKb9e/+udds9St8L/mLX5XsauGOjV/uSH+lK9zE6m/tCRtxN7e62f3rkHFUocYlJSNyYldfNaP6f7I5QIOclbb8G0aV6rs670Hh/vfVmy4sV/reA3erT36ebly/96ZMaQId5TCdWqwccfe35+9llISrpwfZ06MHGi5+c+fWDLll/XnfgpgXrFNjGk0l8A+P2uv7I384YLXt+oeBLPVXwNgN/tfJ0jZ8pesL55ie/oX+FtT/tp73DqbLEL1rcstYje138IQM+fEigRd2F8vva9ffugUiVITYXHH/de//zz0L49bN4Mfft6rx86FFq39nxuzz7rvX7MGLjtNli+HF5+2Xt9jn0vm3fegbp1YdYsGDvWe/3kyVC9OkydCm+/7b1++nSoUAEmTYJ3PvH+7nz0yAaKh5wlYU0VZm+q4LX+827rvRsVL/5I5FWB1Gz304BmFz/JGNMH6ANQo0YNP2zWv1Jeud/zZcrhz8/zPcv/+mXa6r1+6BPlf/0y7fBeP2bAbb9+mdLyGFCtmF9/fno11AVmAbtzeO5zyVAdmAocyGH973dABWAScCSH9X/YAyWAt3IPJyIhggOJPTm2p+UFjwdde4qIBM9fwn1r+5K+58Jf/TXpR4hIeA6APet/x4k9jS5Yn/TLXiISPHv7dif/npN76mWta3JDk9wDAko0bcr1kU2pP64HAGUeg+MXfb4VWjSl/l/6AFD6Yfjl4IXrK97VlPp/fAaAUvdB8MkL11du15T6L/zes724S4bDzi2HvR77JmE3P3+3h/STIezc0sBr/VlrOHE6g4Qh29i5xXsQ8NU/fuKnrw6y90gJdm6p67UeyuUaT2ryej79YwIrKmwi5UA9UpN7ej3no99P5JtyO9iytxGpyV291n/w7JtUum4nG3Y1JXXzQ17rJw4YS9kSB0j6KYbUbW2pfot3sj4vNKys12NVR99OiRJQ7i0IzWEQUe2VGO8H/WjEiBFXtf2C2qax1vrWgDGPAG2stU+cu/840NRa+3Rur4mOjraJiYk+bVcKVkRC7l/Qq2V9T3eNxv7R7z8Fvs2nJtyZ67qxXdoVYCQez09115Wk3ZbIjTGrrbXRFz/uj52daXjGg+dVA3b5oV0REckDfyTyVcDNxpjaxphrga7ATD+0KxLwpn97M9O/vdnpMMTlfJ4jt9ZmGmMGAl8DwcAH1toNPkcmUgSkHSzldAgSAPxyQpC1dg4wxx9tiYhI/ugUfRERl1MiFxFxOdVaEXFQpTInnA5BAoASuYiDurXc7HQIEgA0tSIi4nJK5CIO+mRxXT5ZnNOp9yJ5p6kVEQftO1rC6dWQVEAAAAbZSURBVBAkAGhELiLickrkIiIup0QuIuJymiMXcVC18ulOhyABQIlcxEGdbs/hSiUi+aSpFRERl1MiF3FQwje3kJDDdV5F8kNTKyIOOvxzqNMhSADQiFxExOWUyEVEXE6JXETE5TRHLuKg2pWPOh2CBAAlchEHPdBsu9MhSADQ1IqIiMspkYs46N15DXh3XgOnwxCX09SKiIN+zghxOgQJABqRi4i4nBK5iIjLKZGLiLic5shFHFS36mGnQ5AA4FMiN8Y8AowA6gNNrbWJ/ghKxI2emnBn/l+T9VOYP0ORIsbXqZUfgI7AEj/EIiIiV8CnEbm1diOAMcY/0YgUMffd57mdO9fZOMTdNEcu4qCTJ52OQALBZRO5MWYBcEMOq/5grf0yrxsyxvQB+gDUqFEjzwGKiMilXTaRW2tb+2ND1tqJwESA6Oho6482RUREx5GLiLier4cfPgS8AVQEvjLGJFlr2/glMpEioF07pyOQQODrUStfAF/4KRaRIueFF5yOQAKBplZERFxOiVzEQXFxnkXEF0rkIiIup0QuIuJySuQiIi6nRC4i4nKqtSLioM6dnY6gaBsxYoTTIfiFErmIgwYMcDoCCQSaWhFx0IkTnkXEFxqRiziobVvP7aJFjoYhLqcRuYiIyymRi4i4nBK5iIjLKZGLiLicdnaKOKhXL6cjkECgRC7iICVy8QdNrYg46MABzyLiC43IRRzUqZPnVseRiy+UyCVP1vdc73QIIpILTa2IiLicErmIiMspkYuIuJzmyEUc1L+/0xFIIFAiF3FQly5ORyCBQFMrIg5KTfUsIr7QiFzEQY8/7rnVceTiC43IRURczqdEbox51RizyRizzhjzhTGmrL8CExGRvPF1RD4faGCtbQhsAYb4HpKIiOSHT4ncWjvPWpt57u53QDXfQxIRkfzw587O3sBUP7YnEvCef97pCCQQXDaRG2MWADfksOoP1tovzz3nD0AmMOUS7fQB+gDUqFHjioIVCTTt2zsdgQSCyyZya23rS603xvQE2gF3WWvtJdqZCEwEiI6OzvV5IkXJ5s2e27p1nY1D3M2nqRVjzL3AYKCltfaEf0ISKTr69vXc6jhy8YWvR628CZQG5htjkowxE/wQk4iI5INPI3Jr7U3+CkRERK6MzuwUEXE5JXIREZdT0SwRBw0d6nQEEgiUyEUc1PqSB/eK5I2mVkQclJTkWUR8oRG5iIOefdZzq+PIxRcakYuIuJwSuYiIyymRi4i4nBK5iIjLaWeniIPGjHE6AgkESuQiDrrtNqcjkECgqRURBy1f7llEfKERuYiDXn7Zc6vjyMUXGpGLiLicErmIiMspkYuIuJwSuYiIy2lnp4iDxo1zOgIJBErkIg6KjHQ6AgkEmloRcdCCBZ5FxBcakYs46E9/8txejSsFPT91tv8blUJJI3IREZdTIhcRcTklchERl1MiFxFxOe3sFHHQO+84HYEEAiVyEQfVret0BBIIfJpaMcaMNsasM8YkGWPmGWNu9FdgIkXBrFmeRcQXvs6Rv2qtbWitjQRmA8P8EJNIkTF2rGcR8YVPidxaeyzb3ZKA9S0cERHJL5/nyI0xfwZ6AEeBVpd4Xh+gz7m76caYzb5uWyRQGON0BOISNXN60Fh76UG0MWYBcEMOq/5grf0y2/OGAMWstcN9iVJERPLnsok8zw0ZUxP4ylrbwC8NiohInvh61MrN2e52ADb5Fo6IiOSXTyNyY8wMoC5wFvgR6Get3emn2EREJA/8NrUiIiLOUK0VERGXUyKXgGSMSTHGeF2uwRjzsjFmhzEm3RiTZoyZeu7xDeceSzfGnDHGZGS7/7Ixppcxxhpj/n5Rew+ee3xSAb01ES9K5FJkGGN6Ao8Dra21pYBo4BsAa224tbbUuceXAgPP37fWjjnXxDagizEm+/kXPYAtBfcuRLwpkUtR0gT42lq7DcBau8daOzEfr98DrAfaABhjrgduA2b6O1CR/FAil6LkO6CHMeZFY0y0MSb4Ctr4CM8oHKAr8CVwyl8BilwJJXIpMqy1HwNP4xlRLwb2GWNeymczXwBxxpgyeBL6R/6NUiT/lMilSLHWTrHWtgbKAv2AUcaYNvl4/UngK2AoUMFa++3ViVQk75TIpUiy1p621n4OrAPyW1biI+B5YLLfAxO5ArpCkASyEGNMsWz3HwN2A0uAn/FMsYQD/81nu4uBu4G1/ghSxFdK5BLI5lx0fyNwGPgYCMZTVqK/tXZZfhq1ntOhv/FLhCJ+oFP0RURcTnPkIiIup0QuIuJySuQiIi6nRC4i4nJK5CIiLqdELiLickrkIiIup0QuIuJySuQiIi73/4eeNNm6FOwAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(len(lstm_first_degree_scores)):\n",
    "    plt.bar(i, lstm_first_degree_scores[i], label=labels[i])\n",
    "plt.plot([-1,6], [0, 0], 'b', linestyle='--')\n",
    "plt.plot([4,4], [-6.5, 6.5], 'b', linestyle='--')\n",
    "ax = plt.gca()\n",
    "ax.set_title('LSTM', y=-0.1)\n",
    "ax.axes.xaxis.set_visible(False)\n",
    "plt.ylim(-3, 3)\n",
    "plt.legend(ncol=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2kAAAD6CAYAAAA7kg/WAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3QU9RrG8SeNBBISCCGVIr0pvUsRkaICFsCCNFFUEBEVBBv2ilexK4jSbSACgqJXUMECKBCQ0KSGXgIJSQiQzd4/cjcQsgkhO9mZ7H4/53hOsjvzm3dmwuO+O83Hlp5oFwAAAADAEnzNLgAAAAAAcA5NGgAAAABYCE0aAAAAAFgITRoAAAAAWAhNGgAAAABYCE0aAAAAAFiIvxkLTU5JMWOxcLP0dB9JUpkyPOXBG/hICg0NNbsMl5BN3oFs8j7kE0oK8sm7FJRNpjRp8A4EjHfx9eXAPEoGssn7kE8oKcgn71JQNpFaAAAAAGAhNGkoNqNGx2jU6BizywCAXMgmAFZFPsHBx5ae6PbjqheeV512yqaTaZnKtHGI15Mk7g2QJFWudNbkSszn7+ejssH+Ci7tZ3YpxcbP11chISFml+ESrvnwDuHR9SVJSQcTTK4E7kI+oaQgn7xLQdlkiWvSTqScVcXwUgrw95GPj4/Z5cAgRw4GSZIqRWWYXIm57Ha7zmbadSTpjEc3aQAAADCGJZo0u6RSAZx5Cc/k4+OjUgE+4jgxAAAACoPOyABjH3/R7BK8zo///VUdr+mtGbPnOn3/2l793VwRAAAAYAxLHEk7X+sZNpfm/3OA+08ne/WlJ9y+TLNsv6tToad1PPVh+3mv1ZiyzJA6Fn3/kyZ/MEG1a1U3ZDwAAADAKizXpBWnAwcP697hj+psZqYa1KutG3t114eTZ8hutyvp+AkNGXSbZn32tQIDA/X5zPd18NCRXNO//up4TZ3xpeZ+vVhZ9iwdPHhYf//5va7t1V/fLZipnjcNVovmjfTLr39o8KBbNaBfb323ZKle+88HatSwvrZs3a7vFsw0ezNY1oX7p0yZMrrtlht08NBhPfvCG/rlv3M1/IHH1L9fb32/ZJn+2bhFLzwzRs+++Ka+WzBTe/bs06uvv6f33n7J7FUBAAAAisyrTnesEF5OX3/5sb5bMFMnU9O0fedu2e12zZr2rrp07qC/127QN3M+UUxMpNZv2JR3+h27NXjALVo4b6patWiicWNG5FnGjb26a/GCmfr8y/mSpLfenaJv503To48M15Ejx9y9yiXKhds7IiJcq1av1arV6xQVWVEnU9N0+OgxtW3TXJ07tdOk919V61bNzC4bQAnTqOEpNWp4yuwyACAP8gkOXnUkLel4skaPfU7JySe1J3Gfqlerqnp1a0mSoqMqqkKF8v//OVInklPyTH/w4GHVqF5VCxf9qFMZGerbu0eeZdSrW1MBAQHy9cnuf/38/BQcXEbBwWUUHl7efStbAl24vQfe0UezPvtaWXa7+vbuoe++X6rIiAr5zm+3c2sOABe37IedZpcAAE6RT3BwuUk7m5mp16d/r0ybTVlZdjWtW1U9OzYxojbDzfl6ka7rfrX63XaT7hn+qK5s01y/LP8j5/3zb/9vt9vzTG+327V12w7NnP21Zk17x+kyLnyEQFZWltLTTyk55aSSko4Xz4p5CGfb++Dho4qNiVKrFk3U/84HNGTQbXnmO51xRpKUsHmru0uGxZWkfALgPcgmABfjcpPm7+enh/p3U1CpANlsWZowfbEa1IxT9bhII+ozVId2rTTsgce0+PulRZ7+nfc/1b79B3VT37sVGRmhKR+9XuAYI+8foh43DtLll9dVxYr5HwWC8+0dHRmh+vVqqUqVOB07dlwtWzTOM1+Xazro2l791axJQ3eWixKgJOUTAO9BNgG4GB9beqJh54idOZupCdMWq9+1bVQtrmK+0yWnpOT6fe+hDFWKCjKqDEvJzMyUv7+/9h84pIdGP6MvZn1gdkluszY+e582aeTdD7N28OS/c0ny8/VVSEiI2WXkqzD5dGE2wTOFR9eXJCUdTDC5EriLlfOpqJ+d4JnIJ+9SUDYZck1aVlaWXpqyUEeOn1TH5nWdhszyNVu0fG326WiDe7RRbFREznv7j5yRn69n3sNk3qIf9cnUz5WWfkqvvfiEx65nQbxxnZ3x9fVVWGjoxScsoVJTU80uwamL5VNB2QTP5sn/HpGbFfPJ1c9O8Gzkk3coKJsMPZKWnnFaH85Zplu7tlJcZP43yfCmI2nejCNpuXn637mVv6mWCpdPfFPtHfim2vtYOZ+K+tkJnol88i4FZZOhhzjKBAWqdpVobdyxz8hhAcBl5BMAKyKbADjjcpN2Mi1D6RmnJWWfV715135FVwhzuTAAcBX5BMCKyCYAF+PyNWnJqematnCFsux22e12Nat3mRrWqmxEbQDgEvIJgBWRTQAuxtBr0gqLa9IubuI7k3VLn16KjYkyu5QiK2nXpKWlpav3bUMVHRWpqR+/mef9Vya8p9atmuqqDm2KNL6n/51b+ZqPwuKaD+/ANR/eh3xCSUE+eZdiv7ujkdotWe7S/Cu6tTeokuwHWkt5H1DtDqMeGOr2ZRZG0nX9Cj1tVcc8570Wvni2YbUYvX/+SdiiNq2a6eknHzZkPADW9MaE/WaXAHg1Vz/rXcjIz35mI5/gYLkmrTg9+Mh4Db93kOrUrqGPPp6pqMiKate2hUY+/JRSU9NUu1Z1vf7qeL0y4T0l7t2vAwcPq2O7VqpXt5a6dumoRd/9pJ27EjVi2OCcMcc9+bL++WezsuxZmvTeq6pUKVZdr++nJo0a6Pc//9KokUM1f8ES7di5Rx+++7Iub1A3zzySNOyBxyVJW7Zu17QpEzVz9lw98tC9+nPlGi39+TelpGTfovOLWR/o+PFkDb57lIKCghQQ4K9h9wxUuytbun17Gs0d++dkarpmfz5Pzz8zRg0ad9LnM9/X8RPJWrduo777YZn27Tsof38/7d9/SI88dK+qV6uqnjcN1sJ5U03bLgCMNXjACbNLAACnyCc4eNUDrHr16KoF3/4gSfpp6Qp16dxeE9/5WKNGDtWCr6cqJCRYq/5aJ0mqWeMyff3FZPW++XrNm/+9JGn+wiW66YbuucYc//goffvNND36yHBNnfGVJOn48RN65KF79eXsj/TEU69o0vuv6T+vjdfM2V87nadSpVgtnDdVLz43Vq1aNlGrlk1yLSMiooK+nP2hYmIitTFhi2bMmqNBA2/Rl7M/1JkzZ4t1m7mTO/ZP3To1tGXrdu3Zs09169bUylVrtWr1OrVs0VhPjB2pW/v21BPjHnTvigMAAADn8aojaR3bt9bb707R4AG3KCQkWMHBZbR12w4998Kb8vHxUVpaupo2uUKS1Khh9jnBleJidPxEspKSTig5+aTiYqNzjfn2e5/o1xV/6uzZTNWuVV2SVKFCeUVWzH7g5GWXVVZQUKBioiJ1Ijkl33mOHz+h8c9O0KeT35DvBQ9/rle3piQpJjpKyckntTtxn7p1vUqSdMXldYthS5nDHfvHx8dHgYGl9OuKlRo6pJ++W7JMSUknNGLYnfrr7/hzM553CqXjtEoAnmHqjHKS+MYagPWQT3DwqibN399flSvH6Z33P9X1114tSapZ8zLd0runGjdqIEnKzMxUwqZtuRql7t2u0sOPPqvu/2+MHJKSTui331dr8fwZWvbL75oz91tJua+R8rngw76zebKysjTy4fF6/pkxKl++XJ66LxyjauU4JWzaprp1ampjwlZdfdWVrm8cC3DX/mnUsL4+/nS2Fsz9VAsW/qDTp88oKCgw17yhoSE6dOio4mJjtGPn7mJcawDu9vCYWEl8CAJgPeQTHLyqSZOkG3p01ZB7HlbC+l8kSQ+PvEcPjXlGKSkn5evrq4mvP5tnnht7dtNTT7+m/7w6Ptfr5cqFKji4jG7ofafq169dqOU7m2flqrVauWqtHn8q+/q0l54fW+AYA+7oo0F3jdLsL+bJ18dX/gGesxvdsX9atWii+QuXKDS0rGJjoxUeXj7PmLf1vUEjRj2hKxrUVWRkhMFrCQAAAOSPW/AXwokTyRr5yHhNn/KW2aVIkrKysiRJvr6+uvWOYXrz9Wcseat+d92C32r7Jz9W/zt3Fbe4RknBLa69D/lkLdzdMX/kk3cpKJu86sYhRbF12w7dPvB+3Xf3ALNLyZGalq4eNw3SNdfepisur2vJBs1drLh/AAAAAFd4znlyxaR2rer6bsFMs8vIJbRsiBbPn2F2GZZgxf0DAAAAuIIjaQAAAABgITRpAAAAAGAhnO6IYlPcNwwBgKLggnwAVkU+wYEjaU7s2bNPv674023L++/S5frhx1/ctrySzqj9M/Lh8bq2V3/tP3Aoz3srflulF1+x9t0iAQAA4JksdyTtg/kNXJp/2A0bXa5hT+I+LV+xUh3atXZ5rMK45uqSc+vYfb3mujR/3ILeLtdg1P75d/tObjoCAAAAy7Fck1acVvy2Sh9Mmq6srCwdP5GsOZ9PUlBgoO4bMU4HDh5WTHSkPnz3FU2b+ZVWrlqr1X/F65s5n+TMP+uzefr8y/lKS0vXk48/qKuvulL3j3xcoWFltXbdP+rSuYMOHTqi1X/H667Bt6t/v5udzjNwyIM6fjxZBw8d1i19eiouNlqZNpuuat9GDzz0pMqVC9OexH2aOfUdxcVG6/4Hn9D+/QcVFxejSnExGjfmfhO3YvEpjv3T9/Z79dVnH+me4Y+qSePLdefAWzV02BjVrVNDGxO26rb+w9WrRxdl2mwaeEcfvTLhPbVr28LErQAYg+cQ5a9T12qSpGU/7DS5EgDIjXyCg1ee7vjZjPfVpXMH/bp8pb5d/JPq1K6hRd9MV906NbXg2x81qH9f3dq3Z64GQJJuuqG7Fs6bqm/mTNG7H0zNef3KNi30/cJZ+uKrBerfr7e+XzhLMz+bm+880z95S7Omv6u42GgNGtA31zJOJKfo08lvaPi9g7Rw0Y/6a816BQaW0ryvpqhWjcuKc7NYhpH7p0yZ0kpLS1dWVpYSNm3Tuvh/1KhhfT0x7kHVr1dLn89834Q1BGCm+PWlFb++tNllAEAe5BMcvOpImiTVq1tLkhQTHank5BQdOnxEja7Ifrp740YNFL8+QZEVKzidd+nPv+mjyTNlt9t15FhSnjGjIiuqXt2aCggIkI+PT77z2O12jXrkaT35+ChFVozItYw6tWvI19dXMdGR2rFzj3bv2asG9WpLki6/vK5W/xVv4NawHqP3T7MmV2jx90tVuVKsDh8+qlWr16lViya55nPsKyl73wAAAABm8rojaed9HpfdblfVKpW1bn32dWzr4jfqsssqyz/AXzZbVp55J749WV/O/lCzpr0j3/MGOn/M8z/w5zfPfyZ+pA7tWql504ZO6svdMFStUkkJm7ZJkjYmbL30FS5hjN4/LVs01jvvf6JWLZuocuU4fbNgiZo2uSLXfKGhZXXo0BFJ0qbN24ppzQAAAIDC8bom7UI9ruuszVu26/obByph0zb1ur6L6tetpZWr12rIPY/kmrZrl47qceNAvfDyWwoLK1uo8Z3N88ZbkzR33mL1vGmwZn8+r8D5mzdtqIzTp3VjnyFK2LRVAQHedfDT1f3TuNHl2vbvTrVq0UStWjaRXXYFB5fJNV+H9q3107IV6jfQM6/1AwAAQMniY0tPdPv5XckpKbl+33soQ5WigtxdRomRmZkpf39/vfXOx6pUKUa9b7re7JIKZW189j7leWnZPP3v3M/XVyEhIWaX4ZILs6kk48Yh+QuPzj6FmucReQ/yyVrIp/yRT96loGzyrsMyJdTIh57S7sR9KhsSoqlD3zS7HAAAAADFiCatBHj/nZfNLqFIKoTbzC4BAPIY2P+42SUAgFPkExxcbtKSUtI0dcFypaSeko+Pj9o1qa3OLesbURtKuCqVz5pdArwc+QRnJr5+wOwS4OXIJuSHfIKDy02an4+P+nRuoSoxFZRx+qxe+mSh6lWLVWzFckbUBwBFRj4BsCKyCcDFuHx3x7CyZVQlJvu5VUGBAYquEKYTJ9NdLgwlX3q6j9LTfS4+IVBMyCc4sy4+SOviPfcmPrA+sgn5IZ/gYOg1aUdPnFTioSRVi4vI897yNVu0fG32c74G92ij2Khz0+w/ckZ+vtZ5GsDuPfu0a3eiOrZvXaT5v138X7Vt01zh5d3zjdi4J1/Si8+OlZ+fn1uWV1hbtpWSJDVvcsbQca2wf/btP6h+g0aoZfPGmvDyk3neH/bAYxr90H2qUb1qzmu+vr4KCw0t8jKtLjU11ewSCpRfPhWUTTjHk/52r+5WSZJkS99rciVwFyvnU1E/O+Ec8gklVUHZZFiTlnHmrCbN/Vm3dGmp0oGl8rzfvmkdtW9aR1L2bWTPv5VsVlaWbFnZDyf+Y1oDl+poM2ijS/NL0q7difpl+R9qd2XLIs3/7eL/qk6dGgoLc09ovPjcOEnK2YbFafyrl35B67zvz/383NjyLtdghf2z4vdVGtCvt+4cdKvT7W6325Vlz8r1XlZWlkfdQvlCVvqi5UIF5VNB2YRzPHG7eOI6wTmr5pMrn51wjiduF09cJ+RVUDYZ0qTZbFmaNHeZWl5eXU3qVr34DCZZ8dsqfTBpurKysnT8RLLmfD5JQYGBum/EOB04eFgx0ZH68N1XNG3mV1q5aq1W/xWvb+Z8kjP//SMfV9nQsopfv1FXdWirsaOH65+NmzV63POy2bI0dEg/tW3dTD8tW6Et23aoV4+uGnn/kJz57xz6sI4cOaZSgaU09eM3FVo2RB0636wrGtTVuvUJeuzREZoxa64OHzmqmVPfUVxsdJ55tm3boWeef0OSFL8hQb//Ml/DRjymeV99rNff/Ej7DxxSYuI+Va4cp7ffeE47d+3R0GGPKioyQmlp6Xr7jedVpUqc27d9YVht//z559/avWev7rj9ZlWv20ab4n/Wou+Wyt/fT2++PVl2u12ZmZla8O2PmvfVx/L391fPmwZr4bypJmw95Kek5BMA70I2ASiIy18t2e12TV/0m6IrhOmaVq4dBXOXz2a8ry6dO+jX5Sv17eKfVKd2DS36Zrrq1qmpBd/+qEH9++rWvj1zNQAO7dq20HcLZurHn36VJL306jua9N5rWjx/uiZNmaWoqIrq3KmdJr3/aq4GQJLee+tFffvNNN3Yq5vmzf9OknTo0FH957Wn9caEp/X6mx/qsxnv6f77BumbBd87nadZ04ZaOG+qRo64S71vuk6V4mJyLaNunRqa99UU7d13QMnJKXr3/al6+YXHNP2Tt3TkaFJxbE7DWWX/tGzRWKv/jteatRt0ZdsWWv1XvFb/vU7t27XSgyPu0ojhd2roXXe4ZZugaEpiPgHwfGQTgItx+Uja9r2HtXLDdsVFltcLk+dLkm7o1ExX1KzkcnHFoV7dWpKkmOhIJSen6NDhI2p0RfZtbxs3aqD49QmKrFjhovOXDsq+qPPAseNKDwvV9vQMlY+J0p879yj57FntOJmmM8knc+az2Wx684U3tH3rdqWnpqlDl47anHxS0ZVjtev0GaWXKaPoqpXl6+ur6KhIbdm2QzabTU8/97oSNm/TyZOpuv7aayRln+734eTpmj3tvfzXL6qiUlJStTtxrxrUqy0/Pz/Vq1vTgC1YvIzePyeSU3KOHFatUinfRtXZti5XLkxJx09o1V/rNPL+u7R8xUrt3ZaoimmByjyULpvNpjPbjsuekakz244ry99ftvSzOrPtuGwpZ3R2V4rOZJ47PdR22q59QxflWm7cgt5F3FIojJKWTwC8A9kE4GJcbtJqVo7Sh08MNqAU9/A572aDdrtdVatU1rr1G9W1S0eti9+oatWqyD/AXzab8+u7fC64WWFIaIgO7N2vilGR2p+4X+XDw+Xnn3f+bZu2KePUKX0w60PN//IbHTl05P/jnTdg7uK04Z/NSk8/pUXfTNe0mV/pwIHDSk8/pYfGPKt33nxegU7OXz9/PLvdrqqVKylh8zY1adRAm7dsL+RWMo/R+ycstKz27NmnmJhI7d6dqIoR4U7nd7atJSkuNlrLV6zUQyOHatLHM1U+LCzPMkODQ3To2DEFBQbqSNIx1zYADFXS8gmAdyCbAFyMoXd3LIl6XNdZ997/o66/caCiIivqwRF36dSpDD3/0kQNuecRfTLpPwXOf/fIe/T06KeVZbOp9x295R/gr1btWuv1Zyfo6u5X66bbb5YkValWRXt379VDd41SZEykKkZVvGhtNWteph279qjP7fcoLjZaMdFRWrjoR/27faeGjXhMkvTxRxMKHGPE8MG6Z/hYVYwIV7mwUPkHlKxd7ur+eezRBzR0+KOy2Wy6687bFRAQoKuvulJjxj2vG3p2052DbpXkfFtLUsvmTXT0aJJ8fHwUHFxGjSvXyrOM/r16aeC4sWrZsKEqlOMZNwAAAA77es01dDxvOQvJx5aeaHf3Qi+8Y83eQxmqFGX8MyE2HTN21epVyPvMr83nndJohLphZQ0dLzMzU/7+/rLZbOres7++WzBD/v7uadQcz0grU8btf2LF5sy2S7+7pcOB03b5P74s12ueFDR+vr4KCQkxuwyXeNLdtNotWW7oeCu6tTd0PDM5nkHUuFGGyZXAXcgnayGf8ueJ+USTlr+CsqlkHVbBJdu1e68efGS80tNPqf/tN7utQZM8qzkD4Dk86cMPAM9CPsGBJs3D1axxmRZ9M93sMgAAAAAUkiWe7uij7JtEwLPsSQzQnsQAs8uwBLskZfE3DljBqNExGjU65uITAoCbkU9wsESTFhDgq5PpNho1D3MsyU/HkvzMLsN0dklp8pHP/lSzSwEgafrM8po+s7zZZQBAHuQTHCxxumOFsAAdSz6rk6mZMrJNS0szcDBJezPzvpZ6yvmt4Iu8jAzPORc5I9MmSdp76KzJlRjHdroIf6FZdvnsT5Xv9PXGFwQAAACPY4kmzc/PR5HheZ/55ao+M2yGjvfngLxHhW5bstrQZXjSHYoaNsp+CHXSwQSTKzHOhQ+jBgCr4A5qAOA5LHG6IwAAAAAgG00aAAAAAFgITRoAAAAAWIglrkmDZ2rU8JTZJQBAHmQTAKsin+BAk4Zis+yHnWaXAAB5kE0ArIp8ggOnOwIAAACAhdCkAQAAAICF0KSh2IRH11d4dH2zywCAXMgmAFZFPsGBJg0AAAAALIQbhwAAAACX4I9pDQwdr82gjYaOh5KPI2kAAAAAYCE0aQAAAABgITRpAAAAAGAhXJMGAG7WeobN0PH+HOBn6HhWwnUfAABvZEiTNn3hCm34d6/KBgdp/D03GjEkPMAbE/abXQK8HNkEZ8gmWAH5BGfIJzgY0qS1aVRTVzWvp6kLlxsxHDzE4AEnzC4BXo5sgjNkE6yAfIIz5BMcDLkmrVaVaJUpXcqIoQDAMGQTAKsinwAUxG3XpC1fs0XL126VJA3u0UaxURFuWOpxQ0cLCw01dDyzluEuk6YES5LuuSvN5EqMs8/g8Txpf6empppdQpGQTdZZhjuEhYZ6ZDZJ5FNByCfP5gl/q4518MR8IpvyV1A2ua1Ja9+0jto3rSNJSk5JUXJKirsWbRh31FwSt0t+hj1QSZJ0a98DJldiXZ60v/18S+bNYskm6yzDHZJTUsimQvKUfS6RT57OE7aLYx3Ip4vzhP3tUFA2cXdHAAAASOLus4BVlMyvlgAAAADAQxlyJO3jeb9o6+6DSj2VoXFvf6meHRrrysa1jRgaAIqMbAJgVeQTgIIY0qTdfVNHI4YBAEORTQCsinwCUBBOdwQAAAAAC6FJAwAAAAAL4e6OKDZJBxPMLgEA8iCbAFgV+QQHjqQBAAAAgIXQpAEAAACAhdCkodh06lpNnbpWM7sMAMiFbAJgVeQTHLgmDcUmfn1ps0sAgDzIJgBWRT7BgSNpAAAAAGAhNGkAAAAAYCE0aQAAAABgITRpAAAAAGAhNGkAAAAAYCHc3RHFZmD/42aXAAB5kE0ArIp8ggNNGorNxNcPmF0CAORBNgGwKvIJDpzuCAAAAAAWQpOGYrMuPkjr4oPMLgMAciGbAFgV+QQHTndEsbm6W3VJUtLBBJMrAYBzyCYAVkU+wYEjaQAAAABgITRpAAAAAGAhNGkAAAAAYCE0aQAAAABgITRpAAAAAGAhhtzdceP2vfryh1XKstt1ZeNa6t62oRHDAoDLyCcAVkQ2ASiIy01aVlaWPvt+pR7s11XlQ8vo5U++VcNaVRRbsZwR9aEEW7pkh9klwMuRT3CGbILZyCbkh3yCg8tN2q79RxUZXlYVy5eVJLWoX03rt+4xLGjCo+vn+94bE/Zr8IATkqSpM8rp4TGx+U5be8KGnJ93T6yp0/tKO50urFWSovrskyRl7A3SnrdqnatlTO5pz/+HtP+TDjrxSz2nYwZVPaLqz32d83vCoHvzrXPqJazT+c/Q6NS1muLXO1+ngf2Pa+LrByRlPyTR8QwOZ5Yu2aHGjTIkSaNGx2j6zPJOp2vU8JSW/bAz53ej9pP11yn7tfGXfau+kWslSV8dbqLndvXId/kbWj6f8/Mt/9ytTekx596MtsI6ZXO2n0r6c1q8JZ8uzCZJqvbMJpWudlSSMfkUfonrZNy/ZXvOTxMfaKZaldZIkt6e+5GWrLrH6Zg14v7W2yOb5/x+/dhzY2hs7mnJp5KZT46fSypvySbJkz87kU0S2VScn51cbtKOn0xX+bLBOb+XCw3Wzn1H8ky3fM0WLV+7VZI0uEcbxUZFuLpolQ4qrbDQrJyfC7Lp/nM7osUsP63Z53y6vvVL6aP/T/v3mgC1fCv/MUOCg7Wh7/WSpHt/LqePf3E+Xf3yYVr9/+kkyW9Q/mNeyjp9ML9Bzs9HTvwlqZnT6RJ2faUP5meH277dTSX9ne+Y67/tq1PrsoPm0NaPJDkPmtRjCfpjWvPzXrE7nU46t05r+jbTkT0DJL2e77Tb7+qU8/PpXT9IauR0ulIBAQoLDZUkhQQH5DueJKU8+ISSymzKHnP3U5J6O50uc9tOJV3X77xX1uU7ZswjV6r+XY2zf54SLD2Q//Lr/3xnzs+l21bIdyLadPsAABRkSURBVFjHOj34xG4dPhTsfKL/+3BaiiKj0iRJf60Lz3e6/QdtOdvpYpz97RV2XklKTU0t9LTuUph8Ko5skszPpy+uaadmTc9KKjn51KXBwQLHbHvr4px1mre5nLTK+XRhFa9Q9wcSz70w1vl0UvZ6bL+rSXad5JNb88nP16/I+eT4ubCslk98dvKsz05kk2dlk1U+O/nY0hPz/3RdCH9v2qWE7fs0oMeVkqQ/N2zXrv1HdFu31vnOk5yS4soivc75gWKEYTdszPPaH9OMXUabQRs1anT2tx6ObzjODxFX1ZiyLM9ruUPCdeGLZxs6nqfz8/VVSEiI2WXkcqn5RDZdGndkU3EozmySPDufxr963LCxnhvr/Fvs4mC1fOKzU/HzhHwimwrPyGyS3JdPBWWTy0fSypcto+Mn03J+P5GSpnIhZVwdFm7WZpDx4eM4jOz4IAS4G/kEZ8gmmI1sQn7IJzi4fAv+qrEROpyUoqMnTirTZtPqhJ1qWLuyEbUBgEvIJwBWRDYBuBiXj6T5+frq1m6t9fZnPyory662jWoqtqL7TmEAgPyQTwCsiGwCcDGGPCftipqVdEXNSkYMBQCGIp8AWBHZBKAgLp/uCAAAAAAwDk0aAAAAAFiIIac7As40anjK7BIAIA+yCYBVkU9woElDsTn/ae0AYBVkEwCrIp/gwOmOAAAAAGAhNGkAAAAAYCE0aSg24dH1FR5d3+wyACAXsgmAVZFPcKBJAwAAAAALoUkDAAAAAAuhSQMAAAAAC6FJAwAAAAALoUkDAAAAAAuhSQMAAAAAC/E3uwB4rjcm7De7BADIg2wCYFXkExxo0lBsBg84YXYJAJAH2QTAqsgnOHC6IwAAAABYCE0ais3UGeU0dUY5s8sAgFzIJgBWRT7BgdMdUWweHhMriUP3AKyFbAJgVeQTHDiSBgAAAAAWQpMGAAAAABZCkwYAAAAAFkKTBgAAAAAW4tKNQ/7etEvf/rpOB4+e0Lg7e6hqbIRRdQGAS8gnAFZENgEoDJeatNiK5XRvn06atfh3o+oBAEOQTwCsiGxCYdWYsszsEmAil5q0mAie44D8JR1MMLsEeDHyCfkhm2AmsgkFIZ/g4LbnpC1fs0XL126VJA3u0UaxURzeN0tYaKjZJbjM2TokuWEZyF9qaqrZJRQJ2WQdZv2ba/rV38W+DM/Jp+OGjeTOdSCf4CpP/UxANjnnrvUoKJsu2qRNnLVEKWmn8rzeq2NTNa5TpdBFtG9aR+2b1pEkJaekKDklpdDzwliesO3dsQ6esJ3cyc/X/fchMiKfyCbrYNsXnidsK3eug7vzic9OnodtXziesp3ctR4FZdNFm7RRd3QztBh4j05dq0mSlv2w0+RK4KnIJxQF2YTiRjahqMgnOLjtdEd4n/j1pc0uAQDyIJsAWBX5BAeXmrS1m3frix9WKjU9Q+9++V9VjgrXyNu7GlUbABQZ+QTAisgmAIXhUpPWpG5VNalb1ahaAMAw5BMAKyKbABSG+6/0BwAAAADki2vSAABAoTw3trzZJQCAV+BIGgAAAABYCEfSUGwG9jf2wYIAYASyCYBVkU9woElDsZn4+gGzSwCAPMgmAFZFPhWNJ56KzemOAAAAAGAhNGkoNuvig7QuPsjsMgAgF7IJgFWRT3DgdEcUm6u7VZckJR1MMLkSADiHbAJgVeQTHDiSBgAAAAAWQpMGAAAAABZCkwYAAAAAFkKTBgAAAAAWwo1DAABFMuyGjWaXAACAR+JIGgAAAABYCEfSUGyWLtlhdgkAkAfZBMCqyCc40KSh2DRulGF2CQCQB9kEeL6Sejo2+QQHmrQSoKQGzYVqTFlmdgkAAACA5XFNGorNqNExGjU6xuwyACAXsgmAVZFPcKBJQ7GZPrO8ps8sb3YZAJAL2QTAqsgnONCkAQAAAICF0KQBAAAAgIXQpAEAAACAhbh0d8e5P63W+m2J8vfzU0S5shrU80qVCQo0qjYAKDLyCYAVkU0ACsOlJq1etVjd2KmZ/Hx99fXSv/T97xt089XNjaoNAIqMfAJgRWQTgMJwqUmrXz0u5+dqsRW1ZvNulwuC52jU8JTZJcCLkU/ID9kEM5FNKAj5BAfDHmb9e/w2Na9fLd/3l6/ZouVrt0qSBvdoo9ioCKMWDYta8+ex//8U6pblJRk8Xlioe+r2FKmpqWaXkK+C8ols8j7uziaJfDKbVfOJz064EJ+dvEtB2XTRJm3irCVKScvb1ffq2FSN61SRJC1eES9fX1+1vLx6vuO0b1pH7ZvWkSQlp6QoOSXlooUDlyJ88WxDx+Nv9NL4+br/PkRG5BPZhJKIv9NL4+584rMTvBV/o5emoGy6aJM26o5uBb7/x/p/teHfvXrojm7y8fG59OoAoIjIJwBWRDYBcJVLXy1t3L5XS/7YoOF9O6tUgGFnTsJDhEfXV3h0fbPLgJcin5AfsglmIptQEPIJDi6lw+dLVioz06a3Zi+RJFWLq6g7rmtrSGEA4AryCYAVkU0ACsOlJu354b2NqgMADEU+AbAisglAYbj/Sn8AAAAAQL5o0gAAAADAQmjSAAAAAMBCaNIAAAAAwEK49yuKzRsT9ptdAgDkQTYBsCryCQ40aSg2gwecMLsEAMiDbAJgVeQTHDjdEQAAAAAshCNpKDZTZ5STxLdCAKzFjGwKXzzbbcsCUHLx2QkOPrb0RLu7F5qckuLuRcIE4dH1JUlJBxNMrgTu4Ofrq5CQELPLcAnZ5B3IJu9DPqGkcHc+JV3Xz9Dx+ELq0hSUTZzuCAAAAAAWQpMGAAAAABZCkwYAAAAAFkKTBgAAAAAWQpMGAAAAABZCkwYAAAAAFsJz0lBsuL01ACsimwBYFfkEB46kAQAAAICFcCQNAAAA8EI8fNq6OJKGYtOpazV16lrN7DIAIBeyCYBVkU9w4Egaik38+tJmlwAAeZBNAKyKfIIDR9IAAAAAwEJo0gAAAADAQlw63XHBz2sUvy1RPpLKBpfWoJ7tVK5sGYNKA4CiI58AWBHZBKAwfGzpifaiznzq9BmVDiwlSVq6OkEHjpzQHde1veh8ySkpRV0kSpDw6PqSeOaHt/Dz9VVISIjZZeQoSj6RTd6BbPI+VsonPjuhIOSTdykom1w63dERMpJ05kymfHx8XBkOAAxDPgGwIrIJQGG4fHfHb5at0coN/6p0UCk9dEd3I2qChxjY/7jZJcDLkU9whmyC2cgm5Id8gsNFT3ecOGuJUtJO5Xm9V8emalynSs7v3/+2XmczberZsYnTcZav2aLla7dKkgb3aKPYqAhX6gZgMampqW4/nciIfCKbAM/n7nzisxOAwigom1y6Ju18x5JT9d4X/9X4e2686LScVw14Hitd83GhwuYT2QR4JqvmE5+dAO9WbNekHUo6FxjrtyYqqkKYK8PBw6yLD9K6+CCzy4CXIp+QH7IJZiKbUBDyCQ4uXZP2zdK/dSgpWT4+PgoPDVa/a9sYVRc8wNXdqkviDkUwB/mE/JBNMBPZhIKQT3BwqUm7t08no+oAAEORTwCsiGwCUBgune4IAAAAADAWTRoAAAAAWAhNGgAAAABYCE0aAAAAAFgITRoAAAAAWIhLd3cECrJ0yQ6zSwCAPMgmAFZFPsHBx5aeaHf3QpNTUi4+EYASxc/XVyEhIWaX4RKyCfBM5BMAKyoomzjdEQAAAAAshCYNxWbU6BiNGh1jdhkAkAvZBMCqyCc4cLojik14dH1JUtLBBJMrgTtwOhFKCrLJ+5BPKCnIJ+9SUDaZcuMQH0m+vhzE83TJhzf//yf2tTfIysoyuwSXkU3egWzyPuQTSgryybsUlE2mHEkDAAAAADhHmw4AAAAAFkKTBgAAAAAWQpPm5abO+FJ+ZSo7/S88pkGuaXbtTjS5WgDI64+Vf+v2gcNVpWYLBYVVV7moemrV7nqNf3aCDhw4lDPd+fnmH1xFkZUb6sa+Q7QxYUuu8XbtTpRfmcr6+NPPnC7v6m591aHzzcW6TgBKDsfnpH+373T6flZWlj6d/oVat++hiLjLVTaitmo1uFK3DxyuVavXSlK+n8XO/6963TaSpGdfeEN+ZSorOLymkpPz3lDm/M92+dUE6zPlxiGwni9mfahKcblv+erv52dSNQBQOG+89ZEeffxFderYVs89PUbVq1VRamq6/vjzL03+dLb+WrNei+fPyJl+UP++uufu/srMzNSGDZv0zItv6LobBih+9Y8qVy7MxDUB4KnGPPa83nn/Uz044i6Nf+JhBQT4a9u2Hfpm4RKt+mudWrZoot9+np9rnt63DlXDK+rp6ScfznktsFSpXNMEBARozrzFumvwbblenzl7rsqWDdHJk6nFt1IodjRpkCQ1blhfNWtUM7sMACi0Zb/8rkcff1Ej7x+iN157Jtd713W/WuPGjNBXX3+b6/W42Gi1btlUktSubUuFhoVq4JCR+v7Hn3Vb3xvcVToAL3Hq1Cm9/9F0jRg2WBNefirn9S6dO2j4fYNz7u7nyCWHwMBSiogIz/P6+W66obtmfTY3V5OWuHe/fln+pwb0661pM78yeG3gTpzuiCKpXreNBgwZqRmz56peo44KDq+pjtfcrG3/7lRaWrruGzFOFStdoZjLmmj0uOeUmZmZM29GRoYefvQZNWzeWaEV6yj2sqbq1ftObd7yb65lOA7X/7lqjfrf+YDKRdVTperN9OAj45WRkeHuVQZgMa/9531FRITrlRced/p+cHAZDR5wS4FjNG18uSQpMXG/4fUBQFraKZ05c0bRUZFO33flsQoD+vXRrytWaveevTmvzZw9V1Uqx6l9u1ZFHhfWQJMGSZLNlqXMzMxc/13suTLLV6zUh5Om6+XnH9Onk9/U9h271bffPep/5wMqWzZYs6e9p6F39tObb0/W5CmzcuY7ffqMTp5M0+NjR2rh11P13lsvKuP0abW96gYdPHg4z3IG3fWgalSrqjmfTdJ9Qwfo/Y+m6ZUJ7xm+DQCUHJmZmfp1xZ+65ur2KnXBKUCXYtfu7A831atXNao0AMgRERGuapdV0X/e+kgfTp6hPYn7DBu7/ZUtdVnVypr9+byc12Z+9rXuuO0m+fj4GLYcmIMmDZKk+o2vUmBotVz/3dBnSIHzpKalafH8GbqxV3fd0runxo6+Xxv+2ayICuGa8PJT6tK5g557eoyaNLpcc+YtypkvLCxUkz+YoNv63qCO7duoV4+u+ubLKbLZbPr8q/l5lnP7LTfq2fGjdc3V7fXkY6PUvWsnp9MB8B7Hjh1XRsZpVakcl+e9C79wOp/dbldmZqZOnz6t1X+t09gnXlDrlk3V6/ou7iodgJeZOfUdBZcpo/sffFzV6rRW5RrNdfd9o3NuGlJUPj4+uuO2mzTzs68lSatWr9XmLf9qwB19jCgbJuOaNEiS5n4+Oc+NQ8qVCy1wntYtmyks7Nw0dWrXkCR17dIx13R16tTQ6r/W5Xrty7kL9eZbk7Rl245cdybasnVHnuVc1/3qXL9fcXld/bRsRYG1AfBsdrvd6esHDx5WXPVmuV47nbJT/v7Z/7t7ecK7ennCuznvXVa1sn767ksFBAQUX7EAvFrrlk21Kf5nLfvld/20bIX+XLlG02fN0bSZX2nKR//RQBeaqgF39NYLr7yl1X+t04zZc9W6ZVPVqllNv/2x2sA1gBlo0iBJurxBnUu+cUj58rnvhOY45ah8uQtfD1BGxumc3xcu+lG3Dxiugf376KnHH1JEhfLy9fVVj5sHKeN03mvNwsPL5fo9sFQpnT59Os90ALxHRES4goIC85w6FBERrpXLs28WMvmT2fr409m53r9z4K26b+gAZZw+raXLftPzL09Uv0H364dFn+WcHuTvl/2/RpvN5nTZNptN/v7c/RZA4QUGBqp7107q3rWTJGnrth3q3P0WjR73nEtNWs0a1dSmVTN9Mu0Lzf1mkZ59arRRJcNkNGlwuy/mLFDNGpfp00lv5rx29uxZJSWdMLEqACWJv7+/2rdrpf8uXa4zZ87kfEnk7++v5s0aSZIWffdTnvlioiNz3m/XtqXsdruee+lNzZm3SH1v7iFJqlChnPz8/HI9Y+18+w8cUrOmDYtjtQB4idq1quuWPj018Z2PdfjwUUVGRhR5rP79euuBh56Uv7+/bu3T08AqYSauSYPbpaefyjn1yGHG7Ln5fmsNAM6MeWiYjh5N0rgnXyryGI8+MkyxMVF6/qU3c06hLF26tFq1aKL53y7JcwOldfEbtWPnHl3Voa1LtQPwDmfPntWhQ0ecvrd5y3aVLh2ksLCyLi3j1j491fP6Lhr7yHCFh5d3aSxYB0fSIElatz5BR48dz/N682L4trhbl6s0f+ESPfzoM7r+2mu0Zu0GvfP+JzxIFsAl6dypnV5+/jE99tTL2vDPZvXv11vVLqusjIzT2rZtp76Ys0DBwWUKvMtZ6dKl9diYEXrg4af09fzv1PvG6yRJzz/zqLr16Kdrrr1V99zdX5EVI7QxYYtefPVt1a9X26XTkwB4piU//KJ10Qm5XrPb7Ro45EH17d1D13brpEpxMTp27Li+mLNA3/+wTGMeHqbAwECXllu+fDl9/cXHLo0B66FJgyTp1jvuc/r6oT3xhi9r6JB+2rt3vz6d/oUmTZmlFs0aaf6cT9X7tqGGLwuAZxvz8DC1bdNcb7/3iZ58+lUdOZqkoKBA1alVXbf07ql77+4vP7+Crx+7e0g/vT7xI730ytu6+YZr5ePjo6s6tNHSJV/qxVfe1v0PPqHU1DTFREeq78099MyTj6hMmdJuWkMAJcXIR57K81q9urX0/NNj9OPSXzX28Rd1+MgxBQUFqkG92vrgnVc0dEg/EypFSeBjS090fossAAAAAIDbcU0aAAAAAFgITRoAAAAAWAhNGgAAAABYCE0aAAAAAFgITRoAAAAAWAhNGgAAAABYCE0aAAAAAFgITRoAAAAAWAhNGgAAAABYyP8AOPTIYOeb0vcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x288 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.style.use('Solarize_Light2')\n",
    "plt.figure(figsize=(15, 4))\n",
    "labels = ['amazing', 'very amazing', 'not amazing', \n",
    "          'not not amazing', 'awful', 'very awful', 'not awful', 'not not awful']\n",
    "\n",
    "\n",
    "\n",
    "# plt.subplot(142)\n",
    "# for i in range(len(elman_second_degree_scores)):\n",
    "#     plt.bar(i, elman_second_degree_scores[i], label=labels[i])\n",
    "# plt.plot([-1,8], [0, 0], 'b', linestyle='--')\n",
    "# plt.plot([3.5,3.5], [-6.5, 6.5], 'b', linestyle='--')\n",
    "# ax = plt.gca()\n",
    "# ax.set_title('Elman (2rd)', y=-0.1)\n",
    "# # ax.axes.xaxis.set_visible(False)\n",
    "# ax.axes.xaxis.set_visible(False)\n",
    "# plt.ylim(-3.5, 3.5)\n",
    "\n",
    "\n",
    "plt.subplot(132)\n",
    "for i in range(len(gru_first_degree_scores)):\n",
    "    plt.bar(i, gru_first_degree_scores[i], label=labels[i])\n",
    "plt.plot([-1,8], [0, 0], 'b', linestyle='--')\n",
    "plt.plot([3.5,3.5], [-6.5, 6.5], 'b', linestyle='--')\n",
    "ax = plt.gca()\n",
    "ax.set_title('GRU', y=-0.1)\n",
    "ax.axes.xaxis.set_visible(False)\n",
    "plt.ylim(-3.5, 3.5)\n",
    "\n",
    "plt.subplot(133)\n",
    "for i in range(len(lstm_first_degree_scores)):\n",
    "    plt.bar(i, lstm_first_degree_scores[i], label=labels[i])\n",
    "plt.plot([-1,8], [0, 0], 'b', linestyle='--')\n",
    "plt.plot([3.5,3.5], [-6.5, 6.5], 'b', linestyle='--')\n",
    "# plt.legend()\n",
    "\n",
    "ax = plt.gca()\n",
    "ax.set_title('LSTM', y=-0.1)\n",
    "ax.axes.xaxis.set_visible(False)\n",
    "# plt.xticks(ticks=range(0, 5), )\n",
    "plt.ylim(-3.5, 3.5)\n",
    "\n",
    "\n",
    "plt.subplot(131)\n",
    "for i in range(len(elman_first_degree_scores)):\n",
    "    plt.bar(i, elman_first_degree_scores[i], label=labels[i])\n",
    "plt.plot([-1,8], [0, 0], 'b', linestyle='--')\n",
    "plt.plot([3.5,3.5], [-6.5, 6.5], 'b', linestyle='--')\n",
    "ax = plt.gca()\n",
    "ax.set_title('Elman', y=-0.1)\n",
    "ax.axes.xaxis.set_visible(False)\n",
    "plt.ylim(-3.5, 3.5)\n",
    "plt.xlabel('Elman (1st-degree)')\n",
    "plt.legend(loc=2, prop={'size': 8}, ncol=2, bbox_to_anchor=(0, 1.0))\n",
    "plt.savefig(\"imgs/negation_intensification_rnn.pdf\", bbox_inches='tight', dpi=900)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check Approximation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from self_defined_lstm_spectral import LSTM2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_errors(rnn_type = 'elman'):\n",
    "    best_model.eval()\n",
    "    test_dg.reset_samples()\n",
    "    errors = []\n",
    "    while test_dg.index<test_dg.data_len:\n",
    "        sent_ids,  label_list, sent_lens = next(test_dg.get_sequential_ids_samples())\n",
    "        sent_ids = sent_ids.to(device)\n",
    "        preds, scores =  best_model(sent_ids, sent_lens.to(device))\n",
    "\n",
    "        seq_embs = best_model.embeddings(sent_ids)\n",
    "        batch_size, max_len, hidden_dim = seq_embs.size()\n",
    "        ##############gru, elman rnn\n",
    "        if rnn_type != 'lstm':\n",
    "            hidden_vecs, final_vecs = best_model.rnn(seq_embs, sent_lens.to(device))\n",
    "        else:\n",
    "            lstm2 = LSTM2(300, 300).to(device)\n",
    "            lstm2.f_cell.linear_hh.weight = best_model.rnn.f_cell.linear_hh.weight\n",
    "            lstm2.f_cell.weight_ih = best_model.rnn.f_cell.weight_ih\n",
    "            hidden_vecs, final_vecs, cell_vecs = lstm2(seq_embs, sent_lens.to(device))\n",
    "\n",
    "\n",
    "\n",
    "        batch_size, max_len, _ = hidden_vecs.shape\n",
    "        hiddens = torch.zeros_like(hidden_vecs).to(device)\n",
    "        if rnn_type == 'lstm':\n",
    "            cells = torch.zeros_like(hidden_vecs).to(device)\n",
    "            cells[:, 1:] = cell_vecs[:, :-1]\n",
    "        hiddens[:, 1:] = hidden_vecs[:, :-1]\n",
    "        approx_hiddens = torch.zeros_like(hiddens)\n",
    "        #predict next hidden state\n",
    "        for j in range(max_len):\n",
    "            x = seq_embs[:, j]\n",
    "            hidden = hiddens[:, j]\n",
    "\n",
    "            ####GRU\n",
    "            if rnn_type == 'gru':\n",
    "                approx_hiddens[:, j] = get_gru_hidden_state(x, hidden)\n",
    "            if rnn_type == 'elman':\n",
    "                approx_hiddens[:, j] = get_rnn_hidden_state(x, hidden)\n",
    "\n",
    "            if rnn_type == 'lstm':\n",
    "                c = cells[:, j]\n",
    "                approx_hiddens[:, j] = get_lstm_hidden_state(x, hidden, c)\n",
    "\n",
    "        #calculate errors    \n",
    "        for i in range(batch_size):\n",
    "            length = sent_lens[i]\n",
    "            for j in range(length):\n",
    "                new_hidden = approx_hiddens[:, j]\n",
    "                hidden = hidden_vecs[:, j]\n",
    "                error = (new_hidden - hidden).norm(2)/hidden.norm(2)\n",
    "                errors.append(error.item())\n",
    "                \n",
    "    return errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# best_model.eval()\n",
    "# test_dg.reset_samples()\n",
    "# errors = []\n",
    "# while test_dg.index<test_dg.data_len:\n",
    "#     sent_ids,  label_list, sent_lens = next(test_dg.get_sequential_ids_samples())\n",
    "#     sent_ids = sent_ids.to(device)\n",
    "#     preds, scores =  best_model(sent_ids, sent_lens.to(device))\n",
    "    \n",
    "#     seq_embs = best_model.embeddings(sent_ids)\n",
    "#     batch_size, max_len, hidden_dim = seq_embs.size()\n",
    "#     #gru, elman rnn\n",
    "#     hidden_vecs, final_vecs = best_model.rnn(seq_embs, sent_lens.to(device))\n",
    "#     batch_size, max_len, _ = hidden_vecs.shape\n",
    "#     hiddens = torch.zeros_like(hidden_vecs).to(device)\n",
    "#     hiddens[:, 1:] = hidden_vecs[:, :-1]\n",
    "#     approx_hiddens = torch.zeros_like(hiddens)\n",
    "#     #predict next hidden state\n",
    "#     for j in range(max_len):\n",
    "#         x = seq_embs[:, j]\n",
    "#         hidden = hiddens[:, j]\n",
    "#         approx_hiddens[:, j] = get_gru_hidden_state(x, hidden)\n",
    "#     #calculate errors    \n",
    "#     for i in range(batch_size):\n",
    "#         length = sent_lens[i]\n",
    "#         for j in range(length):\n",
    "#             new_hidden = approx_hiddens[:, j]\n",
    "#             hidden = hidden_vecs[:, j]\n",
    "#             error = (new_hidden - hidden).norm(2)/hidden.norm(2)\n",
    "#             errors.append(error.item())\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([225, 7, 300])"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_embs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rnn_hidden_state(x, hidden):\n",
    "    gi = F.linear(x, best_model.rnn.f_cell.weight_ih)\n",
    "    #print(hidden.shape)\n",
    "    #print(self.weight_hh.shape)\n",
    "    gh = best_model.rnn.f_cell.linear_hh(hidden)\n",
    "    g = torch.tanh(gi)\n",
    "    f1 = 1 - g**2\n",
    "    f2 = -g*f1\n",
    "    hidden = g + f1*gh# + f2 * gh**2\n",
    "    return hidden\n",
    "\n",
    "def get_gru_hidden_state(x, hidden):\n",
    "    gi = F.linear(x, best_model.rnn.f_cell.weight_ih)\n",
    "    #print(hidden.shape)\n",
    "    #print(self.weight_hh.shape)\n",
    "    gh = best_model.rnn.f_cell.linear_hh(hidden)\n",
    "    i_r, i_i, i_n = gi.chunk(3, 1)\n",
    "    h_r, h_i, h_n = gh.chunk(3, 1)\n",
    "    g_r = torch.sigmoid(i_r)\n",
    "    f_r = g_r*(1-g_r)\n",
    "    g_z = torch.sigmoid(i_i)\n",
    "    f_z = g_z*(1-g_z)\n",
    "    g_n = torch.tanh(i_n)\n",
    "    f_n = 1-g_n**2\n",
    "    g = (1-g_z) * g_n\n",
    "    hidden = g + f_n*(1-g_z)*g_r*h_n-f_z*g_n*h_i+g_z*hidden\n",
    "    return hidden\n",
    "\n",
    "def get_lstm_hidden_state(x, hidden, cell):\n",
    "    gi = F.linear(x, best_model.rnn.f_cell.weight_ih)\n",
    "    #print(hidden.shape)\n",
    "    #print(self.weight_hh.shape)\n",
    "    gh = best_model.rnn.f_cell.linear_hh(hidden)\n",
    "    i_i, i_f, i_g, i_o = gi.chunk(4, 1)\n",
    "    h_i, h_f, h_g, h_o = gh.chunk(4, 1)\n",
    "    gx_i = torch.sigmoid(i_i)\n",
    "    fx_i = gx_i*(1-gx_i)\n",
    "    gx_f = torch.sigmoid(i_f)\n",
    "    fx_f = gx_f*(1-gx_f)\n",
    "    gx_o = torch.sigmoid(i_o)\n",
    "    fx_o = gx_o*(1-gx_o)\n",
    "    gx_c = torch.tanh(i_g)\n",
    "    fx_c = 1 - gx_c**2\n",
    "\n",
    "    g_c = gx_i*gx_c   \n",
    "    g_c_tanh = torch.tanh(g_c)\n",
    "    g_h = gx_o * g_c_tanh\n",
    "    B = gx_f*cell\n",
    "    D = gx_c*fx_i*h_i + gx_i*fx_c*h_g\n",
    "    E = gx_o * (1-g_c_tanh**2) * gx_f * cell \n",
    "    FF = gx_o * (1-g_c_tanh**2) * D + fx_o * g_c_tanh * h_o######original\n",
    "\n",
    "    hidden = g_h + E +FF\n",
    "    cell = g_c + B + D\n",
    "    return hidden#, cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# errors = []\n",
    "# batch_size, max_len, _ = hidden_vecs.shape\n",
    "# hiddens = torch.zeros_like(hidden_vecs).to(device)\n",
    "# hiddens[:, 1:] = hidden_vecs[:, :-1]\n",
    "# approx_hiddens = torch.zeros_like(hiddens)\n",
    "# #predict next hidden state\n",
    "# for j in range(max_len):\n",
    "#     x = seq_embs[:, j]\n",
    "#     hidden = hiddens[:, j]\n",
    "#     approx_hiddens[:, j] = get_rnn_hidden_state(x, hidden)\n",
    "# #calculate errors    \n",
    "# for i in range(batch_size):\n",
    "#     length = sent_lens[i]\n",
    "#     for j in range(length):\n",
    "#         new_hidden = approx_hiddens[:, j]\n",
    "#         hidden = hidden_vecs[:, j]\n",
    "#         error = (new_hidden - hidden).norm(2)/hidden.norm(2)\n",
    "#         errors.append(error.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tarlor series lstm linear simplified\n",
      "tarlor series lstm linear simplified\n",
      "tarlor series lstm linear simplified\n",
      "tarlor series lstm linear simplified\n"
     ]
    }
   ],
   "source": [
    "lstm_errors = get_errors('lstm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#gru_errors = get_errors('gru')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "elman_errors = get_errors('elman')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAO/ElEQVR4nO3df2zU933H8dfdGTD3y3fGxsaAXQLGDFLbM0PCgIkmSkIio1jd2rCsTeiiUjU/FClZqixS6bR/WqXLpEVrpKpRtyXZWKmaSCQjBBYxZCAFMswAk+KEmB/GcY1jn89nYwK+2x+Z3Di+4wv2/TBvno+/+PL95D4fS5dnvnzC9/t1jQxdSAgAcMtz53oBAID0IOgAYARBBwAjCDoAGEHQAcCIvFxN3B+N5mpqwJHL5VIiwV8Aw9TjkhQMBpOe4wodSCIYCOR6CUBSbnfqbBN0ADCCoAOAEQQdAIwg6ABgBEEHACMIOgAY4fj30F95c79OfNShgC9fW7c0jTt/6OQZ7X7vpCRpxrQ8PXhvveaVFKZ/pQCA63K8Qq+vWaQnNq1Peb4oFNBT39qgH373ft23pkav7TyY1gUCAG6M4xV6ZXmpeiIDKc8vnDd79NcL5harLzqUnpUBafKTn/5Mz7/wUkbn+MHTj+rZZx7L6ByAk7Te+n/gfz/UnQvnpvMjgUl79pnHbjq2haXL1NvVmqEVAZmRtqCfPvuJDh77UH/90L0pxzQfPa3mljZJ0ubGepWVFKVreiDtClI8LwPIpVgslvJcWoLe8ftevfqfB/XEpq/J781POa6hrkoNdVWSPn84Fw/owlTG9xNTkSeTz3Lp7Y/p57/Zq+/c36CSWQWT/TgAwAS5nF4S/fIb+9R2rkuxy8MK+mZq49pajYzEJUlrly/Rq28dUMvpcyoM+iR9/iSw5x7Z6DgxVz+YythDx1Tlcbvl9/uTnnMMeqYQdExlBB1T1fWCzp2iAGAEQQcAIwg6ABhB0AHACIIOAEYQdAAwgqADgBEEHQCMIOgAYARBBwAjCDoAGEHQAcAIgg4ARhB0ADCCoAOAEQQdAIwg6ABgBEEHACMIOgAYQdABwAiCDgBGEHQAMIKgA4ARBB0AjCDoAGAEQQcAIwg6ABhB0AHAiDynAa+8uV8nPupQwJevrVuaxp1PJBLavvuwTp7p0PRpeXq4cY3K58zKyGIBAKk5XqHX1yzSE5vWpzx/8sxFdfdG9Xff/7r+8r56/fuu99K6QADAjXEMemV5qbwzp6c8f7ztvFZWL5TL5dIdc2fr8vBn6h8YSusiAQDOHLdcnEQGhhQO+kaPQ0GfIgNDKgh4x41tPnpazS1tkqTNjfUqKyma7PRAxhQEg7leAjBOLBZLeW7SQU8kkvymK/nYhroqNdRVSZL6o1H1R6OTnR7IGL6fmIo87tQbK5P+Wy7hoFd90cHR40h0UCH/+KtzAEBmTTro1ZXz9dvjZ5RIJPTxxW7lz5iedLsFAJBZrpGhC8k2TUa9/MY+tZ3rUuzysIK+mdq4tlYjI3FJ0trlS5RIJPQf7xxS65mLmj7No4cb16iizHlvnD/OYiorLF2m3q7WXC8DGMfjdsvv9yc95xj0TCHomMoIOqaq6wWdO0UBwAiCDgBGEHQAMIKgA4ARBB0AjCDoAGAEQQcAIwg6ABhB0AHACIIOAEYQdAAwgqADgBGTfsEFkE13LKlXJJKdB7sVli7L6OeHQkF9/DvewYv0Iei4pUQi0aw8BbEgGMz4E0Ez/R8M3H7YcgEAIwg6ABhB0AHACIIOAEYQdAAwgqADgBEEHQCMIOgAYARBBwAjCDoAGEHQAcAIgg4ARhB0ADCCoAOAETf0+NzWMx3avvuw4omEVtdWasOq6jHne/tj+pc39+vy8GeKJxJq+tPl+uqieRlZMAAgOcegx+Nxbdt1SE8+eLfCQa9+/Mu3VF1ZrrLi0OiYnfuPa/kffUV3LV+izksR/dOv9uirj38jowsHAIzluOVytrNHswsDKg4HlOfxaMXSBTredn7MGJdLGr5yVZI0fOUzhfzezKwWAJCS4xV638CQwgHf6HEo6FP7xUtjxjQ21Ooft+3W3vc/0GdXr+nJB+9O+lnNR0+ruaVNkrS5sV5lJUWTWTtuUwXBoJl5svWzwI5YLJby3IReQedyjT0+cqpd9dWLtH7lnfq4o1v/vKNZW7c0yf2lgQ11VWqoq5Ik9UejGX/FF2zKxvcmG6+gk7Lzs8AWjzv1xorjlks44FXfwODocSQ6OG5L5cCxD7V86QJJ0h3zZuvatRHFhoYnul4AwAQ4Br2irEjdvVH1RAZ0bWRER061q3rx/DFjCoM+/a69U5L0SU9EV6+NKODNz8yKAQBJOW65eNxuPXDPSr24bY/i8YRW1SxSWXFYO/a1qGLOLNUsLteffW2FXtt5UO8ePiWXpIc3rpHry/syAICMco0MXUjkYmL2DjERhaXL1NvVmvF5srGHnq2fBbZ43G75/f6k57hTFACMIOgAYARBBwAjCDoAGEHQAcAIgg4ARhB0ADBiQs9yAXKluKlRa95pzvUy0qK4qTHXS4Ax3FiEWwo3FuF2x41FAHAbIOgAYARBBwAjCDoAGEHQAcAIgg4ARhB0ADCCoAOAEQQdAIwg6ABgBEEHACMIOgAYQdABwAiCDgBGEHQAMIKgA4ARBB0AjCDoAGAEQQcAI27oJdGtZzq0ffdhxRMJra6t1IZV1ePGvH+qXW81H5NLLs0rCeuRprvSvlgAQGqOQY/H49q265CefPBuhYNe/fiXb6m6slxlxaHRMb/vjeqdgyf0zEP3yTdzhqKDlzO6aADAeI5bLmc7ezS7MKDicEB5Ho9WLF2g423nx4zZ39Kmu5YvkW/mDElS0DczM6sFAKTkeIXeNzCkcMA3ehwK+tR+8dKYMd29/ZKk5/91pxLxuBrX1mrZwnnjPqv56Gk1t7RJkjY31quspGhSi8ftqSAYNDNPtn4W2BGLxVKeu6E99C9zucYex+MJdfdG9fS3NqhvYFB//8rb2rrlfnnzZ4wZ11BXpYa6KklSfzSq/mh0ItPjNpeN701BMJiVefh3ADfL4069seK45RIOeNU3MDh6HIkOKuT3jhkTCnhVs3i+PB63ikIBlcwqUHfvwCSWDAC4WY5BrygrUndvVD2RAV0bGdGRU+2qXjx/zJjaqnK1neuSJMWGhtX9ab+KQv7MrBgAkJTjlovH7dYD96zUi9v2KB5PaFXNIpUVh7VjX4sq5sxSzeJyLb1jrk593Km//fkbcrtc+vq6P5Hfm5+N9QMA/p9rZOhCIhcTs3eIiSgsXabertaMz5ONPfRs/SywxeN2y+9PvgPCnaIAYARBBwAjCDoAGEHQAcAIgg4ARhB0ADCCoAOAEQQdAIwg6ABgBEEHACMIOgAYQdABwIgJveACyKXC0mW5XkJahEK8rQjpRdBxS8nW0wl5EiJuRWy5AIARBB0AjCDoAGAEQQcAIwg6ABhB0AHACIIOAEYQdAAwgqADgBEEHQCMIOgAYARBBwAjCDoAGEHQAcCIG3p8buuZDm3ffVjxREKrayu1YVV10nH/88FZ/eL1/9bffKdRFWVFaV0oAOD6HK/Q4/G4tu06pMc3rdePvtekI63t6rwUGTdu+MpV7T3ygRYQcgDICcegn+3s0ezCgIrDAeV5PFqxdIGOt50fN27HvqO6u/5O5eV5MrJQAMD1OW659A0MKRzwjR6Hgj61X7w0Zsz5rk/VFx1SdeV87fntyZSf1Xz0tJpb2iRJmxvrVVbC1TymroIgr4jD1BOLxVKem9Ar6FyuP/w6nkjo13sO6+GNaxz/uYa6KjXUVUmS+qNR9UejE5keyAq+n5iKPO7UGyuOQQ8HvOobGBw9jkQHFfJ7R4+vXLmqzksR/cNruyRJ0dhlvfTrd/XoN9bxP0YBIIscg15RVqTu3qh6IgMKBbw6cqpdjzStHT0/M3+6XnjqL0aPX3j1bf35uhXEHACyzDHoHrdbD9yzUi9u26N4PKFVNYtUVhzWjn0tqpgzSzWLy7OxTgCAA9fI0IVELiZmfxJTWWHpMvV2teZ6GcA4Hrdbfr8/6TnuFAUAIwg6ABhB0AHACIIOAEYQdAAwgqADgBEEHQCMIOgAYARBBwAjCDoAGEHQAcAIgg4ARhB0ADCCoAOAEQQdAIwg6ABgBEEHACMIOgAYQdABwAiCDgBGEHQAMIKgA4ARBB0AjCDoAGAEQQcAIwg6ABhB0AHAiLwbGdR6pkPbdx9WPJHQ6tpKbVhVPeb8fx1q1f5jbfK43fJ78/VQ42rNKvBnZMEAgOQcr9Dj8bi27Tqkxzet14++16Qjre3qvBQZM2Z+SaGe+6uN+uF371fdkgq9/u77GVswACA5x6Cf7ezR7MKAisMB5Xk8WrF0gY63nR8zpuorczR92ucX+wvmFqtvYCgzqwUApOQY9L6BIYUDvtHjUNB33WAfOPah7lw4Nz2rAwDcsBvaQ/8ylyv57x86cUbnP+nRU9++N+n55qOn1dzSJkna3FivspKiiUwPZEVBMJjrJQDjxGKxlOccgx4OeNU3MDh6HIkOKuT3jhv3QXun3j5wXE99e4Om5XmSflZDXZUa6qokSf3RqPqjUcfFA7nC9xNTkcedemPFcculoqxI3b1R9UQGdG1kREdOtat68fwxY853fap/2/mevv/NdQr6Zk5+xQCAm+Z4he5xu/XAPSv14rY9iscTWlWzSGXFYe3Y16KKObNUs7hcr7/7vq5cvapf/GavJKmwwK9Hv7ku44sHAPyBa2ToQiIXE/PHWUxlhaXL1NvVmutlAON43G75/cnv8+FOUQAwgqADgBEEHQCMIOgAYARBBwAjCDoAGEHQAcCICT3LBbiV/OSnP9PzL7x00/9cYemyGx77g6cf1bPPPHbTcwDpxI1FQBIFwSDfUUxJ3FgEALcBgg4ARhB0ADCCoAOAEQQdAIwg6ABgBEEHACMIOgAYkbM7RV2S3Nd52SmQS5909yjgzc/1MoBx4vF4ynM5C3owGMzV1ICjF3+1V889sjHXywBuCpfIAGAEQQcAIwg6kETDHy/O9RKAm5azpy0CANKLK3QAMIKgA4ARvLEI+IJX3tyvEx91KODL19YtTbleDnBTuEIHvqC+ZpGe2LQ+18sAJoSgA19QWV4q78zpuV4GMCEEHQCMIOgAYARBBwAjCDoAGMGdosAXvPzGPrWd61Ls8rCCvpnauLZWq2t5DABuDQQdAIxgywUAjCDoAGAEQQcAIwg6ABhB0AHACIIOAEYQdAAw4v8AF5Kk+dWvcWsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = plt.boxplot(lstm_errors)#, histtype='step')\n",
    "# plt.ylabel('Frequence')\n",
    "# plt.savefig(\"imgs/lstm_rnn_approx_error.pdf\", bbox_inches='tight', dpi=900)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD4CAYAAAAD6PrjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAca0lEQVR4nO3deXhU5b0H8O/MhCwzmclM9kwWCGSBQBbCYhYCV5FNg0SLYm1dWlCrtWpp7bXUpbd68VZrveW22vXWWisVigtSZBG5GNmRQCCBBEMCZCH7ZGayJ2fuH9gDQ7aTZc4s+X6eh+fJmfPOzG/mBb45y/u+it62SzYQEREBUDq7ACIich0MBSIiEjEUiIhIxFAgIiIRQ4GIiERezi5gNFrMZmeX4FAKhQI2G28Oc1fsP/fl6X2nAKDT6frdxyMFF6bTap1dAo0C+899eXrfKZUD/9fPUCAiIhFDgYiIRAwFIiISMRSIiEjEUCAiIhFDgYiIRAwFIiISMRSIiEjk1iOa3d1/vfIbvPzq6yN+/o9+8Ciefuq7Y1gREY13CndeZMfTp7kIDJ+OpstFzi6DRihAp/P4v6OeytP7TqVUwt/fv999PH1EREQihgIREYkYCkREJJLlQvNbH32OU19WQqvxxXMP5fXZf/h0GXYdPA0A8JnghXuWZSIqLFCO0oiI6BqyHClkpsbhe3cvGnB/sF6Ltd9cimcfXIFb5qXi7e0H5CiLiIiuI8uRQnxMOBpMlgH3T4kKFX+OjQxBs7lNjrKIaJwa7e3ggOfeEu5y4xT2nzyHGVMiB9yff7wE+QWlAIAHcjNhDAuWqzSnCBhgdSRyD+w/1/TSCz/GSy/8eMD9KnU0etsuyViRvKxW64D7XCoUSipqcODEOfzwvmUDtslJT0ROeiKAK+MUPPleYsDzx2J4Mk+/193TeXLfqdxh5bXK2ib89Z8H8MidN8Ff7evscoiIxiWXCIWmFit+t2UvvrUiB2FBAc4uh4ho3JLl9NEf39+H0guXYW3vwNMbNmH5/DT09goAgPmzpuKf+SfR2t6JjR8fBHBlUel1q5fLURoREV2Dcx+5MM595N54TcF9efq/Pc59REREkjAUiIhIxFAgIiIRQ4GIiEQMBSIiEjEUiIhIxFAgIiIRQ4GIiEQMBSIiEjEUiIhIxFAgIiIRQ4GIiEQMBSIiEjEUiIhIxFAgIiIRQ4GIiEQMBSIiEjEUiIhIxFAgIiIRQ4GIiEQMBSIiEjEUiIhI5CXHm7z10ec49WUltBpfPPdQXp/9NpsNm3YdwemySnhP8ML9ufMQExEkR2lERHQNWY4UMlPj8L27Fw24/3RZFeqazPjZI3fgG7dk4p0dB+Uoi4iIriNLKMTHhEPt5z3g/sLSi8hImQKFQoHJkaFo7+hCi6VNjtKIiOgaspw+GorJ0gaDTiNu63UamCxtCNCq+7TNP16C/IJSAMADuZkwhgXLVqczBOh0zi6BRoH95748ue+sVuuA+1wiFGy2fh5U9N82Jz0ROemJAIAWsxktZrPjCnMBnv75PFmATsf+c2Oe3Hcq5cAniVzi7iODTo1mc6u4bTK3Qu/f9yiBiIgcyyVCISU+GocKy2Cz2XC+qg6+Pt79njoiIiLHkuX00R/f34fSC5dhbe/A0xs2Yfn8NPT2CgCA+bOmYkZcFE6XVeHZ19+D9wQV7s+dJ0dZRER0HVlCYc3tCwbdr1Ao8PWlGXKUQkREg3CJ00dEROQaGApERCRiKBARkYihQEREIoYCERGJGApERCRiKBARkYihQEREIoYCERGJXGKWVCKisTJt5iLU1lSP+nUCw6eP6vlhEUacKdg96jrkxlAgIo9SW1ONhFcKnV0GSp9KcXYJI8LTR0REJGIoEBGRiKFARESiIUNBEAS8+teP0d3TK0c9RETkREOGglKpRKPJClu/CykTEZEnkXT66NacNLyz4yAaW6wQBAGCzSb+ISIizyHpltS3/7kfAHD41PmrD9psgEKBN9bd75DCiIhIfpJC4cXHVjq6DiIicgGSQiEowB8AINhssFjbofX3g1KhcGhhREQkP0mh0N7Zhb/vPIxjReUQBAEqlRKzk2KxavEN8PP1dnSNbolD7YnIHUkKhXd3HUZXVw+ee2gFAgP80dRixYf/dxzv7jqMB27LcXSNbolD7YnIHUm6+6i4rArfWpGDsKAATPBSISwoAPctn4ei81WOro+IiGQk6UjBy0sFS1uHeG0BAKxtHfBSqSS/UVFZJTbtOgLBZkN2WjyWZtn/BtvUYsWbH32O9o4uCDYb8m6cheS4KMmvT0REoycpFOalJeBX7+zCzXOTxNNHe44UI2dmgqQ3EQQBG3ccxhP3LIZBp8ZL/7sNKfExMIboxTbbPy/ErGmTsGDWVFTXm/Drd3cj+bE7R/apiIhoRCSFwrLsFAT4q3G06DxM1jbo/dVYnDkDWanxkt6koroBoYFahBi0AIA5SbEoLL1oFwoKBdDR2Q0A6Ojsgt5fPdzPQkREozRkKAiCgG35J7EsOwXZadJC4HrNljYYtBpxW6/ToLyq3q5Nbk4afrVxF/YeO4Ou7h48cc/ifl8r/3gJ8gtKAQAP5GbCGBY8oprGkwCdztkleJz/ePGX+Nn610b1Gs+t+z6ef2btGFVErshV/+1ZrdYB9w0ZCkqlEvu+OIvc+WljWtT1wxyOFpcjMyUOizJm4HxlHf68NR/PPZTXZzxETnoictITAQAtZjNazOYxrcsT8Tsae08+vgZPPr5m0DaB4dPRdLlo0DbsG8/mqv2rUg58j5Gku48ykqfgsy9KRlyAQatGs6VV3DaZW/ucHtp/4hxmJcUCACZHhaKnpxfWto4RvycREQ2fpGsKFdUN2HvsDHYdOoVAnQbA1d/ef3jfsiGfP9EYjLomMxpMFui1ahwtLsfqvPl2bQJ1Gpwtr0ZWajxqGkzo7umFVu07vE9DRESjIu3uo5kJmCfxTqP+qJRKrFqSgQ0bd0MQbMhKjYMxxICt+wowMSIIqQkx+NrNc/D29gPYc6QYCgD3L58HBafSICKSlaQLzfXNFizLTsEEL+njEq6XHBfVZ9zBbQtmij8bQ/T40f23jPj1iYho9CQtsrPvi7NQqbhyJxGRp5PlQjMREbkHWS40ExGRe5DlQjMREbkHSaGQmRLn6DqIiMgFDBoK7+48jFVLbhC3958oRXba1SOG3/1jLx5eeaPjqiMiGqaQvFx4hR5wdhkIyct1dgkjMmgoHCz80i4Utuw5ZhcKZ8pHv7IYEdFYqv9gGwzZ651dBuo/+A7w2587u4xhG/TuIxtsctVBREQuYNBQUIAjiomIxpNBTx/1CgJKKmrE4wVBsNlv23gkQUTkSQYNBa3GF29t2y9ua/x87LY5YR0RkWcZNBTWczlMIqJxhRMaERGRiKFAREQihgIREYkYCkREJJIcCta2Dhw6VYadB08BAEyWNjSbW4d4FhERuRNJoVB64TKe/+37OHK6DNs/PwkAqGsy452PDzq0OCIikpekUNi8+wgevH0BHv/6YqiUV54SGxmMipoGhxZHRETykhQKjS1WTI012j2mUqkgCBzRTETkSSSFQniwHkVlVXaPnS2vhjFE75CiiIjIOSQtsrPy5tn4zbt7kBwXhe6eXvxt+wEUnruER+5c6Oj6iIhIRpJCYXJkKJ5ZcxuOFJ1Hlnc8DDo1nv5WLgw6jaPrIyIiGUkKhUuXGxEdHoQlmckjfqOiskps2nUEgs2G7LR4LM1K6dPmWHE5tuWfgAIKRIUZsDpvwYjfj4iIhk9SKPxq4y74q30xd/pkzJ0xGcF67bDeRBAEbNxxGE/csxgGnRov/e82pMTH2F2TqG0yY+eBU3jqvlug8fOBubV9eJ+EiIhGTVIovPzEKhSVVeFocTle/MNWRIToMXf6ZMxKmgSdxm/I51dUNyA0UIsQw5UwmZMUi8LSi3ah8HlBKRbMmgqNnw8ASHpdIiIaW5JCQalUIjk+Gsnx0ejq7sHJ0ov47HgJtuw5il8/fd+Qz2+2tMGgvXr9Qa/ToLyq3q5NXVMLAODlv2yHTRCQOz8N06dE9Xmt/OMlyC8oBQA8kJsJY1iwlI8wrgXodM4uYdzidz++uWr/W63WAfdJCoV/6e7pwakvK3GsuAIXahoRFx024qIU1630KQg21DWZ8YNvLkWzpRW/eOtjPPfQCqh9feza5aQnIic9EQDQYjajxWwecQ3jBb8j5+F3P765av//axByfySFwqkvK3G06DwKSy8hIjgAs6fH4p5lGQjwV0sqwKBVo9lydZ4kk7kV+uueq9eqERsZApVKiWC9FmFBAahrsmCS0ef6lyMiIgeRFApb9hzFnKTJ+MmaNIQYhn84NNEYjLomMxpMFui1ahwtLsfqvPl2bdISY3C0qBxZqfGwtnWgrrEFwXr/Yb8XERGNnKRQ+OnDt4/qTVRKJVYtycCGjbshCDZkpcbBGGLA1n0FmBgRhNSEGCRNjkTx+Wr89HfvQ6lQ4I6Fs+HPNaCJiGQ1YChs//wkbpmXCgDYuq9gwBe4bcFMSW+UHBeF5Dj7C8fXPlehUODORXPBVaGJiJxnwFAwWdrEn7luAhHR+DBgKNyzLFP8+f7l82QphoiInEvSLKlrX32n38efeu3vY1oMERE5l6QLzb2C0PexXgGCjespDCQkLxdeoQecXQZC8nKdXQIRuZFBQ+EXb20HoEB3Ty9+8dbHdvtMllZMjgxxZG1urf6DbTBkr3d2Gaj/4DvAb3/u7DKIyE0MGgrZaQmw2Wy4UNOA7LR48XEFAK3GD1MnRTi6PiIiktGgoZCZEgcAmBwZgvBgrrJGROTpJF1TCA/Ww2xtR0V1A6ztHbj2UsK1RxBEnmLazEWorake9esEhk8f8XPDIow4U7B71DUQDYekUDhRcgF//jAfoYE6VNebYAzRo6q+GXHRYQwF8ki1NdVIeKXQqTWUPtV3ISoiR5MUClv3FeC+5fMwa9okrH31HfxkzW04cPIcqutNjq6PiGhYwiKMLhGoYRFGZ5cwIpJCoamlFbOmTbJ7LCMlDv/+3+9i5c1zHFEXEdGIjMUpt8Dw6Wi6XDQG1bgfSYPXtBpfmK1XlscMCvDH+co61DdbOE6BiMjDSDpSmJeWgC8ra5E+dRIWzk3CL9/eAYVCgUU3jPwiGhERuR5JobAkK1n8OSMlDgkTw9HZ3YMI3qZKRORRhrUc578EBnDxGyIiTzRgKPx4w6a+Cyn346XvcQUEIiJPMWAofGvF/IF2ERGRhxowFBImhstZBxERuQBJ1xS6e3rxz/yTOFZ8Htb2Tvz3D7+B4vNVqG0048Y50xxdIxERyUTSOIXNu4+gur4Z314xH/+6yhARrMdnx0scWBoREclNUiicKLmI1XnzMTkqFIqvLj4bdBqYLFy7mYjIk0gKBS+VEoJgP3rZ0toBjZ+vQ4oiIiLnkBQK6dMm4c2P8tFgsgAAWixt+PvOQ5idFCv5jYrKKvH8G+/h2de3YMeBgWef/OJMBb7zn2/iQnWD5NcmIqKxISkU8m5MR1CAP174/Ydo7+jCc2+8hwCtGrfmpEp6E0EQsHHHYTx29yI8/3AejhaV9zvDakdnN/YePYNYY/DwPgUREY0JSXcfealUuGvxDbhr8Q2wtHbAX+0jXluQoqK6AaGBWoQYtACAOUmxKCy9CGOI/TQZW/cdx+LMGdh96PQwPgIREY0VSUcK19JqfKFQKFBZ24Tfb9kr6TnNljYYtBpxW6/ToNnSZtfm4uVGNJvbkBIfPdySiIhojAx6pNDV3YMd+wtxqbYJoYE65M5Pg7WtE//45CjOllcj46s1nEfi2gMNwWbD5t1HcP/yeUM+L/94CfILSgEAD+RmwhjGU01DCdDpnF0CjRD7znk8+bu3Wq0D7hs0FDbuOIRLlxuRNCUSRWVVqKpvRm1DCzJS4nDvrVnwV0u7+8igVaP5mttXTeZW6P3V4nZnZzeq60345ds7AABmazte37wHj965EBOvu76Qk56InPREAECL2YwWs1lSDeMZvyP3xb5zHk/+7lXKgU8SDRoKxeer8JM1t0Gn8cONs6dh3f9sxtp7lyE+JmxYBUw0BqOuyYwGkwV6rRpHi8uxOu/q3Ep+vt54de3Xxe1X//oxVi6c0ycQiIjIsQYNhc6uHug0fgCuDFbz8Z4w7EAArqTSqiUZ2LBxNwTBhqzUOBhDDNi6rwATI4KQmhAzsuqJiGhMDRoKvYKAkooaXDts7frtqZMiJL1RclwUkuOi7B67bcHMftv+4N5lkl6TiIjG1qChoNX44q1t+8VtjZ+P3bZCAbz43ZWOq46IiGQ1aCisf4wL6BARjSfDHqdARESei6FAREQiSdNcEI03IXm58Ao94PQaiOTGUCDqR/0H22DIXu/kGr4D/PbnTq2Bxh+ePiIiIhFDgYiIRAwFIiISMRSIiEjEUCAiIhFDgYiIRAwFIiISMRSIiEjEUCAiIhFDgYiIRAwFIiISMRSIiEjEUCAiIhFDgYiIRAwFIiISMRSIiEjEUCAiIpFsK68VlVVi064jEGw2ZKfFY2lWit3+Tw4X4fMTpVAplfBX++K+3GwEBfjLVR4REUGmIwVBELBxx2E8dvciPP9wHo4WlaO63mTXJjosEOu+vRzPPrgC6VMn4r09x+QojYiIriFLKFRUNyA0UIsQgxZeKhXmJMWisPSiXZvESRHwnnDlwCU2MgTNljY5SiMiomvIcvqo2dIGg1Yjbut1GpRX1Q/Yfv+Jc5gxJbLfffnHS5BfUAoAeCA3E8aw4LEt1gMF6HTOLoFGiH3nPJ783Vut1gH3yXZN4XoKRf+PHz5Vhos1DVh777J+9+ekJyInPREA0GI2o8VsdlSJoxIWYUTpUylDN5ShDlf9jmho7Dvn8eTvXqUc+CSRLKFg0KrRbGkVt03mVuj91X3anSmvxsf7C7H23qWY4KWSozSHOVOwe9SvERg+HU2Xi8agGiIiaWS5pjDRGIy6JjMaTBb09PbiaHE5UhKi7dpcvNyIv20/iEfuWgidxk+OsoiI6DqyHCmolEqsWpKBDRt3QxBsyEqNgzHEgK37CjAxIgipCTF4b88xdHZ34w9b9gIAAgP88ehdC+Uoj4iIviLbNYXkuCgkx0XZPXbbgpniz09+Y4lcpRAR0QCcdqGZyJW5wo0CYRFGp74/jU8MBaJ+8EYBGq849xEREYkYCkREJGIoEBGRiNcUiGjc+a9XfoOXX3190DaB4dMH3f+jHzyKp5/67liW5RIUvW2XbM4uYqQ8eRg6wAuV7o79574CdDqP/v9FpVTC37//pQl4+oiIiEQMBSIiEjEUiIhIxFAgIiIRQ4GIiEQMBSIiEjEUiIhIxFAgIiIRQ4GIiESc5oJoBKRMkwAMPlWCp06TQO6N01y4ME6T4N48faoET+bpfcdpLoiISBKGAhERiRgKREQkYigQEZGIoUBERCLZbkktKqvEpl1HINhsyE6Lx9KsFLv93T29eHNrPi5eboTGzwdrbl+AYL1WrvKIiAgyHSkIgoCNOw7jsbsX4fmH83C0qBzV9Sa7NvtPnIPa1xsvPPo1LJybhPc//UKO0oiI6BqyhEJFdQNCA7UIMWjhpVJhTlIsCksv2rUpPHcRmSlxAID0aZNwtqIGNpvbDqEgInJLspw+ara0waDViNt6nQblVfV2bUyWNhh0V9qolEr4+Xijtb0T/mpfu3b5x0uQX1AKAHggNxPGsGAHV+84//HiL/Gz9a8N2mawEbHPrfs+nn9m7ViXRWMoQKdzdgk0Qp7cd1ardcB9TpvmQqGw35Z6UJCTnoic9EQAV0Y0u/OowycfX4MnH18z4H4poyrd+fN7Ok8fFevJPL3vVMqBTxLJcvrIoFWj2dIqbpvMrdD7q+3b6NRoNl9p0ysIaO/sgsbPR47yiIjoK7KEwkRjMOqazGgwWdDT24ujxeVISYi2a5MSH42DhV8CAI6fqUDipAgorj+cICIih5Ll9JFKqcSqJRnYsHE3BMGGrNQ4GEMM2LqvABMjgpCaEIPstHj8+cN8PPv6Fqh9r9ySSkRE8uIsqS7M089rejr2n/vy9L7jLKlERCQJQ4GIiEQMBSIiEjEUiIhI5NZrNCsAKAcZhOHuauoaoL1uRDe5D/af+/L0vhMEYcB9bh0KOg8ehg4AG97di3Wrlzu7DBoh9p/7Gs9957m/ZhMR0bAxFIiISMRQcGE5MxOcXQKNAvvPfY3nvnPrEc1ERDS2eKRAREQihgIREYnc+pZUd/DI+r8gMkQvbs+eHoulWSl49a8fY+XCOZhodN+V48Yrs7Udmz85gvKqeqh9faBSKbE4YwbUft54Y/OnCNFr0dXTg+S4aKy8eQ4A4KPPCuDjPQGLM2aIr7Pu15ux7tvL+6wuSGPviZffxq9+9E27xy43tuBv2w+gvaMLPb0C4qLDMHPqRLz/6TEAQH2zBXqtGhO8VIgMC0RWahxee3sn7r01C9lpV645XLzciPV/+gh3LJxt17fujKHgYN5eKjzz4Apnl0FjxGaz4Y1/fIrM5ClYnXdlevfGFisKSy9B7eeN+OgwfHfVzejq7sF//mkr0hJjEBcd5uSqqT+bdh3GwrnTkZYYAwCoqmtGZKgB06dEAkCfX9xKLtQgMtSAY8UVYigcKy5HVKjBOR/AQRgKLuCJl9/GgtlTcba8Bmpfb6z4t3S89+kxNJtbceeiuUhNiEGDyYI3t+ajs6sHAHD30gxMiQpFyYUabPvsBPzVvqiub0ZMeDC+vSKHCxQ5SElFDbxUSsyfNVV8LCjAHzfOmYaSCzXiY94TvBAdFgiTpc0ZZZIELdZ2GHRXV4CMlPCfe6BOg/bObpit7dBqfFFUVoUZcVGOLFN2DAUH6+rpxYt/+FDcXpqdgtlJsXZtOrt7kBATjjtumo03Nn+KrfsK8OQ9S1DTYMKbW/ORmhADncYPT9yzGBO8vFDbZMaf3t8njri8VNuE5x/KQ4BWjVf+sh1llXX87dRBqhtMiA4PGrJda3sn6posiI9hP7iqhXOT8NrfdmJKZCimTTYiKzUOat+hlwBOnzYRX5ypQHR4IGLCg+Cl8qxLswwFB5Ny+shLpRQPWSNDDfBSKaFSKWEMNaCxxQoA6O0V8Pedh3CptglKhQK1TVcXAJlkDIZBpwEARIcFotFkZSjIZOOOQ/jyUi28VErcsXA2zl2qxQt/+BC1jS1YkpmMgK/WIleAR26uJis1HkmTI1FUVoWTpReRX1CCZ9aswAQv1aDPmzVtEv74/j5cbmzBnOmxKKusk6lieXhWxLkplVIpnu5RKACvr/5SKhUKCMKVYSSfHCmCVuOHZx5cgR+vXo7e3qsTWk1QXf1LrFAo0DvIZFc0OsZgPS5dbhS3v740A9//xhJY2joAAPHRYXj2wRV49sEV+Ox4idhW4+eDtvYuu9fq7OqB2tdbvuKpD71Wjey0eDx610KolEpU1zcP+ZwAfzVUSiXOlldj6qQIGaqUF0PBTbR3diPA3w9KhQKHT5VBsHHMoTMkTopAd08v9n1xVnysq7unT7uwoAAsyUrGzoOnAQDxMWEoPHcRHZ3dAICCsxcQFWrw6Fl+XV1RWaX4y1WLtQ3Wtk7oteohnnXF8gUzcftNszyy/3j6yMGuv6YwfUokbr9p9rBfZ8Gsqfj9lr04fqYCCZMi4DOBXecMCoUCj6y8CZs/OYJdB0/DX+0LH28v3NFPn85PT8TuQ0VoMFkQFRaIf5s9Da+8tR0KAFqNH755a7b8H2Cc6uruwdMbNonbN98wHc3mVmzadUQ8Mv/awtni6b6hTIkKdUidroDTXBARkcjzjn2IiGjEGApERCRiKBARkYihQEREIoYCERGJGApERCRiKBARkej/ASWZLd0WNypzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.style.use('Solarize_Light2')\n",
    "_ = plt.boxplot([elman_errors, gru_errors, lstm_errors], patch_artist=True, labels=['Elman', 'GRU', 'LSTM'])\n",
    "plt.ylabel('Relative Error')\n",
    "plt.savefig(\"imgs/elman_gru_lstm_approx_error_sst.pdf\", bbox_inches='tight', dpi=900)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore Ngram Polarity Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import ngrams\n",
    "from collections import Counter\n",
    "texts, labels = zip(*train_dg.data_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1092, 6889, 5]"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0][-3:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 2\n",
    "ngram_label_pair = []\n",
    "ngram_set = []\n",
    "for i, text in enumerate(texts):\n",
    "    label = labels[i]\n",
    "    if len(text)<n:\n",
    "        continue\n",
    "#     ###Find all possible ngrams\n",
    "#     ngram_list = list(ngrams(text, n))\n",
    "#     ngram_set += ngram_list\n",
    "#     label_list = [label] * len(ngram_list)\n",
    "#     ngram_label_pair += list(zip(ngram_list, label_list))\n",
    "    \n",
    "    ###Find the last ngrams\n",
    "    ngram_list = text[-n:]\n",
    "    ngram_set += [tuple(ngram_list)]\n",
    "    ngram_label_pair += [(tuple(ngram_list), label)]\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ngram_label_pair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get_phrase_polarity([16, 9, 39])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'get_phrase_polarity' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-153-3475b4859daa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0mphrase\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0;31m#print(diff)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0mngram_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_phrase_polarity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mphrase\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0mpos_ngram_scores\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mngram_score\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mrate\u001b[0m\u001b[0;34m<\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m0.8\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mdiff\u001b[0m\u001b[0;34m<\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'get_phrase_polarity' is not defined"
     ]
    }
   ],
   "source": [
    "ngram_label_freq = Counter(ngram_label_pair)\n",
    "ngram_set = set(ngram_set)\n",
    "pos_ngram_scores = []\n",
    "neg_ngram_scores = []\n",
    "for item in ngram_set:\n",
    "    item_pos = (item, 1)\n",
    "    item_neg = (item, 0)\n",
    "    diff = ngram_label_freq[item_pos] - ngram_label_freq[item_neg]\n",
    "    sums = ngram_label_freq[item_pos] + ngram_label_freq[item_neg]\n",
    "    rate = diff*1.0/sums\n",
    "    if rate>0.8 and diff>3:\n",
    "        #print(item, diff)\n",
    "        phrase = list(item)\n",
    "        #print(diff)\n",
    "        ngram_score = get_phrase_polarity(phrase)\n",
    "        pos_ngram_scores.append(ngram_score)\n",
    "    if rate<-0.8 and diff<-3:\n",
    "        phrase = list(item)\n",
    "        #print(diff)\n",
    "        ngram_score = get_phrase_polarity(phrase)\n",
    "        neg_ngram_scores.append(ngram_score)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Freq Density')"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEJCAYAAABlmAtYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3gTZdo/8G+SaUiTaZI2gbIcFCi6HASBVrcUEXpYX5RVsOKRi1W7AgrFLR5WQVx91UpVhJVWEXmhHLayZfGEL+q6XawFClrQoiz62gDyE6nWJG3TnJpMJr8/uhlIkyYB2iSd3p/r4qLJTJ65J3cmd+b0PBKv1+sFIYQQEoI01gEQQgiJf1QsCCGEhEXFghBCSFhULAghhIRFxYIQQkhYVCwIIYSExcQ6gJ5y5syZWIdw0ViWhdVqjXUYUUPrK159aV2B3ru+gwYN6nJa1IpFfX09ysvLwfM8cnNzMXv2bL/px44dw5YtW3Dq1CkUFRUhMzNTmGY0GvH666/DZDIBAJYtW4YBAwZEK/SYUSgUvfIDd6FofcWrL60rIM71jUqx4HkeGzduxIoVK6DT6bBs2TJkZGRgyJAhwjx6vR6LFi3C+++/H/D6srIy5OfnY/z48XA6nZBIJNEImxBCyH9EpVgYDAYMHDgQqampAICsrCzU1dX5FQvfnkLnQnD69Gl4PB6MHz8eQEfFJoQQEl1RKRZmsxk6nU54rNPp0NDQENFrz5w5A5VKhVWrVqGpqQnjxo3D3LlzIZX6n5uvqqpCVVUVAKCkpAR6vd5vusPhgNFohEwmA8dx6NevX8Cy2tvbwTAMOI6DTCYDw/i/PR6PBxzHCfOEa4NhGMhkMr/pHMfB4/FE1MbPP/8Mj8cTtA2e58OuS0JCAtxud5dx8DwPqVQKj8cT0IbX64XL5QrZhtvthtfrhVQqBc/zkMvlIdtISEgIyJvb7QbQ8SPh559/DliPc9twuVxQKpXQ6/V+Pyra2tqENnieh1qt9mvD4/GgubkZWq0WLS0tSE5ODlgXi8UCqVQKX+83SUlJAe9XS0tLyDZaW1vBMAx4ng/ahtvtRmtrq9AGwzABn9OWlhbI5XJwHAepVAqWZf2mu1wuWCwWoQ2dThfwA6u5uRkKhQIulwsMw0ClUvlNb29vh9VqhVqtRmtra0AMQMc2q1Qq4XQ6IZfLoVQq/aY7nU7Y7XawLAuLxRKyDYfDAalUGjCPrw2VSgWr1er3HeFjMpnAsixsNhuUSmXAj0WHwwGn04nExETY7XakpKQEtGE0GqFWq2G1WoO2Ybfb4XK5oFAoQrah0WhgsVjAsmzA9mKz2cBxHORyOZxOZ0BuvV4vTCaTkDe1Wh2wvVitVvA8D4Zh4HK5oNVq/abzPA+z2Sy0odFokJCQ4DePb1uQSqXgOA4ajcZveudtQavVBnzPdUUSjb6hDhw4gCNHjuD+++8HANTU1MBgMKCgoCBg3ldffRXp6enCOYuDBw9i3bp1ePHFF6HX67FmzRpMmjQJOTk5IZfZ+QS3w+FAQkJCxG9MPPB9CfcVkawvx3Fwu91ITEyMUlQ9R6lUwm63xzqMqOhL6wr03vUNdYI7KpfO6nQ64eQ00PFrITk5OaLXpqSkYPjw4UhNTYVMJsPVV1+NEydOnHcMvordm/SlQgFEtr7n/nLv7VwuV6xDiJq+tK6AONc3KsUiLS0NjY2NaGpqAsdxqK2tRUZGRkSvHTlyJGw2GywWCwDg6NGjfuc6ItUbT4p3PmQjdpGub2/MZTB96fxbX1pXQJzrG5XDUADwxRdfYMuWLeB5HtnZ2cjPz0dlZSXS0tKQkZEBg8GAVatWwWazISEhAVqtFqtXrwYAfPXVV9i6dSu8Xi9GjBiBhQsXht1L6HwYym63Bxx3Jb2TWHLZlw4z9qV1BXrv+oY6DBW1YhFt4YqFZ/5N3bo82YZdYecZOnQoRo0aBY/Hg5EjR+KVV14JeezddxL8XI888ggWLFiAyy+/HGvXrsWDDz4oTLvpppuwa1f4OLpiNptx++23AwB++eUXyGQy4WTf7t27A07IAcDJkyexYMEC/POf/7zg5foEW99gxFIs9Ho9jEZjrMOIir60rkDvXd+4uCmPdOya+r5UCwsLsXXrVixcuPC82li1apXwd2lpqV+xuJhCAXScH/LF9/LLL0OlUgkXJRASTKQ/ugKvcwsukh9dJDb61kHxOHL11Vfj+++/BwCsX78eOTk5yMnJwYYNGwB0/HqeO3cu8vLykJOTg/feew8AMGfOHBw5cgTPP/88nE4nfvvb36KwsBAAcNlllwEA7r//fvzrX/8SllVUVITdu3fD4/Hg2WefxQ033IC8vDxs27btvGJ+7bXXhDg3bdoUMP3kyZO47rrr8PXXX4PjODz99NOYOXMm8vLy8OabbwLouBLutttuw3333YepU6f6FTtCSPyiPYsY4DgOn3zyCaZPn46vvvoKO3bswP/+7//C6/Xid7/7HSZPnoxTp04hNTUVW7ZsAQDhBL/P8uXLUV5eHvTwz6xZs7Br1y7k5ubC5XJh3759WLlyJbZv346kpCR88MEHaG9vx+zZszFt2jRccsklYWP+8ssv8fbbbwtFZ+bMmcjMzBQOozU0NGDx4sV45ZVXMHr0aGzevBl6vR67d+9Ge3s7brzxRkybNg1Ax0UKn3zyCfR6PW688UYcPnwY6enpF/u2EkJ6EO1ZRJFvT+D666/H4MGDceedd+Lzzz/HjBkzoFQqoVKpcP311+Ozzz7DqFGjUFNTg+LiYnz22WcBN5qFkp2djf3796O9vR2ffPKJ8KX+6aefYufOnfjtb3+L3/3ud2hubsbJkycjavOzzz7DzJkzkZiYCJZlMWPGDHz++ecAOm5Y+sMf/oDXXnsNo0ePBtCxB1FZWSksy2KxCJc8T5o0SbgUeuzYsTh9+vR5vpOEkGijPYsoOvechU9X1xekpaXhww8/xJ49e7By5UpMmzYNS5cujXg5kydPxqeffopdu3Zh1qxZwrTnnnsO06dPv+B1CEatViM1NRV1dXUYOXIkgI71ev755zF16lS/eWtqavxOlPvuNAUQ0cltMemNJ0BJZMSYWyoWMZaZmYmlS5eisLAQXq8XH330EdauXYuffvoJOp0Ot9xyC1QqFXbs2BHw2nO70ehs1qxZ2L59O44cOYI1a9YAAKZNm4atW7diypQpSEhIwPHjx/GrX/0qoiuLfvOb3+Cxxx7D/fffD57n8Y9//APr1q0DAMjlcpSXl+POO++ESqXCTTfdJCxr8uTJYBgGBoMBgwcPDrkMmUwGj8cTydsmChqNBq2trVFdZndfBUiCi0Vue1qfLRbxctXFuHHjcOutt2LmzJkAgDvvvBNXXHEFqqur8dxzz0EikSAhIQErV64MeK3vBPi4ceNQVlbmN23atGn44x//iOuuu074JX/XXXfhhx9+wIwZM+D1epGSkhL0RHUwEydOxOzZs4U4f//732P06NHCYSyVSoUtW7bgzjvvhFKpxLx583DmzBlcd911ADru4i8vLw+5jL5UKIDA81BEPMSY2z57n0Vv0Ftv7LlQka5vb8xlMElJSULHb9ES73sW8fIj7mLFIrfdIeZ9Q5ELQ919iFuw3oKJOIgxt31r6ySEEHJBqFgQQggJi4oFIYSQsKhYxLG+dnVQX1tfm80W6xBIDxFjbqlYEBIjfe0mxL5EjLnts/dZzKr4tlvbe2/uqLDzDB48GAsWLMBTTz0FAHj99ddhs9nw8MMPB51fIpF0eYd3KL216/ILXd/eSi6XC2OQE3ERY25pzyKK+vXrhw8//BBmszmi+S/0HovS0lK/x93Vdfk///lPzJs3D/PnzxceBysUF6ov3VMCdPQVRsRJjLmlYhFFMpkMc+fOxRtvvBEwzWQyYf78+bjhhhtwww03oK6uDgzDwGQy4Y477sB//dd/4U9/+hOuvvpqodgUFBRgxowZyM7Oxl//+lcA6NVdl99+++19quvySMehJ72PGHMbtWJRX1+PP/7xj1iyZAnefffdgOnHjh3DY489hjvuuAMHDx4MmG6327Fw4UJs3LgxGuH2mHvuuQfvvPNOQHcAf/7znzF//nx88MEH2LBhAx555BEAwOrVqzFlyhT84x//wPXXX48ff/xReM3LL7+Mjz76CB988AE2bdoEs9mM5cuXCx0Wdu4CxNd1OQCh6/KcnBy/rst3796NN998E//v//2/iNbn3K7Ld+3ahS1btuDYsWPC9IaGBixcuBCvvPIKxo0bh7/+9a9C1+W7d+/G5s2bhXX6+uuvUVxcjOrqahgMBhw+fPj832BCSI+IyjkLnuexceNGrFixAjqdDsuWLUNGRgaGDBkizKPX67Fo0SK8//77QduorKzEmDFjohFuj0pKSsKcOXOwceNGvyFV9+7di++++054bLVaYbVa8fnnnwsFMjs7G1qtVphn06ZN+PDDDwF0dG9y8uRJ4VxCMNnZ2XjyySfR3t6O6upqv67Lv/nmG+zevRsA0NbWhpMnT0Y0zsW5XZcDELounzZtmtB1+aZNm4TeaGtqatDQ0CAM5tTW1iZ0XZ6eno7U1FQAELoup3EuCIkPUSkWBoMBAwcOFL4IsrKyUFdX51csBgwYAKDjJGdnJ06cQGtrKyZMmIDjx49HI+Qedd9992HGjBnCSWOgo6Du2rXLr4AwDNPlCd/a2lrs3bsX77//PhITEzFnzhy0t7eHXG5v7bqcEBJ7USkWZrMZOp1OeKzT6dDQ0BDRa3mex9atW1FYWIijR492OV9VVRWqqqoAACUlJdDr9X7TfR0L9lTnfAzT8VZyHCcsQyqVBvR3JJVKkZycjJtuugnbt2/HXXfdBYZhMH36dGzduhWLFy8Gx3E4duwYxo0bh9/85jfYvXs3lixZgurqarS0tEAikaCtrQ1arRaJiYk4efIkvvjiC8hkMjAMI3RdrlAohHsXfPHl5+ejoqIC9fX1+Mtf/gKpVIpp06Zh27ZtmDZtmtB1eWpqKhQKhdBtuG9dfP8YhoHH40FmZqbQdblUKsXHH3+M9evXg2EY9OvXD+Xl5bjrrrugUqlw8803IycnB9u2bcPUqVPBMAz+7//+D4MHDxZ+JPjilEqlkMlkwvLP/b9fv35++fV12CaRSMDzfMBAUR6PB83NzdBqtWhpaUFycjJkMpnfPBaLBVKpVCjOSUlJftM5jkNLS0vINlpbW8EwjPD56tyG2+1Ga2ur0AbDMAGf05aWFsjlcuFzxLKs33SXywWLxSK0odPpAn5gNTc3Q6FQwOVygWEYqFQqYVqkY2HHSuf3w2QygWVZ2Gw2KJVKKBQKv+kOhwNOpxOJiYmw2+1B96yNRiPUajWsVmvQNux2O1wuFxQKRcg2NBoNLBYLWJYN6PvJZrOB4zjI5XI4nc6A3Hq9XphMJiFvarU64OIQq9UKnufBMAxcLpffUQSg47vQbDYLbWg0moDhCXzbgu/Hlkaj8ZveeVvQarXCNhdOVIpFsF/HwfYggvn4448xceLEgA9RZ3l5ecjLyxMedx58xJdI34YcyaWu5+PcX8G+ZfA8H1CYfI8XLFiATZs2ged5cByHZ555BsuXL8f06dPBcRx+85vf4IUXXsDSpUuxaNEivPvuu8jMzERqaiqUSiWmT5+Obdu2IS8vDyNGjMCkSZPg8XjAcVzQrst98V1zzTUoLCzEddddh4SEBPA8L3RdnpeX59d1+bnFxrcuvn++9iZMmIBZs2YJXZfPmzcPl19+OU6ePAmv1wuVSoXNmzcLXZf7lpWTkwPgbNflXq8XXq9XaJfneXg8HmH55/7f3t4ecnCZrqa1tLQA6PgyDaervbRwbZx7uWS4Nn766aeg08/9LHV1VY2vDZPJFHS676Ywt9sNh8MRdJ54FCx3vi9A36HZYHzPd5V73znCi2nDNz5FW1tblz3K+vLfVW59eQvVhbnL5QoZh6+NSMbLCNeG73+fUL3ORqWL8u+++w5///vf8cQTTwAA3nnnHQDAzTffHDDvq6++ivT0dGRmZgLouGfgm2++gVQqhdPpBMdxuO666zB37tyQyxRDF+UymQx2u13YYzh06BCWLVt2Xvcu9CaRDn7UG3MZjO/XXTRRF+XREYvcdodQxSIqexZpaWlobGxEU1MTUlJSUFtbG/GlkefOV11djePHj4ctFGLh8Xjw448/CqPTyeVyvPTSS7EOq8f0te4+euOXCYmMGHMblWIhk8lQUFCA4uJi8DyP7OxsDB06FJWVlUhLS0NGRgYMBgNWrVoFm82Gw4cPY8eOHVi9enU0wotbUqkUI0aMwMcffxzrUKKirw32pFarRTmiGhFnbvvMSHk2m83vRF9vwDBMr74iyGDq/mPlI3WJvTKXwej1+pDnXnoCHYaKjljktjvQSHmgSzHFwneVECEkuvpMR4IKhQJOpxPt7e0RX4kVa/369Qt770Q8+7/G8FdrnA+vFxis1AZc+kgI6Xl9plhIJBK/G956g966K+vzxpHIugw5H78b96tub5MQEh7tz8exrq4HJ+JA+RUvMeaWikUc60tXBvVFlF/xEmNuqVjEsUhvwye9E+VXvMSYWyoWccx32z8RJ8qveIkxt1Qs4ljnjsSIuFB+xUuMuaViQQghJCwqFoQQQsKiYkEIISQsKhZxTIyX35GzKL/iJcbcUrGIY2azOdYhkB5E+RUvMeaWikUcE+MVFeQsyq94iTG3VCzimBgHUCFnUX7FS4y5pWIRxzoPtk7EhfIrXmLMLRWLOJaQkBDrEEgPovyKlxhzS8WCEEJIWFHr7aq+vh7l5eXgeR65ubmYPXu23/Rjx45hy5YtOHXqFIqKipCZmQkA+P7777FhwwY4HA5IpVLk5+cjKysrWmETQghBlIoFz/PYuHEjVqxYAZ1Oh2XLliEjIwNDhgwR5tHr9Vi0aBHef/99v9fK5XIUFhbiV7/6FcxmMx5//HFceeWVohiDmRBCeouoFAuDwYCBAwciNTUVAJCVlYW6ujq/YjFgwAAACBjy9NwBxFNSUqDRaGCxWPpEsWhra4t1CKQHUX7FS4y5jUqxMJvN0Ol0wmOdToeGhobzbsdgMIDjOKHonKuqqgpVVVUAgJKSEuj1er/pTqcTdrsdLMvCYrEETPfFqVQq4XA4oFAoAoZh9bWhUqlgtVr91snHZDKBZVnYbDYolcqA8aIdDgecTicSExNht9uRkpIS0IbRaIRarYbL5QLLsgFt2O12uFwuKBSKkG34CivLsujXr5/fdJvNBo7jIJfL4XQ6kZyc7Dfd6/XCZDJBq9WipaUFarUacrncbx6r1Qqe58EwDFwuV1SuLddqtQFjBfg2TIlEAp7noVar/aZ7PB40NzcL65KcnAyZTOY3j8VigVQqhdfrBQAkJSX5Tec4Di0tLSHbaG1tBcMwwt27ndtwu91obW0V2khKSgqYp6WlBXK5HBzHQSqVgmVZv+kulwsWi0VoQ6fTBfzAam5uhkKhgMvlAsMwfj+sfkZ867xdduf2ZLVag7bRE9uTRqPxu4u7O7YnnudhNpuFNjQaTcCJdN+2IJVKwXFcwFVZnbeFYNtTVyRe39bRgw4cOIAjR47g/vvvBwDU1NTAYDCgoKAgYN5XX30V6enpwjkLn+bmZjz99NNYvHgxLr/88rDLPHPmTPcEH0OJiYlwOByxDuOCzar4ttvbfG/uqG5vM1ZikV/P/JuiurzzJduwK9YhdIveuu2eeySns6hcDaXT6WAymYTHJpMp4JdsKHa7HSUlJbjjjjsiKhRiwXFcrEMgPYjyK15izG1UikVaWhoaGxvR1NQEjuNQW1uLjIyMiF7LcRxWrVqFa6+9FpMnT+7hSOOLGG/sIWdRfsVLjLmNyjkLmUyGgoICFBcXg+d5ZGdnY+jQoaisrERaWhoyMjJgMBiwatUq2Gw2HD58GDt27MDq1atRW1uLb775Bm1tbaiurgYALF68GMOGDYtG6IQQQhClcxaxIIZzFnq9HkajMdZhXLCeOGfR3WJ5DiQW+aVzFtHRW7fdmJ+zIIQQ0rtRsYhjHo8n1iGQHkT5FS8x5paKRRxrbm6OdQikB1F+xUuMuaViEcfEOIAKOYvyK15izC0VizgmxgFUyFmUX/ESY26pWMQxMf46IWdRfsVLjLmlYhHHIu2zhfROlF/xEmNuxbdG5IL0hnsiCCGxE9Gexbm9JxJCCOl7IioWCxYsQHl5OY4fP97T8RBCCIlDER2GWr58Ofbu3YsXXngBKpUKU6dOxbXXXht0TAjSfcQ4gAo5i/IrXmLMbUTFYsSIERgxYgTmzZuHr776CjU1NXj44YcxYsQITJ06FVlZWQEDihBCCBGP87oaSiqVYtCgQRg8eDDUajXMZjP27duHBx54ADU1NT0VY5/VeQQ0Ii6UX/ESY24j2rOwWq04cOAAampq8OOPP2Ly5MkoLCzEr3/9awAdw50WFxfj2muv7dFg+xq6sEDcKL/iJcbcRlQsHnjgAYwdOxbXX389rrrqqoBxX0eOHBnxYEYkcmq1uld2c0wiQ/kVLzHmNqJisXbt2qDDoPoG/AY6BiQihBAiThGdsygqKgr6/NKlS7s1GEIIIfEpomIRbDA9u90OqZR6CyGEkL4g5GGoBx54AADgcrmEv32sViumTJkS8YLq6+tRXl4OnueRm5uL2bNn+00/duwYtmzZglOnTqGoqAiZmZnCtOrqarz99tsAgPz8fEyfPj3i5fZmYhxAhZxF+RUvMeY2ZLFYsmQJvF4vVq5ciSVLlvhN02q1IcdrPRfP89i4cSNWrFgBnU6HZcuWISMjA0OGDBHm0ev1WLRoEd5//32/11qtVuzcuRMlJSUAgMcffxwZGRlgWTaiZfdmYhxAhZxF+RUvMeY2ZLEYM2YMAGDjxo3o16/fBS/EYDBg4MCBSE1NBQBkZWWhrq7Or1gMGDAAQOD1yfX19Rg/frxQHMaPH4/6+npcc801FxxPb6HVakXZLz7pQPkVLzHmtsti8fbbbyM/Px8A8O6773bZwO233x52IWazGTqdTnis0+nQ0NAQUYCdX5uSkgKz2RwwX1VVFaqqqgAAJSUlAV2ROJ1O2O12sCwLi8UStKsSs9kMpVIJh8MBhUKBxMTEoG2oVCpYrVa/uHxMJhNYloXNZoNSqQy4s93hcMDpdCIxMRF2ux0pKSkBbRiNRqjValgsFrAsG9CG3W6Hy+WCQqEI2YZGoxHa6FzsbTYbOI6DXC6H0+kMeH1f4fscWCwWSKVS4fxcUlKS33wcxwlX/7W0tCA5ORkymcxvntbWVjAMI1xj37kNt9uN1tZWoQ2pVBrwOWxpaYFcLgfHcZBKpQF70C6XCxaLRWhDp9MF/MBqbm6GQqGAy+UCwzBQqVTCtJ/P9w2Kss7vR3duT1arNWgbPbE9MQzjty5erxcmk0nIm1qthlwu92vDarWC53kwDAOXyxUwJgbP8zCbzUIbGo0m4DYGXzcjUqkUHMdBo9H4Tfd4PGhubhba0Gq1EXen3uVcJpMp6N8XItgJ8ou5wzHYa/Py8pCXlyc87uoaZ4vFEnK61WoF0JF8m80WdB5fQrpqwzfdarUK7XW1nFBxJicno7m5+YLbaG1tFeLpqq8at9sd9Pm+oqv3rr29Pejzvl+LXR1mOPf9DNeG1+sNunyO44S/uyrkvja62jZ9n1232w2HwxF0nngU7P3oru3pYts4n+3J96XcmS9vvniCcblcIePwteGLJ5RwbXTe+wl1aqHLYjF//nzh70WLFoUNKhSdThdQfILdtxFMSkoKjh07Jjw2m83C4TGx6/zLlYgL5Ve8xJjbiK59PX36tFCBnE4nduzYgZ07d3b5y6mztLQ0NDY2oqmpCRzHoba2NuI7vidMmIAjR44IvwiOHDmCCRMmRPRaQggh3SOig1WvvPIKli5dCq1Wi61bt6KxsREJCQl44403Aq6SCkYmk6GgoADFxcXgeR7Z2dkYOnQoKisrkZaWhoyMDBgMBqxatQo2mw2HDx/Gjh07sHr1arAsi1tuuQXLli0DAMyZM6dPXAlFCCHxJKJi8csvv2DQoEHwer2oq6vDyy+/DLlcjsLCwogXNGnSJEyaNMnvuXNPjo8cORKvv/560Nfm5OQgJycn4mURQnonz/yburU92YZd3dpeXxZRsUhISIDD4cDp06eh0+mgVqvh8Xj6/MnRnhbqJBjp/Si/4iXG3EZULKZMmYJnnnkGDocDM2bMAACcPHlSuDeC9AzqTkXcKL/iJcbcRlQs7rnnHhw5cgQymQxXXHEFgI7LV+++++4eDa6vC3bJMREPyq94iTG3kd2NAeDKK6/0e5yWltbtwRBCCIlPERWLpqYmbN++Hd9//33ATULr1q3rkcBIxx3AkV6eTHofyq94iTG3EV86m5qait///vcX1UcUIYSQ3imiYnH69Gk8++yzojxpQwghJLyIvv1Hjx6N77//vodDIYQQEq8i2rPo378/iouLcfXVVwf0hBhJr7PkwpzboRzpGbMqvu32Nt+bOyqi+cLlt7tvUCPRI8ZtN6Ji0d7ejvT0dHg8novugZZETmz94RN/lF/xEmNuIyoWF9vrLLkwYhxAhZxF+RUvMeY24vssTp8+jYMHD6K1tRV/+MMfcObMGbjdblx66aU9GV+fJrYPG/FH+RUvMeY2ohPcBw4cwFNPPQWz2YyamhoAHSNUbd26tUeD6+siHfOD9E6UX/ESY24j2rPYsWMHnnzySQwbNgwHDhwAAFx66aV0hVQPE+MAKuQsyq94iTG3Ee1ZtLa2BhxukkgkFzU0KiGEkN4jomIxYsQI4fCTz/79+zFy5MgeCYoQQkh8iegw1L333ovnnnsOe/bsQXt7O4qLi3HmzBmsWLGip+MjhBASByIqFoMHD8Zf/vIXHD58GOnp6dDpdEhPT4dCoYh4QfX19SgvLwfP88jNzcXs2bP9prvdbpSVleHEiRNISkpCUVERBgwYAI7j8Prrr+PkyZPgeR7XXnstbr755vNby16qtbU11iGQHkT5FS8x5jbizp4kEglGjx6NGTNmYMqUKedVKHiex8aNG7F8+XKsWbMG+/fvx+nTp/3m2bNnD1QqFUpLSzFz5kxUVIRPhwoAAB8WSURBVFQAAA4ePAiO4/Dyyy+jpKQEVVVVaGpqinjZvRnDRHxlM+mFKL/iJcbchl2jo0ePoqKiAidPnoTX64VEIsHw4cNx1113Ydy4cREtxGAwYODAgUhNTQUAZGVloa6uDkOGDBHmOXToEG699VYAQGZmJjZt2iQMIOJ0OuHxeOByucAwDJRK5XmvaG/E83ysQyA9iPIrXmLMbchicfz4caxcuRK5ubmYO3cuUlJSYDab8dlnn+GFF17A008/HdFJbrPZDJ1OJzzW6XRoaGjoch6ZTAalUom2tjZkZmbi0KFDWLBgAVwuF+6++26wLBuwjKqqKlRVVQEASkpKoNfr/aY7nU7Y7XawLAuLxRIw3ReDUqmEw+GAQqFAYmJi0DZUKhWsVqvfOvmYTCawLAubzQalUhmwB+ZwOOB0OpGYmAi73Y6UlJSANoxGI9RqNVwuF1iWDWjDbrfD5XJBoVCEbEOj0cBisYBl2YCu5W02GziOg1wuDxijhFwc33udlJTk97zb7UZra6twd29SUlLAPC0tLZDL5eA4Do6oRSxeer1e2J6sVmvQbbIntieNRuNXMLxeL0wmk5B7tVoNuVzu14bVagXP82AYBi6XK6AfPp7nYTabhTY0Gg0SEhL85mlrawPQMawrx3HQaDR+0z0eD5qbm4U2tFptxHtBIefatWsXZs2ahdtuu014btCgQbjiiiugVquxa9cuPPTQQ2EXEmyIwc6X3XY1j8FggFQqxfr162Gz2fDnP/8Z48aNE/ZSfPLy8pCXlyc8NhqNQWPxDaTe1XSr1QqgI/k2my3oPL6EdNWGb7rVahXa62o5oeL0fdAvtA3fcdO2tjYhps7cbnfQ58mF8w1609XgN+fe3Rssd2LshC5WfO+vb7u/mG3yfLYnnueDtuPLvS+eYFwuV8g4fG1Ecl4kXBud7zQfNGhQl22FPGfx3Xff+X0Bnys3NxffffddyEB9dDqdXweEJpMp4A7Hc+fxeDzCXsC+ffswYcIEMAwDjUaDX//61zh+/HhEyyWEENI9QhaLrnbJACAlJQV2uz2ihaSlpaGxsRFNTU3gOA61tbXIyMjwmyc9PR3V1dUAOk5qjx07FhKJBHq9HkePHoXX64XT6URDQwMGDx4c0XIJIYR0j4s6ZR/pHdwymQwFBQUoLi4Gz/PIzs7G0KFDUVlZibS0NGRkZCAnJwdlZWVYsmQJWJZFUVERAGDGjBl47bXX8PDDD8Pr9SI7O5s6LySEkCgLWSycTiceeOCBLqefz4DkkyZNwqRJk/yeO3fgJLlcHvT8h0KhiOi8iBjR+QRxo/yKlxhzG7JYPPXUU9GKgwQhxht7yFmUX/ESY25DFosxY8ZEKw4ShBgHUCFnUX7FS4y5jfgObhJ9YvuwEX+UX/ESY26pWMSxrq5EI+JA+RUvMeaWikUck0opPWJG+RUvMeZWfGtECCGk20V0n0VpaWlE91QUFhZedECEEELiT0R7FiqVCnV1deB5HikpKeB5HnV1dVAqlUhNTRX+EUIIEaeI9iwaGxvx+OOPY/To0cJz3377Ld566y0UFBT0WHB9nRivqCBnUX7FS4y5jWjP4rvvvsNll13m99zIkSMj7kiQXJjOXRgTcaH8ipcYcxtRsRg+fDi2b98udJ3rcrnwt7/9DcOGDevJ2Po86qpa3Ci/4iXG3EZ0GGrRokVYu3atMPCQ1WpFWloaHnzwwZ6Or08T4+V35CzKr3iJMbcRFYsBAwbgueeeg9FoRHNzM5KTk4OONEe6F8uyNIqdiFF+xUuMuY24/LW1teHYsWM4duwY9Ho9zGaz34BGhBBCxCuiYnHs2DEUFRVh7969eOuttwAAP/30EzZs2NCjwRFCCIkPERWLzZs3o6ioCE888QRkMhmAjquhaHhTQgjpGyIqFr/88gvGjRvn9xzDMPB4PD0SFOngu/qMiBPlV7zEmNuIisWQIUNQX1/v99zXX3+NSy65pEeCIh0sFkusQyA9iPIrXmLMbURXQ82bNw8vvPACJk6cCJfLhTfeeAOHDx/Go48+GvGC6uvrUV5eDp7nkZubi9mzZ/tNd7vdKCsrw4kTJ5CUlISioiIMGDAAAHDq1Cm88cYbcDgckEgkWLlypShveulMjAOokLMov+IlxtxGVCwuv/xyvPTSS9i7dy8UCgX0ej2ef/556HS6iBbC8zw2btyIFStWQKfTYdmyZcjIyMCQIUOEefbs2QOVSoXS0lLs378fFRUVWLp0KTweD0pLS1FYWIhhw4ahra0NDBNR2L2e2D5sxB/lV7zEmNuwh6F4nsfTTz+NpKQkzJo1C/fddx9mz54dcaEAAIPBgIEDByI1NRUMwyArKwt1dXV+8xw6dAjTp08HAGRmZuLo0aPwer04cuQILrnkEuFu8aSkJFHe8BLM+bzHpPeh/IqXGHMb9ie6VCpFU1MTvF7vBS/EbDb7vXk6nQ4NDQ1dziOTyaBUKtHW1obGxkZIJBIUFxfDYrEgKysLs2bNClhGVVUVqqqqAAAlJSUBNw06nU7Y7XawLAuLxRL0pkKz2QylUgmHwwGFQoHExMSgbahUKlit1qAfCJPJBJZlYbPZoFQqoVAo/KY7HA44nU4kJibCbrcHHVHLaDRCrVZDIpGAZdmANux2O1wuFxQKRcg2NBoNLBYLWJZFv379/KbbbDZwHAe5XC66m4dizfdeJyUl+T3vdrvR2toqHKJISEgI+By2tLRALpeD4zg4ohaxeOn1emF7slqtQbfJntieOufW6/XCZDIJuVer1QGH0q1WK3ieB8MwcLlc0Gq1ftN5nofZbBba0Gg0SEhI8Junra0NQMf3Nsdx0Gg0ftM9Hg+am5uFNrRabcRHaiKaa86cOdiwYQNuu+22gC/ISH7lBys0ncfH6Goej8eDb7/9FitXrkS/fv3wzDPPYMSIEQFXZ+Xl5SEvL094bDQag8biO/HU1XSr1QqgI/k2my3oPL6EdNWGb7rVahXa62o5oeLU6/UX1UZra6sQjy+mztxud9DnyYVrb2/3+78z3yEKjuOC5k6M/QrFiu/99W330dqeusqtL/ehToD7rqTqKg5fG754QgnXRufDZYMGDeqyrYiKxfr16wEANTU1AdMqKyvDvl6n0/nd7W0ymZCcnBx0Hp1OB4/HI+wF6HQ6jBkzBmq1GgAwceJEnDx5MqBYEEII6TkRFYuysrKLWkhaWhoaGxvR1NSElJQU1NbWBnRCmJ6ejurqalx++eU4ePAgxo4dC4lEgiuvvBK7du1Ce3s7GIbBN998g5kzZ15UPIQQQs5PyGLhO6bVv3//i1qITCZDQUEBiouLwfM8srOzMXToUFRWViItLQ0ZGRnIyclBWVkZlixZApZlUVRUBKCjQ66ZM2di2bJlkEgkmDhxIiZNmnRR8fQWzc3NsQ6B9CDKr3iJMbcSb4gz13fffTe2bNkiPF61ahUeeeSRqAR2sc6cORPrEC6aSqXq8rxJd5tV8W1UltMXvDd3VETzhcuvZ/5N3RVSnyXbsCsmy43mttudQp2zCHl2unMd+fe//909EZGIiLHLAHIW5Ve8xJjbkMWi8xVLJLr6ys2HfRXlV7zEmNuQa+TxeHD06FHhMc/zfo8B4IorruiZyAhUKhUcDrraXqwov+IlxtyGLBYajQbr1q0THrMs6/dYIpFc9JVShBBC4l/IYvHqq69GKw5CCCFxrG90skQIIeSiULGIY111F0HEgfIrXmLMLRWLONZVHzZEHCi/4iXG3FKxiGO+/rCIOFF+xUuMuaViEcci6VWS9F6UX/ESY26pWMSxYGNuEPGg/IqXGHNLxYIQQkhY4rsnvY+gjv/iV7flZvqLAIC3q//UPe0RchFoz4IQQkhYVCwIIYSERYeh4pjZbI51CIT0aj0xJkgkY2SIcdulPYs4plQqYx0CIeQCiHHbjdqeRX19PcrLy8HzPHJzczF79my/6W63G2VlZThx4gSSkpJQVFSEAQMGCNONRiOWLl2KW2+9FTfd1DdGEHM6nbEOgRByAcS47UZlz4LneWzcuBHLly/HmjVrsH//fpw+fdpvnj179kClUqG0tBQzZ85ERUWF3/TNmzdj4sSJ0Qg3bsjl8liHQAi5AGLcdqNSLAwGAwYOHIjU1FQwDIOsrCzU1dX5zXPo0CFMnz4dAJCZmYmjR48Kw7p+/vnnSE1NxZAhQ6IRbtwQ464sIX2BGLfdqBQLs9kMnU4nPNbpdAEngM6dRyaTQalUoq2tDU6nE++99x5uvfXWaIRKCCEkiKics/DtIZyr8/jeXc2zY8cOzJw5EwqFIuQyqqqqUFVVBQAoKSkJuN3e6XTCbreDZVlYLJagt+ObzWYolUo4HA4oFAokJiYGbUOlUsFqtfoVQB+TyQSWZWGz2aBUKgPidjgccDqdSExMhN1uR0pKSkAbRqNR6IiMZdmANux2e8j3ghDSc5KSktCvXz+/52w2GziOg1wuh9PpBMMwft8xXq8XJpMJWq0WLS0tUKvVAYeqrFYreJ4HwzBwuVzQarV+03meh9lsFtrQaDRISEjwm6etrQ0AIJVKwXEcNBqN33SPx4Pm5mahDa1WG/F44VEpFjqdDiaTSXhsMpmQnJwcdB6dTgePxyN8sRsMBnz22WeoqKiAzWaDRCKBXC7HjBkz/F6fl5eHvLw84bHRaAwai8ViCTnd17WwzWaDzWYLOo8vIV214ZtutVq77KrY93yoOPV6fcg2CCHR19bWJmzjnbndbgAAx3FBt+2WlhYAZ7+HgnG5XAC6/m7wtRFJZ4Xh2vD97zNo0KAu24pKsUhLS0NjYyOampqQkpKC2tpaPPjgg37zpKeno7q6GpdffjkOHjyIsWPHQiKR4JlnnhHm2bFjBxQKRUChECsxXlFBSF8gxm03KsVCJpOhoKAAxcXF4Hke2dnZGDp0KCorK5GWloaMjAzk5OSgrKwMS5YsAcuyKCoqikZocY0ONxHSO4lx25V4g50sEIEzZ87EOoSLplaru9xdpY4E+w7qSDC+RHIHd6htN56FOgxFd3DHsd74YSOEiHPbpWIRx8Q4gAohfYEYt10qFoQQQsKiYkEIISQsKhaEEELComJBCCEkLCoWcUyMA6gQ0heIcdulYhHHxNhzJSF9gRi3XSoWcczhcMQ6BELIBRDjtkvFIo6F62mXEBKfxLjtUrGIY527SCeE9A5i3HapWBBCCAmLigUhhJCwqFgQQggJi4pFHBPjACqE9AVi3HapWMQxMQ6gQkhfIMZtl4pFHFOpVLEOgRByAcS47VKxiGNWqzXWIRBCLoAYt92ojMENAPX19SgvLwfP88jNzcXs2bP9prvdbpSVleHEiRNISkpCUVERBgwYgK+++goVFRXgOA4Mw2DevHm44oorohV2TOl0OhiNxliHQQg5T2LcdqOyZ8HzPDZu3Ijly5djzZo12L9/P06fPu03z549e6BSqVBaWoqZM2eioqICAJCUlITHHnsML7/8MhYvXozS0tJohEwIIeQcUSkWBoMBAwcORGpqKhiGQVZWFurq6vzmOXToEKZPnw4AyMzMxNGjR+H1ejF8+HCkpKQAAIYOHQq32w232x2NsAkhhPxHVA5Dmc1m6HQ64bFOp0NDQ0OX88hkMiiVSrS1tUGtVgvzfPbZZxg+fDgSEhICllFVVYWqqioAQElJScAYuE6nE3a7HSzLwmKxBB0j12w2Q6lUwuFwQKFQBNyy72tDpVLBarX6rZOPyWQCy7Kw2WxQKpUBfcQ4HA44nU4kJibCbrcLhfBcRqNRWG+WZQPaEOOVFoT0FklJSejXr5/fczabDRzHQS6Xw+l0gmEYv+8Yr9cLk8kErVaLlpYWqNVqyOVyvzasVit4ngfDMHC5XNBqtX7TeZ6H2WwW2tBoNAHfhW1tbQAAqVQKjuOg0Wj8pns8HjQ3NwttaLVaMExkZSAqxcLr9QY8J5FIzmueH374ARUVFXjiiSeCLiMvLw95eXnC466OF1oslpDTfSembDYbbDZb0Hl8CemqDd90q9Xa5Yku3/Oh4tTr9SHbIIREX1tbm7CNd+Y76sFxXNBtu6WlBcDZ76FgXC4XgK6/G3xttLa2ho01XBu+/30GDRrUZVtROQyl0+lgMpmExyaTCcnJyV3O4/F4hL0A3/yrVq3C4sWLMXDgwGiEHBfOfc8IIb2HGLfdqBSLtLQ0NDY2oqmpCRzHoba2FhkZGX7zpKeno7q6GgBw8OBBjB07FhKJBDabDSUlJbjzzjsxatSoaIQbN3zFkhDSu4hx243KYSiZTIaCggIUFxeD53lkZ2dj6NChqKysRFpaGjIyMpCTk4OysjIsWbIELMuiqKgIAPDRRx/hp59+wltvvYW33noLALBixYqAY3Fi1NVhMEJIfBPjtivxBjtZIAJnzpyJdQgXjWXZLs9XzKr4NsrRkFh5u/pPsQ6BnEO2YVfYeUJtu/Es1DmLqN2UR86fQqHolR840r3yp7/Y7W1SAepZYtx2qbsPQgghYVGxIIQQEhYdhooCOr9ACOntaM+CEEK6mcPhiHUI3Y6KBSGEdDMxjpRHh6EIIeQ8eObfFH6e82gvkktx4wHtWRBCCAmLigUhhJCwqFgQQggJi4oFIYSQsKhYEEIICYuKBSGEkLCoWBBCCAmLigUhhJCwqFgQQggJi+7gJoSQGIrkjvDz0VN3hFOxCIJ6iSWEEH9RKxb19fUoLy8Hz/PIzc3F7Nmz/aa73W6UlZXhxIkTSEpKQlFREQYMGAAAeOedd7Bnzx5IpVLce++9mDBhQrTCJoQQgigVC57nsXHjRqxYsQI6nQ7Lli1DRkYGhgwZIsyzZ88eqFQqlJaWYv/+/aioqMDSpUtx+vRp1NbWYvXq1Whubsazzz6LV155BVIpnW4h5ELRUK3kfEXlG9dgMGDgwIFITU0FwzDIyspCXV2d3zyHDh3C9OnTAQCZmZk4evQovF4v6urqkJWVhYSEBAwYMAADBw6EwWCIRtiEEEL+Iyp7FmazGTqdTnis0+nQ0NDQ5TwymQxKpRJtbW0wm8247LLLhPlSUlJgNpsDllFVVYWqqioAQElJCQYNGnTB8dY9euGvJaTPevRQrCMgPSgqexZerzfgOYlEEtE8wZ4PJi8vDyUlJSgpKbmwIOPQ448/HusQoorWV7z60roC4lzfqBQLnU4Hk8kkPDaZTEhOTu5yHo/HA7vdDpZlA15rNpuRkpISjbAJIYT8R1SKRVpaGhobG9HU1ASO41BbW4uMjAy/edLT01FdXQ0AOHjwIMaOHQuJRIKMjAzU1tbC7XajqakJjY2NGDlyZDTCJoQQ8h+yp59++umeXohUKsXAgQNRWlqKjz76CFOnTkVmZiYqKyvhdDoxaNAgXHLJJdi3bx/efPNNfP/991iwYAFYloVGo4HVasX69euxb98+FBQUXNT5iN5mxIgRsQ4hqmh9xasvrSsgvvWVeCM9KUAIIaTPopsVCCGEhEXFghBCSFjUN1QcOnDgAP7+97/jxx9/xPPPP4+0tDRhmti7PtmxYwf+9a9/Qa1WAwDuvPNOTJo0KcZRda9wXd+IzeLFi6FQKCCVSiGTyUR1eTsAvPbaa/jiiy+g0Wjw8ssvAwCsVivWrFmDX375Bf3798fSpUvBsmyMI71IXhJ3fvjhB++PP/7ofeqpp7wGg8Hv+UceecTrcrm8P//8s7ewsNDr8XhiGGn3q6ys9L733nuxDqPHeDweb2Fhofenn37yut1u7yOPPOL94YcfYh1Wj1q0aJG3tbU11mH0mH//+9/e48ePex966CHhuW3btnnfeecdr9fr9b7zzjvebdu2xSq8bkOHoeLQkCFDgl7xRV2f9H6RdH1DepcxY8YE7DXU1dVh2rRpAIBp06aJIsd0GKoXibTrk97uH//4B2pqajBixAj8/ve/7/277+eIpOsbMSouLgYA/Pa3v0VeXl6Mo+l5ra2two3HycnJsFgsMY7o4lGxiJFnn30WLS0tAc/fcccduOqqq4K+xiuSq5xDrft1112HOXPmAAAqKyuxdetWLFq0KNoh9phgOezc9Y3YPPvss0hJSUFrayuee+45DBo0CGPGjIl1WOQ8UbGIkSeffPK8XyOWrk8iXffc3Fy88MILPRxNdEXS9Y3Y+D6jGo0GV111FQwGg+iLhUajQXNzM5KTk9Hc3CxcsNGb0TmLXqQvdH3S3Nws/P35559j6NChMYym+0XS9Y2YOJ1OOBwO4e+vvvoKl1xySYyj6nkZGRn49NNPAQCffvppl0cLehO6gzsOff7559i0aRMsFgtUKhWGDRuGJ554AgDw9ttv45NPPoFUKsU999yDiRMnxjja7lVaWorvv/8eEokE/fv3x4IFC0T3y/uLL77Ali1bwPM8srOzkZ+fH+uQeszPP/+MVatWAejoIPSaa64R3fr+5S9/wbFjx9DW1gaNRoPbbrsNV111FdasWQOj0Qi9Xo+HHnqo1597o2JBCCEkLDoMRQghJCwqFoQQQsKiYkEIISQsKhaEEELComJBCCEkLCoWhKCjt9u1a9de8OvfeOMN7Ny5sxsjIiS+0B3cRFQWL16MlpYWSKVSKBQKTJw4EQUFBVAoFD263AULFgh///vf/0ZpaSlef/31C2rLZrNhy5Yt+PLLL9He3o7k5GRkZ2eLvitzEt+oWBDReeyxxzB+/HiYzWYUFxfjrbfewty5c3tseTzPQyrtvp30LVu2wOl0Ys2aNVAqlWhsbMQPP/zQbe0DHTfIyWSybm2TiBsVCyJaKSkpmDBhgvBFazabsWHDBnz77bdgWRazZs3qsgfU1atX45tvvoHL5cKwYcNw3333CV2PvPrqq5DL5TAajTh27BgeffRR7N27FzqdDrNnz8bzzz8PjuMwb948AMArr7yCBx98EOvWrUNSUhIA4MSJEyguLsb69evBMP6b4fHjx3H77bcLd/wOHjwYgwcPFqb/8MMP2Lx5M06cOAGGYXD99dcjPz8fbrcbFRUVOHDgAABg8uTJmDt3LhISEoS9nRkzZmD37t0YP348lixZgsOHD+Nvf/sbfvnlFwwZMgTz58/HpZde2o1ZIGJB5yyIaBmNRnz55ZcYNmwYgI4vbZ1Oh/Xr1+Phhx/G9u3b8fXXXwd97YQJE7B27Vr8z//8D4YPHx5wPmPfvn24+eabsWXLFowaNUp4XqFQYPny5UhOTsa2bduwbds2pKSkYOzYscKXOADU1NRgypQpAYUCAC677DL87W9/wyeffILGxka/aQ6HA88++ywmTJiA9evXY+3atRg3bhyAjq5gGhoa8OKLL+Kll16CwWDAW2+9Jby2paUFVqsVr732GhYuXIgTJ05g3bp1WLBgATZt2oS8vDy8+OKLcLvd5/dGkz6BigURnZdeegn33HMP/vznP2PMmDHIz8+H0WjEt99+i7lz50Iul2PYsGHIzc1FTU1N0DZycnKQmJiIhIQE3HrrrTh16hTsdrsw/aqrrsKoUaMglUohl8vDxjRt2jTs3bsXQMdhq/379+Paa68NOm9BQQGuueYafPTRR3jooYewZMkSfPnllwCAw4cPQ6vV4sYbb4RcLkdiYqIwxsm+fftwyy23QKPRQK1WY86cOcIygY6u0G+77TYkJCRALpfjX//6F/Ly8nDZZZdBKpVi+vTpYBimT4yvQc4fHYYiovPoo49i/Pjxfs81NzeDZVkkJiYKz+n1ehw/fjzg9TzPY/v27Th48CAsFosw3oTFYoFSqQQAvwGMIpGRkYENGzbg559/RmNjI5RKZZc9BsvlcuTn5yM/Px92ux3vvvsuVq9ejXXr1sFkMiE1NTXo68xmM/r37y887t+/v9/gWGq12q+wGY1GfPrpp/joo4+E5ziOE+WAWuTiUbEgfUJycjKsViscDodQMIxGY9DxQPbt24dDhw7hySefRP/+/WG323Hvvff6zRNqwKJg0+RyOSZPnoy9e/fizJkzXe5VdKZUKpGfn493330XTU1N0Ol02L9/f9B5U1JS8MsvvwjnVjqvX+e4dDqdUJQICYcOQ5E+Qa/X49e//jXefPNNuFwunDp1Cp988gmmTp0aMK/D4QDDMGBZFu3t7di+fft5LUuj0aCtrc3vsBUAXHvttfj0009x6NChoMv12blzJwwGAziOg8vlwgcffACVSoVBgwYhPT0dLS0t2L17N9xuNxwOh3DYaMqUKXj77bdhsVhgsViwc+fOkMvJzc3FP//5TzQ0NMDr9cLpdOKLL74Qxp8g5Fy0Z0H6jD/+8Y/YsGEDFi5cCJZlceuttwYcrgI6zi8cOXIE999/P1iWxe23346PP/444uUMHjwYU6ZMQWFhIXiex+rVq5GSkoJRo0ZBIpFg+PDhGDBgQJevl0gkWLduHYxGI6RSKS699FI8/vjjwr0iK1aswObNm7Fz504wDIOZM2fisssuEw5bPfLIIwCAzMzMkHsNaWlpWLhwITZt2oTGxkbI5XKMGjUKo0ePjnhdSd9B41kQEkX//d//jWuuuQa5ubmxDoWQ80KHoQiJEoPBgJMnTyIrKyvWoRBy3ugwFCFRUFZWhrq6Otx7771+V2QR0lvQYShCCCFh0WEoQgghYVGxIIQQEhYVC0IIIWFRsSCEEBIWFQtCCCFh/X9TI4cSuQf7lwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#pos_token_sentiment_score, neg_token_sentiment_score\n",
    "plt.hist(pos_token_sentiment_score, label='Positive Token', density=True)\n",
    "plt.hist(neg_token_sentiment_score, label='Negative Token', density=True)\n",
    "plt.grid(linestyle='-.')\n",
    "plt.legend()\n",
    "plt.xlabel('Polarity Score')\n",
    "plt.ylabel('Freq Density')\n",
    "#plt.savefig(\"imgs/sst_token_polarity_score_gru.pdf\", bbox_inches='tight', dpi=900)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pos_ngram_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Freq Density')"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEJCAYAAABlmAtYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3gT1b438G/SNE2TNGmTSnm4S0VRQFGqVlCgULegKGxEUdkclSMgyF0UZePliAhsBbZQL4iAiIgoAnI26N5WLFfBFgVF5IUCKki1tGmb5tZ0MvP+0TfzNk3ShDaTTFZ+n+fxkWQma9a3K8nK3NZSCIIggBBCCGmGMtYVIIQQIn/UWRBCCAmJOgtCCCEhUWdBCCEkJOosCCGEhESdBSGEkJBUsa6AVC5cuBDweb1eD5vNFuXaSIfyyBtreQD2MlGe/69du3ZBlyXcnoVGo4l1FSKK8sgba3kA9jJRnvAkXGdBCCHk0lFnQQghJCRmz1k0JQgCXC4X/vzzT9TV1cW6OhFDeVpGEAQolUpoNBooFArJt0dIvEuYzsLlciE5ORnJyclISkqKdXUiRqlUUp4W4jgOLpcLqampkm3D4XBIVnassJaJ8oQnYQ5D8TwPlUoFnudjXZWIojwtF433g9vtlrT8WGAtE+UJT8J0Ft5DDUolW5EpT+tIfQiKtSttAPYyUZ7wsPVNEwb6JS5vrOVh7RAHwF4myhOehDln4aVSqcBxHDzj74louUmrtodcp2PHjujevTs8Hg+uuOIKvP7665d8vHz27NmYMGECrrzySixfvhyzZs0Cx3EAgHvuuQfbt4euRyirV6/G+++/j169eqGgoKDV5V0Kb/uwwmQyoaKiItbViCjWMlGe8ChYnfyo6R3cDocDWq02pp1Ft27dcOrUKQDAlClT0KtXL0ycOLHF2+zWrRvOnj0b8S/X/v3744MPPkCnTp1aVY4gCOJVR+ES28fjicqJbu/7QiqZmZlMfREBiNhnJ5zPTDSw1katyUN3cMvQTTfdhF9++QUAsHLlSgwaNAiDBg3CqlWrADR8iY0dOxb5+fkYNGgQPvvsMwDAqFGjcPToUbzyyitwuVwYNGgQpkyZAqCh8wCAxx9/HF999ZW4rRkzZmDHjh3weDyYP38+7rzzTuTn52P9+vV+9ZozZw5+++03PProo3jnnXdQVVWFcePGIT8/H8OGDcPx48cBAEuWLMHbb78tvm7QoEE4d+4czp07hwEDBuDZZ5/FHXfc4ddpO51OTJw4Efn5+Xj88ccxbNgwHD16VKz/4sWLMWzYMBw+fBjLli3DnXfeiUGDBuHpp5+G93fNqFGj8MILL2DkyJEYMGAAjhw5gsceewz9+vXD4sWLW902hBB/CXcYSg44jsPXX3+NgQMH4ocffsDHH3+Mf/3rXxAEAcOGDcMtt9yCX3/9FW3bthW/0K1Wq08Zc+fOxdq1a7Fr1y6/PYvhw4dj+/btGDx4MNxuN/bt24eFCxdi48aNSEtLw86dO1FXV4cRI0ZgwIABPnsQixcvRlFRET755BOYTCbMmzcPPXv2xJo1a7Bv3z5Mnz4dX375ZbP5Tp8+jaVLl2LhwoV+y9atWwej0YjCwkKcOHECf/nLX8RlDocD3bt3x5NPPgmgofOYOXMmAGDq1Kn48ssvxfXVajW2bNmCd999F+PGjcPnn3+O9PR09O3bF+PHj4fJZAq3OQghYaA9iyhyuVy4/fbbMXToULRv3x4PPvggvv32WwwZMgRarRY6nQ5Dhw7FoUOH0L17d+zduxcLFizAoUOHYDAYwt5OXl4e9u/fj7q6Onz99dfIzc1Famoqdu/ejc2bN+P222/HsGHDUFVVhbNnzzZb1rfffot7770XAHDrrbeiqqrKr+NqqkOHDujTp0/Q8oYPHw4A6N69O66++mpxWVJSEoYNGyY+PnDgAIYNG4bBgwfjwIEDOHnypLjM22l0794dV155JbKyspCSkoLOnTsHHUSSENJyCbdnEcuTpxqNxu9XebBTRtnZ2fj888+xa9cuLFy4EAMGDBB/ZTcWKI9Go8Ett9yC3bt3Y/v27eKXMwC8/PLLGDhwYNh1DlQ/hUKBpKQknyuXGt913fgcwOeff46lS5cCAF577bWgeQEgJSVFXO5yuTB37lzs3LkT7du3x5IlS3y2oVarATRcauv9t/exx+MJO5/UWDoWzirW2kiqPAm3ZyG3u51zc3Px73//G06nEw6HA1988QVuvvlm/PHHH0hNTcW9996Lxx9/HD/++KPfa5OTk4Neajp8+HBs2rQJhw4dEjuHAQMG4P3330d9fT2AhsNFoS6zy83NxZYtWwA0/NI3mUxIS0tDx44dxTr9+OOP+O233wK+fujQofjyyy/x5Zdf4rrrrsNNN92E//3f/wUAnDx5EidOnPBZ39s+3o7BZDLBbrdjx44dzdZTroxGY6yrQEJgrY2kypNwexbeX51yuRKjV69euO+++3DXXXcBAB588EH07NkTRUVFePnll6FQKJCcnBzw+P+YMWOQl5cX8BLXAQMGYPr06fjLX/4i/vJ+6KGHcO7cOQwZMgSCIMBkMmHNmjXN1m/WrFmYNWsW8vPzodFo8M9//hMAcOedd4qHtHr37o2uXbuGlffhhx/G9OnTkZ+fjx49euDqq69GWlqauNzbPkajEQ899BDy8/PRoUMHXHfddWGVLzehDtmR2GOtjaTKk3CXziqVSqZu/Iq3PB6PB/X19dBoNPjll18wevRo7N271+ewUjTzSH3pbFpaGmprayUrPxZYu3SWtTZqTZ7mLp1NuD2LePtyDSXe8jidTtx3333iobCFCxf6nXOIpzyhpKSkMPVFxCLW2kiqPAnXWZDY0uv1+Pzzz2NdDULIJUq4E9yEEEIuHXUWhBBCQkq4zkJO1+BHAuWRN7vdHusqkBBYayOp8iRcZ0FINLE0gi6rWGsjqfIk3AluhUIBQRAwfMOJ0Ctfgs/GdA+5Tvv27TFhwgS88MILAIC3334bdrtdHAupJbx5Glu+fDmmTZsmPo6nocsD5YlnarVavPKLyBNrbSRVnqjtWRw5cgTTp0/H1KlTsW3bNr/lx48fx5w5c/DAAw/g4MGDPsuKioowbdo0TJs2DUVFRa2qRywvy0xJScHnn38Oi8USsTID5VmxYoXP40h0FEDDIIDr169vdUchCELQdgjVPvF2mMrlcsW6CiQE1tpIqjxR6Sx4nsfq1asxd+5cLFu2DPv378f58+d91snMzMTkyZNx6623+jxvs9mwefNmvPLKK3jllVewefNm2Gy2FtdFpYrdzlRSUhLGjBmDd955x29ZZWUlxo8fjzvvvBN33nkniouLxecfeOAB3HHHHXj66adx0003iZ3NuHHjMHToUOTl5eGDDz4AAHHo8ttvv52poctfffXVuBy6PCMjIybbJeFjrY2kyhOVzqK0tBRt27ZFVlYWVCoV+vbtK34ZerVp0wadO3f2mxP5yJEjuPbaa6HX66HX63HttdfiyJEj0ai2JB555BFs3brV75b8559/HuPHj8fOnTuxatUqzJ49GwCwdOlS9OvXD//+978xdOhQ/P777+JrlixZgi+//BI7d+7EmjVrYLFYMHfuXHHAwqZ7AN6hywGIQ5cPGjTIZ+jyHTt24MMPP/Qb62nx4sXIysrCJ598ggkTJmDJkiXo2bMnCgsL8cwzz2D69Okhs58+fRqjRo3Cf/7zH3To0MFnmXfo8qKiIsyYMQM//PCDuMzhcOCqq67Cv/71L9x000145JFHsHPnTuzatQtOp9NncEbv0OVjx47FuHHjsGDBAuzatQsff/xxRPfoCEk0UfmZbbFYYDabxcdms1mcMe5SX2symQJ+6AsLC1FYWAgAWLRoETIzM32WSz1stXePheM48S5kpVLpN0uc0WjEqFGjsGbNGmg0GiiVSqhUKuzdu1f8mwiCAJvNBpvNhuLiYqxduxYqlQq333470tPTxQ517dq14g1uFy5cwG+//YY2bdqI20pKShIP23hf//zzz8Pj8YhDl+t0OuzevRsnTpzAzp07ATSMLfPrr7+iU6dOYhlKpVIcbValUqG4uBirVq2CQqFA//79UVVVBYfDIWZWqVTiL37v36NDhw64+eabff4e3vp9++23eOyxxwAAPXv2xDXXXIOkpCTxv3vuuUfMdOjQIRQUFMDpdKK6uhpXXXUV7rjjDgANQ5erVCr06NEDV111Fdq3bw9BENC5c2f8+eefMJlMSEpKEv+GKSkpyMzMhNVqhVKpFOvceLwqb7tWV1cjPT0d1dXVyMjI8BuUsqamBiqVSjyUlpaWBpVKJb4X6+vrUVNTI5ZhMpn83h/V1dVQq9Xi+0iv1/ssd7vdsFqtYhlms9nvB1ZVVRU0Gg3cbjdUKhV0Op3P8rq6OthsNhgMBtTU1Ph9VoCGz51Wq4XL5YJarfYZEuVPv7VbRq/Xw+FwQKfTwWaz+XzOvSorK6HX62G326HVaqHRaHyWO51OuFwupKamwuFwBJzHpKKiAgaDATabLWAZ3verRqNptgyj0Qir1Qq9Xo+UlBSf5Xa7HRzHQa1Ww+Vy+f26FwQBlZWVYrsZDAafkQuAhqMoPM9DpVLB7XYjPT3dZznP87BYLGIZRqMRycnJPuvU1tZCoVAgNTUVHMf5DSro8XhQVVUllpGenh720ZaodBbBhrluqUCvzc/PR35+vvi46TC93oaUSuMrELxfFjzP+x2D53kejz32GIYMGYLRo0dDEARwHAee5/HZZ5/5zcnN8zw8Ho9P+YIg4MCBA9izZw927NgBtVqNUaNGweFw+KzX+Pg+x3FQqVTIzc3FV199JQ5d7q3f/PnzAw5d7i2D53kIgiDWxfvY+593XYVCAY7jxHrU1dWJ29BqteLzgYYu95bDcZy4LY/H4/PBtNvtmDNnjt/Q5d7Xer9oBUFAcnKyuD2lUgm32+33d6mrqws4pHPj4dAbq66uBtDwhRxI4xOLdXV1Aae49JYRbE+ncRsGO/7sLaOysjLgcu/lk/X19XA6nQHXqampARB8SGvv4V6O40KOTtwS3vK9Q1MEq4d3ufcHVHNlBSvDuycfqIzMzExwHBeyDO/fq7a2NuhwGt72D1aGt92aG+zP+z4NVYa3Pk0JgiC2eagyvP/3ivm0qmaz2edNXVlZGfZxNZPJ5PNai8XSqmNycrjSJiMjA3fffTc2btwoPjdgwAC899574uNjx44BgM+Q3rt37xYbt7a2FkajEampqSgtLcV3330nvjY5OTno1RByH7pcEISAQ5d7xdvQ5XJ4v5HmsdZGUuWJyp5FdnY2ysrKUF5eDpPJhAMHDvhc2tmc3r17Y+PGjWKvf/ToUTz00EMtrov3V2U4l7pKaeLEiVi7dq34eP78+Zg7dy7y8/PBcRxuvvlmLF68GLNmzcLkyZOxfft25ObmIisrCzqdDgMHDsT69euRl5eHrl274oYbbhDLGjNmDPLz8+Ny6PK8vLyAQ5d7xdvQ5cF++RP5YK2NpMoTtSHKv/vuO6xbtw48zyMvLw8jR47Epk2bkJ2djZycHJSWluK1116D3W5HcnIy0tPTxcMUu3btwtatWwEAI0eORF5eXsjtBRuivPFx/HhQV1cnnicoKSnBs88+63NCN97yBOMdulyn0+H06dN+Q5dLReohyr3HhlnC2hDlrLVRa/I0dxgq4eaziDdnzpzB448/Dp7noVar8corr6B3796xrlbE2Ww2n6HL586di0GDBkm+3Xh9X8QSa50F+f+oswBNfhQvWJv8yGAwxHwmtkiPVhCOLUVPh1xHLp2FHNooklqTJ+YnuOWg8WWcLKE8rSP1byWpD6OR1mOtjaTKw9Y3TTOUSiVzA4aR1vHey0AICS1hBhLUaDRwuVxQKBRBr6GPRykpKZSnBQRBgFKp9LtBixASWMJ0Ft67GgPdJBXPKA8hJBoSbh+8NYMQyhHlkTfW8rCItTaSKk/CdRYsXTkEUB65Yy0Pi1hrI6nyJFxnEcshyqVAeeSNtTwsYq2NpMqTcJ2Fd5AuVlAeeWMtD4tYayOp8iRcZ9F02N94R3nkjbU8LGKtjaTKk3CdBSGEkEtHnQUhhJCQqLMghBASUsJ1FnSZnLxRHhJtrLURXTobIcGmsoxXlEfeWMvDItbaSKo8CddZ0JUP8kZ5SLSx1kZ0NVSEsDQjFkB55I61PCxirY2kypNwnYXRaIx1FSKK8sgba3lYxFobSZUn4TqL5OTkWFchoiiPvLGWh0WstZFUeRKusyCEEHLpqLMghBASEnUWhBBCQkq4zqK2tjbWVYgoyiNvrOVhEWttJFWehOssCCGEXLqE6yyUSrYiUx55Yy0Pi1hrI6nysPVXCgPHcbGuQkRRHnljLQ+LWGsjqfIkXGdBN+DIG+Uh0cZaG9FNeYQQQmKGOgtCCCEhqaK1oSNHjmDt2rXgeR6DBw/GiBEjfJbX19ejoKAAZ86cQVpaGmbMmIE2bdqA4zi8/fbbOHv2LHieR//+/fHXv/41WtUmhBCCKO1Z8DyP1atXY+7cuVi2bBn279+P8+fP+6yza9cu6HQ6rFixAnfddRc2bNgAADh48CA4jsOSJUuwaNEiFBYWory8vMV18Xg8rcoiN5RH3ljLwyLW2kiqPFHpLEpLS9G2bVtkZWVBpVKhb9++KC4u9lmnpKQEAwcOBADk5ubi2LFjEAQBAOByueDxeOB2u6FSqaDValtcl6qqqha/Vo4oj7yxlodFrLWRVHmichjKYrHAbDaLj81mM06dOhV0naSkJGi1WtTW1iI3NxclJSWYMGEC3G43Hn74Yej1er9tFBYWorCwEACwaNEiZGZm+ix3uVxwOBwwm824ePGi33JvHbRaLZxOJzQaDVJTUwOWodPpYLPZfDJ5VVZWQq/Xw263Q6vVQqPR+Cx3Op1wuVxITU2Fw+GAyWTyK6OiogIGgwE2my1gGQ6HA263GxqNBsnJyUhKSgpYhtFohNVqhV6vR0pKis9yu90OjuOgVqvhcrmQkZHhs1wQBFRWViI9PR3V1dUwGAxQq9U+69hsNvA8D5VKBbfb7TfpCs/zsFgsYhlGo9FvREzv3aZKpRIcx8FkMvn8MvJ4PKiqqhLLSE9Ph0qlCliGQqEAz/MwGAw+y5uWkZGR4fc3s1qtUCqV4g+UtLQ0n+Ucx4nbD1ZGTU0NVCqVOK1lWloakpKSxDz19fWoqakRyzCZTH7XxFdXV0OtVoPjOCiVSr/3utvthtVqFcswm81QKBQ+61RVVUGj0Yg/ruRKr9fL4vMkCAJqamqg0WiaLSNePk86nQ5OpxMcx/ldGRXO5ymYqLyTvB/Axpq+wYOtU1paCqVSiZUrV8Jut+P5559Hr169kJWV5bNufn4+8vPzxccVFRUB63Lx4sVml9tsNgANjW+32wOu4/1yClaGd7nNZhPLC7adYGVYrdawywimpqZGrE+wIQDq6+ubrYd3IhVvfQJxu91hleGtT3P+/PPPZssIZ2KXUPUI55dXXV1di8rw/j3DKSPY9JeNr5N3uVzNllFZWRlwufe927g+cuN9/8rt8xSsjHj5PDV+34Uqo+nnqV27dkHrFZXDUGaz2edNXVlZ6dfzNl7H4/HA4XBAr9dj37596N27N1QqFYxGI6666iqcPn26xXWhKRTljfKQaGOtjeJ6WtXs7GyUlZWhvLwcHMfhwIEDyMnJ8VmnT58+KCoqAtBwUrtHjx5QKBTIzMwUz1+4XC6cOnUK7du3b3Fd5Lxb3hKUR95Yy8Mi1tpIqjxR+SslJSVh3LhxWLBgAXieR15eHjp27IhNmzYhOzsbOTk5GDRoEAoKCjB16lTo9XrMmDEDADBkyBC8+eabePLJJyEIAvLy8tC5c+doVJsQQsj/E1ZnwfN8qwenuuGGG3DDDTf4PDd69Gjx32q1GrNmzfJ7nUajCfg8IYSQ6AmrB5gwYQLWrl3bqnMFhBBC4ldYexZz587F3r17sXjxYuh0Otx2223o379/wMtP5Y4mOpE3ykOijbU2kipPWJ1F165d0bVrV4wdOxY//PAD9uzZgyeffBJdu3bFbbfdhr59+/pdu0wIIYQdl3QiQqlUol27dmjfvj0MBgMsFgv27duHSZMmYc+ePVLVMaKa3t8R7yiPvLGWh0WstZFUecLas7DZbPjmm2+wZ88e/P7777jlllswZcoUXHXVVQAahvNYsGAB+vfvL0klI8l7dy0rKI+8sZaHRay1kVR5wuosJk2ahB49emDo0KG48cYb/W4xv+KKK/zum5Arg8EQ9K7GeER55I21PCxirY2kyhNWZ7F8+XK/O64BiGOLAMATTzwR2ZoRQgiRjbDOWXhvkGtq5syZEa0MIYQQeQqrswg0yJ/D4Wj1jXqEEELiQ7OHoSZNmgSgYRRE77+9bDYb+vXrJ13NJEITncgb5SHRxlobSZWn2c5i6tSpEAQBCxcuxNSpU32WpaenNzucrVzRRCfyRnlItLHWRjGZ/Oiaa64BAKxevdpvso945Z30gxWUR95Yy8Mi1tpIqjxBO4stW7Zg5MiRAIBt27YFLaDxYIDxgKU3BUB55I61PCxirY2kyhO0s2g6WRErMjIymNrtpDzyxlqeSPKMv6fVZSSt2t7qMlhrI6nyBO0sxo8fL/578uTJEd9wrASarzqeUR55Yy0Pi1hrI6nyhHVT3vnz56HX65Geng6Xy4Xt27dDqVTi7rvvZuZcBiGEkODCulHi9ddfh8PhAAC8//77+Pnnn3Hy5Em88847klaOEEKIPIS1Z3Hx4kW0a9cOgiCguLgYS5YsgVqtxpQpU6SuHyGEEBkIq7NITk6G0+nE+fPnYTabYTAY4PF4UF9fL3X9Is5qtca6ChFFeeSNtTwsYq2NpMoTVmfRr18/vPTSS3A6nRgyZAgA4OzZs2jTpo0klZISa0OUUB55Yy0Pi1hrI6nyhNVZPPLIIzh69CiSkpLQs2dPAA0TbDz88MOSVEpKgca5imeUR95Yy8Mi1tpIqjxhdRYAcN111/k8zs7OjnhlCCGEyFNYnUV5eTk2btyIX375BS6Xy2fZW2+9JUnFpJKWloa6urpYVyNiKI+8sZaHRay1kVR5wuosXn/9dWRlZeG//uu/6L4KQghJQGHflDd//nzmTgQRQggJT1jf/ldffTV++eUXiatCCCFErsLas7jsssuwYMEC3HTTTeKc217xNuosx3GxrkJEUR55Yy0Pi1hrI6nyhNVZ1NXVoU+fPvB4PHE/Ai0NRyxvlIdEG2ttFPUhyhuLxKizR44cwdq1a8HzPAYPHowRI0b4LK+vr0dBQQHOnDmDtLQ0zJgxQ7zp79dff8U777wDp9MJhUKBhQsXQq1Wt6geNNGJvFEeEm2stVHUJz9q6vz58zh48CBqamrw3//937hw4QLq6+vRuXPnkK/leR6rV6/GvHnzYDab8eyzzyInJwcdOnQQ19m1axd0Oh1WrFiB/fv3Y8OGDZg5cyY8Hg9WrFiBKVOmoEuXLqitrYVKFXa1/bD0pgAoj9yxlodFrLWRVHnCOsH9zTff4IUXXoDFYsGePXsAAE6nE++//35YGyktLUXbtm2RlZUFlUqFvn37ori42GedkpISDBw4EACQm5uLY8eOQRAEHD16FJ06dUKXLl0ANFxD3JqrsjIyMlr8WjmiPPLGWh4WsdZGUuUJ6yf6xx9/jOeeew5dunTBN998AwDo3Llz2FdIWSwWmM1m8bHZbMapU6eCrpOUlAStVova2lqUlZVBoVBgwYIFsFqt6Nu3L4YPHx7WdgOhiU7kjfKQaGOtjWI6+VFNTY3f4SaFQgGFQhHWRgKNVdL0tcHW8Xg8OHHiBBYuXIiUlBS89NJL6Nq1K3r16uWzbmFhIQoLCwEAixYtQmZmps9yl8sFh8Mh/iGbLgcaOiytVgun0wmNRoPU1NSAZeh0OthsNp8O0KuyshJ6vR52ux1arRYajcZnudPphMvlQmpqKhwOB0wmk18ZFRUVMBgMsNlsActwOBxwu93i84GyVFRUwGg0wmq1Qq/X+91MabfbwXEc1Go1XC6X368RQRBQWVkpHv80GAx+54lsNht4nodKpYLb7fa7Uo7neVgsFrEMo9GI5ORkn3Vqa2sBNAx+xnEcVCqVTx6Px4OqqiqxjPT0dL/DkN4yFAoFeJ6HwWDwWd60jIyMDL8PlNVqhVKpFN+HaWlpPss5jhO3H6yMmpoaqFQq8DwvltE4T319PWpqasQyTCaT315ydXU11Go1OI6DUqmEXq/3We52u2G1WsUyzGaz32epqqoKGo0Gbre7VYds44FOp2v150mpVEKlUkGj0TRbRrx8nhQKBVJTU8FxHIxGo8/ycD5PwYS1VteuXbFnzx4MGDBAfG7//v244oorwtqI2Wz2m9O76R/Tu47ZbIbH44HD4YBer4fZbMY111wjfgFcf/31OHv2rF9nkZ+fj/z8fPFxRUVFwLp4PJ5ml9tsNgANjW+32wOu4/1yClaGd7nNZhPLC7adYGV4hxkOVYZGowlaRk1NjVgfb52a8g4zH6wM7/HP5oY9drvdYZXhrU9zOI4LWI63jHCOx4aqRzjzEwcbLiFUGY2H7a+rq0NmZqZffbxlWCyWgGU0vvSx6fA6TcsIdnWi970bj9MIXApvztZ8njIzM8FxXMgy4uXzJAgCnE5nWGU0/Ty1a9cuaL3COvj/6KOP4qOPPsILL7yAuro6LFiwAJs2bQp71Nns7GyUlZWhvLwcHMfhwIEDyMnJ8VmnT58+KCoqAgAcPHgQPXr0gEKhwHXXXYfffvsNdXV18Hg8+Pnnn31OjBNCCJGeQghzPNu6ujocPnwYFRUVMJvN6NOnj9/uXHO+++47rFu3DjzPIy8vDyNHjsSmTZuQnZ2NnJwcuN1uFBQU4OzZs9Dr9ZgxYwaysrIAAHv27MG2bdugUChw/fXX429/+1vI7V24cCHg88nJyUz92qI88iaHPMM3nIj6NrcUPR2V7SSt2t7qMuTQRpHUmjzN7VmE3Vm43W7Y7XbodLoW3+MQTcE6i9TUVHEXjQWUR97kkIc6i+bJoY0iqTV5mussQp6zOHbsGDZs2ICzZ89CEAQoFApcfvnleOOjBGEAAB13SURBVOihh/zOG8QD78lHVlAeeWMtD4tYayOp8jTbWZw+fRoLFy7E4MGDMWbMGJhMJlgsFhw6dAiLFy/Giy++GPZJbkIIIfGr2c5i+/btGD58OO6//37xuXbt2qFnz54wGAzYvn07Zs2aJXklI4kmOpE3ykOijbU2kipPs1dDnTx50udy1MYGDx6MkydPRrxChBBC5KfZziLYDSoAYDKZ4HA4JKkUIYQQeWnV1Hfh3sFNCCEkvjV7zsLlcmHSpElBl8fjcT6WrqcGKI/csZaHRay1kVR5mu0sXnjhBUk2GkvhDDkRTyiPvLGWh0WstZFUeZrtLK655hpJNhpLNNGJvFEeEm2stZFUeVp1ziIesfSmACiP3LGWh0WstVFMJz9iSbCru+IV5ZE31vKwiLU2kipPwnUWrZllT44oj7yxlodFrLWRVHnY+isRQgiRRFiTH61YsSKseyqmTJnS6goRQgiRn7D2LHQ6HYqLi8HzPEwmE3ieR3FxMbRaLbKyssT/CCGEsCmsPYuysjI888wzuPrqq8XnTpw4gU8//RTjxo2TrHJSoCsf5I3ykGhjrY1iejXUyZMn0a1bN5/nrrjiirgcSDAeJm66FJRH3ljLwyLW2kiqPGF1Fpdffjk2btwoTiTudrvx0UcfoUuXLpJUSkocx8W6ChFFeeSNtTwsYq2NpMoT1mGoyZMnY/ny5Xj44Yeh1+ths9mQnZ2NadOmSVIpKdFlcvJGeUi0sdZGUuUJq7No06YNXn75ZVRUVKCqqgoZGRnIzMyUpEJS0+v1cLlcsa5GxFAeeWMtD4tYayOp8oTdBdXW1uL48eM4fvw4MjMzYbFYUFlZGfEKEUIIkZ+wOovjx49jxowZ2Lt3Lz799FMAwB9//IFVq1ZJWjlCCCHyEFZn8d5772HGjBn4+9//jqSkJAANV0OdPn1a0soRQgiRh7A6i4sXL6JXr14+z6lUKng8HkkqJSXvFV2soDzyxloeFrHWRlLlCauz6NChA44cOeLz3I8//ohOnTpJUikpWa3WWFchoiiPvLGWh0WstZFUecLqLMaOHYsVK1agoKAAbrcb77zzDt5880387W9/k6RSUkpPT491FSKK8sgba3lYxFobSZVHIQiCEM6KFosFe/fuxcWLF5GZmYnbbrsNZrNZkkpFwoULF2JdBUJkYfiGE1Hf5paip6OynaRV26OynUTRrl27oMtC7lnwPI8XX3wRaWlpGD58OB577DGMGDFC1h1Fc+K13sFQHnljLQ+LWGsjqfKE7CyUSiXKy8sR5g6I7IUz1Ho8oTzyxloeFrHWRlLlCeucxahRo7Bq1SpcvHgRPM/7/BeuI0eOYPr06Zg6dSq2bdvmt7y+vh7Lli3D1KlTMXfuXJSXl/ssr6iowNixY7F9O+12EkJItIU13MfKlSsBAHv27PFbtmnTppCv53keq1evxrx582A2m/Hss88iJycHHTp0ENfZtWsXdDodVqxYgf3792PDhg2YOXOmuPy9997D9ddfH051CSGERFhYnUVBQUGrNlJaWoq2bduKEyT17dsXxcXFPp1FSUkJ7rvvPgBAbm4u1qxZA0EQoFAo8O233yIrKwspKSmtqgchhJCWabazqK6uRnp6Oi677LJWbcRisficdDGbzTh16lTQdZKSkqDValFbWwu1Wo3PPvsMzz33XLOHoAoLC1FYWAgAWLRokd9Ahy6XCw6HA/X19QAQcCBEi8UCrVYLp9MJjUaD1NTUgGXodDrYbLaAJ5IqKyuh1+tht9uh1Wqh0Wh8ljudTrhcLqSmpsLhcMBkMvmVUVFRAYPBAJvNFrAMh8MBt9sNjUaDmpqagFkqKipgNBphtVqh1+v9Olq73Q6O46BWq+FyuZCRkeGzXBAEVFZWIj09HdXV1TAYDH7j5NtsNvA8D5VKBbfb7XfJHs/zsFgsYhlGoxHJyck+69TW1gJoODfGcRwUCoVPHo/Hg6qqKrGM9PR0qFSqgGUoFArwPA+DweCzvGkZGRkZ4kgEXlarFUqlUjw3l5aW5rOc4zhx+8HKqKmpgUqlEg/PpqWl+eSpr69HTU2NWIbJZPIbIbS6uhpqtRocx0GpVEKv1/ssd7vdsFqtYhlms9nvGHVVVRU0Gg3cbrff34o1Op2u1Z8nl8sFlUoFjUbTbBnx8nmy2+1ITU0Fx3EwGo0+y8P5PAXT7FrTp0/HunXrxMevvfYaZs+eHVbBjQU6Od70DR5snY8//hh33XWXXwM3lZ+fj/z8fPFxRUVFwPW8Y70HW26z2QA0NL7dbg+4jvfLKVgZ3uU2m00sL9h2gpXhvbEmVBk6nS5oGTU1NWJ9vHVqytt5BivDO+tWczf6eO8YDVWGtz7NcTgcAf/u3jLCmQUsVD2qqqpCllFXV9eiMrx/T28ZOp3OL4+3DIvFErCMxvMRBBs91FtGsME8vdtsXB8WeXO25vOk0+nAcVzIMuLl89T4PReqjKafp+YunW22s2j6Bf7TTz81t3pQZrPZ501dWVnp1/N61zGbzfB4PHA4HNDr9SgtLcWhQ4ewYcMG2O12KBQKqNVqDBkypEV1oVv75Y3ykGhjrY2kytNsZxGpS7Cys7NRVlaG8vJymEwmHDhwwG/ipD59+qCoqAhXXnklDh48iB49ekChUOCll14S1/n444+h0Wha3FEADWNasfRri/LIG2t5WMRaG0mVp9nOwuPx4NixY+Jjnud9HgNAz549Q24kKSkJ48aNw4IFC8DzPPLy8tCxY0ds2rQJ2dnZyMnJwaBBg1BQUICpU6dCr9djxowZLYzUPJ1OB6fTKUnZsUB55I21PCxirY2kytPscB9PPPFE8y9WKFp9pZRUgg33kZmZGfQ4XjyiPPImhzw03Efz5NBGkdSaPC0+Z/HGG2+0aIOEEBINnvH3tL6QrQdaX0YCYGumckIIIZJIuM4i2CWR8YryyBtreVjEWhtJlSfhOotg9yzEK8ojb6zlYRFrbSRVnoTrLJre3RvvKI+8sZaHRay1kVR5Eq6zCOcu4nhCeeSNtTwsYq2NpMqTcJ1FoHGU4hnlkTfW8rCItTaSKk/CdRaEEEIuHXUWhBBCQqLOghBCSEjUWRBCCAkp4TqLYHMIxCvKI2+s5WERa20kVZ6E6yy0Wm2sqxBRlEfeWMvDItbaSKo8CddZBJt5LF5RHnljLQ+LWGsjqfIkXGfRdN7beEd55I21PCxirY2kypNwnQXtcsob5SHRxlob0WEoQgghMdPs5EeEkMiJxYx1hEQK7VkQQggJKeE6C7ryQd4oD4k21tqIroaKEIfDEesqRBTlkTfW8rCItTaSKk/CdRZ6vT7WVYgoyiNvrOVhEWttJFWehOssrFZrrKsQUZRH3ljLwyLW2kiqPAnXWdBEJ/JGeUi0sdZGNPkRIYSQmKHOghBCSEjUWRBCCAmJOgtCCCEhJVxnQROdyBvlIdHGWhtJlSdqY0MdOXIEa9euBc/zGDx4MEaMGOGzvL6+HgUFBThz5gzS0tIwY8YMtGnTBj/88AM2bNgAjuOgUqkwduxY9OzZs8X10Gq1sNlsrY0jG5RH3ljLwyLW2kiqPFHZs+B5HqtXr8bcuXOxbNky7N+/H+fPn/dZZ9euXdDpdFixYgXuuusubNiwAQCQlpaGOXPmYMmSJXjiiSewYsWKVtXF6XS26vVyQ3nkjbU8LGKtjaTKE5XOorS0FG3btkVWVhZUKhX69u2L4uJin3VKSkowcOBAAEBubi6OHTsGQRBw+eWXw2QyAQA6duyI+vp61NfXt7guGo2mxa+VI8ojb6zlYRFrbSRVnqgchrJYLDCbzeJjs9mMU6dOBV0nKSkJWq0WtbW1MBgM4jqHDh3C5ZdfjuTkZL9tFBYWorCwEACwaNEivxtTXC4XHA4H9Ho97HZ7wBtXLBYLtFotnE4nNBoNUlNTA5ah0+lgs9l8MnlVVlaK29BqtX4N53Q64XK5kJqaCofDIXaEjVVUVMBgMMBmswUsw+FwwO12Q6PRBKyntwyj0Qir1Qq9Xo+UlBSf5Xa7HRzHQa1Ww+VyISMjw2e5IAiorKxEeno6qqurYTAY/Gbgstls4HkeKpUKbrcb6enpPst5nofFYhHLMBqNfm1XW1sLAFAqleA4DmlpaT55PB4PqqqqxDLS09OhUqkClqFQKMDzvM97JlAZGRkZSEpK8lnHarVCqVRCEAQADXu0jXEcJ24/WBk1NTVQqVTgeV4sQ6VSBWwfIh86nQ51dXXQaDTNfibj5fOk1WrB8zw4joPRaPRZHs7nKZiodBbeD2BjCoXiktY5d+4cNmzYgL///e8Bt5Gfn4/8/HzxcUVFRcD1PB5Ps8u9x/rsdjvsdnvAdbxfTsHK8C632WxBjx16nw9WhveW/VBlaDSaoGXU1NSI9fHWqSnvXlqwMqqrq33qE4jb7Q6rDG99msNxXMByvGV4/9+cUPWoqqoKWUZdXV2Lymi811tXV4fMzMyg9WHZyIH/iMl2txQ9fcmv8X6xhvpMxsvnSRAE8VBUqDKafp7atWsXtF5ROQxlNptRWVkpPq6srPTreRuv4/F4xL0A7/qvvfYannjiCbRt2zYaVSaEENJIVDqL7OxslJWVoby8HBzH4cCBA8jJyfFZp0+fPigqKgIAHDx4ED169IBCoYDdbseiRYvw4IMPonv37tGoLiGEkCaichgqKSkJ48aNw4IFC8DzPPLy8tCxY0ds2rQJ2dnZyMnJwaBBg1BQUICpU6dCr9djxowZAIAvvvgCf/zxBz799FN8+umnAIB58+b5HYsLF010Im+Uh0Qba20kVR6FEOhkAQMuXLgQ8HmlUimegGQB5ZG3xnloDm7pteScRfLqfzH7nrtUMT9nISc6nS7WVYgoyiNvrOVhEWttJFWehOssWLpTE6A8csdaHhax1kZS5Um4ziLQvRHxjPLIG2t5WMRaG0mVJ+E6C0IIIZcuagMJEiIXdKKZkEtHexaEEEJCoj0LQkhC+/OvfVtdRtKq7RGoibwl3J5F42FHWEB5CCGNSfUZSrjOwjveFCsoDyGkMak+QwnXWQQbSTZeUR5CSGNSfYYSrrPQarWxrkJEUR5CSGNSfYYSrrOgWbHkjbU8hESbVJ+hhOssCCGEXDrqLAghhIREnQUhhJCQEu6mPO/ctKyI5zw07AYhkSfVd0LC7VnQrFiEEJZJ9Z2QcJ1FampqrKsQUazlIYS0jlTfCQnXWTgcjlhXIaJYy0MIaR2pvhMSrrMwmUyxrkJEsZaHENI6Un0nJFxnQQgh5NJRZ0EIISQk6iwIIYSERJ0FIYSQkBLupryKiopYVyGiWMtDSDzyjL8nIuVEYsY9qb4TEq6zMBgMsFqtsa5GxEQiD91JTQg7pPqOS7jDUDabLdZViCjW8hBCWkeq74SE6yxYm1yHtTyEkNahyY8ihLXJdVjLQwhpHam+E6J2zuLIkSNYu3YteJ7H4MGDMWLECJ/l9fX1KCgowJkzZ5CWloYZM2agTZs2AICtW7di165dUCqVePTRR9G7d+9oVZsQQgii1FnwPI/Vq1dj3rx5MJvNePbZZ5GTk4MOHTqI6+zatQs6nQ4rVqzA/v37sWHDBsycORPnz5/HgQMHsHTpUlRVVWH+/Pl4/fXXoVRKt1MUqxO+n43pHpPtEsKKkQP/EZPtbil6OiLlROSqqq0HWl9GAFHpLEpLS9G2bVtkZWUBAPr27Yvi4mKfzqKkpAT33XcfACA3Nxdr1qyBIAgoLi5G3759kZycjDZt2qBt27YoLS3FlVdeGY2qRxVdlURIfIr3TiocUeksLBYLzGaz+NhsNuPUqVNB10lKSoJWq0VtbS0sFgu6desmrmcymWCxWPy2UVhYiMLCQgDAokWL0K5du6D1aW4ZABQ/1fxyQgiRhadKAj4d6juuJaJyglsQBL/nFApFWOsEej6Q/Px8LFq0CIsWLWp2vWeeeSas8uIF5ZE31vIA7GWiPOGJSmdhNptRWVkpPq6srERGRkbQdTweDxwOB/R6vd9rLRYLDctNCCFRFpXOIjs7G2VlZSgvLwfHcThw4ABycnJ81unTpw+KiooAAAcPHkSPHj2gUCiQk5ODAwcOoL6+HuXl5SgrK8MVV1wRjWoTQgj5f5JefPHFF6XeiFKpRNu2bbFixQp88cUXuO2225Cbm4tNmzbB5XKhXbt26NSpE/bt24cPP/wQv/zyCyZMmAC9Xg+j0QibzYaVK1di3759GDduXKuPx3Xt2jVCyeSB8sgba3kA9jJRntAUQrgnBQghhCSshLuDmxBCyKWjzoIQQkhICTVE+fbt2/HBBx/g3XffhcFggCAIWLt2Lb7//nukpKRg8uTJcXPs8qOPPkJJSQkUCgWMRiMmT54Mk8kUt5nWr1+Pw4cPQ6VSISsrC5MnT4ZOpwMQn8O9fPPNN/jkk0/w+++/45VXXkF2dra4LB7zAKGH7IkHb775Jr777jsYjUYsWbIEQMMorcuWLcPFixdx2WWXYebMmdDr9TGuaXgqKirwxhtvoLq6GgqFAvn5+bjzzjulySQkiIsXLwovv/yyMGnSJKGmpkYQBEE4fPiwsGDBAoHneeH//J//Izz77LMxrmX47Ha7+O8dO3YIK1euFAQhfjMdOXJE4DhOEARBWL9+vbB+/XpBEATh3LlzwuzZswW32y38+eefwpQpUwSPxxPLqobl3Llzwu+//y688MILQmlpqc/z8ZjH4/EIU6ZMEf744w+hvr5emD17tnDu3LlYV+uS/fTTT8Lp06eFWbNmic+tX79e2Lp1qyAIgrB161bxvRcPLBaLcPr0aUEQBMHhcAjTpk0Tzp07J0mmhDkMtW7dOowZM8bnZsCSkhL0798fCoUCV155Jex2O6qqqmJYy/A1Hoa4rq5OzBWvma677jokJSUBAK688krxLv1gw73IXYcOHQJetReveRoP2aNSqcQhe+LNNddc4/cLu7i4GAMGDAAADBgwIK5yZWRkiEcOUlNT0b59e1gsFkkyJURnUVJSApPJhC5duvg8b7FYkJmZKT42m80BhxKRq40bN2LSpEnYt28fRo8eDSD+MwENg0p6D800HSom2HAv8SJe8wQasice6h2Ompoa8SbhjIyMuJ1Js7y8HGfPnsUVV1whSSZmzlnMnz8f1dXVfs8/8MAD2Lp1K+bNm+e3TAhjGJJYai7TjTfeiAcffBAPPvggtm7dii+++AL333+/rDOFygMAW7ZsQVJSEm677TYAgdtILsLJ05Sc8zRHzu8rArhcLixZsgSPPPKIZJMfMdNZPPfccwGf/+2331BeXo6nnnoKQMNQI3PmzMHChQthNpt9JjcPNAxJLAXL1NStt96KRYsW4f7775d1plB5ioqKcPjwYTz//PPiF5Gch3sJt30ak3Oe5oQzZE+8MhqNqKqqQkZGBqqqqmAwGGJdpUvCcRyWLFmC2267DTfffDMAaTIxfxiqU6dOePfdd/HGG2/gjTfegNlsxuLFi5Geno6cnBzs2bMHgiDg5MmT0Gq1cfMBKCsrE/9dUlIiHh+P10xHjhzBZ599hjlz5iAlJUV8nrXhXuI1TzhD9sSrnJwc7N69GwCwe/fuoHuFciQIAt5++220b98ew4YNE5+XIlPC3cH9xBNPYOHCheKls6tXr8bRo0ehVqsxefJkn0sc5ey1115DWVkZFAoFMjMzMWHCBPHS2XjMNHXqVHAcJ5587NatGyZMmACg4dDU119/DaVSiUceeQTXX399LKsalm+//RZr1qyB1WqFTqdDly5d8Pe//x1AfOYBgO+++w7r1q0Dz/PIy8vDyJEjY12lS/bPf/4Tx48fR21tLYxGI+6//37ceOONWLZsGSoqKpCZmYlZs2bFzaWzJ06cwPPPP49OnTqJe+MPPvggunXrFvFMCddZEEIIuXTMH4YihBDSetRZEEIICYk6C0IIISFRZ0EIISQk6iwIIYSERJ0FIQA+/vhjLF++vMWvf+edd7B58+YI1ogQeWHmDm5CgIb7aKqrq6FUKqHRaHD99ddj3Lhx0Gg0km7Xe08IAPz0009YsWIF3n777RaVZbfbsW7dOnz//feoq6tDRkYG8vLy4nJIcMIO6iwIc+bMmYNrr70WFosFCxYswKeffooxY8ZItj2e56FURm4nfd26dXC5XFi2bBm0Wi3Kyspw7ty5iJUPAB6PRxzll5BwUGdBmGUymdC7d2/xi9ZisWDVqlU4ceIE9Ho9hg8fjvz8/ICvXbp0KX7++We43W506dIFjz32GDp27AgAeOONN6BWq1FRUYHjx4/jqaeewt69e2E2mzFixAi88sor4DgOY8eOBQC8/vrrmDZtGt566y2kpaUBAM6cOYMFCxZg5cqVUKl8P4anT5/G6NGjxTtu27dvj/bt24vLz507h/feew9nzpyBSqXC0KFDMXLkSNTX12PDhg345ptvAAC33HILxowZg+TkZHFvZ8iQIdixYweuvfZaTJ06FYcPH8ZHH32EixcvokOHDhg/fjw6d+4cwVYgrKBzFoRZFRUV+P7778Wh6V9//XWYzWasXLkSTz75JDZu3Igff/wx4Gt79+6N5cuX491338Xll1/udz5j3759+Otf/4p169ahe/fu4vMajQZz585FRkYG1q9fj/Xr18NkMqFHjx7ilzgA7NmzB/369fPrKICGoU4++ugjfP311z5jgAGA0+nE/Pnz0bt3b6xcuRLLly9Hr169ADQMI3Lq1Cn84x//wKuvvorS0lJ8+umn4murq6ths9nw5ptvYuLEiThz5gzeeustTJgwAWvWrEF+fj7+8Y9/oL6+/tL+0CQhUGdBmPPqq6/ikUcewfPPP49rrrkGI0eOREVFBU6cOIExY8ZArVajS5cuGDx4MPbs2ROwjEGDBiE1NRXJycm477778Ouvv8LhcIjLb7zxRnTv3h1KpRJqtTpknQYMGIC9e/cCaDhstX//fvTv3z/guuPGjcOtt96KL774ArNmzcLUqVPx/fffAwAOHz6M9PR03H333VCr1UhNTUW3bt0ANHRg9957L4xGIwwGA0aNGiVuE2gYUvz+++9HcnIy1Go1vvrqK+Tn56Nbt25QKpUYOHAgVCoVTp06Fd4fmiQUOgxFmPPUU0/h2muv9XmuqqoKer0eqamp4nOZmZk4ffq03+t5nsfGjRtx8OBBWK1WcYA2q9UqzhXQeCKgcOTk5GDVqlX4888/UVZWBq1WG3S0WbVajZEjR2LkyJFwOBzYtm0bli5dirfeeguVlZXIysoK+DqLxYLLLrtMfHzZZZf5TFBkMBh8OraKigrs3r0bX3zxhfgcx3HMTGpEIos6C5IQMjIyYLPZ4HQ6xQ6joqIi4FwS+/btQ0lJCZ577jlcdtllcDgcePTRR33WaW7in0DL1Go1brnlFuzduxcXLlwIulfRlFarxciRI7Ft2zaUl5fDbDZj//79Adc1mUy4ePGieG6lab6m9TKbzWKnREgodBiKJITMzExcddVV+PDDD+F2u/Hrr7/i66+/Fmfka8zpdEKlUkGv16Ourg4bN268pG0ZjUbU1tb6HLYCgP79+2P37t0oKSkJuF2vzZs3o7S0FBzHwe12Y+fOndDpdGjXrh369OmD6upq7NixA/X19XA6neJho379+mHLli2wWq2wWq3YvHlzs9sZPHgwvvzyS5w6dQqCIMDlcuG7776D0+m8pLwkMdCeBUkY06dPx6pVqzBx4kTo9Xrcd999foergIbzC0ePHsXjjz8OvV6P0aNH4z//+U/Y22nfvj369euHKVOmgOd5LF26FCaTCd27d4dCocDll1+ONm3aBH29QqHAW2+9hYqKCiiVSnTu3BnPPPOMeK/IvHnz8N5772Hz5s1QqVS466670K1bN/Gw1ezZswEAubm5ze41ZGdnY+LEiVizZg3KysqgVqvRvXt3XH311WFnJYmD5rMgJIr+53/+B7feeisGDx4c66oQcknoMBQhUVJaWoqzZ8+ib9++sa4KIZeMDkMREgUFBQUoLi7Go48+6nNFFiHxgg5DEUIICYkOQxFCCAmJOgtCCCEhUWdBCCEkJOosCCGEhESdBSGEkJD+L7ljOCOdtwOVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(pos_ngram_scores, label='Positive four-gram', density=True)\n",
    "plt.hist(neg_ngram_scores, label='Negative four-gram', density=True)\n",
    "plt.grid(linestyle='-.')\n",
    "plt.legend()\n",
    "plt.xlabel('Polarity Score')\n",
    "plt.ylabel('Freq Density')\n",
    "#plt.savefig(\"imgs/sst_4gram_polarity_score_gru.pdf\", bbox_inches='tight', dpi=900)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.024874087423086166"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ngram = [16, 9, 39]\n",
    "ngram_score = get_phrase_polarity(ngram)\n",
    "ngram_score.item()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RuXWeGUICyO2"
   },
   "source": [
    "## Statistics\n",
    "Let us do an experiment, check whether the labels have strong relationship with certain words like the sentiment indicator \"good\", \"bad\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fgv4qVlYCyO2"
   },
   "outputs": [],
   "source": [
    "texts, labels = zip(*train_dg.data_batch)\n",
    "text_length = [len(t) for t in texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2CRTATPWCyO4"
   },
   "outputs": [],
   "source": [
    "texts2, labels2 = zip(*dev_dg.data_batch)\n",
    "text_length2 = [len(t) for t in texts2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kCJdH7v3CyO7"
   },
   "outputs": [],
   "source": [
    "texts3, labels3 = zip(*test_dg.data_batch)\n",
    "text_length3 = [len(t) for t in texts3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pV0bZn6pCyPL"
   },
   "outputs": [],
   "source": [
    "#Calculate the times that a token appears in a specified label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YpVb053gCyPO"
   },
   "outputs": [],
   "source": [
    "word_label_pair = []\n",
    "word_doc_freq = {}\n",
    "vocab = []\n",
    "for i, text in enumerate(texts):\n",
    "    for w in text:\n",
    "        word_doc_freq[w] = word_doc_freq.get(w, 0) + 1\n",
    "        \n",
    "for i, text in enumerate(texts):\n",
    "    label = labels[i]\n",
    "    for w in text:\n",
    "        word_doc_freq[w] = word_doc_freq.get(w, 0) + 1\n",
    "    word_label_pair += [(id2word[item], label) for item in text]\n",
    "    vocab += [id2word[item] for item in text]\n",
    "vocab = set(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9E9bMJSBCyPS"
   },
   "outputs": [],
   "source": [
    "word_label_pair_freq = Counter(word_label_pair)\n",
    "#Get the ratio, a word appears in positive sentences: a word appears in negative sentences\n",
    "token_freq_pos = {}\n",
    "token_freq_neg = {}\n",
    "type0_tokens = []\n",
    "type1_tokens = []\n",
    "type2_tokens = []\n",
    "neutral_tokens = []\n",
    "total_diff_freq = 0\n",
    "#Note here, 1 means positive and 0 means negative\n",
    "for w in vocab:\n",
    "    freq_2 = word_label_pair_freq[(w, 2)]\n",
    "    freq_1 = word_label_pair_freq[(w, 1)]\n",
    "    freq_0 = word_label_pair_freq[(w, 0)]\n",
    "    total = freq_1+freq_2+freq_0\n",
    "    if total == 0:\n",
    "        continue\n",
    "    if freq_2/total>2/3:\n",
    "        type2_tokens.append(w)\n",
    "    if freq_1/total>2/3:\n",
    "        type1_tokens.append(w)\n",
    "    if freq_0/total>2/3:\n",
    "        type0_tokens.append(w)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(neutral_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from textblob import TextBlob\n",
    "pos_tokens_selected = []\n",
    "neg_tokens_selected = []\n",
    "for w in type2_tokens:\n",
    "    s = TextBlob(w)\n",
    "    if len(s.tags) == 0:\n",
    "        continue\n",
    "    if s.polarity>0.2 and s.tags[0][1]=='JJ':\n",
    "        pos_tokens_selected.append(w)\n",
    "for w in type0_tokens:\n",
    "    s = TextBlob(w)\n",
    "    if len(s.tags) == 0:\n",
    "        continue\n",
    "    if s.polarity<-0.2 and s.tags[0][1]=='JJ':\n",
    "        neg_tokens_selected.append(w)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-sK28pJhCyPd"
   },
   "outputs": [],
   "source": [
    "# pos_tokens_index = [word2id[w] for w in pos_tokens]\n",
    "# neg_tokens_index = [word2id[w] for w in neg_tokens]\n",
    "# neutral_tokens_index = [word2id[w] for w in neutral_tokens]\n",
    "# pos_tokens_index_label1 = [token_freq_pos[w] for w in pos_tokens_index]\n",
    "# pos_tokens_index_label0 = [token_freq_neg[w] for w in pos_tokens_index]\n",
    "# pos_tokens_index_diff = [token_freq_pos[w]-token_freq_neg[w] for w in pos_tokens_index]\n",
    "\n",
    "# neg_tokens_index_label1 = [token_freq_pos[w] for w in pos_tokens_index]\n",
    "# neg_tokens_index_label0 = [token_freq_neg[w] for w in pos_tokens_index]\n",
    "# neg_tokens_index_diff = [token_freq_pos[w]-token_freq_neg[w] for w in neg_tokens_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualization Negating Adjectives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_dim = 300\n",
    "pos_token_ps = []\n",
    "negation_pos_token_ps = []\n",
    "neg_token_ps = []\n",
    "negation_neg_token_ps = []\n",
    "negation_id = word2id['not']\n",
    "for w in pos_tokens_selected:\n",
    "    bigram = [negation_id] + [word2id[w]]\n",
    "    score1 = gru_phrase_polarity_multi([word2id[w]], best_model, hidden_dim)\n",
    "    score2 = gru_phrase_polarity_multi(bigram, best_model, hidden_dim)\n",
    "    pos_token_ps.append(score1)\n",
    "    negation_pos_token_ps.append(score2)\n",
    "for w in neg_tokens_selected:\n",
    "    bigram = [negation_id] + [word2id[w]]\n",
    "    score1 = gru_phrase_polarity_multi([word2id[w]], best_model, hidden_dim)\n",
    "    score2 = gru_phrase_polarity_multi(bigram, best_model, hidden_dim)\n",
    "    neg_token_ps.append(score1)\n",
    "    negation_neg_token_ps.append(score2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_token_ps = torch.stack(pos_token_ps).detach().cpu().numpy()\n",
    "neg_token_ps = torch.stack(neg_token_ps).detach().cpu().numpy()\n",
    "negation_pos_token_ps = torch.stack(negation_pos_token_ps).detach().cpu().numpy()\n",
    "negation_neg_token_ps = torch.stack(negation_neg_token_ps).detach().cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'whiskers': [<matplotlib.lines.Line2D at 0x7f2c140eac10>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140ea490>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140ea610>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140e7290>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140de890>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140de2d0>],\n",
       " 'caps': [<matplotlib.lines.Line2D at 0x7f2c140ebe50>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140e8690>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140e7fd0>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140e1590>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140de210>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140db550>],\n",
       " 'boxes': [<matplotlib.lines.Line2D at 0x7f2c140ebe90>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140e7110>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140e1650>],\n",
       " 'medians': [<matplotlib.lines.Line2D at 0x7f2c140e8750>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140e2410>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140db3d0>],\n",
       " 'fliers': [<matplotlib.lines.Line2D at 0x7f2c140e7910>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140df6d0>,\n",
       "  <matplotlib.lines.Line2D at 0x7f2c140d87d0>],\n",
       " 'means': []}"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAQtUlEQVR4nO3df4hc13nG8efpeJMtjRtLlVo7tolcasKk058ZTENE8TQxOKFEbpsQqdA61QRBqbfpfzIMxN7Agrd/FNq1aVAzwnYp45YkOGrqYmIywR1atxoZ26y9NVFNg7c2eGOptky77Wr79o8dKav1aDW7c3Xv7JzvBwbN3Dk750WDHt197z33OiIEABh/P1J0AQCAfBD4AJAIAh8AEkHgA0AiCHwASMQ1RRdwOXv27Il9+/YVXQYA7CinTp36QUTs7ffeyAb+vn371O12iy4DAHYU29+/3Hu0dAAgEQQ+ACSCwAeARBD4AJAIAh8AEkHg56zVaqlSqahUKqlSqajVahVdEoBEjOxpmeOo1Wqp0Wio2Wxq//796nQ6qtfrkqRDhw4VXB2AcedRvTxytVqNcTsPv1KpaG5uTrVa7eK2drutqakpzc/PF1gZgHFh+1REVPu+R+Dnp1QqaXl5WRMTExe3raysaHJyUqurqwVWBmBcbBb49PBzVC6X1el0LtnW6XRULpcLqghASgj8HDUaDdXrdbXbba2srKjdbqter6vRaBRdGoAEcNA2RxcOzE5NTWlhYUHlclkzMzMcsAWQC3r4ADBG6OEDAAh8AEgFgZ8zVtoCKAoHbXPESlsAReKgbY5YaQvgamOl7YhgpS2Aq42zdEYEK20BFInAzxErbQEUiYO2OWKlLYAi0cMHgDFCDx8AQOADQCoyCXzbx22/YbvvyeS2b7f9lu3neo8vZTEvAGBwWR20fVjSg5Ie3WTMP0TEr2c0HwBgizLZw4+IpyWdyeKzAABXR549/I/aft7239v+2X4DbB+x3bXdXVpayrE0ABh/eQX+s5I+GBG/IGlO0uP9BkXEsYioRkR17969OZUGAGnIJfAj4u2IeKf3/AlJE7b35DE3AGBNLoFv+3rb7j2/rTfvm3nMDQBYk8lZOrZbkm6XtMf2oqT7JE1IUkR8RdJnJP2+7fOS/lvSwRjVJb4AMKYyCfyI2PRiMBHxoNZO2wQAFISVtgCQCAIfABJB4ANAIgh8AEgEgQ8AiSDwASARBD4AJILAB4BEEPgAkAgCHwASQeADQCIIfABIBIEPAIkg8AEgEQR+zlqtliqVikqlkiqVilqtVtElAUhEJtfDx2BarZYajYaazab279+vTqejer0uSTp0aNNbCgDA0DyqN56qVqvR7XaLLiNTlUpFc3NzqtVqF7e1221NTU1pfn6+wMoAjAvbpyKi2vc9Aj8/pVJJy8vLmpiYuLhtZWVFk5OTWl1dLbAyAONis8Cnh5+jcrmsTqdzybZOp6NyuVxQRQBSQuDnqNFoqF6vq91ua2VlRe12W/V6XY1Go+jSACSAg7Y5unBgdmpqSgsLCyqXy5qZmeGALYBc0MMHgDFCDx8AkE3g2z5u+w3bfc8t9Jo/s33a9gu2fzmLeQEAg8tqD/9hSXdu8v4nJd3aexyR9OcZzQsAGFAmgR8RT0s6s8mQA5IejTXPSLrO9g1ZzA0AGExePfwbJb267vVib9slbB+x3bXdXVpayqk0AEhDXoHvPtvedXpQRByLiGpEVPfu3ZtDWQCQjrwCf1HSzete3yTptZzmBgAov8A/Iel3e2fr/IqktyLi9ZzmBgAoo5W2tluSbpe0x/aipPskTUhSRHxF0hOSPiXptKT/kvR7WcwLABhcJoEfEZteGyDWlvP+QRZzAQC2h5W2OeOOVwCKwsXTcsQdrwAUiYun5Yg7XgG42rjj1YjgjlcArjauljkiuOMVgCIR+DnijlcAisRB2xxxxysARaKHDwBjhB4+AIDAB4BUEPgAkAgCHwASQeADQCIIfABIBIEPAIkg8AEgEQQ+ACSCwAeARBD4AJAIAh8AEkHgA0AiCHwASATXw79KbGfyOaN6+WoAOw+Bf5VcKahtE+YAcpVJS8f2nbZftn3a9r193v+87SXbz/UeX8hiXgDA4Ibew7ddkvSQpDskLUo6aftERLy0YehfR8Q9w84HANieLPbwb5N0OiJeiYj/lfSYpAMZfC4AIENZBP6Nkl5d93qxt22j37L9gu2v2b653wfZPmK7a7u7tLSUQWkAgAuyCPx+p6NsPBr5t5L2RcTPS3pK0iP9PigijkVENSKqe/fuzaA0AMAFWQT+oqT1e+w3SXpt/YCIeDMi/qf38i8kfSSDeQEAW5BF4J+UdKvtW2y/R9JBSSfWD7B9w7qXn5a0kMG8AIAtGPosnYg4b/seSU9KKkk6HhEv2v6ypG5EnJD0h7Y/Lem8pDOSPj/svACArfGoLv6pVqvR7XaLLuOqYeEVcHWkvsrd9qmIqPZ7j5W2AMYKq9wvj4unAUAiCHwASASBDwCJIPABIBEEPgAkgsAHgEQQ+ACQCAIfABLBwitgg9RXamJ8EfjABoMEdcqrNbFz0dIBgEQQ+ACQCAJ/m3bv3i3b235IGurnbWv37t0F/y0A+ePf3vbRw9+ms2fPFt7DzergIrCT8G9v+9jDB4BEsIcPYEeJ+35cuv/9xdewAxH4SM7u3bt19uzZoT9n2F/rd+3apTNnzgxdR2o8/fZItHTi/kJL2BYCH8kZhR6wtHP7wNi56OEDQCIIfABIBIEPAIkg8AEgEZkctLV9p6Q/lVSS9NWIeGDD+++V9Kikj0h6U9LnIuLfs5gb2KpROK3vYh1AjoYOfNslSQ9JukPSoqSTtk9ExEvrhtUlnY2In7F9UNKspM8NO3eRRiE0CIztGYXT+qSde2ofdq4s9vBvk3Q6Il6RJNuPSTogaX3gH5B0f+/51yQ9aNsxCv/qtmkUQoPAALAVWfTwb5T06rrXi71tfcdExHlJb0n6iY0fZPuI7a7t7tLSUgalAQAuyGIPv9/qkY27voOMUUQck3RMkqrV6o7d+8foG4VFT7t27Sq6hB2r6O9vp353WQT+oqSb172+SdJrlxmzaPsaSe+XxJpyFCKLVhx3vCrOsH/vKX93WbR0Tkq61fYttt8j6aCkExvGnJB0d+/5ZyR9Zyf37wFgJxp6Dz8iztu+R9KTWjst83hEvGj7y5K6EXFCUlPSX9o+rbU9+4PDzjsK+LUSwE6SyXn4EfGEpCc2bPvSuufLkj6bxVyjgl8rAew0rLQFgEQQ+ACQCAIfABLBDVCADQY9GH+lcRyjwagh8IENCGqMK1o6AJAIAh8AEkFLB8BYGeQYzCBjxrG1R+ADGCvjGNRZoaUDAIkg8AEgEQQ+ACSCwAeARBD4wBa0Wi1VKhWVSiVVKhW1Wq2iSwIGxlk6wIBarZYajYaazab279+vTqejer0uSTp06FDB1QFX5lE9halarUa32y26jG3L6uYoo/r9pKhSqWhubk61Wu3itna7rampKc3PzxdYGfBDtk9FRLXve6MaKDs98DF+SqWSlpeXNTExcXHbysqKJicntbq6WmBlwA9tFvj08IEBlctldTqdS7Z1Oh2Vy+WCKgK2hsAHBtRoNFSv19Vut7WysqJ2u616va5Go1F0acBAOGgLDOjCgdmpqSktLCyoXC5rZmaGA7bYMejhA8AYoYcPIHmsoaClAyABrKFYM9Qevu3dtr9t+3u9P3ddZtyq7ed6jxPDzAkAWzUzM6Nms6laraaJiQnVajU1m03NzMwUXVquhurh2/5jSWci4gHb90raFRFH+4x7JyLet5XPpocPICspraG4mj38A5Ie6T1/RNJdQ34eAGSuXC5renr6kh7+9PR0cmsohg38n4qI1yWp9+dPXmbcpO2u7WdsX/Y/BdtHeuO6S0tLQ5YGAGtqtZpmZ2d1+PBhnTt3TocPH9bs7Owll8lIwRVbOrafknR9n7cakh6JiOvWjT0bEe/q49v+QES8ZvunJX1H0scj4t82m5eWDoCsVCoV3XXXXXr88ccvrqG48HrcroN01a6lY/tlSbdHxOu2b5D03Yj40BV+5mFJ34qIr202jsAHkBV6+GuGbemckHR37/ndkr7ZZ/Jdtt/be75H0sckvTTkvAAwMK6DtGbYwH9A0h22vyfpjt5r2a7a/mpvTFlS1/bzktqSHogIAh9AbrgO0pqhFl5FxJuSPt5ne1fSF3rP/1HSzw0zDwAMg+sgreFaOgAwRriWDgCAwAeAVBD4AJAIAh8AEkHgA0AiCHwASASBDwCJIPABIBEEPgAkgsAHgEQQ+ACQCAIfABJB4ANAIgh8AEgEgQ8AiSDwASARBD4AJILAB4BEEPgAkAgCHwASQeADQCIIfABIxFCBb/uztl+0/X+2q5uMu9P2y7ZP2753mDkBANsz7B7+vKTflPT05QbYLkl6SNInJX1Y0iHbHx5yXgDAFl0zzA9HxIIk2d5s2G2STkfEK72xj0k6IOmlYeYGAGxNHj38GyW9uu71Ym/bu9g+Yrtru7u0tJRDaQCQjivu4dt+StL1fd5qRMQ3B5ij3+5/9BsYEcckHZOkarXadwwAYHuuuIcfEZ+IiEqfxyBhL63t0d+87vVNkl7bTrEAsF2tVkuVSkWlUkmVSkWtVqvoknI3VA9/QCcl3Wr7Fkn/IemgpN/OYV4AkLQW9o1GQ81mU/v371en01G9XpckHTp0qODq8jPsaZm/YXtR0kcl/Z3tJ3vbP2D7CUmKiPOS7pH0pKQFSX8TES8OVzYADG5mZkbNZlO1Wk0TExOq1WpqNpuamZkpurRcOWI0W+XVajW63W7RZQAYA6VSScvLy5qYmLi4bWVlRZOTk1pdXS2wsuzZPhURfddFsdIWwNgrl8uanp6+pIc/PT2tcrlcdGm5IvABjL1arabZ2VkdPnxY586d0+HDhzU7O6tarVZ0abki8AGMvXa7raNHj+r48eO69tprdfz4cR09elTtdrvo0nJFDx/A2KOHv4Y9fABjr1wuq9PpXLKt0+nQwweAcdNoNFSv19Vut7WysqJ2u616va5Go1F0abnKY+EVABTqwuKqqakpLSwsqFwua2ZmJqlFVxI9fAAYK/TwASSPa+nQ0gGQAK6ls4aWDoCxV6lUNDc3d8lCq3a7rampKc3PzxdYWfY2a+kQ+ADGHufhr6GHD2DscR7+GgIfwNjjPPw1HLQFMPY4D38NPXwAGCP08AEABD4ApILAB4BEEPgAkAgCHwASMbJn6dhekvT9ouu4ivZI+kHRRWDb+P52rnH/7j4YEXv7vTGygT/ubHcvd+oURh/f386V8ndHSwcAEkHgA0AiCPziHCu6AAyF72/nSva7o4cPAIlgDx8AEkHgA0AiCPyc2T5u+w3b43VftQTYvtl22/aC7Rdtf7HomjA425O2/8X2873vb7romvJGDz9ntn9V0juSHo2IStH1YHC2b5B0Q0Q8a/taSack3RURLxVcGgZg25J+LCLesT0hqSPpixHxTMGl5YY9/JxFxNOSzhRdB7YuIl6PiGd7z89JWpB0Y7FVYVCx5p3ey4neI6k9XgIf2Abb+yT9kqR/LrYSbIXtku3nJL0h6dsRkdT3R+ADW2T7fZK+LumPIuLtouvB4CJiNSJ+UdJNkm6znVRblcAHtqDX+/26pL+KiG8UXQ+2JyL+U9J3Jd1ZcCm5IvCBAfUO+jUlLUTEnxRdD7bG9l7b1/We/6ikT0j612KryheBnzPbLUn/JOlDthdt14uuCQP7mKTfkfRrtp/rPT5VdFEY2A2S2rZfkHRSaz38bxVcU644LRMAEsEePgAkgsAHgEQQ+ACQCAIfABJB4ANAIgh8AEgEgQ8Aifh/+qRCdTYjKuEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.boxplot([negation_neg_token_ps[:, 0], negation_neg_token_ps[:, 1], negation_neg_token_ps[:, 2]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fd2ec8842d0>]"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD4CAYAAADy46FuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAevklEQVR4nO3deVRTd6IH8G8SdkLCathVNisIilURFa1aba2odLGLtVOdTp0uM22nPju209c3Zzqvm8+e1mk7fWM7nem02tpn1VatdasKLrihKCKooCIgiwgBwhKSvD+0UQZyveINNwnfzzmeAzfh5nt/JHy9u8JkKLOAiIjIBqXcAYiIyLGxKIiISBCLgoiIBLEoiIhIEIuCiIgEuckdQGoNer3cEURRKBSwWHjAmVQ4ntLhWErLWcZTAUCj0XT7GNcoZKLx85M7gkvheEqHYyktZxlPpdJ2HbAoiIhIEIuCiIgEsSiIiEgQi4KIiASxKIiISBCLgoiIBLEoiIhIEIuCiIgEudyZ2Y7grSUf4p2lH93yfF5a+AwWL3pWgkRERD2ncLUbFznLJTwCQ5NQd7FA7hguQ6vROM3v3tFxLKXlLOOpUiqhVqu7fYybnoiISBCLgoiIBLEoiIhIEIuCiIgEsSiIiEiQwxwe+/n3OTh2+gL8fL3w2oIsAEBzSxuWr9mBS/VNCPJX48l774Cvt6fMSYmI+haHWaNIHxqH3z48pdO0TXuO4bYBYXj9mftx24Aw/Lj3mEzpiIj6LocpivjoUPh4e3Sall98HunJcQCA9OQ4HC06L0c0IqI+zWE2PXVH39wCrZ8PAEDr54NGQ2u3z8s+XITsvGIAwLzMdITrgnst463Q2rg/LfUMx1M6HEtpOcN4NjU12XzMoYtCrIzhg5AxfBCAK2dmO8NZkIDznEXuDJzl7FdnwLGUlrOMp8pZ75mt8fVGQ6MBANDQaICfj5fMiYiI+h6HLoqUhCjsPXYaALD32GmkJETLnIiIqO9xmE1Pn6zZieJzF9HU0orFy1ZhxvhhuCs9GcvX7MTuI6cQqFVjwX13yB2TiKjPcZii+NW9E7qd/rtH7+rlJEREdD2H3vRERETyY1EQEZEgFgUREQliURARkSAWBRERCXKYo56cweDUKaiqrJBsfoGhSZLMRxcWjsK8LZLMi4jo37EobkJVZQUSluTLHaOL4kUpckcgIhfGTU9ERCSIRUFERIJYFEREJIhFQUREglgUREQkiEVBRESCWBRERCSIRUFERIJYFEREJIhFQUREglgUREQkiEVBRESCWBRERCSIRUFERIJYFEREJIhFQUREgnjjopsQkpUJt3575I7RRUhWptwRiMiFsShuQs3a9QgY+4bcMbqoWfsU8PHbcscgIhfFTU9ERCSIRUFERIJYFEREJIhFQUREglgUREQkiEVBRESCWBRERCSI51HcBF1YOIoXpcgdowtdWLjcEYjIhbEobkJh3hbJ5hUYmoS6iwWSzY+IyF646YmIiASxKIiISBCLgoiIBDnFPopXPvgGXh7uUCoUUCqVeOWJGXJHIiLqM5yiKADgxbl3Q+3jJXcMIqI+h5ueiIhIkFOsUSigwPsrNkOhUCAjNQEZwwd1ejz7cBGy84oBAPMy0xGuC5Yj5k3TajRyR3ApHE/pcCyl5Qzj2dTUZPMxhclQZunFLD1S32iAv58P9M0teH/FZjx8Vxrio0O7fW6DXt/L6XqG51FIS6vROM3v3tFxLKXlLOOpUiqhVqu7fcwpNj35+/kAADS+3hg2KBqlFbUyJyIi6jtEF4XJZMap81U4eKIUANDWbkRbu9FuwX7W1m5Ea5vR+nVhSQUiQvzt/rpERHSFqH0U5dWX8dGqbXBzU6Jeb8CIxIEoPl+Fffmn8eR9d9g1oL65FR//33YAgNlswcikgUiKjbTraxIR0TWiimLFD3sxY0IqRifH4sWlKwAACdE6fLFhj13DAUBIgB/+88lZdn8dIiLqnqhNT5W19UgbEtNpmqeHO4wdHXYJRUREjkNUUQRp1ThfeanTtNLyGoQEOP4hX0REdGtEbXqaMSEVH6zaivGpg2AymbFpdz52HS7C3Olj7J2PiIhkJmqNIiU+Cr99aAoaDa2Ijw7FpYYm/PqBiUiMibB3PiIiktkN1yjMZjM+X78bj94zBnOmpfdGJiIiciA3XKNQKpU4UVIBpULRG3mIiMjBiNr0NDktEd/vyoPJZLZ3HiIicjCidmbvOFCIhuYWbM0tgJ+PF3Dd2sWbv51tt3BERCQ/UUUxf9Z4e+cgIiIHJaooEvp3f6VWIiJyfaKKwmQyY2POUeQeP2O95HfakFhMG5cCN5XK3hmJiEhGoopi9faDOFtRiznT0hGoVaOuoQkbc46ipd2IB6eMsndGIiKSkaijng4XnsUzsychMSYCoUFaJMZE4KkHJuHQibN2jkdERHITVRQWm/fAc/ib4xER0S0Stenp9sH98dGqbZieMQyBWt+rm57ycfvgAXaOR0REchNVFPdNHoGNOfn46sd91p3ZIxMHYtq4ofbOR0REMhNVFG4qFWZOSMXMCan2zkNERA5G1D6KTXvycbaittO00vIa/Lj3mF1CERGR4xC1RrF9fyEmjhjcaVpYiD8+/r/tuCs92S7BnNlbSz7EO0s/uuHzAkOTBB9/aeEzWLzoWaliERH1iLgT7sxmqFSdVz7cVEoYO0x2CeXsFi969oZ/4LUaDRr0+l5KRETUc6I2PUWHBmHnoZOdpu06XITo0CC7hCIiIschao1i9pSReH/FZuQeK0FwgB9q6vTQN7fi+TlT7Z2PiIhkpjAZykSdNdfabsSxU2W4rDcgQOOD5PgoeHm42zvfTXOWzTnc9CQtjqd0OJbiid0fKYbc+yRVSiXUanW3j4kuip8ZWttQW9+E0CAtPNxFrZD0Kmd5g/PDKC2Op3Q4ltIKDE1C3cUCuWPckFBRCP6l37z3OEIC/JB6W38AQMGZC1j+7U60tRvh4+2J3zx0JwZGhEifmIiIHIbgzuzdR4sRHuJv/f7rzftxx4jBeG/Ro5g8KhFrfzpk94BERCQvwaJoaGqBLkgLAKiu06OuoQl3j02Gp4c7poweggvVl3slJBERyUewKDzc3NDS1g4AOF1WhYh+AdYd2EqFAmaz2f4JiYhIVoJFMSQuEl9u3IOjxeexNbfAuq8CAC5U1SFA42v3gEREJC/Boph950h4uLlh3Y7DGBgRgsmjrl1yoqCkHCMSB9o9IBERyUvwqCdvLw/8Ysa4bh+7h5cYJyLqE0RdwoOIiPouFgUREQlyvFOriYhkNjh1CqoqKySb341uKSCWLiwchXlbJJnXzRBVFGazGUolVz6IqG+oqqxAwpJ8uWN0UbwoRZbXFfXX//fvr8LXm3Nx7t/uckdERK5P1BrFbx6+E/uPl+DDVdvg4+WBtORYpA2JQaC2+wtIERGR6xBVFP3DgtE/LBj33zkShSXlyD1egj8tX4fo0CCkDYnFiMQB8LTjJccLzlzAqs37YbZYMHZYPO4eI8/qFxFRX3RTO7OVCgV0QVqEBmlRWl6D+kYD9heUYPW2A3hwahpGJ8dKHtBsNmPlplw8P2cqAjQ+ePPv65ESH93pYoVERFIKycqEW789csfoIiQrU5bXFVUUzS1tOFR4FrnHzuDipQbcPngA5s3MQGxkPwDA2YpaLFu52S5FcbaiFv0C/RAS4AcAGJk4EPnF51kURGQ3NWvXI2DsG3LH6KJm7VPAx2/3+uuKKoqX//INBvUPxcSRgzE0IRrubqpOjw8ID0ZKQpRdAl5uNCDA79o1pfw1vigtr7H5/MJLFszf2P3FChenKZCVcGX//dpiM97KtX3Ppn2PXVvGxzeYUFTX/fNmxSnwcvqVeZ68ZME8G68NAP+4R4nbghQAgNd+asY3J0zdPm9QIPDP6ddef/S/un8e4FjL9OZeM9ad7v71uUzOs0xrH7n2vassU09+T45q9L9MdnnvHXjc9nKLKorXn74PWj+fLtMbmgzQqq9MnzcjQ8ysJKFQdP4++3ARsvOKAQCTJo23+XPe3t7Qajyvft0GwGDzuVqNxvq1SqUH0P0vwsPDHdqrF0dUt3YAaLQ5T7WvL7San4e82ebzVCpVp9cHbF/O3ZGWycOjGUB7t8/rnWVSwfWWqfd/T50zuMYy9eT35Mjs9d6zRdStUF9Y8iXeW/Rol+kvLl2BdxfOuekXvRklF6qxPvsInntkKgBg0+4rxzbfPbb7HdrOcgtH3m5SWhxP6XAsr5wg56jnUdjrtqo9vhXqzyzo2iUtbe1Q/Pt/7e2gf3gwquv0qK1vhL+fDw6cKMUTWbbXGoiIbpUuLFy2k9uE6MLCZXldwaJ4edkqQKGA0WjCy3/5ptNjzS1tGNkLlxlXKZV46K7RWLZyC8xmC8YMjUN4SIDdX5eI+i4pL5MRGJpkt7WA3iJYFPNnjYcFFnzw1VbMn3ltH4RCAfj5eiP06m1S7S05LhLJcZG98lpERNSZYFEk9A8FACx98RF4uPP6gUREfZHNv/4bc45ab060ac8xmzOYOSFV+lREROQwbBZFfeO1Q6gu67s/lNP+u7KJiEhuNotizrR0AIDZYkFacixiI/t1OdGOiKgve2vJh3hn6Uc3fJ6Y+1G8tPAZLF70rBSxJCfqPIrnl3yB9xfN7Y08t8xZjv/mserS4nhKh2MpLWcZT6HzKESdqx4fFYqS8mpJQxERkXMQdShToNYXH3y1FSkJUQj08+10DQ3uzCYicm2i1iiMHSYMTYiGAgpcbjTgsr7Z+o9ITqvXbMCYCbPgru6PMRNmYfWaDXJHInI5otYoHp8xzt45iG7a6jUb8Oc3l2HZu3/CXVPuwI9bduC5F18DANx/73SZ0xG5jpu6nm5rmxG19Y2ouXztH5Fclr73Nyx790/IGJcGd3d3ZIxLw7J3/4Sl7/1N7mhELkXUGkVFTT0+W7cLF6rqruyfsFis+yn++srjdg1IZEvxqRKMThveadrotOEoPlUiUyIi1yRqjWLlpr1I6B+KpQsfgbenO95dOAcZqQmYx01SJKOE+Bjsyz3cadq+3MNIiI+RKRGRaxJVFOXVl3HfpBHw8fKExWKBt5cH7p88At/tzLN3PiKbFr6wAM+9+Bqyc3JhNBqRnZOL5158DQtfWCB3NCKXImrTk5tKBZPZDJVKCbWPF+oamuDj5YnmljZ75yOy6ecd1r//wxu498FfISE+Bq++/Bx3ZBNJTNSZ2X/7dgeGxEZgzNB4rNl+EPmnLsBNpUSg1hdPz57cGzlFc4YzIAHnOVtTbmIvkSCGI18iwZHwvSktZxlPoTOzRRXF9cwWC/YfL0FbuxGjk2Ph6eEuSUipOMMvBHCeN4+zcIWbwzgKvjel5Szjecu3Qr2eUqHA6OTYWw5FRETOwWZRfLZuV6dLddhy/Z3viIjI9dgsipAATW/mICIiB2WzKDLHD+vNHERE5KBE76MoOluJ3GNnUN9kgL/aB6OSY3HbgDB7ZiMiIgcg6oS7nLxifLJmJzRqbwwb1B8atQ/+vnYnsvOK7Z2PiIhkJmqNYvO+43h+zlRE6gKt00YkDsD/rv4JGakJdgtHRETyE1UUzS1tCAv27zRNF6SFobXdLqGobxicOgVVlRWSzU/MfYnF0IWFozBviyTzInIFoooiNrIfvtm6H/dNGgEPdze0tRux9qfDiIkIsXc+cmFVlRVIWJIvd4wuihelyB2ByKGIKopHp6Xjk7U78cL/fAlf7yvXeIqN7IcnsibYOx8REclMVFFo/Xyw8LFpuKxvRn2jAf5+PgjQ+No7GxEROQDBomg3dmBjzlFU1NQjKjQQd49JYUGQZEKyMuHWb4/cMboIycqUOwKRQxEsiq827cPZylokxUYi7+Q5NLe04eG7RvdWNnJxNWvXI2DsG3LH6KJm7VPAx2/LHYPIYQieR3H8TDmef2Qq7p88Ar95eAqOnbrQW7mIiMhB3HDTk9bPBwAQqPFFSxsPhyXp6MLCHfIII11YuNwRiByKYFGYzGYUna3EzzesMJstnb4HwMt4UI9Jea4C70dBZD+CReHn64XP1++2fu/r7dnpe4UC+POzD9gvHRERyU6wKN74zezeykFEvYC3lqWeuOlboTo6Z7jlIOA8t0eUG/+w9T5uxpOWs3zWJb1ntqNzhl8I4DxvHmfB8ZQOi0JazvLeFCoKUZcZJyKivkv0jYvk8P2uPOTknYKfjycAYNbE25EcFylzKiKivsWhiwIAJqclYuroIXLHICLqs7jpiYiIBDn8GsWOg4XIPXYG/UODcP+dI+Hr7Sl3JCKiPkX2o57e+/JH6JtbukyfOWE4YiJCoPbxBBQKfLcjD/omA34xY1yX52YfLrLev3teZjrCdcF2z03kqlQ+UTAZyuSOQb2sqanJ+Q+Pra1vxEertuG1BVmCz3OGw9AA5zlkzllwPKXDw2Ol5SzvTac9PLah0WD9+kjReYSH+As8m4iI7MGh91F8u/0gyqrqoFAoEKRV49Fp6XJHIiLqcxy6KObPGi93BCKiPs+hNz0REZH8WBRERCSIRUFERIJYFEREJIhFQUREghz6qCciEm9w6hRUVVZIMq/A0CRJ5qMLC5f03ugkDxYFkYuoqqxAwpJ8uWN0UrwoRe4IJAFueiIiIkEsCiIiEsSiICIiQSwKIiISxKIgIiJBLAoiIhLEoiAiIkEsCiIiEsSiICIiQSwKIiISxKIgIiJBLAoiIhLEoiAiIkG8eiyRiwjJyoRbvz1yx+gkJCtT7ggkARYFkYuoWbseAWPfkDtGJzVrnwI+flvuGHSLuOmJiIgEsSiIiEgQi4KIiASxKIiISBCLgoiIBLEoiIhIEIuCiIgE8TwKIhehCwtH8aIUuWN0ogsLlzsCSYBFQeQiCvO2SDKfwNAk1F0skGRe5Bq46YmIiASxKIiISBCLgoiIBLEoiIhIEIuCiIgEsSiIiEgQi4KIiATJfh7FocKzWL/rCC7W1mPx/Ez0Dw+2PrZpdz52Hz0FpUKBB6emISk2QsakRER9k+xrFOEh/vj1AxMRF63rNL2iph4HTpTitQVZ+O0jU7By0z6YzWaZUhIR9V2yF0VYsD9Cg7RdpucXn8fIxIFwd1Mh2N8P/QL9cLaiVoaERER9m+ybnmy53GhATESI9Xt/P19cbjR0+9zsw0XIzisGAMzLTEe4Lrjb5zkarUYjdwSXwvGUDsdSWs4wnk1NTTYf65WieO/LH6FvbukyfeaE4Rg2KFr0fBSK7qdnDB+EjOGDAAANej0a9Poe5exNWo3GKXI6C46ntDiW0nGW96ZKaXsDU68UxQuP3nXTPxPg54PL+mbr9/WNzfBX+0gZi4iIRHDYTU8pCVH4dO0uTE5LQkOTAdV1egwId45NSkSO6q0lH+KdpR/d8HmBoUk3fM5LC5/B4kXPShGLHJzCZCizyBkg7+Q5fL05F02GVnh7eSBKF4jnHpkKANiYcxR7jp6GSqnA7CmjMCQu8obzc4ZVPMB5VkedBcdTOhxLaTnLeKqUSqjV6m4fk70opOYMvxDAed48zoLjKR2OpbScZTyFikL2w2OJiMixsSiIiEgQi4KIiASxKIiISBCLgoiIBLEoiIhIEIuCiIgEudx5FHq9HkqBa5Y4ikZDK/x8vOSO4TI4ntLhWErLWcbTbDZDY+PihQ57CY+esrWgjmbZ1z/hlSdmyB3DZXA8pcOxlJYrjKfj/9ebiIhkxaIgIiJBLAqZZKQmyB3BpXA8pcOxlJYrjKfL7cwmIiJpcY2CiIgEsSiIiEgQi4L6tB9253f6/p1/bJApCVFXR4rOoaKm3vr9dzvzUFha0es5WBTUp236t6J4ad50mZIQdXWk+Dwqa68VxcwJqRg8MLzXc3Bntp3U1jfiL19twcDwEJRV1aFfoAbzZ2ag5EI1Vm87CJPZjAFhwXhkWjrc3VRYs/0gjp4qg0qpxOCB4XjgzpFyL8JNscfyPv/OF5g4MhHHTpfBw02Fp2dPhkbtjcbmVqz4YS/q9E0AgNlTRiEuSofG5lZ8um4nmg1tGBAejIIz5XjliRlQ+3jhr99sw2W9AcYOEyaNHIyM4YOwZvtBbN5XgIgQf4SFBOCJrPF4/p0v8P5Lc2GxWPDt9oMoOFMOALhn3FCMSByIonOVWL/rCNQ+XqiouYzo0GD8clYGFApFr473raitb8QHX21FbFQ/lFyogb+fD56ePQkNTQas3LQPTYY2eLirMPeeMQgN9kfNZT3+vnYXzBYLkmIjsS23AO+/NFfuxRDNXsv7j++z4e3hgXOVtdA3t+DeSSNw++ABAIDNe4/jUGEpOkxmDEuIxowJqQCADdlHsb/gDAI0vlB7eyE6LAhTRw9Bdl4xcvKK0GEyo1+ABvNnZaCsqg4ffr0V3p4e8PZ0x68fmIgNOUeRHBeF2wcPwMnSim4/W6988A3Sk+OQf6oMJrMZC+67A6HB/rc0hi53ZrYjqbqkx2PTxyIuSofPv8/B1twCZB8uwguP3gVdkBaffZeNXYdPYnRyHI4Unccfn7oXCoUChtY2uaP3iNTL22bsQExECLImDsfqbQeRc6QY94wbilVbcjE5LRFxUTrUNTRh2cot+ONT92J99hHc1j8Md49NQcGZC8jOK7bO6xeZ4+Dr7Yl2Ywfe+mw9Um/rj3snjcCOgyfx6pOzurx2XtE5lFXV4dVfzUSToQ1vfrYe8VE6AEBZVR3+a0EWtH4+WPLPjThzoRpxVx9zFtV1ejyRNR6PTR+Lv327A3knz2FP/mnMmZYOXaAGpeU1WLlpH343926s2rwfk0YlYmRSDHYdOil39B6x1/I2NBnwH4/fg4u1DfjrN9tw++ABOFFSjuo6PRbPz4QFwF9XbcOp8xfh4eaGvJNn8YcnZsJstuC/P/0O0WFBAIDUQdHWw2jX7TiM3UdOYeLIwUhJiLIWw/WMHR345/c5XT5bk0clAQDUPl74w69mYsfBk9iyrwCPZY69pfFjUdhRgMbX+gdkVHIsNuYcRZC/H3RBWgBAenIsdhw6iTtGDIabmwr/2rAHyXGRSI6PlDN2j0m9vG4qpfWx/mFBKCy5sm32ZGklKq/bbtvSZkRrmxFnyqrw1OxJAICk2Ej4eHlYn7P9wAkcKToPAKjTN6P6sh5qgevvnC6rxsjEGCiVSmjU3kiI1uFsZS28PN0xIDwYARpfAECULhCX6pucriiC/NWICr3yR6p/aBAuNTSh5EI1lq/+yfqcDpMZAFBSXmMd15FDYrB628HeD3yL7LW8QwdFQ6lQIDzEH/rmFgDAiZIKnCgtx39/8h2AK//hqa7To7XdiKEJ0fBwv/JnNyU+yjqfipp6fLfzMAyt7Whr70BijPDmpapL+m4/Wz8XxbBB0VeWNSwIR4rOiRwl21gUdiR2Y4RKqcTi+Zk4ebYSB0+UYsfBQvxu7t12zWYPt7K8z8+Zijc+/R4AkJIQjZkTUqFSKq2bdBQKBUyWKx9ks8WCl+ZNt37gfmZrG2rRuUqcLK3E76/+zNJ//QBjh0k4pMX2Fll3lcr6tUKhgMlsvsESOx53t+uWQalAc3MbvD09ul27cgVSLO/anw7j+OkyALD+nNt174Xr34B3j0nB+OGDOv381twCm/P+5/c5eHr2JETqArHn6CkUn78omMUi8P4Eri2vUqL3J3dm21GdvhklF6oBAAcKSnDbgDBcamhCdZ0eALDv+BnER4eitd2IlrZ2JMdF4sEpo1BWVSdn7B67leVVKpV49clZePXJWZh5dXuuLYkx4dhx8NomgbKLlwAAcVE6HDpxFgBwoqQchtZ2AEBLqxE+Xh7wcHfDxdp6lJbXWH9WpVLCZOr6QYqPDsWhwlKYzWY0Nrfi1PkqDAgP7vngODgvT3cE+6txqPAsgCt/iC5cfR8ODA9B3skr/ys9WFAqV0RJ9WR5syYOt75HhSTGhGPP0VNobTcCAC7rm6FvbkFclA75p8pg7OhAa7sRx09fsP5Ma7sRGrU3TCYz9heUXMvp4Y62q/O5XmiwttvPlr1wjcKOQoO12Jt/Gl/+sBf9AjR4aGoSYiJCsPzbHdYdUOOHD4KhpQ0ffbMdHSYTLBYLZk8ZJXf0Humt5X1oahpWbtqH15evg8lsRnyUDo/eMwbTM4bi07W7cPBEKeL7h0Kr9oanhzuSYiOQfbgIry9fB12gBgMjQqzzGpeagNeXr0NUaBCeyBpvnT5sUDRKyqvx56ubD+6bPAJatQ8uXmqQZrAc0C+zxmPFD3uxMecoTGYzRiYORKQuELOnjsJn63Zhy74CJMdFwsvT48YzcwL2Wt7EmAhU1jZYD7X29HDHL2dlYEB4MFISovDn5d8hUKtGdFgwvK/Oe+aEVLz92QYEan0R0S/AWjIjEgfii4178NOBQiy4/w7ra7i7ueHxzLFdPlv2wqOe7KS2vhEfrdqG1xZkyR2lVzjC8ho7TFAqFVAplSi5UI0VP+x12U0pvand2AF3NxUUCgUOFJTgQEEpnnlwstyx7Maey9vaboSXhzvajR34n89/wNx7xlh3aDsyrlGQy6jTN+OTb3fAbLHATaXE3Om3dqQHXXGu8hK++nEfAMDb0wO/uMUjaBydPZf3y417UFlTD6PJhPTkOKcoCYBrFEREdAPcmU1ERIJYFEREJIhFQUREglgUREQkiEVBRESC/h+qz9Hs+LOQ8AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.style.use('Solarize_Light2')\n",
    "_ = plt.boxplot([pos_token_ps, negation_pos_token_ps, neg_token_ps, negation_neg_token_ps], patch_artist=True, labels=['pos', 'pos-negation', 'neg', 'neg-negation'])\n",
    "plt.ylabel('Polarity Score')\n",
    "plt.plot([0.5, 4.5], [0, 0], linestyle='--')\n",
    "# _ = plt.boxplot([pos_token_ps, negation_pos_token_ps], patch_artist=True)\n",
    "#plt.savefig(\"imgs/mr_gru_polarity_score_negate_adjective.pdf\", bbox_inches='tight', dpi=900)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualizing Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_dim = 300\n",
    "# sent = \"not to be dismissed\".split()\n",
    "# sent = \"the movie is hardly a masterpiece\".split()\n",
    "\n",
    "sent = \"hardly an objective documentary\".split()\n",
    "# sent = \"never loses its ability to shock and amaze .\".split()\n",
    "# sent = \"not always a narratively cohesive one .\".split()\n",
    "#sent = \"hardly a film that comes along every day .\".split()\n",
    "# sent = \"never achieve the popularity of my big fat greek wedding\".split()\n",
    "#sent = \"never rises to its full potential as a film\".split()\n",
    "#sent = \"never succeed in really rattling the viewer\".split()\n",
    "# sent = \"hardly seems worth the effort .\".split()\n",
    "# sent = \"does n t add up to much\".split()\n",
    "#sent = \"hardly the most original fantasy film ever made\".split()\n",
    "#sent = \"does n't wrap the proceedings up neatly\".split()\n",
    "#sent = \"does n't completely survive its tonal transformation from dark comedy to suspense thriller\".split()\n",
    "\n",
    "\n",
    "sent = \"never fails to fascinate\".split()\n",
    "# sent = 'hardly seems worth the effort'.split()\n",
    "#sent = 'not in a good way'.split()\n",
    "#sent = \"what antwone fisher is n't , however , is original\".split()\n",
    "#sent = \"however , is original\".split()\n",
    "#sent = \"holds its goodwill close , but is relatively slow to come to the point\".split()\n",
    "#sent = \"it 's never boring\".split()\n",
    "#sent = \"same song , second verse , coulda been better , but it coulda been worse\".split()\n",
    "#sent = \"want nothing else than to show us a good time\".split()\n",
    "#sent_example = 'this is possibly not horrible'.split()\n",
    "lengths =  torch.LongTensor([len(sent)]).to(device)\n",
    "sent_example = [word2id[w] for w in sent]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "polarity_score = []\n",
    "for i in range(len(sent_example)):\n",
    "    temp = rnn_phrase_polarity_new(sent_example[i:], best_model, hidden_dim)\n",
    "    polarity_score.append(temp)\n",
    "polarity_score.reverse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdAAAADNCAYAAAAIYiHxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3gVRReH39nb9tYA0puIgIKIvYJiwQYC0ot0UZqIYENRQMFeEBT5UEFRpEoRERs2bGBDkd57Cy25vezu98eGhEhQCIF7k8z7PDwh2XLPwsn8dmZOEVpom4FEIpFIJJITQkm2ARKJRCKRFEakgEokEolEkg+kgEokEolEkg+kgEokEolEkg+kgEokEolEkg+kgEokEolEkg+sJ3JyRmZmgX64EALDkFk0EukLkhykL0gOkwq+IACfz5fnsaTOQH1ebzI/XpJCSF+QHEb6guQwqeALinJsmZRLuBKJRCKR5AMpoBKJRCKR5IMT2gOVSCT5xDAgFEYcPIRyMANx8BAkEmCxgKKYfywWDEX842cKhsUCQsFI82KULgU2W7KfRiKRIAVUIjk5whEsGzahrN+Esicdse8Ayu69KHv3IfYfRBzKQPgDiEDQPN9ux7Bacq4XR/7FML/mETQhDAM0DaIxcKroJdMwSp+BXrE8epVK6BXLoZctjVG2TNbX0hg+Lwhx1L0kEknBIAVUIjkeMv1Y1m3EsnYjlpVrsCxbgWXDZsTBDAynaupgNIaIx//9PuEwJy1pwRCWYAi274I/lwNg2GwYDjsIBaHrEIuB1YJWvRraRXVJXHQ+Wu1aaOfUALfrZC2QSCRIAZVIcmMYKJu2Yl38G9Y/lmFZvhpl4xZEKAxOFUPTUELhXJdkzy7zup0iwOEwl2eFAoaOSCQgFkfoOgaAIkAo5iRUCECYX4U44mdkLeUK0HRENIrQtBwb4vGjxTsex7piNdYVq7HPno9hsSLCYYySJdBq1yJx6QVodWuj1amFXrWyuWQskUiOGymgkuKNYaBs3IL1x1+wLfwO6+LfEfE4BgIlnFso8QdyzR4Nq9WcfWoaxOIYJdMwSvjQS5bEKF0KvXxZjLKl0UuVxChVAqNkCfSsr0bJEmC3ga6bS7N61hKtYZj303TzmJ51LJFAyfQj9qSby8N796Hs2IWybYe5ZLzvAOJQBiQ0cNhNsT1C7EUokm27SN+Pkv4z1h+WYLidCE2HeAKt5lnEb76exHX1SVx8Ptjtp+W/QCIprEgBlRQvDANl/SZTML9aZApmIgGACEeyT8sllA47ht1uzvAMA71KJRJ1aqFdcB56zbPRzjkbvXJFM/jnRLFYjrruWGnjGkCdc/79fqEwSvp+xN50LOs3Y/lrOdalf2NZvwkSCfM5IhFEPIHQdYQ/Z/ZsXbkWy9oN8PZkiMXR6tUh1rgRiWuvRKtzjpyhSiT/QJxIQ+2CrkSU5vMV+D0lhZNT6QviwCFsny7EtmAh1l/+MGdchpFLMA9zeC9RxOJoNc5CO7822vl10GpVR6t1Nka5MoUzMMcwEOn7sKxci2XlGqy//oVlxWqUnbvMJWaAYOio/VnDYQer+Z4dv+Ji4rfcQOLaK9GrVT1l/w5yXJAcJhV8waIoeDyePI9JAZWkBAXtC+LgIWyffoVjymwsy1aA1WruY/6DXIJZrw6xmxqSaHAFWr06xSNdJB5H2bAFy8rV2L5fgu2bHxAHDoHNigiGjjrdcKpgmF/jN9Qn1qY5iQZXZItsQSDHBclhUsEXpIBKUp6C8AVxKAPbZ19jnzIL65/L8xRNw27LXo5N1DuP+M0NSdTPEswCFIHCjNi5G9v3i7F9/g3WH5cgYubStYhEc51nAIbHjQBiTW8h2v4OtMsuOumlXjkuSA6TCr4gBVSS8uTbFzL92D/9CvvU2Vj/WAY221EzJ8PtMoNk6tUhduv1pmCeX1sK5vGQtWds+34xtk+/wvrbnxgWCyIWM4X18GmKYkYp22zEWjUh1vYO86UkH8u8clyQHCYVfEEKqCTlOSFfSCSwffY1jnemYv11ad6i6XJCIkHi4nrEOrQkfusNGGl5d1SQnACahuXvVVi/+wnH7E9QNm8DDEQ0ln2KYbGYkcAeN9G2zYm1aYZ+To3j/gg5LkgOkwq+IAVUkvIcjy+I9P04Jk3H8fZkM+/xWKJ5YV1iHVuZolki7VSaXexRtu7ANucTHFNmo+xNh4SWKx/VsFrBZkUvXYpYh5ZE72xtBmL9C3JckBwmFXxBCqgk5fk3X7As/Rt17ERsX34LCEQ0Zy9OdzkRCQ2tXm2iHVsRv+1GM8dSctpR1m7APms+jmlzEX4/xGKIRE6xB8Nh5pXGr6tPpH9PtEsuyHOJV44LksOkgi9IAZWkPEf5QjSG/ePPUce8hbJth1kmT9cBMCwK2OxoNaoR7dqeeONGGGeUTJLlkqMwDCwrVmOfOQ/7zI/NF55wJOf/TwhQVfQKZYkMuIdY89tAdWRfLscFyWFSwRekgEpSnsO+IHbvxTHhAxzvTgNdRzlimdZQVTAMYs1uIdK3O3rtWkm0WHJcGAaW3//CMX0u9tmfmNG8R0RGG24XCEGkSzuiPe/EqFhejguSbFLBF6SASlKetA1bSAx7Htv3PwPkCkrR3S5wOYn06UasYyu5r1lYiUSxz12AOvpNlD3pZnu3rEOG3QZCEG9wBdbBA8ioe27hLFghKVBSQSOkgEpSFmXVOlxPvoh1ye8QiZptu8Bs+WW1op1fm/B995C48RpZSq6oYBhYfl2KOuZt84XJMLJTYgwAlwu97BlE7rubWIsm4FSTaq4keaSCRkgBlaQcyobNOJ96Cdt3P2V3JgEwnE5zmbZNUyK9uqLXOCvJlkpOJWLXHhwTPkB9dxqGYeResnc5MWxWIg/2I9qlHThkcfviRipohBRQScqgbNmO+vQo7F98A/G4WZcWcy/McKpEBvYm2u4O8LiTbKnktBKJYv/oU9Qxb6Ps2mP2TT28GuFyYtjthB/pT+zOVrJLTDEiFTRCCqgk6Ygdu3A+9xr2eZ9BIpHdy9JwqhgOBzz5MBktGsvBsbhjGKSt2UDiudHYvvnB7H2a1S3HcDkxnCrhwQOItb+jeNQqLuakgkZIAZUkDbE3HeeLY7HPmAealjMYZpV9izzQh2jX9qSVLSN9QQLkjAvKtp2oz43BPv+L3C9dLieG20V4yEBirZvKkoxFmFTQCCmgktNPph/ni2NxvD8DdD0nSERVwaIQHnAP0Z6dwOUEpC9IcvinLyhbtqE+Mxr7Z18fLaReD+EnHiDWonH++rFKUppUGBekgEpOH4aB/cN5OIc8i4hGs9NRDIcDLAqRPt2J9Ol21B6n9AXJYY7lC8rGLTifHoXtq0W5989dTvQSaYSHPkC82a0yWrsIkQrjghRQyWlBWbUO94DHsKzflJ0sbzjsoChEenQk2r/nMXM4pS9IDvNfvqCs24hz5CtHR3C7XeilSxF6+SmzR6mk0JMK44IUUMmpJRDE+cyrOKbMMkvuGYZZrs1hJ9qxFZFBfTBKl/rXW0hfkBzmeH1BWb0e54iXsf34C0SPyCF2qsTrX07ouScwKlc81eZKTiGpMC78m4DKtQ5J/jEMbHMXkHZpIxwfzEJkFUIwXE60C+uS+cVMws8M+U/xlEjyg35uDYIfjMO/YAqJa6/CUFUMQIQj2L75kbRrmqK+8Dr8oxG4RFJQSAGV5Atl3Ua8TTriHjQU5VAmIhrFcDjQS/gIvvIU/gVT0WudnWwzJcUArc45BKa/hX/uJLTaNTFcToSmIcIR1HHvkHbpTdg+/QqM415sk0iOCymgkhMjGEId9gK+m1pjWfo3IhTGEAJDdRC9syUZvy0kfkdjWcdUctrRLqyL/6vZhJ59HN3nxXA4EOEIyr79uPs9gqd5Z5R1G5NtpqQIIQVUctzYFiwk7bKbUCdNy71ce35tMj+dTviZx2UFIUlyURRi7e4g4/eFRDu1xlAdGIqCCIWx/vYXvpta4xzyDPgDybZUUgSQAir5bzL9uO8ehLvfIygHDiEiUQyHHd3nJfT8E/g/n4Feu2ayrZRIcvB6CD/9GJlffkjikgvMZV1dR0SiOCZ/SNoljbBPnwtZEbwSSX6QAir5V6w//kLaVbdh+/wbRDiCARiqg1jb5mT++gWxNs3lcq0kZdFrVicw732CY59HL30GhlNFRKMomX5cj47Ee2NLlFVrk22mpJAiBVSSN5EozseexnNnb5T9BxGxGIZTRa91Nv75HxB6cThGmi/ZVkok/40QxG+7kYxfvyDSu5tZRtJqQYTCWFatw3dbe9QXXoN4PNmWSgoZUkAlR2FZsRrftU1xTJmNyEoBMFQHkXu6kPnVLLS6tZNsoUSSD5wqkUf6k/H9x8Svq2/ORgERiaKOexffdXegrFqXbCslhQgpoJIcNA3Hq+PxNumIsnUHIhLBUB3oFcrhnzOJyKMDZAcMSaHHqFyR4ORxBN4fi162NIaaFa27YTO+29qhvjgWspoeSCT/hhRQCWD26fTe2g7n6LfMCFvMwu+xlk3I+PETtIvOT7aJEkmBkmhwJRk/f0qsdVMM9YjZ6BsT8V3fAmX1+mSbKElxpIAWdwwD+5RZ+K6/A8uKNYhwGMNmQy/hIzBxNKFXRmR3TJFIihxuF6GXniQw5X/oZY6Yja7fhO/WtqgvvyFno5JjIgW0GCP2H8TT/h5cjz+LCIURuo7hdBJveBWZP31K4oYGyTZRIjktJK6+jIzFWbNRp4owDHM2+vpEfDe0RFkjZ6OSo5ECWkyxLP0b3zVNsf70q1lNyGLBcLsIvTiU4PtvYJQqkWwTJZLTy+HZ6ORx5mzU4UCEwyjrNuK7pR2OUePlbFSSCymgxRD7pOl4W3RDOXAQEY+b1YTq1SHju4+ItW4m8zolxZpE/cvJWLyAWKsm5t6oYSAiEZxj3jJno2s3JNtESYogBbQ4EYni6vcIruEvICIRwAwUCg/qg/+TKbL1k0RyGLeb0CsjCEx+wyzAkGs22hb75JmyOL1ECmhxQWzfie+m1tg/+dKsKGS1opfw4Z/5NtF77wJFuoJE8k8SDa4gY8mnxFo0ztkbDUdwPfE87p4DIRhMtomSJCJHzWKAddHPZpL4xs1mHVunila7JpmL5qFddlGyzZNIUhu3m9CrI8280VIlMWw2RDiMbeF3+K5pKksBFmOkgBZlDAPHq+PxdOmHEggiNB3DqRJr0wz/gqkYZcsk20KJpNCQaHAlmT/OJ3HFxRhOJyIaQ9m5B1/jDtgnTZdLusUQKaBFlUAQz519sgsjABhOleCLwwm9MExWFJJI8oFRsgSBmRMIP9jXzBkFc0l3+Au4ewyAgFzSLU5IAS2CKOs24mvYDOuPS8zCCHYbetnS+D+ZQrx102SbJ5EUboQg2q8H/tnvopcuhWG3IcIRbF9/T9o1TbGsWJ1sCyWnCSmgRQzb/C/w3dIWZeceRDSG4XSSuPRCc7+zzjnJNk8iKTJoF9cj84f5JK68NGdJd9cevE3uxP7uVLmkWwyQAlpUMAzUl9/Afe+jZlUhw8BQVSK9OhP4cCJGibRkWyiRFDmMEmkEpr9F+OF7MVQHACISwfXUy7i73gv+QJItlJxKpIAWBRIJXAOGoL4+weygIgSG20Xg7VeIDB4gU1QkklOJEET7dMM/ZxJ6mTPMJd1QGNt3P5HW4HYsf69KtoWSU4QcWQs7wRCedndj//hzM7/TbkOvXJHMr2aRaNQw2dZJJMUG7aLzzSXd+ldguLKWdPek423aySy8IClyWJNtgCT/iPR9eFt2R9m63dzvVB1o552Df+qb4PMm2zyJ5N/RNESGH7SEuV9oYH71BxF+v/l3RcEoWQKylkdTHSPNR2DK/3C8+R7OZ8cgIhFzSfeJ57AsX0145KNglcNuUUFooW3HvdOdkZlZoB+e5vMV+D2LC8r6TXhbdEUczEAkEhhOlfiN1xJ843mw25Nt3gkjfaGIEQxhWbMey7qNKLv3InbtQdmxC2VPOmL/QZSMTAiFTTFRjqy9LLJqMWcNSwYQi4HFguH1YKT50M8oiVHmDPQK5dArlkevcRZarbPRz6wMFksSHjZvLH8ux9O5H+JQhllz2qmSqFeH4HtjMdJ8yTavUJAK44JFUfB4PHkekwJaCLH88gfejr0hGDKDhZwqke4diDzxQKEtBC99oZASj6Ns2IJl9Vosy9dg/eMvLGs3IA5lYKhO0HVENILQ9FPy8YbFYi6XahrEYugVyqHVqoF24Xlo59ZEO7cm+tnVkhYHIA4ewt25H9YVq7O2WOzopUsR+HAievUzk2JTYSIVxgUpoEUI27zPcA8YggjnFIMPDX+QWLcOSbbs5JC+UEhIJLAsW4lt0c/YPvsay8o1YLODIPuF7r8wLBYMh/3o2WLWtUKI7L8bAIqC0HVIaBCNHt9nKIoprLoOQpC4uB7xmxqSuPoyM53rdApqIoFr8Ajss+abIioEuJwEJrxK4rr6p8+OQkgqjAtSQIsIjrETcb74eq7KQoG3XikSwULSF1IUXUdZsz5bMK1L/8awWhHRGCIez/MSw2Y124DF44Awiw2ULoVergx6pQroFcub35cpjXFGSbBZQQhTWBB4PW78wZApyrqOcjADceCgufS7/yDK7j2I3XtR0vcj9qSj7E0HhwNDUcwULk3L2y6HA6wW0HUSF9YlftN1JBpcgXZ+7dOycmN/dyqu4S8d0QnJQfix+4ne3bnQrhydalJhXJACWtjRNFyPjsQ+c575BqsoGF4PgRlvo11wXrKtKxCkL6QQhoFl6d/YZ3yEfc4niIQGCQ0RjR51qu52IQwgHkOvVAGt7rkkLq6HVrsW2rk1McqXPWFxOGFfSCRQtmzHsnaDue/653Isq9eh7NhllqzM6qBy1GM67GC1YthtxJrfRqxlE7RLLzyls1PrT7/i7novIhhC6Fm1qW+/mdDLTxbK2IVTTSqMC1JACzPhCJ4eA7Au/s0UT5sVvWxpArMnmUETRQTpC8lHWbUW+8x5OGbMQ4TDEDl679JQVRBg2O0kGlxB/JbrSVxyAXq1KgUWwFNgvqDrKBs2Y/3pV2xfLcK65HdELI4hBEoonOtUQ1HA5cRQFOK330SsVVMSV15ySoKSlC3b8bTugbI3PSt6XkU752wCU8abM3JJNqkwLkgBLawEgnhbdjODMiJRM02l5tkEZrxlhvYXIaQvJAdl207sM+ZinzIb5cBBiCcQiUT2ccNqBYcdDIPE5RcTu/V6EtdcZQbAnKJlx1PmC4aBsnkr1p9/NwX1518RoQjoGiKWsxx9WEwBYrfdSLR7B7SLzi/Y5/UH8PS4D+tvf2W/GBsl0vDPmIBeu2bBfU4hJxXGBSmghRCRkYnnji5YNmxBxGJmmkqDKwm+9UqhyYk7EaQvnEYMA+vi31FHv4n151/NJc5/CohTxXCqRNs0I96iCVrdc09b4M3p9AVl3Ubs8z7DPmMeyp695r9FNJZ93FAUUB3o5coS6duNWMvbwe0qmA/XNNSnXkJ9b4YpogAuJ8FxLxK/5fqC+YxCTiqMC1JACxni4CG8TTuZBRJiZv5YtH0Lwk8/VmTL8klfOA1EotjnLkB9dTzK3n0QCnN4TmUIAW4XhqIQa9GYWLs7Cn7WdZwkyxeUTVuwzfscx4yPULbvAsi172u4nKAbxFo2JnJ3lwKbKdpnfITrkaeOiKx3EL6/F9EB9xT74KJUGBekgBYixL4DeG/viLJjd3bydeSeLkQG31ekf5mkL5w6xJ50HG9PxvHOVIRuIEKh7GPG4aXKxo2IdWh5yvb9ToRU8AVl6w5s87/AMWkaSvp+CEey02cMiwI2G9rZ1Yj07UG86c0nHQBk+WMZno69EP4gQtPM4KIWjQm9ODzp/x/JJBV8QQpoIUHsTcfbuKNZuSWrulDk3ruIPNA32aadcqQvFDxi526cz7yK/ePPze+PXJp0u9BLlSBy3z3EWjcFp5osM48ipXzBMLD8sQx13LvYvvwWhJKdhgJmFDIOB+EH+xK7s7W5X5xPxM7deNv2RNm+04x5cKrEr76c4MTRJ3Xfwkwq+IIU0EKA2LUHX+MOiPR9iIT5Bhp+oC/Re+9KtmmnBekLBYfYdwD1pbE4ps4x681mBQUZVitYLCQuvYDIwF4k6l+RkqsaqeoL4uAh7FNno45/DxEIIoK5Z/KG3UZkYG+iXdrl/4UkFMbTrT/WX5ea+6KqA612LfzT3yqW9a1TwRekgKY4yradeJt0QOw/gNDM3LDsBOtigvSFk0dkZOIY8xbqxCmmcGYFBhlOFQyItW1GpE839LNSu4RcyvuCYWD9YQnquHew/vgrJBLZxRsMlxPDYiFy391Ee3QAt/vE759I4Lr/ceyfLESEw9kdlvxzJ2GULVPAD5PapIIvSAFNYZTNW/E2uRNx8JCZWK06CD35CLGu7ZJt2mlF+sJJEAyZ3T9ee9usPXu4UpXDARaFSL8eRO7pAt68B4FUozD5grJlG+oLr2GfvzDXbF93qgiLhUif7kR6dQHPCQqpYaA+PQp1wgfmTNRqwShVEv/c94pVDd1U8AUpoCmKsn4T3qZ3Ig5lmkXhVQeh554g1r5Fsk077RR3X8gXhoF95jycjz9rFnU/HMVpt5vC2b0D0QH3FLrOH4XRF8T2nThfegP7nAXmS0xWmUNDVTHsNsJPDCLWsdUJBwTZJ0zGNXJUdg3d7ApkF9Y9FY+RcqSCL0gBTUGU1evxNu+MyPRni2fwlRHEWzZJtmlJoTj7Qn5Q1qzHfd9jWNZtRGRV1TGsVrBaiLVtTvihfhhlSifZyvxRmH1B7N6LOup/OKbPNYX08DK6y4letjShl54k0eCKE7qnbd5nuO8bklND1+UkMHF0sShEnwq+8G8CWmSSCjdu2kKLNndR+axLOe+iG3h93Lv/ev6suQto3LwzVWtcTplK9fI8Jz19P337P8rZta+mao3LufbGluzavfekbbWsWG3OPDNyZp7B158rtuIpOQGCIZxDn8d3S1ssy1YiQmGzNrLqINb8FjJ+mE/ohWGFVjwLO0b5soSfH0rGr18Q7doeQ3WYxfdDYSybt+Hp3BdPu54om7Yc9z3jzW4l8MG47JQjEQrj6XYftlnzT9VjSI6TIiGgmqbRocu91KpZnbUrvmfKpNcZ8/oEZs/99JjXlEjzcVe39jzz1OA8j0ciUe5ocxc2u41ffviEzWsXM37sC7hPsgqJZeUaPM27oPgDCMwlnuCbLxO//eaTuq+kiGMY2BYsJO2ym3G8Nx0RMdt6GS4niYvrkblwFqGxL2BUrphsSyWAUaY04RGDyfj5U+K33mDWEAZEOIL1hyX4rm+B87GnERnHN7tK1L8c/8eT0Uv4zK4zkQjuB4bheOOdU/kYkv+gSCzhfv/DEtp37sua5YvwZEW9Pf38GJYs+YN5s9/912t/+PEXWrTtSfqOZbl+/u57M3jp1fEsXfIZNpvtpG0EUDZuwXtbO5QMP5DVjmzCaBI3NCiQ+xdmUmGpJlVRtmzHNegJrH/8bRZ5xwwQMpwOQs8PJd7s1pRMR8kvRdEXLEv/xvXgMCybtuYsuTscYLMSGvoAsU5tjqvKmLJlu7n1s+9Adq54tHNbwk8+XKR84DCp4AtFfgl3+co1nF39zGzxBLjg/DosX7km3/f8/sdfOKdWdQY+9CRn176aKxrcztj/vZvv+4kdu8xl28wAkCWe742V4ik5NoaBfdJ0fNc1z+rGE8bALPUWbX8HGb9+Sbz5bUVy4CxqaBedj3/hLIKjRqCfUQrDqSKiUUQgiGv4i3gbdziuZV39zMpkLpyFflZVDIcdEY7geH8m7l4PwDH6s0pOHUVCQAOBID5v7iTjNJ8Xvz+Y73seOHCQb779ibrnncOqv77lf68/x6gxbzEzH/sOYt8BvE07IQ5lmMtuTtUMArjmynzbJynaiPT9eNr2xDX8RUTYbCtmuJxotWvin/8B4eeHFpq0FEkWQhBvfhsZvy8k3P9uDKeKYbOZ+6PLVuK7viWOMW/DEd1w8sIoXYrMz6aRuOA8s3F5OIzty+/wdOwNR1Sbkpx6CqWAzpw1nyrVL83+4/G4yfT7c52TkenH681HEnMWHo+bChXK0fvuztjtdi66sC5tWt3Ogs++PqH7iIxMvM07o+zdZw6CqkpwzDMkrpczT0ne2L74Bt/VjXNmnRYLhttF6MmH8X81G61u7WSbKDkZVAfRQb3JWPIZ8ZsamrNRXUdEIjhHjcN3fQss/7V65nYT+HAi8RsamNeHI1h/XYqnwz0QObrxueTUUCgFtE2r29m28bfsP3XrnMOGjVsIHlFaa9nfq6hb55x8f0bd885F5LE0ltfPjkkojKdVd5RtO7L3K0LPPEa86S35tktShAmGcN33GO5eD5pBZvFEVpDQ+WT8+Amxzm2LbDee4ohRtgzBiaMJvPUKeskS2UuyyrqNeBt3QB3x8r/PKO12gm+PItq2uSmikSjWP5bhadsTwpFjXycpMIrEb+PVV11K5coVGfHsaMLhCH8vX8Wk92fStUvbY16jaRqRSJRY1r5BJBIlEoliZHVc6NjuDg4ePMTbE6egaRrLV6zmw1mfcHvjRsdnVCyGp0MvM08vqyVZ+OH+ZjK1RPIPLH8sI61+E+zzPstJmlcdhB++l8BH72OUL5tsEyWniESjhmT88jmxlrebS7KAiERRJ07Bd3VjLL/9eeyLFYXwc08Q7dw2R0T/WoG3dQ/IClaSnDqKRBQumHmggx56kl9//4s0n5c+vbrSv2/37ONtOvSiSpWKvPLCMACmTJvDvfc/ftR9/vzlC6pWrQSYEbpDhj3Phg1bKF++DL3v7kzPHh3/2xhNw93lXmw/LsnuqhDp1YXI4AEF8qxFkVSItksKhoFj1P9wjnkrpwSfU0UvX47Au2PQz6mRZANPP8XWFwDr4t9x93kIcTAjp3CCmjV+PNQPrNa8LzQM1JGvoE6cYr6AORzmfvmsifmrx5sipIIvyEpEpxPDwNX3YeyffW06stNJtF1zws8+LqMl/4Ui6Qv/hT+A5+5BWH/5Iye1QVWJdmlL+PGBJ91jsrBSLH3hSCJR1OfGoL47DSIRM1/c6USrUQu71TIAACAASURBVI3AO2P+NddXfW602S0mHMFw2NFqno1/7qQTr8WbIqSCLxT5NJaUwTBwPjryCPFUid16vRRPyVEo6zaa6Sk//2pWE7Lb0M8oSWDqeMJPPVJsxVMCqA4iwx8yCyecWRlDdSDCYSwr15DWsDm2+V8c89LI4AFE+vbISpOJYVm7AW+zTpDpP+Y1kvwjBbQAUZ9/Dcf0udl9/OJXX0bo9eekeEpyYVuwEN8tbVF27kFEYxhOlcQVl5D50wISV12abPMkKYJ2fm0yv/2IWNNbTUHUdEQwhLv/o7gGDDlmoFDkoX6E77vHvCYWw7J+E76mnY676pHk+JECWkA4xr2LOn5S9tJJ4oLzCE4cc8LdFyQFw4qVayhX5QJatPnvhuTfLvqZmxp3oEr1S6lRpz4PPvJU9rG58z7n6obNOeucqzjrnKu4rVknfvzp1/wZpWmoT76Iu+8jiFA4uw5ypE83AtPfKnRdUySnAadK6LVnCL4ywuw1KgQiHMH+0af4rm2KsmpdnpdFB/Yi/EDfLBGNo2zagrdJR8TBQ6f5AYo2cg+0ALBPm4Nr8EhEJIJht5n7DvPeh5Osm1ucKEhfSCQS3Ny4Iz6fByEEc2ZOOOa5P/z4C52638fol5/k1puvxzAM1qzdwAX16gCwfccurFYr5cuVQdd1Pvr4C/rf/zgr/vyatBMQPHHwEO5u/bEuW5kdZYvLSWD8SyQaNTzpZy5KFJVxoaBRNm3B06kvyo5dZnAigKoSemIQsR4d81zpcoyfhPPZMebYZMtqzD3/A4wzSp52+/NDKviC3AM9hVi/X5wjnlYreqWKBGa/I8UziYwa8xYXXViXK6+45D/PfeqZV+nepS3Nm96Cw2FHVR3Z4glQuVIFypcrA4BhGFgsCqFwmB07dx+3Pcqa9fgaNs+qZWtGSOpVK5H51SwpnpLjRj/rTDK/nmOmuzgPp7tEcI0chbvnwDyXdKO9uhIa+oCZHhOPo2zbgbdxe0T6/tP/AEUQKaAngbJ2A55u/U3xVBSMMmfgn/e+XIpLIitXrWXq9I8Y9sSg/zw3GAzxx9K/cTgcXHdTa2rUqU/TFt1Y+ufyXOdt376TarWupFyVC+nWcyAtmt9Gndq1jsse68+/4WvcAbF3HyJu5gPHr7uazK/noFermq9nlBRjHHZCrzxFcMyzGC6X2ZklHMa2cBG+Rq0Q23cedUmsR0dCTz1simgigbJjN97b2iPS9yXhAYoWUkDziUjfh7dVd8iqfmR43PjnvodR5owkW1Z8SSQS3DvgcZ4Z8Qi+46gTeygjE13Xef+DDxk7+mlW/vkN1193Ne3u7EPGEQEXlStXZPPaxWxZt4TXXx3JNfUvPy57bHMW4OnQCxEMZbWucxB+sB/Bd1+TKxSSkyLe9GYyv5mNXuMsUxijUZRNW/Fd3wJrHnv0sS7tCD3zmBnRm0ig7N6D95Z2iL3pSbC+6CAFND+Ewnhb32UmO5PVWWXam+hnVk62ZcWKf9ZEHjN2ItWrV+XWm68/ruu9WblxHdvfwXl1zsFutzPwvruJJxL88uvR1V/cbhcd27fgf2+9z1ff/HDsGxsGjtfexj3wiZxkeJeTwDtjiPbrIaOyJQWCfmYVMhd+SLRd8+x6uoo/gKdjbxxvvgdG7vCWWMdWZrN11YFIaCh70/He3kkGFp0EUkBPFF3H03MgyuZtZn1b1UFwzDNoF9dLtmXFjn/WRP7m25/4YuEiatSpT4069XntjYn8tPg3atSpz8E8Bgmfz0vVKpXyFrR/ETlN09i48RitpzQN18NP4nxlXPbSvl7Ch3/e+7KBgKTgsdsJPz+U4LgXMdyu7GbbzmdH4+r94FGF5WNtm5sRvYdFdNduvM27QCD/nauKM1JATxDn0OfM5PeoWaIvPKiPLA6fIrzz1iv8vGge3y2cxXcLZ9GtS1suubge3y2cdcyI2R7d2jNl2lxWr1lPIpHgtbETUR12rrjsQgCmzfiIjZu2oOs6/kCQF15+g+07dnFNgyuOvlk4gqdTX+wffmwGC9ls6BXLk7lwluygIjmlxG+9gcxvP0KrWR3D4TBTXT7/Bt/NbRD/CHiLt2ySNRNVEfEEyuat5naULEB/whyjsKIkL+zvTMXxwaycKkNNbyHav2eyzZJkUbp0qVzfez0eHHY7lSqWz/7ZP2si9+/bnUAgyB2texCJxqhX91xmTPkfPp/ZX3bDxs088/xrHDhwCKdT5bw6tZg2eRzn/qNGrdh/EE/bu7Bs2GymGKgOtFpnE5jxNkaJtFP85BIJ6FUq4v9sOp7u92Fd8rvZ2WXDZnzX30Fgyni0Sy7IPjfWtjn4/bhGjkKEI1hWr8NzZ28C096UVbBOAJkHepxYv/oez10DzMHRbidxUV0Cs945dnFnyQlRmHzhn4g96Xgbd0DZm262IHM6iV9zBcG3RoFDDkYnSmH2hZRA03ANHpG9EgJmnEZw/EvE/xEf4Bg1HueYN7Oqp6nEG1xOcNLrKVMAJhV8QeaBniSWlWvw3D3QFE+Lgl6hHIHJ46R4SkzxvK09yp69WeKpEu3Y0oy0leIpSQYWC6EXhpmViFQVABGO4L7nAeyTZ+Y6NTqwF5HuHbJaoUWw/fgLrn6DQdeTYXmhwzJsyKDhx3tyNFqwnc5Vh6PA71nQiN178Ta5EyWrGLOR5sW/YCqUlukqBUlh8IV/ki2ee9MRCQ1DVQkP7k/kkf4y0vYkKIy+kHIIgXb5xWjVqmBb+D0ikUAkNGyLFkM8TuLqy7J9NHHtVSjbd5q9i8MRLFu2oezcTfymhkn341TwBUUI7MdY1pYz0H8jGMTbqnt2EWbDqRKYMQGjUoUkGyZJNnmJZ+ipR4j27pZs04o1mqYx9MmXqFmnAVXPvowudw1g//6Dxzx/4dffc9W1zahY7WKubticr7/9MfvY+g2b6XrX/Zx34fVUPfsyrrq2Ge998OHpeIwCI35HYwJTx5sRupiVi9Q33sH1wFDQNPMkIQi9MIzYzdeZM9FwBPuHH6M+PSqpthcGpIAeC03D060/yradCE0z01X+9xLaEWXeJMWTY4lnrEvbZJtW7Hn1tbf59POv+fLTqSxf+jUAve8dnOe5m7dso+td9zPwvrvZvHYxA++7my7dB7B16w7ALLRxTf3LWfjZdLas/4VRLw5j6JMv8fEnX5625ykIElddSuan0zBKl8KwWkyBnLMAd5d+OWkuikJo7PPEr7osq31aBHXCBzhGv5lc41McKaDHwDV4BNbf/kLEzHZT4cH3Eb/l+BL0JUUXsXsvvlvb5RbPEVI8U4VJ78/kvnvvotqZVfD5vDz5xAN89c0P2aJ4JFOnf8QF9erQtnVT7HY7bVrdTr16tZk64yMALr24Hj17dKRC+bIIIbjyiku47tqr+PHn3073Y500eq2zyfxqNnrVyhgOOyJs7nd6m3fO6RVqtRJ8ZwyJC+tmn+N8dTz2d6Ym1/gURgpoHtjfn5GTy+dUibVuKpfmJKZ4ZtUQzRbPkYOJdZbimQpkZvrZvmMXFx6xSnRWtap4vR5WrFpz1PkrVq7J1TgAoN75dVix8uhzAUKhML/+/hfn1Tm+OsiphlGuDJlfzCRx0flZQUNRLKvWmrmiu/eaJznsZspLrRoYdhsiHMH15EvYZ85LrvEpihTQf2D5YxmuJ57P6et56QWEnnsi2WZJkszhmafYmyWezizx7NQm2aZJssj0BwDweb25fp7m8+L3H11pJxAIHuPcwFHnappG7/6DObNqJdq3aVaAVp9mPG4CMycQu/WGnF6h23biu7EVyrqN5jkuJ/7Z76CfWQXDajU7vjw0HNunXyXX9hRECugRiPR9eO7sY5ZgEwK9TGkC77yWMjlRkuQgdu0xxTN9f/Z+eGiEFM9U43Bt40y/P9fPMzL9eL3uo873eNzHODd3zl88Hqdn74fYs2cf095/A5vNVsCWn2ZsNkJvvEDkni6miGoa4sBBvLe1x/L7X+Y5Xg/+j95Hr1AOw2JBRKK4+zyM9bufkmt7iiEF9DDxOJ6OfRCH3z5dTgIz3gbP0b94kuKD2LUna9n2sHiqhJ5+TIpnCpKW5qNypQr8tWxV9s82b9mG3x/gvNrnHHX+eXXOYdnfq3L97O/lqzivTs65kUiUzt0HsG/fAWZNfzO7QlWhRwgijw4gNPJRs6SfYaAEgnhb98D65bcAGKVKZDffPlxj19OtP5Z/tPsrzkgBzcL12NNY1m/MKhCvEvjfi+jVz0y2WZIkcvTMUyX0zGPE7mydbNMkx6Br5zaMGTuBLVu2k+kPMHzEK9xwfX2qVq101Lnt2zbjz79WMGvOJ8TjcWbN+YS//lpJh7bNAQgEg7Tt2ItYPM6MKf/D4y56L9OxO1sTmDgaw+UEzIILnrsHYc9K1zHKlcE/fwpGms9MgwlH8LTtibJlWxKtTh2kgAL2aXNyBQ1FenclcdN1yTZLkkTE3nRTPPcdyC2eHVsl2zTJv3B//57cctN13HhbO+peeD26rjP+9eeBnPZ3hzmrWlUmTXiVl0aN58yaV/DSqPG8987obLH9eP6X/PDTryz55Q9qndcgu23eoIefTMqznSoSNzQwl2tL+LJmmlFcQ57NrlqkV62E/6P3slfjRCCI946uiAOyDVqxr4VrWbYSb/POOUFDl19MYPpboMh3i9NJKvhCNqGwmaqycXNOtO2zQ4h1aJlsy4oFKeULBYymwa7dVrZutbNlq40tW22sXq2yYZOdnTutRKIKdpuB3W7gcOjY7QaqaqCqOm6XTp3aUS69JEy98yOcXT1WoMOUsm0nnhZdUPbsQ8Tj5l7/i8OJZQVNWZb8jrf9PdmdhrRa1fHPnwJOteCM+Aep4Av/Vgu3WAuo2H8Q37VNUfYfNIOGKpQj89u5UFT2OQoRyfaFbDQNz519slrWxaR4JoGU8YUCYsNGO59/6WH2XB/L/nZisxnYrAaaBqGwgmEcf7k8IQzcbh1DF8QTUKN6jMsuDXHZpWEuvCBC7XOjJ1V9Txw8hLdJR5StO7K3s4Jjnibe7FYAbPM+wz1gSNaEw0H86ssITn7jlAVapoIvSAHNi0QCb7POWP5eaRYBdznJ/Gw6eq2zk2NPMScVflEAnI89jWPq7Ozl/PCD/Yj265Fss4oVqeIL+SUWg58Wu5m/wMsnC7xkZlowDIhE/226aOBy6lisYBhmLXddE2g6JBLiP0XW5dIAgdercWf7Q7Rrk0HNGrF82S/2H8R7azuUXbuzVmAcBMe/nF1IxjF2Is6XxubkybdpRiirPWBBkwq+IAU0D3INlKpKcOxzxJvclBRbJKnxi2Kf8AGuka/kDAwtGhN6+amkF9QubqSCL+SHP/9SeemV0nz9rQebzSAYUtD13L7j8WimmEYUXC6dihXinFUtxrnnRDnrrDgl0jRiMUE4rBCNCsIRQTSqEAoJDh6y8NcyJ2vWONANzM8IHv0ZNpuOxQKVKsbp2ukgLVtkUrFC4oSeRexNx3dLO8TedISmm4GVE0eTuKEBAM5HnsIx46NT/qKZ5vMx8KFhfP7ld+zYuRuP28VNja5l+OODKFmyRJ7XfLlwEa+Pe4cVq9aiaTq1z63BE4/ez1VXXgKYNY5HPPMqv/3+F35/gEqVKtCnVxe6HCM4UAroP7DN/gT3A0NzgoZ6dCTyxAOn3Q5JDskeNK0Lv8PTc5CZA+ywk7jkAjONSbasO+0k2xdOBMOAb75z88zzZVi1WiUSyT1btFoNVIfZGuyGGwI0a+LnvDoRqlaJ43Qe99B71Gfu2GFl2XKVP5Y6WfyLi5WrHEQjCgkN4vGcma6q6hgGnHtOlLu6H6Btq4zj7pctdu3Bd0tbM5BO181mGpPHkah/Oeg67s59sf3wCyJqNpAPjn6aePPb8vVMxyLN5+PBwU/RvOkt1D63BhkZfvr0fxS73cYHk17P85qZs+bjdKpcU/9y3G4X703+kGFPvcTP339M5UoV+O2PZfz553KaNG5E+XJlWPLLH7Tv3I/XRo2gaR6TKCmgR2BZsRrv7Xea4mk3Kw0FZk6QxRKSTDIHTcuK1XibdkKEwma/16pVyFw4E4pg2kJhoDAIaCIBcz7y8ewLZdmbbiEUyhk/XE4NTRdUrBCnebNMGt8a4OILw6c8LnHdejvTZ6bxwbQS+P0WolGBpuWIucul4XIaPP7oXjq0O8Tx1INQtu3Ee2tbxIFDCMPAcDnxT3sT7fKLIRLF26QjlrXrs3vhBqa+SSJrplcQ5OULX3z5HXf3fZgt65Yc931q1mnAqJeGc3vjRnke79ZzIOXLl+W5kY8edUw21M5CHDyEp30vUzwBo0QawYmjpXgWY8SuPXha34UIhQEwfF78s9+R4inJk0hEMG58KWrXq8WghyuweYs9Wzzdbo2KFeMMH7qX3xev5/fFGxj6WDqXXnzqxROgZo0Yjz+azso/1zF/zma6dzlIWpqGx6MhhEEoZGHffitDhpaj7sW1+GBqGon/WNnVq1Q0iykczgMNhfG2vwfL0r9BdRCYOQG9zBlm+ks4gufO3ihrN5zS5/zuh8UnVI94xco1HDh4iNrn1szz+MnUOC4+M1BNw9OiK9aly80QbadK5oJp6LXz/keVnF6SMusIBPHd3AZly3Yz19PlxD/vfbS6tU+vHZJcpOoM9PsfXPS+txIZGRZC4cOKaOByGVQ7M8aQwXu55aZASmXAaRr8vNjFpMkl+ORTH1pCEE+Ys1K3S8Pj1Rk2ZC9tWmX86zxCWbfRjM7N6txiuF34P3oPrW5tlC3b8d7UGiXTb05MSpci86vZGOXKnLT9//SFefO/4N4BQ/h4zqSjGgHkRXr6fho378ztjRsx7PFBRx3XNI3u9wxi374DfPThxDzLNMoZKKA+Nwbr36uyxTP46kgpnsWZRAJPl34oO3Zl17cNvPmyFE/JURw4YKFn70q071SVXbtthMIKFouOqupc0yDIh9O28MM3G7ntltQSTzAX1xrUD/HWuJ388sN6WrXIQHXoWK0GwZCFPXtsPDS4PBdcWpO587wYx5hO6TWr4//oPfTDxRSCITwtuqGsXo9+ZmUCM9426+oC4lAG3pbdIHB0Af+TYe68z7n/wWF8MOn14xLPXbv30qxVd65veDVDhww86nhB1DhOsf/uU4P1hyWob03ODhqKdmpT4JvdkkKEYeB6+EmsS/82cz2dKuHHB5Fo1DDZlklSCMOAGR/6uPiKGsxf4CUcMYdLVdVp2sTP159v5KMPt3Ll5eEkW3p8VK6c4I3XdrLkhw20aJ6Bw6FjsZhCunOXjXvvr0SLNlXZtTvvwDm9di0Cs9/FcLsAEP4A3uadUTZsRruwLoG3XjGbcSc0lO078XTqA/F4gdj+wdQ5DHp4OFPeG8s1Da74z/O3bt1Bk+ZdaHTDNbzw7OOIf0TSF1SN4yIvoCJ9P+6e95vRlVYLWq2zCQ97MNlmSZKIY9y72OcsyHmhat+CaM9OyTZLkkJs3mKjcbNqDHq4Apl+C7GYmXZyVrUY8+dsZuKbOzj3nPzlWSabKlXijB+7k8WLNtC8mTkjtVh0QiGFnxa7ubz+2bw/JS3P2ahWrw7+GW9juJzmbDPTj/f2O1G2bCPRqCGhpwabM9FoDOtfK3ANGMIxp7XHyfi3JzP0qRf5cOqbXHn5xf95/tp1G7mteWdatbiNEcMfOup4QdY4Ltp7oLqOp2U3rL/9hUgk0L0eMhfNw6hQ7tR9piRfnK59L9uChbj7PpKVruIgceUlBKb8TwaSpRDJ3AM1DHh9XCmee7EssRhomoKiGDjsBg8OTOfevvuPK3q1MLFps41HHy/P9z+6CWft7bpcOhecH2b82B1Urnx0pJF18e94OvRChMMYQmCcUZLMz6ZjVK6IOvIV1Akf5KQJ9upCZPCAfNmW5vNhcVXBarXisOf+h9+28TfATFsZ9NDw7O/7DRjC1OlzcWcVyD/MKy8Op02r25k6fS79BgzB6VRRjpiZtmndlFfyKAhRbNNYHKPfxDn6TTM9QVUJvD2KRKNrT9nnSfLP6Rg0LX8ux9uiq/mLbbWgn3UmmZ9Nh6wlKUlqkCwBDYUEd/epxHeLPNlBQi6XzvnnRRj3+g6qnVkwy5GpysefeOk/sCKRiCAWM/d5HQ54cuhuenQ9dFQ9Eeuin/F0vdf8fVIUjDJnkPn5DIxyZXD3egDbF9+ZL6pOleDLTxFv2eSEbUqFgLJiGURk+f0vnKPGm+LpVIne2UqKZzFG2bYTT7u7c1KY0nz4Z02U4ikBYOtWG9feWJ2vvzXF027X8Xo0Xn5hJwvmbS7y4gnQtImfpUvWcctNAZxOHU1TCIUUhj1VnpubVGPLltwzwMS1V+Xse+o6Yt8BvE06IvYdIPj6cyTqnothtyHCEdyDhmL5Y1mSnuzUUSQFVBzKMN+MIuabkV61stz3LM5k+vG07HZEs3QX/lnvYpQ9+TB7SeFn0Q8urrmhOpu32IlGFVxOnSuvCLH0l/W0a51ZrCo5liypM2nCdia+uZ2SJRLY7ebe6NI/nTS4oTpfLMw9E0s0akjwjRfMptyahrJnr1moJhAi+ME4M0dUCLMZd4d7EDt2JenJTg1FT0ANA3efhxGZWYOlUyXw3liOu36VpGgRj5vJ3Xv2muXIVJXAO6PRz62RbMskScYwYOy4UrTvVBV/wIKuC5yqzj099zN7+lZKldKSbWLSuOWmAEt/WU/zppk4nTq6LggGLXTvWZkXXymdKy4o3rgRwVEjTBFNaCg7d+FtZgblBT6ceETUbhBv6x4QLNj0lmRS5ATUMeEDrIt/R8RiOfmeZ1ZOtlmSZGAYuO5/HOvy1YiYmf8bGvEIiYZXJ9sySZKJRAR39arEsy+UJRJRONwNZdzrOxg6JD3l8jmTgc+nM37sTia/u40SaRoWi044ojD6tdJ07FKFYDBnah5v0ZjQc4+by7nxBMrW7XhadUevUJ7ge2NNcdV1lB278XQfYLabKQIUKTexLF+Fc+QoMzJMdRBrdivxprck2yxJkrBPnIJ9wcKcaMCu7Yh1bptssyRJJj3dwvU3n8VnX3gJhRVsNp1yZRN8sWATzW73J9u8lOP6hkF+/G4DtWrGUFWdUFjhu0VuGjaqztatOfuisfYtCA1/yBTLWBzL+k247x5I4spLCI3MSm+JxbD+uhTnsOfNi3Qd9bkx2OYsSNLTnRyWYUMGDT/ek6PRaIF+uOpwFNw9g0F8TTuhHMowm2NXrkBg8jiwyW4ahYEC9QXMlylP74fMfXDVQfyaqwi/OlK2JisEFLQvHMmevRZuanwWW7fZicUUnE6dCy+I8Om8zVSpcmItv4oTXo/One0PsW69gy1b7IQjChmZFiZPKcGlF4c5s6oZZKVdeD6GxYL1tz8RkSjKrj2I/QeI3t8bkb4fy+p1iHAEy6q1GG4Xzudfwz53AdalfxO9u9NRv5+n0heOF0UI7MfYAiwyAuruNxjL8lUITQOnin/OpAKpxSg5PZzyl6lZ74C9iCXwFVFO1aC5a7eVRrdVZ88eK4mEKZ5dOh3k7f/twOU6uWT/4oDVakbqOp0aPy12E48rxGIKH83z4XRqXHpJBCFAu/wilO07sazbmCWW69C9HqIDe2P9YQnK3nREJIr1+8Vm0+5YHJFIEL++AUb5srk+M9UFtEgs4dqnzcG28Lvssmyhp4eg16yebLMkScI94HHEgYPmN6qDwPtvgFNNrlGSpLJjp5VGt57Fnj0WUzxVnaeG7ubZEXtkDY0TQAjo2+sg0ydvxevRUBSDcEThmefL0fveimZ3FyEIPT+UxGUXYTjsiHAE14iXsX21iMCk19FLZ0XmJhKIaFY1p2gUx6RpSX22/FDoBVRZvwnXoyPNfS6HnfiN1xLr0CLZZkmShH3qbGxfLcp5mXr2cfkyVczZvt0Uz73pFjTNFM9nR+7mru6Hkm1aoeWaBiG+/3oj1c6MoTrMVJf5C3x06l6FWAywWAi8+xp6tSoYVisiEsXd60HU0W+i7D+I+Ed5P6Hp2Od+CpHkzjZPlMItoJGoWbA46x/dKFWS4Gi5z1VcUdZtxPXY01kvUw7ijRoSay9fpoozW7fauOHW6qTvs2aL54vP7aJLJymeJ0vVqnEWfbWR6xqahRfCYYXvv3fTpmNVIhEBLif+D9/BKFnCnHGGI6ivvY2IRPK8nyEEts++Ps1PcXIUagF1PfY0yu69Zqd0VSXw/ljZCLm4Eo7kepnSzyhB8NURSTZKkkw2bbZx421nceCAmeOpqjqjXtpJx/YZyTatyOByGUx+dzutW2aYIhpR+PU3F81bn0kwKDDKnIF/zru5Kn4Zx5jfKMEQ6lvvnx7DC4hCK6DWb37APucTRCSK4XQSHnK/7OVYjHENHoGyJz37ZSo4eZx8mSrGbNxk46Z/iOdro3bStnXqNeou7CgKvPrSLrp2PojTqROJKCz7W6Vx82pkZiroNc4iMHmcmd4CIBSzAL3laPmxLF+F2LXntD9DfimUAioOZZjVhsIRDJuVxKUXyHZUxRjbR59in/dZ1suUSmjYA2h1zkm2WZIksX+/hdvvqMbBQxYMw6wuNO61HbRqIcXzVCEEPP3kHu7tsw+nUycaVViz1sHNTapx4ICFxJWX5FQr0nWw2TBUlbxin+3T5px2+/NLoRRQ1/2PI0JmE1tDVQm+8YLc9yymKFu24R441HyZstuIX30ZsW4dkm2WJElEIoKWbauyf781WzzfHLed5k1lgYRTjRDw6MP7eOTBdJxOnVhMYdNmOzdmBXDFWzQmPOCe7IIKIhLl8Kh9WEhFNIY6afpJ9xA9XRQ6AbXNWYDtu5+yS/WFRj+NUeaMZJslSQaxGJ5OfSErKMFI8xGSL1PFFl2Hu3pVYt0GB/GEwOnUee3VnTS5LZBs04oV9/Xbz1NDd6OqOvG4wo6dVm64uTo7dlqJ9u6KUSINwMzZz0IARlb9RJHpE/0A4wAAEc1JREFUx7L072SYfsIUKgEVu/fifnBYdspK7ObriDdulGyzJEnCOewFlG07s4rEOwhMeh0jzZdssyRJYvjIsny7yEMkYnZUub//PlreIZdtk8Fd3Q/x8vO7UFWdREJhz14L7W924bypA+LgMSKgFQW9ZAkIR3C8WzhyQgtPnTvDwN3rQciqSmF43IReHJ5cmyRJw/bFNzimzTFL9bmchAf2Rru4XrLNkiSJ9yaXYMI7pQiHzQpDTRpn8uDAfck2q1jToV0GLpdO3/6VCEcU+u57Buu+jQiOUTLRohDt0gZxKBPbNz+eXmPzSaERUPukaVj/XolIaOa+55uvgM+bbLMkSUDs3I2r7yNZQWQ2EhfWJdqvR7LNkiSJbxe5Gfx4eSIRsxF23fMivP7qTrmSnwI0b+rH4djOXfdU5v7Iq+wRFRjMs6hKLNcSLpj7n44pc8hY9i3hJNl7ohSKJVxl0xZcT76ECJldVqLt7yBR//JkmyVJBokEni73IsJZ+54eF8G3RiH7TxVPVq+x07l7ZSIRBUUxKF8uwcwpW7HJsscpw603BxgzaidW1cpTxhNcYF3Or2oDdKfz6JNDIayLfy80cQypP+poGu4e90NWzUS99BmEhz+cZKMkyUJ9djSWDZsQmoahOghOHINxRslkmyVJAnvTLTRrVY1g0BzGvF6dj2dvwecrGr0mixKtWmQy8sndOJ066+LVaZj4mtbRKaRTmhA5dapFKIz9nalJtPTESHkBVV8dj2Xz1uxAkeCEV0F1JNssSRKwLvoZdcIHOf09+3QncdWlyTZLkgTicWjV7kwOHbIAZsTtrGlbqFIlnmzTJMege9dDPDjQTHGJRBXm6HdQjc1M5C5CmLNRYRjYv/gGgsEkW3t8pLSAWpavMmsnhiMYTieRPt3RLjgv2WZJkoBI34f77oFmsQSrBa12LSIP9k22WZIkMfTJcmzcZCeRMKsMjR+7g4svyrvGqiR1uOG6/7d35/FNlfkexz85ada2aWlrW2gLhbKUgohWFEFlZFxYxdeVGbkioggqLhcF57oMjjOK1/GOqJdxrhsICiMjLxVEReAysimL0iogmzAUbMtWutA2W5Oc3D/SFoUWSkhyUvp7/0VfOTn5pTwn3+Y5z2JH/VkHgYNYHuY1BrGWElNn/BYzqH6MS1doV+Q5iN4AdbmJnTAl8IGp06F27IBr2mStqxJaUNVAW7DXL55hsVA7dxayD1XbtHJVHO8taIfTqWC1+nj0P44zYpgslBDt1q63MvyWbNzu02NnC/3I9e/i8NhJ4PNhmvN3DSo8d1EboJYZM1GOlQd+MJmofWdWYEdX0eaYX3mDmB/2oPN6AyOw35wpm6W3UaWHYph0f2BahMGg0rePi2mPyHSV1uCTT22cMvD2F+pUA8/yDNVrP6Fu1NDIFXYeojKRYjZuwbTgw8C3T6sF55NTUHOytS5LaED/3fb6LZAC69y67rwN7+CrtS5LaMDrhTvGZ+FwBkZoxsWqzJtdIgOwW4mX//sIQ2+qZep/plNZqcfh+GUPkserMG9+O6prBrD9h1/jXKjDaFRon57IwKvsXJ7vZMBVjqjqeNL5HMUtXnTwRHVoV/VIsNlOP2etnYT+Q1GOl+OP0ePtezG1S+fLNIULXJNtweXGNnA4+tLD+BUFX25XalYsQuYoXNiabAvAM8+mMrthsQSzyqL3f2LgAIcGFYrz4fHAO++2Y8YLqXg8Ourqfv7Z7gdOn8JiMKiYTH6MRj/3Taxg0oQKEhMjM9paryjExcU1+VjUpZL18T+hq6m/n2EyYX9rpoRnG2X5r1dRyisDP5iM2OfOkvBso75cE8vsd06uNPTg5HIJz1bKYID7Jlby5fL9pCR74Rd7sjQ9/9PjUait1VNREcMrs1K4rH83VqxsOtQiKaqSKWb1VxiX/ROdO7BQvP3FP+DvkK51WUID+sJtmN5bdHKpvulTUTtlaV2W0MCRozFMuDcTp0shJsZPrzwXjz9WpnVZ4jxs2mzh+mFdKCsz0FxoNsflUqiq0jPhvkymTGv/i1G9kRY9AVprJ/ahJ07u8TmgH55bR2hdldCCy03sxEcD4alX8PXoivtu2aKsLfL5YNzdmTgcgY8qq1Vl/tziqLoPJs7Nxk1Wbh3TiZoaPR5v8CsOOZ0KH36cwL0PZGgWolEToNbpL6CrrZ88azJhf3VGq1nOSYSW5flXUCrqu26NRuxvvCTd+G3Uiy9dxM5dZrzewN6ec98uIS31DEM5RVQrPRTDb8dm4XSG5np2OhWWr4jnlVkpITnfuYqKT6WYDd9iXPJFfdetBfufn8Z/kTa/EKEtfcFWTPMXNY7Adjw9DbVTptZlCQ1sKbTw2uvJjfc9J91TwXWDWscKNeJ0fj/c+0AGbndovxg5nAovv5rC7j3GkJ63JbQPUIeT2PsfC3TXxejx5veRrtu2yuUmdtLUQHjqFXy53ai7a4zWVQkNOJ067p54cpH4bl3dTH/ymNZlifOwZKmNrdvMeL2hjx2XW8f9D2WE/Lxno3mAWp6bia66ftSt0Yj9ry9I120bZZkx82TXrcmE/U0Zgd1W/fG5VMorAjc6zSY/780pkXVUWrmZ/5Ny2tzPUPH7dezdZ2LbdvPZDw4hbT+dvinEtHDxye66557A3z5N05KERjYX/GLxDMcfpqFmddC6KqGBrzcamf9+O1wuBatV5U/PHKFjR1kkvjXbsdNEUVF4u1jr6uB/30wK62ucSrsAdbnRjZvcONLSm9eDuttv1awcoSGnC93Y+xu7br09u1M3Xrpu2yK7XcdtY5NxuRT0epXeeS4mjK/Suixxnr5cE3fGZfxCwedTWPVlZOeGahagxs9WwpH6expGE47X/yJdt22UcfEyKDu57rHjjZekLbRRiz5MoLw88LFkMsHsN0ukKVwAvt5gPWXFofCorVEoL4/cHCfNArRu1BD8T0zBH6PH+ftHpLuuDasbPQL/tAfw6xUcf3hM2kIbdvuYEzw6pQZF8fPCjMNkZni1LkmEwI6dkbk3abb42bU7cvtFa74WbvVPxfgTbPKNo42TtiAaJNhsFB2wk5Qk8z0vFDk9u1NZGf5RYLZ4H2/8rZQhN9aG7JxnWgtX83Ft/sQErUsQUULagmgg4Xlhidhgel1ktwmWOQJCCCHCKrBofPj5VUi9KHLd/hKgQgghwurKfs6IvI7TpdAz1xWR1wIJUCGEEGHW/0oHsbHh75bv1KkOYwRX9JMAFUIIEVZDb6rB5wvv4ECLxcdd4yrD+hqnkgAVQggRVgkJKiOHV6PXh2/fMVXVMXZMZBfdkAAVQggRdr+behyDITzntphVJoyvJDExshuDSoAKIYQIu645dTz+2DGs1tCHXFKSj6efivxuPRKgQgghIuKhyRX0znNhMoUuRC0WlffeKcZsbvGaQCEjASqEECIi9Hr48IOD5PZwYw5BiFosKgvmFXNp38hNXfk5CVAhhBARExfr5/MlBxg6pAaLJbgQtZhV0lI9LPnwINcNsoe4wpaTABVCCBFRVqufOW+WMu/tElKSvcS1cI6oyaRiMqmM+W0VWzbuo19+ZBZoaI7ma+EKIYRom264vpadW39k+cp4Zv0tma3bLI33R1VVh04HesWP06WjXaKP8eOquHt8Jelp0bFLjwSoEEIIzcTEwIhhNYwYVoPHA3t+NLF7jwmHU8EWbyYhoYZLLnZF5QYDEqBCCCGigsEAvXu56d3LDQS2tjtRrd09zrORe6BCCCFEECRAhRBCiCBIgAohhBBBkAAVQgghgiABKoQQQgRB53MUt3gBwerqahQldJlb43ARbzWH7Hyi9ZK2IBpIWxANoqEtqKqKzWZr8rFzmsbS3EmCNeuD1Tx1z8iQnlO0TtIWRANpC6JBtLcF6cIVQgghgiABKoQQQgRB0wC95tLuWr68iCLSFkQDaQuiQbS3hXMaRCSEEEKIAOnCFUIIIYIgASrC7stvd/LHNxYzZ8m6Jh8/eOg4H6zYDMCGrXtZuHxTJMsTGnK43KzZslvrMsR5Ots1fi7WFexm07Z9QT33eFUN3/yw/7xraKlWtxuLT1XRh3Auqgi/tQW7eXjMDaQkxjf5eKcOKXTqkBLhqkQ0cLjqWFe4m19dnqt1KeI8nO0aPxfX5gffFspP1PLtjv1c0bvLedfREiG5B3q8qobX/rGKnKxU9peUkRhvZfJvBnOi1sHC5ZuodbgxGvTcMWwACXFWnpv9CTMeHI2i01Hn8fLM6x8z48HRVFTXnnZ8ekoi8z5dT6zZRPHRCjqmJzP6+n6heO8iAv6+bAMbtu4jLdnGlb1z2PrjT3i8Pgwxeu4ceTXpyQnsOXiYVZt28OBt17Nh614OHi7n34f0p2DXAT5f/z06nQ6Lychjdw7V+u2IEJu9eA1bfywmLclGzy4dANjxr1IAhl19CZfnddayPNECLbnGD5VV8u6nX+Pz+fAD9956HWlJNjZt28f/bd4BQGZqO+4edS2frvsOk9HAjf17M3P+F3TOuIg9B47gdNcxbvhAunVM43hVDfOWrsddF9hYe8yQ/uRkpvLi3M84XH6ClIQ4+vfpyuB+PVm8uoAfDx7B61MZlJ/LtZf1CNl7D9k30GMV1dxzy7WMGz6Qtz5ew3e7D7Jh2z5uH3oVaUk2ikrLWLh8E4/eMYTM1CT2HjxCj+z2bNtbTF6XDPR6hQXLNjZ5fMP5H7n9xpCuhCTCb+ywAezYX8rUO4ag1ytc378XekVhV9EhPlldyH2jr2v2uZ+v/56Hx9xAO1ssDpc7glWLSLnlunwOlVUxfdIoCncfYF3hHqZPvJlah5sX5n5Gt6w0EuKtWpcpzqAl1/i6wj0MvqInV/bOwevzoap+DpVV8sXX2/jd+GHEWc3YnU1f46rq58kJI9i+r4TP13/PI2NvwhZrYcrtN2KIieFoRTVzFq/lqXtGcsvg/MY/xgHWF+7BYjLy5ISReLw+/vLuMvK6dAjJN2UIYYAmJ8aRlZ4MQKf0ZMpP1LK/5Bhvf7S68RivTwXg8rzObNlZRI/s9ny7o4hB+bm46jzNHg9wWc9sCc9Wzun2MG/pV5RVVgOB7vgzyclM5d3PviK/Z2cu7dExEiUKDe0rPka/vC4oioItzkL3jmkcOHycS+Ll/761aO4a75KRyhdfb6Oq2kHf3E6kJdnYc+AIl+ZmE1e/VF+sxdTkOfvWX/sNuQLg86n8Y8Umio9WoOh0HK2obvK5O4sOUXqsksJdBxrrO1ZRHX0BaojRN/5bp+iw291YTEamTxp12rF9umexZHUBdqebn46Uk5udjtvjbfZ4AKOh1d2uFadYuraQHtnpTP7NYI5X1fDyguVnPH7ssAEUlZaxfV8Jz89eyu8n3tx4sYkLkF9m1LV2zV3jV/TuQueMFLbvK+GvC1dyx/CB+PGj0539nAZ9IFsURdcYyKu+2UF8rIXpk0bh9/t5+M/zm36yH2678Up65WSE5P2dKmxf6cwmAymJcRTUJ7/f76fkaEXgMaOB7A4pLFq5mYu7ZqIoChaTsdnjxYXB6fKQWN8dt7EFo+zKKqvpnHERNw+6lDirmcpqe7hLFBFmNhpwuT0AdOuYTsGuIlRVpcbuYu9PR8mWwWWtSnPXeFllDSmJ8Qzul0efblmUHqsgN7s9BbsOUOtwATTbhdvk67g9JMRZUHQ6Nm//F2r9H19mowFXnafxuLwuHVhXuBtffW/m0fITuH/2+PkK69e6Cbdcy/tfbGTZV1vxqSr98jqTmZYEQH5eZ97+eA1T6+9xnu140frddFVv5n36Fas27yS3U/pZj//on1s4Vt810yO7vbSFC1Cc1UxOVirPvrWEXjkZZKS2Y8bspQD8268vJyFO7n+2Js1d4wU7i9j8w370ig5bnIXh1/Ql1mJi6MA+vLxgOTqdjqz0JO4aeU2LXmdQfi5vfbSawl0H6J7dHlN9D2VmahKKovDc259wVZ+uDL4ij/ITtTw/J9Cm4qxmJo8eHLL3KysRCSGEEEGQUTlCCCFEECRAhRBCiCBIgAohhBBBkAAVQgghgiABKoQQQgRBAlQIIYQIggSoEEIIEQQJUCGEECII/w+F4U1Stq+6PwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x230.4 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.patches as patches\n",
    "index = len(sent)-1\n",
    "temp = polarity_score#s[index]\n",
    "#plt.style.use('ggplot')\n",
    "plt.figure(figsize=(8, 3.2))\n",
    "plt.xlim(-1, index+1) \n",
    "plt.ylim(0, 1.0) \n",
    "plt.xticks(range(index+1), sent, rotation=0) \n",
    "\n",
    "style = \"Simple, tail_width=2, head_width=8, head_length=12\"\n",
    "\n",
    "\n",
    "for i in range(len(sent)):\n",
    "#     if i == 1:\n",
    "#         continue\n",
    "#     if i == 3:\n",
    "#         continue\n",
    "    if temp[i]>0:\n",
    "        kw = dict(arrowstyle=style, color=\"b\")\n",
    "    else:\n",
    "        kw = dict(arrowstyle=style, color=\"r\")\n",
    "    a = patches.FancyArrowPatch((index-i, 0.1), (index, 0.1+0.05*i),\n",
    "                                 connectionstyle=\"arc3,rad=-0.7\", **kw)\n",
    "\n",
    "    plt.gca().add_patch(a)\n",
    "    if i == 0:\n",
    "        plt.text(index-i, 0.4+0.09*i, s=str(round(temp[i], 2)), fontsize=13)\n",
    "    else:\n",
    "        plt.text(index-i+0.1, 0.4+0.09*i, s=str(round(temp[i], 2)), fontsize=13)\n",
    "    \n",
    "circle = plt.Circle((len(sent)-1, 0.1), 0.1, color='b')\n",
    "plt.gca().add_patch(circle)\n",
    "plt.axis('equal')\n",
    "\n",
    "ax = plt.gca()\n",
    "# ax.axes.xaxis.set_visible(False)\n",
    "ax.axes.yaxis.set_visible(False)\n",
    "#plt.savefig(\"imgs/sst_gru_polarity_score_negation_example2_largefont.pdf\", bbox_inches='tight', dpi=900)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "RuXWeGUICyO2",
    "ixc1hrcbCyP2",
    "x4iARfG-CyP8"
   ],
   "name": "rnn_approx_classification_analysis.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}