extracting_motifs_esm_2.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Motif and Substructure Discovery with ESM-2 and Persistent Homology"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's start by importing necessary libraries\n",
    "from transformers import EsmModel, AutoTokenizer\n",
    "import gudhi as gd\n",
    "import numpy as np\n",
    "import torch\n",
    "from scipy.spatial.distance import pdist, jensenshannon, squareform\n",
    "from sklearn.cluster import KMeans, AgglomerativeClustering, DBSCAN\n",
    "from sklearn.metrics import silhouette_score\n",
    "from gudhi.hera import wasserstein_distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = [\"MAVESRVTQEEIKKEPEKPIDREKTCPLLLRVFTTNNGRHHRMDEFSRGNVPSSELQIYTWMDATLKELTSLVKEVYPEARKKGTHFNFAIVFTDVKRPGYRVKEIGSTMSGRKGTDDSMTLQSQKFQIGDYLDIAITPPNRAPPPSGRMRPY\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "def compute_output(sentence, layer, head):\n",
    "    # Load pre-trained model\n",
    "    tokenizer = AutoTokenizer.from_pretrained(\"facebook/esm2_t6_8M_UR50D\")\n",
    "    model = EsmModel.from_pretrained(\"facebook/esm2_t6_8M_UR50D\", output_attentions=True)\n",
    "\n",
    "    # Tokenize input and convert to tensor\n",
    "    inputs = tokenizer(sentence, return_tensors=\"pt\")\n",
    "\n",
    "    # Forward pass\n",
    "    # Specify `output_hidden_states=True` when calling the model\n",
    "    outputs = model(**inputs, output_hidden_states=True)\n",
    "\n",
    "    # Obtain the attention weights\n",
    "    attentions = outputs.attentions\n",
    "\n",
    "    # Obtain the attention weights for the specific layer and head\n",
    "    S = attentions[layer][0, head]\n",
    "\n",
    "    # Obtain the value vectors\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        hidden_states = outputs.hidden_states[layer]\n",
    "        all_W_v = model.encoder.layer[layer].attention.self.value.weight\n",
    "        num_heads = model.config.num_attention_heads\n",
    "        head_dim = model.config.hidden_size // num_heads\n",
    "        W_v_heads = all_W_v.view(num_heads, head_dim, model.config.hidden_size)\n",
    "        W_v = W_v_heads[head]\n",
    "        V = torch.matmul(hidden_states, W_v.t())\n",
    "\n",
    "    # Compute the output O\n",
    "    O = torch.matmul(S, V)\n",
    "\n",
    "    return O\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at facebook/esm2_t6_8M_UR50D were not used when initializing EsmModel: ['lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight']\n",
      "- This IS expected if you are initializing EsmModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing EsmModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of EsmModel were not initialized from the model checkpoint at facebook/esm2_t6_8M_UR50D and are newly initialized: ['esm.pooler.dense.bias', 'esm.pooler.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 155, 16])\n",
      "tensor([[[ 0.6743,  0.5545,  1.2208,  ...,  0.0626, -0.9014, -0.7621],\n",
      "         [ 0.6362,  0.4581,  1.1848,  ...,  0.0766, -0.9643, -0.7803],\n",
      "         [ 0.6222,  0.4346,  1.1726,  ...,  0.0467, -0.9650, -0.7802],\n",
      "         ...,\n",
      "         [ 0.7480,  0.3212,  0.9947,  ..., -0.2329, -0.6257, -0.7537],\n",
      "         [ 0.7485,  0.3842,  1.0374,  ..., -0.2265, -0.6053, -0.7752],\n",
      "         [ 0.6429,  0.5328,  1.0885,  ..., -0.3019, -0.6538, -0.7992]]],\n",
      "       grad_fn=<CloneBackward0>)\n"
     ]
    }
   ],
   "source": [
    "# Set the layer and head to use for computation\n",
    "layer = 3\n",
    "head = 2\n",
    "\n",
    "# Compute the context vectors for each text in the corpus\n",
    "context = [compute_output(t, layer, head) for t in text]\n",
    "print(context[0].shape)\n",
    "print(context[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Barcode for text 0:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGzCAYAAACy+RS/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAaa0lEQVR4nO3dP3NbR7on4JfS6HqsKsvHVJXX9ljBQOFklJTdrQ0GTCcCrU9AMropMYomlKloazci9QksInJKJLd2M4nIJiQcuMY1qypRsFSlO74e+2ygAi7/E4fE3+7nqZoaETwA+jRh9A/dbx8slGVZBgCQrWvTbgAAMF3CAABkThgAgMwJAwCQOWEAADInDABA5oQBAMicMAAAmRMGACBzwgAzaWVlJe7duxcLCwuxsLAQKysrR/63vLwc6+vr0ev1ptbGXq8Xd+/ejSdPnkytDbOo3W7HvXv34pNPPomVlZVpN2dscjlP8vCbaTcATrOzsxMREQsLC7G0tDT4+bD19fX45JNPYnd3N+r1+qSbGAcHB9HtduP58+eXfoxmsxmbm5sjbNX01ev12Nvbi3v37k27KWOVy3mSB2GAubW1tRXtdjtWVlbiu+++i6IoJvr8tVotrvrVHt1ud0StmT2Li4vTbsJE5HKepM0yAXOtXq9Hr9eLdrs97aZU1mq1prrMAdAnDMAUdLvdWF1dnXYzACLCMgFzrj8jcLhmoNfrRbPZjLt378arV6+i2+3Go0ePYmlpaXCfZrMZ3W431tbW4uHDh9Fut2N3dzfW19ej0WhEt9uNra2tuHv37uDT+/7+fqyvr8fS0lJ0u91YX1+PFy9eRK1Wi729vcHzX3TfVqsV33zzTUREvHjxYlB8VqvVjtQPXHQenU4nVldXo9vtRr1ej6dPn8b29nYURRG7u7snHu+w7e3t2NvbGyytLC8vn6i7uOj5h9XpdAZ/p1evXkVEnNqufr/dvn37zOOG+dtVOceIOFIA+urVq7h7926sra2dei5PnjyJ/f39uHv3bhRFEbVa7czzHlX/wUSUMMMiolxaWjr1d5ubm2VElLu7u4Pb9vf3y6Ioyr29vXNvK8uyrNVq5draWrm5uTn4udFolGVZnvqc9Xr9xGPU6/UTx1a5b71eP/XcqpxH/3H659FXFMWJ28qyLBuNRrm2tnbkto2NjUv341nq9XpZFEW5s7Nz5Patra1T++i0Np31tz/vbzfsOZbl+7/V8fZtbGwceazDx25sbBy5bXd3tyyK4sTxo+g/mCRhgJkWEWVRFOXGxsaR//Xf7Pf3948cX6/XT30jbzQaJwbe/mD1+vXrsizLwf/v7e2VtVrtxGPs7OyceCNfW1s7MmBVue95YaDKeTQajTIiTu2L44Pp1tZWGRGDc+07PphWef6znPb8h5/v8MC6u7t7Iti9fv26jIgTg3X/sU/725Xl8Oe4trZ25rkcDzEbGxtlURSnHru0tHSir0bRfzBJwgAz7byZgeP6g8dpn4a3trZOvJmfNVj1H6der5/4JHnc8TBQ5b5nhYGq57G2tnbqQHW8bWVZnvoptizfD1L9wa/q85/lvLBzvM37+/tlvV4/EWjOasd5QWOYc+w/9tbW1qmP0Wg0joS6oihOzDQcbsvh5xtV/8EkqRkgGS9evIiI9+vz29vbJ35/2jr1advCiqKInZ2dWF1djeXl5YiIWFpais3NzQuvZ3CV+476PI7r9XrR6/VOXec+fB2Hyzx/VUVRDNrTX3vf3d0dtLPb7Q62XfbrB4477ZyHPcdOp3PmY/Rv7z9/t9sdXGBqGJPoPxg1YYBk9N/Yl5eXjxSSneesaxM0Go1oNBqD4rRWqxXLy8tDXeDosvftD4yjPI/DDg4OIiLi9u3b5x53mecfhVarFVtbW7G0tBQPHz6MRqNx7nmd9rthz7F/3DhMq//gKmwtJBn9Ku2rXsin3W4f2aWwubkZ+/v70Wg0Tr0S4qju++LFi+h2uyM7j+P6n5b39/fPPW5cz39YP/j0B/Tt7e1YXV2Nra2t2NzcvHTF/bDneP/+/Yg4+xwPDg4GjzXsY/ZNov9g1IQBkrKxsRFbW1un/m59fX3oxznr8sfDfKIc9r7HP9n2er3Bp8pRncdx/RmL0xye0h7V85/VX8+ePYtHjx4Nfm42m/HVV1+dmN4/fFGmYb8DYphzLIoiGo3GYIvnca1WK5rN5uDnjY2NMx/z4ODgxMWjxvX3g3ERBph5Va7St7m5GUVRnBg4Wq3WqV8mc9Zjb29vn/hkt7u7Gw8fPrzwMYa97/Ly8mB9OeL9J8l+QKhyHqcNRmfd/vTp04iIIwNdxPs19MPr51X78Sy1Wu3EINpsNuP+/fuxsbExuO3wGv3h5+pfYfI0Z90+7Dk+ffo0er3eiXX99fX1qNfrR6410O+PVqt15Nj+3/p46BlV/8GkLJTlFS+uDmPQv6BPv9CrXq8PCvGG0R8I+mvH/ftHvB8UHj9+PHhjbzQa8eDBg8Hg1G63o9PpDIrc+mq12mANuNvtRrPZjHa7Hb1eLxqNRjx69CgODg4uvO/xdnY6nVheXo6lpaUTNQXnncdpbVhfX49arXbk9nq9fuKCPM1mc1AU1y/gO62e4bznv0j/S5j6tRO3b9+O/f39uHfv3omL+vR6vVhdXY1erzcovOy3t3+xpkePHkW32z33b3f8+aue41ntO3xsvy9evXoVDx8+jGazObj41NOnT4/0z1X6DyZJGACAzFkmAIDMCQMAkDlhAAAyJwwAQOaEAQDInDAAAJkb6rsJfv311/jhhx/io48+ioWFhXG3CQAYgbIs4+3bt/HFF1/EtWtnf/4fKgz88MMPcefOnZE1DgCYnO+//z6+/PLLM38/VBj46KOPBg9269at0bQMABirN2/exJ07dwbj+FmGCgP9pYFbt24JAwAwZy5a4ldACACZEwYAIHPCAABkThgAgMwJAwCQOWEAADInDABA5oQBAMicMAAAmRMGACBzwgAAZE4YAIDMCQMAkDlhAAAyN9RXGM+L1xt/nnYTAGBmvPnpp6GOMzMAAJlLamYAACbpkydfT7sJ57r+5k3E//qfFx6XVBiY9T8KAMyipMKAmgFgFvmgwqxLKgzApHmTB1KggBAAMmdmAMInfCBvZgYAIHNmBhg7n7oBZltSYcCgAwDVJRUGbC2cLmEMYD4lFQaYPAEAYP4pIASAzAkDXJpZAYA0CAMAkDlhgEsxKwCQjqQKCA1QAFBdUmHA1sLxE7gA0mOZAAAyJwwAQOaEAYZmiQAgTUnVDBisAKC6pMKAAsLxELIA0pZUGDBoAUB1SYUBMwOjJ2ABpE8BIQBkThgAgMwltUzAaFgaAMhLUmHAIAYA1SUVBhQQXo4QBZC3pMKAQQ0AqksqDJgZqEZ4AiDCbgIAyJ4wAACZS2qZgNNZDgDgPEmFAYMeAFSXVBhQQHg6IQmA8yQVBgx6AFBdUmHAzMBJAhIAF7GbIGGCAADDEAYAIHPCAABkLqmaAdPiAFBdUmFAAeF/EYwAGJZlAgDInDAAAJkTBhJkiQCAKoQBAMhcUgWEuTMjAMBlJBUGDIYAUF1SYSD3rYXCEACXkVQYyJEAAMBVKSAEgMwJA3PMrAAAoyAMAEDmhIE5ZVYAgFFJqoDQAAkA1SUVBnLZWij0ADBKlgkAIHPCAABkThiYM5YIABg1YQAAMicMAEDmktpNYAodAKpLKgykvrVQ2AFgHCwTAEDmhAEAyJwwAACZEwYAIHPCwJxQPAjAuCS1m8CACQDVJRUGUt5aKOgAMC6WCQAgc8IAAGROGJgDlggAGCdhAAAyJwzMOLMCAIxbUrsJDJwAUF1SYSDFrYUCDgDjZpkAADInDABA5oSBGWaJAIBJSKpmwOAJANUlFQZSKyAUbgCYhKTCgMETAKpLKgykNDMg2AAwKQoIASBzwgAAZE4YAIDMJVUzYJ0dAKpLKgwoIASA6pIKAwZQAKguqTCQysyAUAPAJCkgBIDMCQMAkDlhAAAyl1TNgLV2AKguqTCggBAAqksqDBhEAaC6pMKAmQEAqE4B4YwRBACYNGEAADInDABA5pKqGTDFDgDVJRUG5rmAUJABYFosEwBA5oQBAMicMDADLBEAME3CAABkThgAgMwltZvAdDsAVJdUGJjXrYVCDADTZJkAADInDABA5oQBAMicMAAAmRMGpkzxIADTltRuAgMrAFSXVBiYx62FAgwA02aZAAAyJwwAQOaEAQDInDAAAJkTBgAgc0ntJlCZDwDVJRUGbC0EgOosE0yRIADALBAGACBzwgAAZE4YAIDMCQMAkLmkdhMoyAOA6pIKA/O2tVB4AWAWWCYAgMwJAwCQOWEAADInDABA5oQBAMhcUrsJVOcDQHVJhQFbCwGgOssEUyIIADArhAEAyJwwAACZS6pmwNQ7AFSXVBiYpwJCwQWAWZFUGDDAAkB1SYUBMwMAUJ0CwikQBACYJcIAAGROGACAzCVVM2D6HQCqSyoMzEsBodACwCxJKgwYZAGguqTCwDzMDAgsAMwaBYQAkDlhAAAyJwxMkCUCAGZRUjUDBlsAqC6pMDDrBYTCCgCzKKkwYLAFgOqSCgNmBgCgOgWEAJA5YQAAMicMAEDmkqoZsCYPANUlFQZmuYBQUAFgVlkmAIDMCQMAkDlhAAAyJwwAQOaEAQDIXFK7CVTsA0B1SYUBWwsBoDrLBACQOWEAADInDEyAJQIAZpkwAACZEwYAIHNJ7SYwHQ8A1SUVBmZ1a6GQAsAss0wwZoIAALNOGACAzAkDAJA5YQAAMicMAEDmktpNoFgPAKpLKgzM4tZCAQWAWWeZYIwEAQDmgTAAAJkTBgAgc8IAAGROGACAzCW1m0DBHgBUl1QYmLWthcIJAPPAMsGYCAIAzAthAAAyJwwAQOaEAQDInDAAAJlLajeBoj0AqC6pMDBLWwsFEwDmhWUCAMicMAAAmRMGACBzSdUMWKcHgOqSCgMKCAGguqTCgAEYAKpLKgzMysyAUALAPFFACACZEwYAIHPCAABkLqmaAWv1AFBdUmFAASEAVJdUGDAIA0B1SYUBMwMAUJ0CQgDInDAAAJkTBgAgc0nVDFirB4DqkgoDCggBoLqkwoBBGACqSyoMmBkAgOoUEI6YIADAvBEGACBzwgAAZC6pmgFT9ABQXVJhYBYKCAUSAOaNZQIAyJwwAACZEwYAIHPCAABkThgYIcWDAMyjpHYTGIwBoLqkwsC0txYKIwDMI8sEAJA5YQAAMicMAEDmhAEAyJwwAACZS2o3gWp+AKguqTBgayEAVGeZYEQEAQDmlTAAAJkTBgAgc8IAAGROGACAzCW1m0ARHwBUl1QYmObWQkEEgHllmWAEBAEA5pkwAACZEwYAIHPCAABkThi4IvUCAMy7pHYTGJgBoLqkwsA0thYKIADMO8sEAJA5YQAAMicMAEDmkqoZsH4PANUlFQYmXUAofACQgqTCgMEZAKpLKgxMamZA6AAgJUmFgXETAgBIkd0EAJA5YWBIZgUASFVSywQGbACoLqkwMOoCQuECgBwkFQYM3gBQXVJhYJQzA4IFALlIKgzMwgD+9bd/Hfz7z3/6wxRbAgDDsZsAADK3UJZledFBb968iY8//jh+/PHHuHXr1iTaNfcOzxAcZ8YAgEkYdvxOaplglhjwAZgXwsCYHZ8hEBIAmDXCwJgZ/AGYdcLAmNldAMCss5sAADInDABA5oSBMTu8NHDedkMAmBY1AxOgVgCAWWZmAAAyJwwAQOYsE0yB7YYAzBIzA1OgqBCAWSIMAEDmLBPMAN9fAMA0CQNTYsAHYFZYJpiir7/9q5oBAKZuoSzL8qKD3rx5Ex9//HH8+OOPcevWrUm0KxtnhQEzBwBc1bDjt5kBAMicmYEZMMxSgZkCAKoyMwAADEUYAIDM2Vo4AywBADBNwsAMsKMAgGkSBmbAn//0h/j6278a/AGYCjUDAJA5MwMzoj87cNrtADBOZgYAIHNmBmaUGQEAJkUYmCECAADTMNIw8Msvv8TPP/88yoecuhs3bsT169cn8lz9mgGhAIBJGsl3E5RlGX//+9+j1+uNo41TVxRFfPbZZ7GwsDD25zpcRCgUAHAVw343wUhmBvpB4NNPP42bN29OZNCchLIs4927d/Hy5cuIiPj888+n3CIAGL0rh4FffvllEARu3749ijbNlA8//DAiIl6+fBmffvrpxJYMIsKFiACYiCtvLezXCNy8efPKjZlV/XNLrR4CACJGeJ2BVJYGTjPJc+vPBPz5T38wKwDARIx1a+E///a3+PXgYJxPMXBtcTF+87vfTeS5xk0IAGCSxhYG/vm3v8X/++//I+Knn8b1FEd98EH8t//z70MHgm63G61WK2q1WnS73VhbW4uiKMbbxorUDAAwCWMLA78eHEwuCERE/PTT++ccMgysrKzE3t5eRLwPBqurq7GzszPOFl7KWV9vfJjAAMBVZPndBN1u98jPtVot2u32lFoDANOV5eWI2+12LC4uHrltcXExOp1OLC0tTalVJ/U/8V80O3De780aAHCRLGcGzrpS4sGEih0BYJZkOTNwllm9nPLhT/cXzRKYCQCgqizDQFEUJ2YBDg4OZm43wWkM9gCMWpbLBPV6/dTb79+/P+GWXN7X3/51qJ0GAHCRLMNArVY78nO324379+/PxcxAX/8KhQIBAFc1tmWCa4uLER98MNGLDl07tkPgPDs7O9FsNuPBgwfx/PnzmbzGAABMwkJZluVFB533fcj/+Mc/4rvvvovf//738dvf/vbI71K5HPF55zhtZ80MqC0A4Lzx+7CxFhD+5ne/G/qKgFyOQR+Aq8qyZiAlCgkBuKostxamxMwAAFclDCTCJYkBuCxhIBHHB/zD4eC0oCAgANCnZgAAMmdmIFEXXZDIlkQA+sYaBv7e+4/ovft5nE8xUNy8EZ8VH07kueaFgR2AYYwtDPy99x/x1f/+v/Gf//x1XE9xxL/85lo8+7d/HToQdDqdWF1djb29vTG3bLq+/vavQgEA5xpbzUDv3c8TCwIREf/5z1+HnoVotVoR8T4QpE4QAOAiWdYMNBqNaTdhYqpckEhwAMhTlmEgJ7M0wOdwpcRZ6m+AYdlaCACZMzPAxFz2U3PVGQWfzgGqEQaYeQZ3gPESBmBMJlEjISgBo5B9GOj1elEUxbSbQYIM1MC8yLKAsN1uR7PZjIiIx48fD647AAA5GtvMQHHzRvzLb65N9AqExc0bQx1br9ejXq/H5ubmmFsFALNvbGHgs+LDePZv/+q7CQBgxo21ZuCz4kMDNADMuCxrBgCA/yIMAEDmRhYGyrIc1UPNnJTPDQCuHAZu3Hhfwf/u3bsrN2ZW9c+tf64AkJIrFxBev349iqKIly9fRkTEzZs3Y2Fh4coNmwVlWca7d+/i5cuXURRFXL9+fdpNAoCRG8lugs8++ywiYhAIUlMUxeAcASA1IwkDCwsL8fnnn8enn34aP/88mesKTMqNGzfMCACQtJFeZ+D69esGTgCYM7YWAkDmhAEAyJwwAACZG6pmoH/RnTdv3oy1MQDA6PTH7YsunjdUGHj79m1ERNy5c+eKzQIAJu3t27fx8ccfn/n7hXKIa+3++uuv8cMPP8RHH300sxcUevPmTdy5cye+//77uHXr1rSbM/P0V3X6rBr9VY3+qkZ/Dacsy3j79m188cUXce3a2ZUBQ80MXLt2Lb788suRNW6cbt265YVRgf6qTp9Vo7+q0V/V6K+LnTcj0KeAEAAyJwwAQOaSCQMffPBB/OUvf4kPPvhg2k2ZC/qrOn1Wjf6qRn9Vo79Ga6gCQgAgXcnMDAAAlyMMAEDmhAEAyNxIv8J43LrdbrRarajVatHtdmNtbS2KorjysSmr2g+dTidWV1djb29vco2cIVX6q9PpRLvdjoiI58+fx9OnT7N7jVXpr35f9Xq9eP78eTx8+DCWlpYm2Nrpu+z7UrPZjEePHnl9XfDfY0TE0tJSdLvd6PV62b2+rqScI0tLS4N/7+/vl41GYyTHpqxKP+zs7JR7e3vlnL0sRqpKf21ubh759+H75qJKfxVFUe7t7ZVlWZZbW1tlrVYbe/tmzWXel/r/Tb5+/XqMLZtNVfprbW2tjIgyIsp6vZ5lf13F3CwTdLvdIz/XarXBJ42rHJuyqv3QaDSyTtJV+qvT6cTjx48HPzcajeh0OiceI2VVX187OztHXl85fso9bNj3pW63G7VabVzNmllV++vevXvx+vXreP36dezu7mb3+rqquQkD7XY7FhcXj9y2uLg4mBq67LEp0w/VVOmvpaWlePr06eDnXq83OD4XVV9f9Xp98O+dnZ1YX18fa/tmzWX+e2y1WtFoNMbdtJl0mf4qikIIuKS5qRnov9ked3BwcKVjU6YfqqnaX4ffpL/55puo1+tZvRFd5vXV6XTim2++ieXl5VhbWxtTy2ZT1f7q9XpZvZ6Ou0x/tVqtiHhfw7O+vp7ljMplzU0YOMtZL5irHpsy/VDNRf3VfxPKtejyuPP6a2lpKWq1WjSbzaw/9R52Vn89e/Ysu8A0jLP663BxYa1Wi+Xl5djf359cw+bc3CwTFEVxIhEeHBycmpyrHJsy/VDNZfur2WxmuUZ52f4qiiJWVlZiZWUlq2Bapb/a7XZ89dVXE2rZbKr6+jpcY9DffZBTDc9VzU0YOLzeeNj9+/evdGzK9EM1l+mvJ0+eRLPZjFqtFr1eL6vBrUp/tdvt+OSTTwY/96dvc3qzrvr6evbsWWxvb8f29nZ0u914/PhxVvU+Vfqr0+nEH//4xxO351TDc1Vzs0xwfO2n2+3G/fv3Bymx0+lEURRRq9UuPDYXVfrsuBzXK6v2V6vVGkx793q97KZ1q/TX4uLikTf3/u9y2r1Spb+OD4Tr6+vZrYFXfc/f3NwcHNtut6PRaGT3HnYVc/VFRd1uN7a2tuLBgwfx/PnzIxfhWFlZiQcPHsTGxsaFx+akSp+12+3Y3d2NJ0+exMbGRjx48CC7Nd1h+6vb7cbdu3eP3Lcoinj9+vUUWj09VV5frVZrMO27u7sbm5ubWQ1uEdX6K+J9KN/e3o5msxlra2uxvr6eVYCq0l/9i4AVRRH7+/tHwgEXm6swAACM3tzUDAAA4yEMAEDmhAEAyJwwAACZEwYAIHPCAABkThgAgMwJAwCQOWEAADInDABA5oQBAMjc/wenUiblXLO4eQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkYAAAHMCAYAAAAwHmdPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABhjElEQVR4nO3deXwTdf4/8NckTdOLNk3lEEUlXXVFPEhbBQ9WIPVaFZAWRHtBaaugCAgNeKwLHgVE19+6ii3rurp40IbiXvpdWvHW1bYB7wMa7pumadrSpE0yvz/aZilNS9JrkvT1fDz60GQmM+9MS/Pqez7zGUEURRFEREREBJnUBRARERH5CwYjIiIiojYMRkRERERtGIyIiIiI2jAYEREREbVhMCIiIiJqw2BERERE1IbBiIiIiKgNgxERERFRGwYjIj+VmpqKhIQECIIAQRCQmpra4Ss5ORl5eXmwWCyS1WixWBAfH4+1a9dKVkMgKS8vR0JCAmJjY5GamtphGY8lkX8IkboAIvKspKQEACAIArRarfvxqfLy8hAbG4uysjLodLqBLhFmsxkmkwkVFRU93oZer8eaNWv6sCr/pdPpUFVVhYSEhE7L+uJYElHvMRgRBbDCwkKUl5cjNTUVu3fvhkqlGtD9azQa9PZ2iyaTqY+qCRxqtbrTc31xLImo93gqjSjA6XQ6WCwWlJeXS12KzwwGg6SnAomITsdgRESSMJlMyMnJkbqMQWUwdueIfMVTaUQBrr1TdOoYI4vFAr1ej/j4eNTU1MBkMmHFihXQarXu1+j1ephMJuTm5mLWrFkoLy9HWVkZ8vLykJKSApPJhMLCQsTHx7u7OtXV1cjLy4NWq4XJZEJeXh4qKyuh0WhQVVXl3v+ZXmswGLBp0yYAQGVlpXsgskaj6TDe6Ezvw2g0IicnByaTCTqdDhs2bEBRURFUKhXKyso6be9URUVFqKqqcp9+TE5O7jRO60z798batWtRXV2N+Ph4qFQqaDSaTut0dyxPPZ5xcXGoqakBgC7fl8lkgl6vR1JSUof9qtVqbNq0CbNmzYJKpTrj99/b/Z7pe6BWq1FYWAiLxYKioiIAQEVFRbffGyJJiUTk1wCIWq3W47I1a9aIAMSysjL3c9XV1aJKpRKrqqq6fU4URVGj0Yi5ubnimjVr3I9TUlJEURQ97lOn03Xahk6n67SuL6/V6XQe35sv76N9O+3vo51Kper0nCiKYkpKipibm9vhufz8/B4fx65otVoxPz+/w3NlZWWiSqVyH+fT34enY+epVk/r1dbWiiqVqsP70Ol07tdXVVWJtbW17mXdff992W/7frr6HuTn53v9vSGSGoMRkZ8D4P5wOfWr/cO9urq6w/o6nc7jh25KSkqnEKLT6USVSuX+sGz/b1VVlajRaDpto6SkpFMoyM3N7fBh6ctruwtGvryPlJQUEYDHY3H6B3lhYaEIoENAEMXOocCX/XuSn58vqlQqj8u0Wq3HbZ9+LEWxNUidHn5ra2tFAGJJSUmHdduD8qm6er+i2PX339f9imL33wNvvzdE/oBjjIgCQPtph1O/SkpKUFhY2OHUTPsg7KSkpE7bSE5ORmVlpcdtt59Oav+vRqOByWRCcnJyh0HdKSkpZzyN1JvX9vR9qNVqj6epPJ220uv1SElJ6XQFn1arxaxZs3q0f0+Kioowc+ZMj8s8XZXWFY1GA51O1+G9tNd++pih6urqLo9BV+OLPH3/fd0v0P33oKvnOfCe/BHHGBEFkfYP7Orqavd4jlN5GtPh6UNapVKhpKQEOTk5SE5OBtAaHNasWXPG+ZJ689q+fh+ns1gssFgsHgPTqfNE9WT/pzKZTO4JG3tLo9GgrKwMQGv9JpPJHUzax/20S0hI6FRve/joKpR2ddx82e+ZtuVLECSSGoMRURBp/wBKTk52D6A9k67mPkpJSUFKSop7UK7BYEBycrJXk0n29LUWi8U9ULiv3sepzGYzACAuLq7b9Xqy//5kMBhQWFjo7mp56ngBwMyZM6HX61FeXu4+zps2bUJ+fn6X2+7uuHm73zNta6Dn1yLqDQYjoiDS3hXo7WXZp17p1v61Zs0apKamoqSkpNtw05vXtl+V1Vfv43TtnaLq6upu1+vt/r3djzeKioqg1+tRVVXlsdN1KpVKhZkzZ6KkpMTd7UlKSuo2GPXFfomCCccYEQWZ/Px8FBYWelyWl5fn9Xa6ugVJe9elL157eifBYrG4uzV99T5O197J8uTU01C93X9+fn6X+zGbzV6Pr9Hr9Zg5c2ancHLq69vvr1ZeXo7k5GQUFha6x6L1JBT5ul+iYMJgRBQAfBmkumbNGqhUqk4fWgaDodONS7vbdlFRUaeOSVlZmXuAcnfb8Pa1pw9kNplM7rDky/voKmh4en7Dhg0AWj/4T2U0GjuMhfH1OJ6u/fUGg6HD8+3HpquAeXq9arW607E0GAzuGc9PpdFoUFBQgPLychiNxg7jgrrS1fffl/0Cvn0P2p8n8keCKPLmPET+qH3CP6PRCKD11FT7IGZvtH/wt4+naX890BoCCgoK3B/aKSkpHU65tH+wqlSqDh9qGo2mw+R/7eNZLBYLUlJSsGLFCpjN5jO+9vQ6jUYjkpOTodVqO51q6+59eKohLy8PGo2mw/M6na7DxIXt220fIN1+1ZSn03zd7d8ber3e/dqamhrMmjULer3efdpww4YN7gkXTz+WWq0WFosFOTk5sFgs7sHs7e+jfcLMFStWdJio0lOnKiUlxb2vM33/AXi9X7PZ7NX3wJfvDZGUGIyIiIJEXl4e4uPjOwWcyspKd8A7fVZtIuqIwYiIKAi03+alq+BjsVgQGxuL2tpaXiVG1A2OMSIiGgQ8TeJIRJ0xGBERBQGdTge1Wt3llWKpqam8aSuRF3gqjYgoiBgMBpSVlXWYdbu6uhqpqalezzxONJgxGBERERG14ak0IiIioja8JYgPXC4XDh06hCFDhkAQBKnLISIiIi+Iooj6+nqMHDkSMln3PSEGIx8cOnQIo0aNkroMIiIi6oH9+/fj3HPP7XYdBiMfDBkyBEDrgY2Ojpa4GiIiIvKG1WrFqFGj3J/j3WEw8kH76bPo6GgGIyIiogDjzTAYDr4mIiIiasNgRERERNSGwYiIiIioDYMRERERURsGIyIiIqI2DEZEREREbRiMiIiIiNpwHqMgI7a0oLmqCvZPP4Pz0GEIKhWUE8ZDOWE8ZFFRUpdHRETk1wK6Y7R27VqpS/Arot2O+pfWw7ruWdg//xzOo0fR8s03aHhpPayr18B54oTUJRIREfm1gA5GNTU13S43mUwoKiqC0WjstMxoNKKoqAgmk6m/yhtwTe+9B/uHH0I2bDhCfnUh5OecgxCNBvLzz0fzt9+i8fW/QRRFqcskIiLyWwEdjLpjMBhQXl4OtVqNnJwcpKamupfp9XqYzWbMnDkTJpPJY3AKNKLNBlv5NgjhEZ1OmQmhoZAPH4Hm7dvh3L9fogqJiIj8X1CPMcrNzQUApKSkICEhAXl5eVCpVMjLy4NGowEA6HQ6GAwGaLXaTq+32+2w2+3ux1arFUDrTWS9uRHdQHLs3w/rvr0QYtWQtdV5KlEQ4Dh6FOYvv0KYBPURERFJpb6+3ut1AyYYWSwWFBQUdHiuvLwcer3e/TguLg75+fkwGo3Q6XQd1n3//fcxevRo5ObmukNRO5VK5XGfBQUFWLlyZafn161bh9DQ0B6+k/7hqq9H8w8/AMpQCCEKD2uIcNVZEbpxI+TlZQNeHxERkVSam5u9XlcQA3jQiV6vx5o1azo93z5u6PQAlJycjPLyclRXV3dYZjAYkJKS0mk7njpGo0aNwnfffed3HSOxpQXWNWvhOHQYIeee22m502yG4HIiesVyyIcOlaBCIiIiadTX12Ps2LGoq6tDdHR0t+sGTMfIFxqNBgaDoUP40ev1KCkpQVFREZKTk1FVVeXuFJnNZo/bUSqVUCqVnZ4fNWrUGQ+sFJpSUtBQWAShpQUytRqCIAAAXA0NcNptCP/tbzEkIUHiKomIiAaW1cMQk64EZTACAK1Wi6KiIlgsFtTU1LjHF+Xn50Oj0WDKlCnQaDTQaDQeu06BKEw3Bc7jx2B79//g/OVnQBEKOFqA0FCETZyIyHvulrpEIiIivxaUp9L6i9VqRUxMjFetOKmIoghHdTXsX34J17HjkEVHIzRBC8XYsRBCgjYHExERdcmXz++ADkYWi6XLgdP9IRCCEREREXXky+d3QM9jNJChiIiIiIJfQAcjIiIior7EYERERETUhsGIiIiIqA2DEREREVEbBiMiIiKiNgxGRERERG0YjIiIiIjaMBgRERFR0HI6nfjDH/7g9fq8RwQREREFpV27diEjIwNffPGF169hx4iIiIiCiiiKWL9+Pa644gocOHAAL7/8stevZceIiIiIgsbBgwcxd+5cbN26FTNnzsRDDz0El8vl9esZjIiIiCjgiaKIN998E/fffz9CQ0Oxfv16XHfddQCAhoYGr7fDU2lEREQU0E6cOIHU1FSkpaVhwoQJKC0tdYciX7FjRERERAHrX//6F7Kzs9Hc3IxnnnkGN998c6+2x44RERERBRyr1Yrs7Gzcfvvt+PWvf43S0tJehyKAHSMiIiIKMB999BEyMjJQU1OD3//+97jzzjshCEKfbJsdIyIiIgoITU1NWLx4MSZNmoRhw4bBYDBgxowZfRaKAHaMgpIoinDu3QtXbS2EiAiExMdDCOG3moiIAldlZSXS09NhMpnw0EMPIT09HTJZ3/d3+GkZZFp2VePkpk1o+eEHiE02CKEKyC+4ABHTp0F59dVSl0dEROSTlpYWPPnkk3jqqadw8cUXo7i4GPHx8f22PwajIOIw7Ub9s8/CeegwZCNGQDbibIh2Oxw7d6H+Ty8CDieU114jdZlERERe+fHHH5GWloavv/4aOTk5yM3NhUKh6Nd9coxREDn5j3/AcfAQ5BdeCFlMDAS5HLK2U2mizY7GkhKIdrvUZRIREXXL5XLhD3/4A8aNGweLxYKNGzdiwYIF/R6KAAajoOE8fhwt23dAPmwYBA/nXOUjR8K5/wBavv9BguqIiIi8s2fPHkyaNAlLlixBamoqNm3ahLFjxw7Y/nkqLUiI9Q0Q7XbIoqM9LheUSsDlhKveOsCVERERnZkoinj11VexaNEiREVF4ZVXXsFVV1014HWwYxQkhOghEJRKiE1NHpeLdjsgl3cZnIiIiKRy5MgR3H777cjOzsaUKVNQWloqSSgC2DEKGvKzzoJi3JWwffABBJWq0+k056FDCBk1CooxYySqkIiIqDODwYB7770XAPDHP/4RkyZNkrQedoyCSMS0qQgZORLOnTvhqquD6HTC1dgIR3U1hDAlIlJTW0+pERERSay2thb33HMPUlNTMW7cOJSWlkoeigB2jIJKyAUXYMjSpThZXIyW73+A68RxCIpQKC66EOF3TocyKUnqEomIiLB161bMmTMH9fX1ePrpp3Hbbbf16ezVvcFgFGQU8RpEL9fDuW9f68zXkZEI0WggyOVSl0ZERINcY2Mjli5dipdffhkTJkzAqlWrMGLECKnL6oDBKAgJgoCQ888Hzj9f6lKIiIgAAJ9//jkyMjJw8OBBPPLII5g1a5bfdIlOxTFGRERE1G/sdjuWL1+O66+/HlFRUTAYDLjrrrv8MhQB7BgRERFRP/n666+Rnp6On376CQ888ADmzJkDuZ8P7WDHiIiIiPqUw+HA008/jaSkJNjtdrz11luYN2+e34cigB0jIiIi6kM7d+5ERkYGvvrqK8yZMwfz589HaGio1GV5jcGIiIiIek0URaxfvx7Lli1DXFwcXnvtNVx55ZVSl+UzBiMiIiLqlQMHDmDOnDkoLy/HrFmzsGTJEkREREhdVo8wGBEREVGPiKKIN954A/fffz+USiVefvllXHvttVKX1SscfE1EREQ+O378OFJSUpCeno7rrrsOpaWlAR+KAHaMiIiIyEf/+Mc/kJOTg+bmZjz77LO48cYbpS6pz7BjRERERF6xWq2YM2cOpk6diksuuQRbtmwJqlAEsGNEREREXvjggw+QlZWFmpoarFq1CtOmTfPb2at7gx0jIiIi6lJTUxMWLVqEyZMnY9iwYdi8eTOmT58elKEI8MOOkclkgsFggEajgclkQm5uLlQqlcd1jUYjAECr1cJkMsFisUCr1Z5xO77sg4iIaLCqqKhAeno6du/ejWXLliEtLQ0yWXD3VPwuGKWmpqKqqgpAa4DJyclBSUmJx3ULCwtRVFQEANDpdB3W6247vuyDiIhosGlpacETTzyBp59+GhdffDGKi4sRHx8vdVkDwq+Ckclk6vBYo9GgvLy8y/UTEhJQW1sLAB06Pt1tx9d9EBERDSY//PAD0tLS8M033yAvLw/z5s2DQqGQuqwB41f9sPLycqjV6g7PqdVq9ykzT1QqVafTYN1tpyf7ICIiCnZOpxPPPvsstFot6urq8MYbb+C+++4bVKEI8LOOkcVi8fi82Wzucn2DwQCg9TxoXl4eNBpNt9vxZR92ux12u9392Gq1dl08ERFRgNq9ezcyMzPx6aefIi0tDQsXLkRYWJjUZUnCr4JRV7oKM6cOmtZoNEhOTkZ1dbXP2+lqWUFBAVauXOlDpURERIFDFEW88sorWLx4MaKjo/HKK68gKSlJ6rIk5Ven0lQqVafOjdls7vKKsVPHC7VfYWYymbrdji/7WLFiBerq6txf+/fv79kbIyIi8jOHDx/GbbfdhpycHNx4443YvHnzoA9FgJ8FI51O5/H5xMTETs8ZjUZMmTKl0/Nqtbrb7fiyD6VSiejo6A5fREREga6kpARjx47FV199hRdeeAErV65EVFSU1GX5Bb8KRhqNpsNjk8mExMREdzfHaDS6u0QajQZr1qxxr1teXo6UlBSoVKput3OmfRAREQUrs9mM2bNnY+bMmUhISEBpaSluuOEGqcvyK343xqikpAR6vR5JSUmoqKjoML9QQUEBkpKSkJ+fD5VKhcTERKxduxYqlQrV1dUd1u1uO90tIyIiCkb/+c9/MGfOHDQ2NmL16tW49dZbg3b26t4QRFEUpS4iUFitVsTExKCuro6n1YiIKCA0NDRg6dKlKCwsxDXXXINVq1Zh+PDhUpc1oBoaGjBhwgSvPr/9rmNEREREfeOzzz5Deno6jhw5gkcffRQzZ85kl+gM/GqMEREREfWe3W5Hfn4+rr/+esTExMBgMGDWrFkMRV5gx4iIiCiI7NixA2lpafjll1/w4IMPIisrC3K5XOqyAgY7RkREREHA4XDgqaeewlVXXYWWlha89dZbyM7OZijyETtGREREAe6XX35BRkYGKioqkJ2dPSjvcdZXGIyIiIgClMvlwksvvYT8/HwMHToUr732Gq688kqpywpoDEZEREQBaP/+/ZgzZw7ef/993HXXXVi8eDEiIiKkLivgMRgREREFEFEUsXHjRjzwwAMICwtzz09EfYODr4mIiALE8ePHceeddyIjIwPXX389Nm/ezFDUx9gxIiIiCgB///vfkZOTg5aWFjz33HNITk6WuqSgxI4RERGRH6urq0NWVhamTZuGSy+9FFu2bGEo6kfsGBEREfmpbdu2ISsrC2azGU888QSmTp3K2av7GTtGREREfubkyZNYuHAhpkyZghEjRqC0tBTTpk1jKBoA7BgRERH5ka+++grp6enYu3cv9Ho97r77bshk7GMMFB5pIiIiP9Dc3IzHHnsM11xzDRQKBYqLi5GWlsZQNMDYMSIiIpLY999/j7S0NHz33XfIy8vDvHnzeEsPiTCGEhERScTpdGLdunVISEhAfX09Nm7cyPucSYwdIyIiIgmYTCZkZmbis88+Q0ZGBh544AEolUqpyxr0GIyIiIgGkCiK2LBhA5YsWQKVSoW//OUvSExMlLosasNgRERENEAOHz6MuXPn4v/+7/8wY8YMLFu2DJGRkVKXRadgMCIiIhoAmzZtwvz58yGXy/Hiiy9i4sSJUpdEHnDwNRERUT8ym8246667cNdddyExMRGlpaUMRX6MHSMiIqJ+8t5772Hu3LloamrCmjVrcMstt3D2aj/HjhEREVEfa2hoQG5uLm699VbEx8dj8+bNuPXWWxmKAgA7RkRERH3o008/RUZGBo4ePYrHHnsMqampDEQBhB0jIiKiPmCz2bBs2TJMnDgRKpUKBoMBM2fOZCgKMOwYERER9dL27duRlpaGXbt2YdGiRcjMzIRcLpe6LOoBdoyIiIh6yOFw4IknnsBVV10Fp9OJt956C3PnzmUoCmDsGBEREfXAzz//jPT0dFRVVSE7O5v3OAsS7BgRERH5wOVy4YUXXsC4ceNw7NgxvP7661i4cCFDUZBgx4iIiMhL+/btQ1ZWFj744APMnj0bixcvRnh4uNRlUR9iMCIiIjoDURTdnaGIiAgUFRVhwoQJUpdF/YCn0oiIiLpx7NgxTJ8+HVlZWbjhhhuwefNmhqIgxo4RERFRF7Zs2YLc3Fw4nU48//zzmDJlitQlUT9jx4iIiOg0FosFGRkZuPPOO3HZZZehtLSUoWiQYMeIiIjoFOXl5cjKykJdXR2efPJJ3HHHHZy9ehBhx4iIiAjAyZMn8cADDyA5ORnnnnsuSktLMXXqVIaiQYYdIyIiGvS+/PJLpKenY9++fVi+fDlmz54NmYy9g8GI33UiIhq0mpub8cgjj+Caa66BUqlEcXEx7rnnHoaiQYwdIyIiGpS+++47pKWl4fvvv8f8+fORnZ2NkBB+LA52jMRERDSoOJ1OrF27FgkJCWhsbMQbb7yBvLw8hiICwI4RERENItXV1cjMzMTnn3+OzMxM3H///VAqlVKXRX6EwYiIiIKeKIooKirCQw89hNjYWLz66qtISEiQuizyQwxGREQU1A4dOoS5c+fiP//5D1JSUrB06VJERkZKXRb5Kb8LRiaTCQaDARqNBiaTCbm5uVCpVGd8nV6vx4oVK9zrdrednu6DiIgCy9tvv4358+cjJCQEL774IiZOnCh1SeTnBFEURamLOFVCQgKqqqoAtAYYvV6PkpKSbl9jNBqRkJCA2tpad8Dpbjs92QcAWK1WxMTEoK6uDtHR0T19i0RE1M9qampw3333oaSkBDfffDMeeeQR/gE8iDU0NGDChAlefX77VcfIZDJ1eKzRaFBeXu7V6zQajVfb6ek+iIgoMLz77ruYO3cubDYb1q5di1tuuUXqkiiA+NXl+uXl5VCr1R2eU6vVMBqNXb7GYDAgJSXF6+34sg+73Q6r1drhi4iI/FN9fT1ycnLw29/+FhdeeCFKS0sZishnftUxslgsHp83m81dru+pNdrddnzZR0FBAVauXOlxfSIi8h8ff/wxMjMzcezYMfzud79DSkoK73FGPeJXHaOudBVmiouLodPper2drpatWLECdXV17q/9+/d7vS8iIup/NpsNDz30EG644Qao1WoYDAakpqYyFFGP+VXHSKVSdercmM1mj12h8vJyzJw50+ft+LIPpVIZsBN/uerq4LJYIERGQn7WWVKXQ0TU54xGI9LS0lBdXY0lS5YgPT0dcrlc6rIowPlVMNLpdCgsLOz0fGJiosf1i4uL3f9vMplQUFCAWbNmdbsdjUbj0z4CjfPoUZz8xz/R/N8vITadhKBQQHH55Qi/7bdQXHyx1OUREfWaw+HA008/jSeeeAIXXngh3n77bVx44YVSl0VBwq+C0alXlgGtYScxMdHdzTEajVCpVNBoNJ1OoeXl5SEvL6/TNk7fzumdodP3EcicR47A+sw6tOzcCZk6DoI6DrDZYP/4E7T8+COGLHoQoWPHSl0mEVGP/fTTT0hPT8f27duRnZ2Ne++9FwqFQuqyKIj4VTACgJKSEuj1eiQlJaGioqLD/EIFBQVISkpCfn6++zmLxYKioiIAwJo1a5CXlwetVtvtdrpbFshO/uOfaNm5EyG/uhBC+80QIyIgxMbCWb0LjRvfgOLJJ/63jIgoQLhcLrzwwgtYvnw5RowYgddffx2XX3651GVREPK7CR79mT9P8OiqrUXtkqUQAciHDu28/ORJuI4dQ8zvHmXXiIgCyt69e5GVlYUPP/wQ99xzDx588EGEh4dLXRYFkICd4JF6zlVrgdjUBJmHUAQAsogIuFqa4aqpGeDKiIh6RhRFvPbaa1i4cCGioqKwYcMGjB8/XuqyKMgFxOX6dGZCeBgQqoBot3tcLra0AIIMQkTEAFdGROS7o0ePYurUqZgzZw4mTZqEzZs3MxTRgGDHKEjIRoyA4pJL0PzllxBiYjrN4eE8cgTyESOguPRSiSokIvJOaWkp8vLy4HK58Pzzz2PKlClSl0SDCDtGQUIQBETccQdksbFwmkwQbTYArZ0ix4EDgNOJ8Km3Q8aOERH5KYvFgvT0dMyYMQNXXHEFSktLGYpowLFjFEQUl47BkAceQOMbb8Kxdy/gdACCAPnw4QifegfCbrxR6hKJiDwqKyvDnDlzUFdXh6eeegq33347Z68mSTAYBZlQ7TgoLhuLlu++g6vWAiEiHIrLLoMsMlLq0oiIOmlsbER+fj5eeuklXH311Xj11Vdx9tlnS10WDWIMRkFIUCgQOm6c1GUQEXXriy++QHp6Og4ePIgVK1bgrrvugkzGER4kLf4EEhHRgGpubsbDDz+M6667DpGRkSgpKcHdd9/NUER+gR0jIiIaMN988w3S09Pxww8/YMGCBZg7dy5COBs/+RHGcyIi6ndOpxOrV69GUlISmpqa8OabbyI3N5ehiPwOfyKJiKhf7dq1C5mZmfjiiy+QlZWF+++/H6GhoVKXReQRgxEREfULURRRWFiIhx56CHFxcfjrX/8KrVYrdVlE3WIwIiKiPnfw4EHMnTsXW7duRWpqKpYuXYoITjBLAYDBiIiI+owoinjrrbewYMECKBQKvPTSS7j++uulLovIaxx8TUREfeLEiROYOXMm7rnnHkyYMAFbtmxhKKKAw44RERH12r/+9S9kZ2ejubkZzzzzDG6++WapSyLqEXaMiIiox6xWK7Kzs3H77bfj4osvRmlpKUMRBTR2jIiIqEc++ugjZGZm4sSJE3j88ccxY8YM3viVAh47RkRE5BObzYYlS5Zg0qRJiIuLg8FgQEpKCkMRBQV2jIiIyGuVlZVIT0+HyWTCQw89hLS0NMjlcqnLIuoz7BgREdEZtbS04Pe//z3Gjx8PmUyGTZs2ITMzk6GIgg47RkRE1K0ff/wR6enp2LFjB3JycpCbmwuFQiF1WUT9gh0jIiLyyOVy4fnnn4dWq4XZbMbGjRvdEzcSBSt2jIiIqJM9e/YgKysLH330EdLS0vDggw8iLCxM6rKI+h2DERERuYmiiFdffRWLFi1CVFQU/vznP+Pqq6+WuiyiAcNTaUREBAA4cuQI7rjjDmRnZ2PKlCnYvHkzQxENOuwYERERNm/ejNzcXAiCgD/+8Y+YNGmS1CURSaLfO0alpaX9vQsiIuqh2tpa3HPPPUhJSYFWq0VpaSlDEQ1q/R6MCgsL+3sXRETUA1u3bsXYsWPxz3/+E08//TSee+45qNVqqcsiklSvT6WtW7cOmzZt8rjMYrHAZDL1dhdERNSHGhsbsWzZMqxfvx7jx4/HE088gREjRkhdFpFf6FUwWr58OYqKipCYmAiNRtNpeU1NDcxmc292QUREfejzzz9HRkYGDh48iIcffhizZs2CTMbrcIja9SoYmUymMwafmTNn9mYXRETUB+x2Ox5//HE888wzuOyyy1BSUoILLrhA6rKI/E6vglFSUtIZ11mzZk1vdkFERL30zTffIC0tDT/99BMeeOABZGVlISSEFyUTedLv/dPdu3f39y6IiMgDp9OJgoICJCYmwm6346233sK8efMYioi60atglJubi3Xr1mHPnj1drsOr0oiIBt7OnTtx3XXX4dFHH0V6ejreeustXHzxxVKXReT3vP6z4aabbvI4nkgURej1eqhUKqjVaqhUKvcyXpVGRDSwRFHE+vXrsWzZMsTFxeGvf/0rxo0bJ3VZRAHD62BUUVHR5dVniYmJHl/Dq9KIiAbOgQMHMHfuXJSVlWHWrFlYsmQJIiIipC6LKKB4HYw0Gg22bt3q8w54VRoRUf8SRRFvvvkmFixYAKVSiZdffhnXXnut1GURBSSvxxht2LChRzvgVWlERP3n+PHjSElJQVpaGq699lqUlpYyFBH1gtcdI1/OUe/ZswdqtRrR0dEYPXp0jwojIqLu/eMf/8C8efPQ0tKCdevW4aabbpK6JKKA16ur0lasWNHpubq6OlRXV6OsrAzr1q3Dtm3berMLIiI6jdVqxdy5czF16lSMGTMGW7ZsYSgi6iO9msyiurq603MxMTGYMmWK+/G6deswefLk3uyGiIjafPjhh8jMzERNTQ1WrVqFadOmQRAEqcsiChq96hh584+xrKysN7sgIiIATU1NWLRoESZNmoRhw4bBYDBg+vTpDEVEfcynjtHy5cthMplQV1cHAKisrOy2fVtZWYnc3FyfCjKZTDAYDNBoNDCZTMjNze0wN9KpysvLAbTOl1RRUYFZs2ZBq9WecTu+7IOISGqVlZVIS0vD7t27sWzZMqSlpfHGr0T9xKdgtHr1agCAwWBAbm4uBEGAKIoe11WpVFi9ejVycnJ8Kig1NRVVVVUAWgNMTk4OSkpKulz3/fffh06ng9lsRmpqqvv0Xnfb8WUfRERSaWlpwZNPPomnnnoKF198MYqLixEfHy91WURBrUdjjFJSUqDVarF8+XIUFxf3WTGnz5Kt0WjcXSFPSkpK3B0iAB06Ql1tx9d9EBFJ4YcffkB6ejq+/vpr5ObmIicnBwqFQuqyiIJej3uxGo0Gs2bN6staUF5eDrVa3eE5tVoNo9HocX2dTuf+/5KSEuTl5Z1xO77ug4hoILlcLjz33HPQarWwWCx44403MH/+fIYiogHSq6vSZsyY0eWyZ555BoIgIDc3F9HR0V5tz2KxeHy+u9uKGI1GbNq0CcnJye7xTN1tx5d92O122O1292Or1dplHUREvbVnzx5kZmbik08+QVpaGhYuXIiwsDCpyyIaVPpt9N6yZcuwdOlSFBQU9HpbXYUZANBqtVixYgWqq6thMBh6vB1PywoKChATE+P+GjVqlJcVExF5TxRFvPLKK7jsssuwa9cuvPLKK8jPz2coIpJArzpG7UpLS2EymVBTU9PheYvF0mlMT3dUKlWnzo3ZbD7jFWMqlQqpqalITk5GbW1tt9vxZR8rVqzAkiVL3I+tVivDERH1qSNHjiA7Oxvvvvsupk+fjvz8fERFRUldFtGg1etglJiY2GF8TnvAsFgsSE5Oxssvv+z1tnQ6HQoLCz3u43Tl5eVITU1FbW0tgNYxT0Dr4OrutqPRaLzeh1KphFKp9Lp+IiJflJSU4N5774VMJsMLL7yAG264QeqSiAa9Xp1KW758OXQ6HWpra+FyuVBSUgKz2Qyz2QyXy+W+pN9b7eGmnclkQmJiojtsGY1GdwdKrVZ3GHxtNBqhUqmg1Wq73c6Z9kFE1N9qa2tx9913Y+bMmUhISEBpaSlDEZGf6HXHqH1uI6A12OzYsQNXXnklgNbB2evWrcPSpUu93l5JSQn0ej2SkpJQUVHRYX6hgoICJCUlIT8/H1qtFrNmzUJRURGA1hm22+cmOtN2ultGRNSf/vOf/2DOnDlobGxEQUEBfvvb33L2aiI/IohdzdDohdNDT11dHVavXt1hwLWvwcifWa1WxMTEoK6uzusr7YiIAKChoQFLly5FYWEhJkyYgFWrVmHEiBFSl0U0KDQ0NGDChAlefX736lTaiRMnAADbtm3Dnj17EBMTg6qqKuzdu9e9Du+VRkSD3WeffYYrrrgCr7/+Oh599FEUFhYyFBH5qV4Fo7y8PNx7773Q6XTuyRVzc3Oh1Wpx33334aabbuLYHSIatOx2O/R6PSZOnIghQ4bAYDBg1qxZPHVG5Md6NcZo9OjRePnll5Gamuoe1JySkgKz2Yzly5cjLi6uw7gfIqLBYseOHUhPT8fPP/+MBx54AHPmzIFcLpe6LCI6g16NMRpsOMaIiM7E4XBgzZo1WLlyJUaPHo2nn34aF198sdRlEQ1qvowx6pMJHomICNi5cyfS09NRUVGBuXPn4r777kNoaKjUZRGRD3p9S5AdO3bgxhtvRFxcHP785z+7n7/33nuxbdu23m6eiMjvuVwuvPjii7jiiitw+PBhvPbaa3jwwQcZiogCUK+C0fbt2zF58mSoVKoO8xkBwMsvv4za2lrs2LGjN7sgIvJr+/fvx4033oj7778fd9xxB0pKStxzuRFR4OnVqbTVq1ejqqoKo0ePBoAOHSPgfxM88pcEEQUbURSxceNGPPDAAwgLC0NhYSGuueYaqcsiol7qVcdo9OjR7lBERDRYHD9+HDNmzEBGRgauv/56bN68maGIKEj0qmN01llndXjs6QK3mpqa3uyCiMiv/P3vf0dOTg5aWlrw3HPPITk5WeqSiKgP9apjtGvXLnz99dfux6dPWrZu3brebJ6IyG/U1dUhKysL06ZNw6WXXootW7YwFBEFoV6PMdJoNEhOTkZSUhKqq6uhVqthMplQWFgIlUqFioqKvqqViEgSH3zwATIzM2E2m7Fq1SpMmzaNs1cTBaledYxUKhUqKythNpuRn5+PwsJCpKSkID8/HzNmzGAoIqKA1tTUhAcffBCTJ0/GiBEjsHnzZkyfPp2hiCiI9XqCR41Gg7KyMtTV1aGyshJqtRrjxo3ri9qIiCTz1VdfIT09HXv37kV+fj7uueceyGS9nvqNiPxcn818HRMTgylTpvTV5oiIJNHS0oJVq1ahoKAAv/71r1FcXOy+FyQRBb9eB6MdO3bAbDZDrVZDo9HwHmJEFLC+//57pKen49tvv0VeXh7mzZsHhUIhdVlENIB61Be2Wq2YNWsW5HI5EhISkJycjISEBMTGxuLmm2/G3r17+7pOIqJ+43Q6sW7dOiQkJKCurg4bN27Efffdx1BENAj53DFat24d9Ho9RFGERqOBSqWCWq2G2WyGxWLB1q1bodFooNfr8fTTT/dHzUQUBERRxN4Tjfh6Xy1qGpoRpQzB2FEqXDRiCELkAzeWx2QyITMzE5999hnS09PdM1kT0eDkUzDasGEDnn76aaxevRq5ubmIiYnptE5dXR02bdqE5cuXIy4uDg899FCfFUtEwcHlEvEP4wF8+OMxNNodUMgFOJwiPvrpGMadH4t7rr0A4aF9NgTSI1EU8ec//xmLFy+GSqXCK6+8gqSkpH7dJxH5P0H0NF21B9u3b0dqairKysq8ug2IxWKBRqPBBx98gCuuuKLXhfoDq9WKmJgY1NXVcSwVUS988vMxvPnZHkSHK6CKULgvf2+0O3CkzoYplw7HXRMu6Lf9Hz58GNnZ2XjvvfcwY8YMLFu2DJGRkf22PyKSVkNDAyZMmODV57fX/erVq1ejsLDQ63ujqVQqFBcX83QaEXXgcLrw8U/HECIXEBsZ2mFOoEhlCGIjFajabUZNvb1f9l9cXIyxY8eioqICL774In7/+98zFBGRm9fBqLq62ufL8XU6HWpra30uioiC19E6G47V2RAbEepxuSoiFHVNLdhzoqFP92s2m3HXXXdh1qxZSExMRGlpKSZOnNin+yCiwOf1SfyezuOhUql69DoiCk4uUYQodr63YjsBAETA5dVJfu+89957mDt3Lk6ePIk1a9bglltu4ezVROSR1x2jnv4SUavVPXodEQWnodFhUEWGoq6p2ePyepsDkWEhOCc2vNf7amhoQF5eHm699VbEx8ejtLQUt956K0MREXXJ62Dk5RjtTvgLiIhOFaaQ45oLz8JJuxONdkeHZc0OF45bbbj0nBicrepdMPr0009x+eWX429/+xsee+wxrF+/HsOHD+/VNoko+PV7x8hsNvfodUQUvCaNGY7xF56FmgY79pxoxNG6JuyracTB2pP49chozLjqvB7/zrHZbFi2bBkmTpyImJgYGAwGzJw5k3+kEZFXvB5jVFJSgt27d/vcOTIajT4XRUTBTamQI+O60bjy/FhUmsw4ZrUhOjwE2gvUGHeBGpHKns1htGPHDqSlpWHnzp1YtGgRMjMzIZfL+7h6IgpmPv32qays9HkH/CuNiDwJkcugvUAN7QW9H4focDiwevVqrFy5EvHx8Xjrrbdw0UUX9UGVRDTYeB2MtFptj4JRYmKiz68hIvLWzz//jIyMDFRWViI7O5v3OCOiXvF6jFFPL9fv6euIiLrjcrnwwgsvYNy4cTh69Chee+01LFy4kKGIiHrF61uCEG8JQuQv9u/fj6ysLGzbtg2zZ8/GokWLEBERIXVZROSnfLklSP/epZGIqA+JoojXX38dCxcuREREBIqKijBhwgSpyyKiIOL1qTQiIikdO3YM06dPR1ZWFn7zm99g8+bNDEVE1OfYMSIiv/fOO+8gJycHTqcTf/jDH6DT6aQuiYiCFDtGROS36urqkJmZienTp+Oyyy5DaWkpQxER9St2jIjIL73//vvIyspCbW0tnnjiCUydOpXzohFRv2PHiIj8ysmTJ/HAAw9Ap9Nh5MiRKC0txbRp0xiKiGhAsGNEJDFzgx3b99bCdKwBMgH41fAhuPL8WMREhEpd2oD78ssvkZ6ejn379mH58uWYPXs2ZDL+/UZEA4fBiEhC3x+w4I3P9uBEvR1yWWtH5MtdNdj2/VGkXTcaF44YInGFA6O5uRmrVq1CQUEBxowZg+LiYk4OS0SSYDAiksgxqw0bP9uDupPNuGBoJGRtp4qcLhH7zSfxt09NeOjWS4K+c/Tdd98hPT0d3333He677z7MmzcPISH81URE0uBvnyAjtrSgubIStk8/g+vwEQgx0VBOmADlNRMgi4qSujw6RdVuM07U26AZGtVh/IxcJmCUOgJ7TzTCuKcWk8YMl7DK/uN0OvHss8/isccew6hRo/DGG29gzJgxUpdFRIMcg1EQEe121L9cCPsnnwAuEUJ4OMR9+9CyYwfsH3+MIYsehPyss6Quk9r8dKgOYSFyj4OK5TIBcpmAXUfrgzIYmUwmZGRk4PPPP0dmZibuv/9+KJVKqcsiImIwCiZN774L+4cfQnb2yA7dIbG5Gc3ffovG117HkCWLeXWPn3CJALr5VghC6y0wgokoitiwYQOWLFmC2NhY/OUvf0FiYqLUZRERufFyjyAh2mywvf8BhPCITqfMhNBQyIePQPOOHXDu2ydRhXS6C4cPga3F6TH8uEQRDqeI0cOC5/TnoUOHcOuttyIvLw8333wzDAYDQxER+R2/6xiZTCYYDAZoNBqYTCbk5uZCpVJ5XNdoNKK8vBwAUFFRgQ0bNrjX7W47vuwjUDiPHIGr5gSEOM+nyoSYGDiPHYVj7z6EnH/+AFdHniRq1Pjk52M4ZGnCSFW4u5MniiIO1p7EWUOU0J6vlrjKvvH2229j/vz5kMvlePHFFzFx4kSpSyIi8sjvglFqaiqqqqoAtAaYnJwclJSUeFy3vLwc+fn5AIC1a9diypQp7td2tx1f9hEwBKH1y+XyvFwUIQgCBLl8YOuiLo2MjcDM8eej+L97YTreiHCFHCJE2FpciIsKxexrLkDckMAed1NTU4P58+ejuLgYN998Mx555JGA/yOEiIKbXwUjk8nU4bFGo3F3hE5nNBpRUFDgDkYpKSnQ6/WdtnH6dnzZRyCRjxwJ+chz4Ni/3+PVZy6zGbKYGIT8Kl6C6qgrSZo4jFSFo9JUg1+O1EMmABePjEaSJg7DY8KlLq9X3n33XcydOxc2mw1r167FLbfcInVJRERn5FfBqLy8HGp1x1MHarUaRqMRWq22w/NarRYbNmxwP7ZYLO71i4uLu9xOZWWl1/uw2+2w2+3ux1artcfvrb8JCgXCbkxGQ1ERnCdOQBYX5z4142pogOvECYTf9lvIhwffFU6B7hx1BM5RR0hdRp+pr6/HkiVL8Oc//xnXXnstVq1ahWHDhkldFhGRV/wqGLWHm9OZzWaPz6ekpLj/f9OmTdDpdFCpVN1ux5d9FBQUYOXKld3W7E/CdFPgPHEctnffg+OXnyGEKACnAwgNRdhvJiIy7R6pS6Qg98knnyAjIwPHjh3D7373O6SkpPAqSCIKKH4VjLrSVZg5dbnBYHCPG+rJdjwtW7FiBZYsWeJ+bLVaMWrUqG73ISVBJkPk7NlQXnUV7F9VwHXsGGTR0QhN0EJx6aUQOJsw9RObzYZHH30Uzz33HMaNG4eXXnrJr/+tEBF1xa8+KVUqVafOjdlsPuNgTb1ej7KyMvd63W3Hl30olcrAm3ROFCHabBAbG+FqaIAgl0NsagKcToDBiPqB0WhEeno6du3ahcWLFyMjIwNyDvInogDlV/MY6XQ6j893N9fJ2rVrodfrodFoYLFYYLFYut1OT/YRKESHA42v/hXWJ59G07/+jZZvv4Nt2wewPrMO1mefg6uuTuoSKYg4HA6sWrUKV199NVwuF95++23MmTOHoYiIAppftRBOv5u2yWRCYmKiu5tjNBqhUqnc6xkMBmi1WncoKi4u9jgn0anb6W5ZoLNtLUPTv9+FLC4OivPOcz8vNjWh+b9fojEmBkMWzJewQgoWP//8M9LT01FVVYV58+bh3nvvhUKhkLosIqJe86tgBAAlJSXQ6/VISkpCRUVFh/mFCgoKkJSUhPz8fJhMJqSmpnZ4rUqlQm5u7hm3092yQCU2N8NWVgaEhkIWG9thmRAeDtmwYWj+8is4p0+HfOTZElVJgc7lcuFPf/oT9Ho9RowYgb/97W+4/PLLpS6LiKjPCGKw3YypH1mtVsTExKCurg7R0dFSl9OBY+9eWFY8DCFWDVlkZKflossF566dGPLQQwi7/joJKqRAt2/fPmRmZuLDDz/E3XffjUWLFiE8PLDnWiKiwaGhoQETJkzw6vPb7zpG1ENnyreCgG7vWErUBVEU8dprr+HBBx9EREQENmzYgPHjx0tdFhFRv/CrwdfUc/Kzz4Zs6FC4upjzSaythRA9BCGa0QNcGQWyY8eOYdq0aZgzZw5uuOEGlJaWMhQRUVBjMAoSglKJcJ0OsNk6XX0m2mxwHjsGZVISQs45R6IKKdCUlpZizJgx+PTTT/H888/jqaeewpAhQ6Qui4ioX/FUWhAJu/kmOA4dgm3bNriOHgXCwoDm1luaKBMTEJmeJnGFFAgsFgseeOABbNy4EZMnT8bvfvc7xMXFSV0WEdGAYDAKIoJCgaiceVBefRXsn38B5+HDkKlUCL3qKiiTEiGEhUldIvWDBlsLzA3NUITIMDw6DDJZz8eSlZeXIysrC3V1dXjqqadw++2385YeRDSoMBgFGUEmQ+iVVyL0yiulLoX6mbWpBVu/PYwqUw0a7A7IZTKcf1YEJo0ZgSvOU/kUaE6ePIn8/Hy8+OKLuPrqq/Hqq6/i7LM5rQMRDT4MRkQBqL6pBRs+2IUfD9VBFR6KuCglHE4Rvxypx57jjZg1/nxcc9FQr7b13//+F+np6Thw4ACWL1+O2bNnQybj8EMiGpz4248oAH2+8zh+PmTFeepInDVEiTCFHFFhITg/rnUOq39tP4j6ppZut9Hc3IyHH34Y1157LcLDw1FcXIx77rmHoYiIBjV2jIgCjMPpwhe7TiA8VI7QkM4hZlh0GPbVNOLbAxZcc6HnrtG3336LtLQ0/PDDD1iwYAHmzp2LEN5kmIiIHSOiQGNrcaLR5kBEqOebtcrbBl9bPXSMnE4nVq9ejcTERDQ1NeHNN99Ebm4uQxERURv+NiQKMKEhcijkMthbnB6Xu0QRIoBwRcfgVF1djYyMDHzxxRfIysrCggULoFQqB6BiIqLAwY4RUYAJDZEhcbQadU0tcLk63wqmtrEZ0WEKjDknBkDrLT1efvllXHHFFdi/fz9effVVLFmyhKGIiMgDBiOiAHTdxcNwrjoCe2oa0Wh3QBRFOFwuHLPaUG9zYOKvh2FodBgOHjyIW265Bffddx9uvfVWGAwGJCQkSF0+EZHf4qk0ogA0LCYM2TfEo7RiP6qPNeCY1QZAgDoqFDdedjamXDocb731FubPnw+FQoGXXnoJ119/vdRlExH5PQYjogA1Ki4SC2+6GHuON+J4vR2hITJcOHwImhosuHv2XTAYDLjlllvwyCOPICYmRupyiYgCAoMRUQATBAGjh0Vh9LAoAMC///1vZGdnw26345lnnsHNN98scYVERIGFY4yIgoDVakV2djZuu+02XHTRRSgtLWUoIiLqAXaMiALcRx99hMzMTJw4cQKPP/44ZsyYwRu/EhH1EDtGRAHKZrNhyZIlmDRpEuLi4lBSUoKUlBSGIiKiXmDHiCgAVVVVIT09Hbt27cJDDz2EtLQ0yOWeZ8ImIiLvsWNEFEBaWlqwcuVKjB8/HgBQXFyMzMxMhiIioj7CjhFRgPjxxx+Rnp6OHTt2YN68ecjLy4NCoZC6LCKioMKOEZGfc7lceP7556HVamE2m/G3v/0N999/P0MREVE/YMeIyI/t3bsXmZmZ+Oijj5CWloaFCxciPDxc6rKIiIIWg1EQctXWwl5ZBVdtLWSRkVBceSVCzhkpdVnkA1EU8de//hUPPvggoqKi8Oc//xlXX3211GUREQU9BqMgIooibOXv42RxMVwnTrQ+5xIhi4lBmG4KImffBYGnX/ze0aNHMW/ePPzrX//C1KlTodfrMWTIEKnLIiIaFBiMgkjzl1+i8dVXIUKAXBMPQS6HKIpw1dSg6Z13IISFIXJmqtRlUjc2b96MvLw8AMD/+3//D5MnT5a4IiKiwYWDr4OE6HKh6d33INqbEXLuuRDaLt8WBAHys86CEDUEtrIyuCwWaQsljywWC9LS0pCSkoIrr7wSpaWlDEVERBJgxyhIOA8dgsNkgmzoUIiiCNFqhWi3QwgJgaBSQTZ0KJymarT89BOUbXPgkH/YunUr5syZg/r6ejz99NO47bbbOHs1EZFEGIyCRXML4HBCbGqC45df4KqrA5xOQCaDEBkJ+QUXQHSJEJtbpK6U2jQ2NmLZsmVYv349xo8fjyeeeAIjRoyQuiwiokGNwShIyIYNhSgT4PjmG0AUIUREACEhgNMJsbERjm+/hXzEcMhHDJe6VALwxRdfID09HQcPHsTDDz+MWbNmQSbjmW0iIqnxN3GQkEVFQVAo4GpogBgaCrGpCc7aWjgbGgCFAq6mJog2G+SjRkld6qBmt9uxYsUKXHfddYiKikJJSQlmz57NUERE5CfYMQoSzuPHW0+ThYVBPHoEosPpXuaSyYDISCAiAo4ffkRoglbCSgevb775Bmlpafjpp59w//33Y86cOQgJ4T9BIiJ/wj9Tg4RY3wDnsWNAYyPgdHVc6HIBTU1wHj8BV0O9NAUOYk6nEwUFBUhMTITNZsObb76JnJwchiIiIj/E38zBIjICzkOHALvd83KHA+Lx4xCdTs/LqV/s2rULGRkZ+O9//4s5c+ZgwYIFCA0NlbosIiLqAoNRkHDV1rZ2i7rT0gLnwcMDU9AgJ4oi1q9fj2XLliEuLg6vvfYaxo0bJ3VZRER0BgxGQcJhMrVenn8GLT//NADVDG4HDhzA3LlzUVZWhpkzZ+Khhx5CRESE1GUREZEXGIyChEsUvVrPWWvp30IGMVEU8eabb+L+++9HaGgo1q9fj+uuu07qsoiIyAccfB0sGk96tZpot/VzIYPTiRMnkJqairS0NFxzzTUoLS1lKCIiCkDsGAULLwdVi152loJZo80Bc6Md4aFyxEUpe337jX/+85+YN28e7HY71q1bh5tuuqmPKiUiooHGYBQs2m4aeybCIL4lyIl6GzZ+ugf/3XUCdU0tkAlA/LAo3Jl0Hm4Y4/uM4FarFYsWLcKrr76KiRMn4ve//z2GDh3aD5UTEdFAYTAKFl42PYSYmP6tw0+dqLfhsZKv8cuRBsgFQKmQweUS8d2BOuw6+gNONNiRctV5Xm/vww8/RGZmJmpqarBy5UpMnz6dN34lIgoCHGMUJEJGjgS8+GAOGTZsAKrxP29/sRc7jzYgSimHOkqJSKUCQ8JDcdaQUDQ7Xdj46W4crWs643aampqwaNEiTJo0CUOHDoXBYMCdd97JUEREFCT8rmNkMplgMBig0WhgMpmQm5sLlUrV5fpGoxE5OTmoqqryeju+7iMQyEaMAEJDu57gEQAEAYqxlw5cUX6i0ebA578chyiKEAQBthYnFHIZ5DIBMkEGVbgCtSeb8d6OQ8j6TXyX26msrERaWhp2796NpUuXIj09nfc4IyIKMn4XjFJTU90hx2QyIScnByUlJR7XbQ83RqPRp+34so9AIQsJAYYM6T4YKRSQD8IxMAdqT+JInQ0tThdqG5sBAHKZgHCFHFFhIQiRyyAC2Gf2fGVfS0sLnnzySTz11FO4+OKLUVxcjPj4rgMUEREFLr8KRiaTqcNjjUaD8vLyLtdPSUnxeTu+7iNQCNHRQHNz9yu5XIBscJ3yaWp2YEvFfrQ4XRABhMhb37/TJaLB7oBLFBEVFgKIQJii8wD2H374Aenp6fj666+Rm5uLnJwcKBSKAX4XREQ0UPzqPEB5eTnUanWH59RqtceOUE+301f78DfOw0cAW9scRTJZx6+QkNbxRy4XHLv3SFrnQDPuqcXu4w2IjQwFxNYx6oIgIKTtVFpTixP1thYoQmS45sKz3K9zuVx47rnnoNVqYbFYsHHjRsyfP5+hiIgoyPlVx8hisXh83mw299l2fNmH3W6H/ZRTU1ar1ac6BlLLblNrR0ihaJ3TyOX630JRbL2c3+VCy48/SlfkABNFEZ//fAyiS0T8sChYm2pha3FBIRfcg6VbHC44XSLGXRCLRE0cAGDPnj3IzMzExx9/jLS0NDz44IMICwuT8q0QEdEA8atg1JWuwkxfbsfTsoKCAqxcubJP9t3fZEOiW7tCDkdrEDqVKLongJSf1i0LRMesNvx0yIpmhwvqqFCMOSem02mwnw5Z8dGPR/HJL8fR7HBhSFgIhseE4XBtE2wtrafV2qkjFFhx+6VQhsjwyiuvYNGiRRgyZAheeeUVXHXVVQP75oiISFJ+FYxUKlWnzo3ZbPb5irHutuPLPlasWIElS5a4H1utVowaNcqnWgZK6JhLAEUI0NLFBI6iCAgCQq9KGtjC+lCLw4W/Gw/gi53HUd/kgCAAMkHACFU4ZiSNwthRKgBApekE3vhsDxrtDihDZGh2uGBrdqLe7oAgCIhUyiFCBETAKYqIUIbg02924bVnHsa///1vTJ8+Hfn5+YiKipL2DRMR0YDzqzFGOp3O4/OJiYl9th1f9qFUKhEdHd3hy18JKhXgOsPtPkQRsnPOGZB6+sPfjQdQ9u1hyGUCRg+NxOihUThbFY6jdU14/VMTfjxYh/98cxgF//gBPx6ywtzQjNAQGQQALghwia2n18JD5Rg6JAzREQpEhytQ++OnyLhtIj7/4gu88MILWLVqFUMREdEg5VcdI41G0+GxyWRCYmKiu5tjNBqhUqk6rQe0ngprX6+77ZzeGTp9H4GquaLif4Ovu3FycylCly0dgIr61nGrDf/deQIx4QqoI5Xu550uEaIo4rv9Fix9wwgIQFOzE0PCQtDsdKHR7oDDJcJub4aI1g7TSbsDAGBvtML07ouo/morRl1xPR5a/ghuGBu4wZGIiHrPr4IRAJSUlECv1yMpKQkVFRUd5hcqKChAUlIS8vPzAbRefVZWVtZhWfsl/N1tp7tlgar5u++9Ws/2VUU/V9I/fjlSD2tTCy4YGul+rraxGTuP1MPW4kRTsxPNThdC5TK0OF1wuOQIV8gRFiKDtan19KJMECACaHGJqN1ViR3Fa+GwN+G3uY8iZswNEJSRXeydiIgGC0Hk7da9ZrVaERMTg7q6Or87rWZ55lk0Pv/8GdeT/fpinP1+4M3b9P73h/Hax7uhDJHhZLMTMgEwNzZDFIFIpRwn6u1ocbggl8tgd7Rekdd6TzQ5IkLlqLc5EB4qh+iw4ed/F2LvF//A+Zcm4pbs5YiKHYpDVieuHx2B60YzHBERBZuGhgZMmDDBq89vv+sYUU+5zrwKAFHeeRJDf+dyifj+QB0OW5oglwlQyGXuLlGYQoZQuYDmFhecAByO/x0Hp9h6Ws3ucCJEEHDC9C1+MKxFc4MZuvTFuHLyNAiCgJPNLijlAkarQ6V7k0RE5BcYjIKEbMgQr9aTh4X3cyV977+7TuCHg1ZEKkPQ4nQhQilHi9MFmUxAs8OFWkcznF28ViYAjuZm7PlwIw59YUDMuRfj9gfX4LzzzgMAWG1ONNhFXH52GEZG858DEdFgx0+CICE7e6RX64Wcf34/V9K3nC4Rn/x8DHJBwMVnR+OXI1bUNzngcLrQfha42fm/s8GCALRdiQ8AsB42ofqdtbCdOIDLb52DSdPugbVZwGFra5SKDJUhcVQYrh8d4Z70kYiIBi8GoyARctZZZ14JgGz4sH6upG/VNjbjSJ0NsZEKRIUpMOacGBy22HDYchJ2R2s4astCrdr+R3Q5ceTzEuz/cCPCh47CFbl/ROoNl+P2MdE4XO9ATaMTchlwTowCMWGBd3qRiIj6B4NRkBBCvRwfExHRv4X00nFrE/65/RB+OlQHANAMjUJTswORoa3hZUiYAkNGKDBKHY5v91twvN4OmUzscAeUppqDqH7nGTQc+gUjr0mBZnIa5AolwkJkEAQBI6MVGBnNe54REVFnDEZBoqW62rv1tu/o30J64f3vjuDFsl9Qd7IZEFonY6wwmSGKIoZGh+GSkdGoO9kCh9MFhVyG0cOiUNPQDKdLhCAATpcLxyv/hb3lryA0So0r5q5D1KgxkMsAZYiAy87m/c6IiKh7DEZBomX/fi/X29fPlfTMT4fq8MetP6G+qQXhyhA02R2wNbvc19rtrzmJAzUnEaaQIUwhhyAIUMgFhIbIoAwR4Gwwo+rtNagzGXH2VbfhVzflQKZQosXZOiH4JcNCcTa7REREdAYMRkGieZd3HSMcOdq/hfTQlsoDsJ5sgQAR5vpmeJpcSwTQ1OKC3eFCpDIEzU4gRCbA+sOHqCp5HrLQMIxNfwrRmgQ4AbQ4ALkMGDNMibu1sQP8joiIKBAxGAULa5136zU3928dPVDb0Iwvdx5HU4t3czG5RMDe4sR5UU58/sZa7DZ+iPgkHa6cvgCysCGosznhcAJDwmS45ddDMP68CMhlvOKMiIjOjMEoSAjhXg6qlvnVfYNRd7IZRR/sxPF6u0+vO/7j5/jqX3+EABduyP498u76LSxNThxvcCCkbbLGX8WFIkzhX++XiIj8G4NRkHAovPxW+tkdYN6pPICfDlk9njrzxGFvxL7/FOL4jq0469fjoctYCsUQNaKVMowdwcHVRETUOwxGQUKsNnm3ot23zkx/OlZnQ4XpBI5bbXB5kYzqdn8N0z+ehaOpAZrbF+OCq26GM0yO6BAZYsI5FxEREfUeg1GQEETRu66LH51K2/rtYXy33wKbo/vKXS127N/2Ko58+Q6GnH8ZxmSuRUTsCMhkgK1FxK9GhXKSRiIi6hMMRkEi5Ne/RstPP515RcXAX7J+rM6Gr/fVwtzQjPBQOS45JwaiKGLrt4fPGIoaDv6M6r+vg732CM67MQ8jrp4KQZAhRAY4XMB5sQpcc4F/T1pJRESBg8EoSLhCvQw8Lu+u/OoLoihi2w9H8d6OQ6hrar0U39rUjJPNTjS3uHCyuatbvwIupwOHPn4TBz99G5Fnx+Oy3BcRPrT1xq8yAAq5gIuGKnH3uBhEs1tERER9hMEoSDjL3/duxQG8XN+4x4x3KvdDIZdhaKQClXvMsNq6DkPtTh7bg+q/r8PJIyacM/FujLzuLsjkrT+qIQIwKlaBK84OQ/JFUVCG+M+pQSIiCnwMRsGizst5jAaIyyXi45+OwekScVaUAh/+eARNLd2fNhNdThz58h3s3/ZXKGPPxqXZzyNq5EUIEQBBAMIVAib/KgpJoyIwLKp19msiIqK+xGAULCIigPp6qatwq2mwY3/NSUSFheD7A3VnDEW22iMw/f1Z1O/7DiPGT8eoSZmQK5RQhgDq8BCowmWYNjYamjjlAL0DIiIajBiMgsV5o4Dvf5C6CreaejsO1J5Ek92JEw1dTxEgiiKOb/8/7N1ahJDwIRiTsQZDLrgcQOvtPEZGK3D1eREYe3YYhkfxx5WIiPoXP2mCRUOj1BW4mRvsMFTsh/VkCwBAEOFxKoHm+hrs/tf/g2XnVxg67iacd2MuQpSRAIBopYCUy2Mw/vwIyHjKjIiIBgiDUbBoaZG6ArcPfjiKvccbMHpoFPbWNEIQ0CkZ1Xz/Mfa8+wIEWQguumslYi+6GqEyICJUQJRSjgXXqDE0auCnFiAiosGNwShYyP3jkvWTdgcqd5sRHaGAOlKJpmYnTtoccDpapwlwNNVjz7t/Qs33H0F9yXW44LcPQBERA4UMOEelgFwQcNV54QxFREQkCQajYHHihNQVAAAabA40NTsxJCwEcpmAi84egqhwObbvqUXNL5Uw/eM5uBzNiJ+uR9zYG9xXlkWGyhChkGHsCCWuHx0p8bsgIqLBisEoWDQ1SV0BACAsVA6FXECzw4VIJSATBAwNE9D06Qb8XL4ZMfEJ0Ny+GKHRZwEAlHJg3DlhuOzscGjiQnFWBC/DJyIi6TAYBQu5HHCeefLE/hYdrsDYc1X49JdjUEUosPfH7Sh+/mE01J6ALmMJVFfcgmYnMGZYKC4ZrsRlZ4dDIeckjURE5B8YjIKFH81jNGnMcHy37wQ2vbwGX//nDYyMvxTTFq2BED0SDpeIW+MjkTSK9zcjIiL/w2AULM6K85tgZD6wE39/Mgum6l3Q3jEPF98wEy1yOVShMiSeG45x54RJXSIREZFHDEZBQnHhRWjZvefMK4aH91sNDocDq1evxsqVK6HRaPD2228hcrgG9XYnFHIBo1QKhPHeZkREkhJFEaLY/d0IApEgCH0yRpXBKEgoxl6Klq1bz7ie7Pzz+2X/v/zyC9LT01FZWYns7Gzcd999UCjaL7nnpfdERP5AFEU4HA7IZMH3R6rL1TotjFzeu4t4GIyChDJBi5NerBd6xeV9ul+Xy4WXXnoJ+fn5GDZsGF577TVceeWVfboPIiLqPVEU4XQ6ER4ejuHDhwfVFcCiKMJms+H48eNwOp0ICel5vGEwChJCiHddGcW55/TZPvfv34+srCxs27YNd911FxYvXoyICA6qJiLyV4IgIC4uDmFhwTfWs/09HT16FKIo9jj4MRgFCUEuB2QyoK2V2BWxD/5CEEURf/vb37Bw4UKEhYWhsLAQ11xzTa+3S0RE/ad9XFFvuin+LiwsDIIgMBgRIBsSBXjxQyCLVvVqP8ePH0dubi7eeecd3H777Vi+fDmio6N7tU0iIho4pwcG56FDEM21A7d/dSzkI0f2z7Y5+JraiU5na8fI6WwNSKdecdD+WCYDnI4e7+Odd95BTk4OnE4n/vCHP0Cn0/VB5UREJBXnoUOoS74RsDcP3E6VoYgp29pv4ai3GIyCRVMTEBICuJyd7mQPQWj9kslal/uorq4OCxcuxOuvv44bbrgBjz/+OM4666y+qZuIiCQjmmsHNhQBgL25db9eBqPdu3djy5YtGD16NHbv3o25c+dCpVL1W3kMRkFCplZDFhUFV734v3FGovi/02syGQRFCOTDz/Zpu9u2bUNmZiZqa2vxxBNPYOrUqUF1JQMREfm3e+65B59//jmA1pC0YMECvPHGG/22v+CbyGCQkp97LuTnnw8hNBRCVBSE0FBAoYCgULgfy4YOheKysV5t7+TJk1i4cCGmTJmCkSNHorS0FNOmTWMoIiKiAbN79+4Oj0ePHo0PPvigX/fJYBQkZBERiJiZCllMDCCXQwgLgywyEkJ4eOvjiAiETZ6MkHjNGbf11Vdf4corr0RRURH0ej02bNiAkX56LpiIiILXtm3boFarOzwXGxuL7du399s+eSotiERMvQOOPXtg//hjiI0n206lAbJQJRSXX46o3NxuOz7Nzc1YtWoVVq9ejUsuuQTFxcXQaM4cpIiIiPpDXV2dx+dra/vvKjoGoyAii4pC9JLFsCclwv7Bh3AePwEhMhJhv5kI5W8mQh4X1+Vrv/vuO6Snp+O7777Dvffei3nz5gX1XBdERBS4ugpMfYGffEFGFh6O8BtvRPiNN0J0OFpPo3XTJXI6nXj22Wfx2GOPYdSoUdi4cSMuvfTSAayYiIjIs5iYGJjN5g7P1dbWIiYmpt/2yTFGQUwICek2FJlMJvzmN7/B8uXLMXv2bGzatImhiIiI/MbkyZM9Pq/Vavttn+wYDUKiKGLDhg1YsmQJVCoV/vKXvyAxMVHqsoiIiDoYPXp0h8e7d++GVqvlPEbUdw4dOoTs7Gz83//9H2bMmIFly5YhMjJS6rKIiEgCgjoWUIYO+MzXgjrW69XfeOMNPProo0hISEBVVRU2btzYj8UBgiiKp8+TLCmTyQSDwQCNRgOTyYTc3Nwuk2F36/Z0WXesVitiYmJQV1cXkPcH27RpE+677z7I5XKsXLkSEydOlLokIiIaIC6XC6Io4vzzz4dSqXQ/H0z3SrPb7di7dy8EQYBM9r/RQg0NDZgwYYJXn99+1zFKTU1FVVUVgNYAk5OTg5KSEp/X7emyYFRTU4P58+ejuLgYN910Ex599NF+bUMSEVHgkI8c6fXtOQYDvwpGJpOpw2ONRoPy8nKf1+3psmD03nvvYe7cuWhqasLatWtxyy23SF0SERGR3/Krq9LKy8s7zXCpVqthNBp9Wreny05nt9thtVo7fAWK+vp65OTk4NZbb0V8fDxKS0sZioiIiM7ArzpGFovF4/Onz2FwpnV7uux0BQUFWLlypcf1/dknn3yCjIwMHDt2DI899hhSU1N5jzMiIiIv+FXHqCtdhRlf1/V12YoVK1BXV+f+2r9/v9d1SMFms2Hp0qX4zW9+g9jYWBgMBsycOZOhiIiIyEt+1TFSqVSdOjdms9njQOHu1u3pstMplcoOI/f92fbt25GWloadO3di8eLFyMjIgFwul7osIiKigOJXHSOdTufxeU+TD3a3bk+XBSKHw4EnnngCV111FVwuF95++23MmTOHoYiIiKgH/KpjdPqd3E0mExITE93dHKPRCJVKBY1G0+26p3d/vF0WaH7++Wekp6ejqqoK2dnZuO+++6BQKKQui4iIAsjROhvqmloGbH8x4QoMjwkbsP35yq+CEQCUlJRAr9cjKSkJFRUVHeYXKigoQFJSEvLz88+4bk+XBQKXy4U//elP0Ov1GD58OF5//XVcccUVUpdFREQB5midDRlFFWh2Dtxcz6FyAa/nJnkdjrZv344FCxbg888/7+fKWvndzNf+zB9mvt63bx+ysrLwwQcf4O6778aiRYsQHh4uSS1ERBQ4PM18/cuReuT9dfuA11KYNQ4XjRhyxvW2bNmCCy64ANdeey1Onjx5xvWDcuZr8kwURbz++utYuHAhIiIisGHDBowfP17qsoiIiPrN9OnTB3yffjX4mjw7duwYpk2bhqysLNxwww3YvHkzQxEREVE/YMfIz23ZsgW5ublwuVx4/vnnMWXKFKlLIiIiClrsGPkpi8WCjIwM3HnnnbjssstQWlrKUERERNTP2DHyQ+Xl5cjKykJdXR2efPJJ3HHHHZy9moiIaACwY+RHTp48ifvvvx/Jyck499xzUVpaiqlTpzIUERERDRB2jPzEf//7X2RkZGDfvn1Yvnw5Zs+e3eFSQyIiosHMYrEMyGTM/OSVWHNzMx555BFce+21CAsLQ0lJCe655x6GIiIiGvS2bduGRx99FACwbt06bNmypd/3yY6RhL799lukp6fj+++/x/z585GdnY2QEH5LiIhoYMSEKxAqFwZ85uuYcO9uXzV58mRMnjwZTz75ZD9X9T/8FJaA0+nEunXr8Lvf/Q7nnXce3nzzTVxyySVSl0VERIPM8JgwvJ6bxHulnYLBaIBVV1cjIyMDX3zxBbKysrBgwQL31OxEREQDbXhMmF8HlYHGYDRARFFEYWEhli5ditjYWLz66qtISEiQuiwiIiI6BYPRADh48CCys7Pxn//8BykpKVi6dCkiIyOlLouIiIhOw2DUz95++23Mnz8fISEhePHFFzFx4kSpSyIiokFMFAduoPVA64v3xmvC+0lNTQ1mzpyJ2bNnY/z48diyZQtDERERSaZ9smCHwyFxJf3HZrNBFMVeTYzMjlE/+Pe//43s7GzYbDY888wzuPnmm6UuiYiICKIooqamBiEhIUF1VwVRFGGz2XD8+HEAYDDyF/X19Vi8eDFeeeUVXHfddVi5ciWGDRsmdVlEREQQBAFyuRxNTU3Yt2+f1OX0ufbTaHK5vFfbYTDqIx9//DEyMjJw/PhxPP7445gxY0ZQpXEiIgp8giC4JxIOtrFGMpmsTz53GYx6yWaz4eGHH8bzzz+PcePGYf369Rg1apTUZREREXnUHh74x7tnDEa9UFVVhfT0dOzatQtLlixBenp6r1t4REREJB1eldYDLS0tWLlyJcaPHw8A2LRpE7KyshiKiIiIAhw7Rj2QnJyMb775BvPmzUNeXh4UCu9uhkdERET+jcHIB+0D1Y4fP47CwkJceumlsNvtsNvtEldGREREXWlsbATg3YBzQQy2Yen96MCBAxxYTUREFKD279+Pc889t9t1GIx84HK5cOjQIQwZMiQgRvNbrVaMGjUK+/fvR3R0tNTlBBQeu57jsesdHr+e47HrnWA+fqIoor6+HiNHjoRM1v3wap5K84FMJjtj0vRH0dHRQfdDPlB47HqOx653ePx6jseud4L1+MXExHi1Hq9KIyIiImrDYERERETUhsEoiCmVSjz++ONQKpVSlxJweOx6jseud3j8eo7Hrnd4/Fpx8DURERFRG3aMiIiIiNowGBERERG1YTAiIiIiasN5jAKcyWSCwWCARqOByWRCbm4uVCpVr9cdDHw9HkajETk5Oaiqqhq4Iv2YL8fPaDSivLwcAFBRUYENGzbwZ8/LY9d+3CwWCyoqKjBr1ixotdoBrNb/9PR3mV6vx4oVK/iz58O/WwDQarUwmUywWCyD42dPpICm1Wrd/19dXS2mpKT0ybqDgS/Ho6SkRKyqqhL5T+Z/fDl+a9as6fD/p752MPLl2KlUKrGqqkoURVEsLCwUNRpNv9fn73ryu6z9329tbW0/Vub/fDl2ubm5IgARgKjT6QbNseOptABmMpk6PNZoNO6/Lnuz7mDg6/FISUkZHH8pecmX42c0GlFQUOB+nJKSAqPR2Gkbg4WvP3slJSUdfvYGc7cD6PnvMpPJBI1G019lBQRfj11CQgJqa2tRW1uLsrKyQfOzx2AUwMrLy6FWqzs8p1ar3e3Pnq47GPB49I4vx0+r1WLDhg3uxxaLxb3+YOTrz55Op3P/f0lJCfLy8vq1Pn/Xk3+7BoMBKSkp/V2a3+vJsVOpVIMmELXjGKMA1v4Bczqz2dyrdQcDHo/e8fX4nfqhtGnTJuh0ukH3y7ZdT372jEYjNm3ahOTkZOTm5vZTZYHB1+NnsVgG7c/a6Xpy7AwGA4DWsYF5eXmDouvGYBSEuvrh7+26gwGPR++c6fi1/6LlAPbOujt2Wq0WGo0Ger2e3Y8udHX8iouLB32YPJOujt2pA7M1Gg2Sk5NRXV09cIVJhKfSAphKpeqU9M1ms8e/jnxZdzDg8eidnh4/vV4/qMYqeNLTY6dSqZCamorU1NRBHeB9OX7l5eWYOXPmAFXm/3z92Tt1TFL7VWyDYWwgg1EAO3XswakSExN7te5gwOPROz05fmvXroVer4dGo4HFYhm0H+6+HLvy8nLExsa6H7efxhgMH05d8fVnr7i4GEVFRSgqKoLJZEJBQcGgHUvoy7EzGo2YMmVKp+cHw9hAnkoLYKef6zWZTEhMTHSnf6PRCJVKBY1Gc8Z1Bxtfjt3pOGbB9+NnMBjcp4MsFsugPr3hy7FTq9UdPszalw3mKyR9OX6nB4G8vLxBM07GE18/M9asWeNet7y8HCkpKYPidx9vIhvgTCYTCgsLkZSUhIqKig6Tl6WmpiIpKQn5+flnXHcw8uXYlZeXo6ysDGvXrkV+fj6SkpIG/TgPb4+fyWRCfHx8h9eqVCrU1tZKULV/8OVnz2AwuE9/lJWVYc2aNYP2g72dL8cPaP1jpqioCHq9Hrm5ucjLyxu04dKXY9c+MatKpUJ1dXWHoBTMGIyIiIiI2nCMEREREVEbBiMiIiKiNgxGRERERG0YjIiIiIjaMBgRERERtWEwIiIiImrDYERERETUhsGIiAJGamoqEhISIAgCBEFw3zus/Ss5ORmpqakeb/lgsVgQHx+PtWvX9lk9RqMRCQkJiI2NRXJycp9tl4ikw1uCEFHAKCkpAQAIggCdTud+fKq1a9ciISEBhYWFHW47YjabYTKZUFFR0aN96/X6TjP/arVaVFVVITk5udPNOYkoMLFjRERBJT8/H1qtFnl5eZ3uDi6Koscw5Y3ubtw62G/RQRRMGIyIKOi03zy0vLy8T7ZnMBhgsVj6ZFtE5N8YjIgo6LR3d/qik2MymZCTk9Pr7RBRYOAYIyIKKiaTCQaDATqdzt05MplMyMvLQ2VlJTQaDaqqqgC0dpT0ej1MJhNyc3Mxa9YslJeXo6ysDHl5eQCATZs2AQAqKyuRmpoKoDVwebrTePtd3AGgoqKiy/WIyH8JoiiKUhdBROSL9sHXZWVl7ucsFos76KSkpHgMJO2DpNuDUbv4+HjodDrEx8cjPz8f8fHx0Gq17vFI7Vecnbq/U7WHrlmzZiE/P9/9fGxsLFasWNHhOSLyb+wYEVFAMplMnS69r66uhk6nQ1JSksfXaDQaj1ePaTQaFBcXY/fu3QCAqqoqqFQqn+oxGo2dBnYnJiZi06ZNDEZEAYTBiIgCkkaj8Rg4LBYLEhISUFBQ0KkzdKbttYchX0NR+2tOH9PUVRAjIv/FwddEFFRUKhUKCwthNBrd44S8oVare7Xf3r6eiPwDgxERBZ32QdftA6G94UuXyNOl+z3pMhGR/2EwIiLyUWVlZbcTPhJR4GIwIqKgYzAYAAApKSl9sr3Tu0EWi4WnzoiCFIMREQUVo9GInJwcaDQabNiwodPyrmaw7m5m6+TkZFRWVrofm0ymDmHJbDZ7fH1XzxOR/+I8RkQUMFJTU2EymWA0GgG0doTaOzftN4kF0Gk+IZPJBL1ej/LyclgsFqSkpGDFihUAgIKCgg4dpqSkJI9Xu+n1ehiNRiQnJ0Or1UKn03ncbl5eHjQaTYfndTod8vLy+qyDRUT9h8GIiIiIqA1PpRERERG1YTAiIiIiasNgRERERNSGwYiIiIioDYMRERERURsGIyIiIqI2DEZEREREbRiMiIiIiNowGBERERG1YTAiIiIiasNgRERERNSGwYiIiIioDYMRERERUZv/D17Yk+MSziGQAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from scipy.spatial import distance_matrix\n",
    "import gudhi as gd\n",
    "\n",
    "def compute_distances_and_plot_barcode(output):\n",
    "    # Detach the output tensor, squeeze, and convert to numpy array\n",
    "    output_np = output.squeeze().detach().numpy()\n",
    "\n",
    "    # Compute the pairwise Euclidean distance matrix\n",
    "    distances = distance_matrix(output_np, output_np)\n",
    "\n",
    "    # Compute the persistent homology of the distance matrix\n",
    "    rips_complex = gd.RipsComplex(distance_matrix=distances, max_edge_length=np.max(distances))\n",
    "    simplex_tree = rips_complex.create_simplex_tree(max_dimension=2)\n",
    "    persistent_homology = simplex_tree.persistence(min_persistence=0.001)\n",
    "    \n",
    "    # Plot the barcode diagram\n",
    "    gd.plot_persistence_barcode(persistence=persistent_homology)\n",
    "    gd.plot_persistence_diagram(persistence=persistent_homology)\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "# Compute the barcode diagrams for each context vector\n",
    "for i, output in enumerate(context):\n",
    "    print(f\"Barcode for text {i}:\")\n",
    "    compute_distances_and_plot_barcode(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Clusters for text 0:\n",
      "Cluster 0: E, S\n",
      "Cluster 1: I, P\n",
      "Cluster 2: K, K\n",
      "Cluster 3: F, R, N\n",
      "Cluster 4: T, N\n",
      "Cluster 5: G, R, Q\n",
      "Cluster 6: R, M, V, A\n",
      "Cluster 7: E, E, L, Y, D, K, L, E, A\n",
      "Cluster 8: G, W, M, L\n",
      "Cluster 9: I, S, K, V\n",
      "Cluster 10: E, L, T\n",
      "Cluster 11: R, K\n",
      "Cluster 12: F, A, V, F, T, D, V, R, G, R, K, E, T, M, S, G, R, G, Q, I\n",
      "Cluster 13: F, V, G\n",
      "Cluster 14: T, L, K, Q, I\n",
      "Cluster 15: A, M\n",
      "Positions of the tokens in cluster 7 of text 0: [45, 55, 56, 59, 63, 67, 72, 75, 80]\n"
     ]
    }
   ],
   "source": [
    "def cluster_and_get_positions(context, sentence, eps):\n",
    "    # Load pre-trained model tokenizer\n",
    "    tokenizer = AutoTokenizer.from_pretrained(\"facebook/esm2_t6_8M_UR50D\")\n",
    "\n",
    "    # Tokenize input and convert to tensor\n",
    "    inputs = tokenizer(sentence, return_tensors=\"pt\")\n",
    "    tokens = tokenizer.convert_ids_to_tokens(inputs['input_ids'][0])\n",
    "\n",
    "    # Squeeze the context tensor to remove the batch size dimension\n",
    "    context = context.squeeze(0)\n",
    "\n",
    "    # Compute the pairwise distance matrix\n",
    "    distances = pdist(context.detach().numpy(), 'euclidean')\n",
    "    dist_matrix = squareform(distances)\n",
    "\n",
    "    # Run DBSCAN on the distance matrix\n",
    "    clustering = DBSCAN(eps=eps, min_samples=2, metric='precomputed').fit(dist_matrix)\n",
    "\n",
    "    # Get the words corresponding to each cluster\n",
    "    clusters = {}\n",
    "    for i, label in enumerate(clustering.labels_):\n",
    "        if label != -1:  # Ignore noise (-1 label)\n",
    "            if label not in clusters:\n",
    "                clusters[label] = []\n",
    "            clusters[label].append((tokens[i], i))  # Store both the token and its position\n",
    "\n",
    "    return clusters\n",
    "\n",
    "# Compute the clusters for each context vector\n",
    "clusters = []\n",
    "for i, output in enumerate(context):\n",
    "    print(f\"Clusters for text {i}:\")\n",
    "    clusters.append(cluster_and_get_positions(output, text[i], 0.05))\n",
    "    for label, words in clusters[-1].items():\n",
    "        print(f\"Cluster {label}: {', '.join(word for word, _ in words)}\")\n",
    "\n",
    "# Assuming you want to get the positions of the tokens in the first cluster of the first text:\n",
    "cluster_number = 7  # Change this to the desired cluster number\n",
    "text_number = 0  # Change this to the desired text number\n",
    "\n",
    "cluster_positions = [position for _, position in clusters[text_number][cluster_number]]\n",
    "print(f\"Positions of the tokens in cluster {cluster_number} of text {text_number}: {cluster_positions}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/amelieschreiber/anaconda3/envs/env_1/lib/python3.10/site-packages/umap/distances.py:1063: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\n",
      "  @numba.jit()\n",
      "/Users/amelieschreiber/anaconda3/envs/env_1/lib/python3.10/site-packages/umap/distances.py:1071: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\n",
      "  @numba.jit()\n",
      "/Users/amelieschreiber/anaconda3/envs/env_1/lib/python3.10/site-packages/umap/distances.py:1086: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\n",
      "  @numba.jit()\n",
      "/Users/amelieschreiber/anaconda3/envs/env_1/lib/python3.10/site-packages/umap/umap_.py:660: NumbaDeprecationWarning: The 'nopython' keyword argument was not supplied to the 'numba.jit' decorator. The implicit default value for this argument is currently False, but it will be changed to True in Numba 0.59.0. See https://numba.readthedocs.io/en/stable/reference/deprecation.html#deprecation-of-object-mode-fall-back-behaviour-when-using-jit for details.\n",
      "  @numba.jit()\n",
      "Some weights of the model checkpoint at facebook/esm2_t6_8M_UR50D were not used when initializing EsmModel: ['lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight']\n",
      "- This IS expected if you are initializing EsmModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing EsmModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of EsmModel were not initialized from the model checkpoint at facebook/esm2_t6_8M_UR50D and are newly initialized: ['esm.pooler.dense.bias', 'esm.pooler.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "hovertemplate": "%{text}<br>x: %{x}<br>y: %{y}<br>z: %{z}<extra></extra>",
         "marker": {
          "color": [
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "red",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "red",
           "red",
           "blue",
           "blue",
           "red",
           "blue",
           "blue",
           "blue",
           "red",
           "blue",
           "blue",
           "blue",
           "red",
           "blue",
           "blue",
           "blue",
           "blue",
           "red",
           "blue",
           "blue",
           "red",
           "blue",
           "blue",
           "blue",
           "blue",
           "red",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue",
           "blue"
          ],
          "opacity": 0.8,
          "size": 6
         },
         "mode": "markers+text",
         "text": [
          "<cls>",
          "M",
          "A",
          "V",
          "E",
          "S",
          "R",
          "V",
          "T",
          "Q",
          "E",
          "E",
          "I",
          "K",
          "K",
          "E",
          "P",
          "E",
          "K",
          "P",
          "I",
          "D",
          "R",
          "E",
          "K",
          "T",
          "C",
          "P",
          "L",
          "L",
          "L",
          "R",
          "V",
          "F",
          "T",
          "T",
          "N",
          "N",
          "G",
          "R",
          "H",
          "H",
          "R",
          "M",
          "D",
          "E",
          "F",
          "S",
          "R",
          "G",
          "N",
          "V",
          "P",
          "S",
          "S",
          "E",
          "L",
          "Q",
          "I",
          "Y",
          "T",
          "W",
          "M",
          "D",
          "A",
          "T",
          "L",
          "K",
          "E",
          "L",
          "T",
          "S",
          "L",
          "V",
          "K",
          "E",
          "V",
          "Y",
          "P",
          "E",
          "A",
          "R",
          "K",
          "K",
          "G",
          "T",
          "H",
          "F",
          "N",
          "F",
          "A",
          "I",
          "V",
          "F",
          "T",
          "D",
          "V",
          "K",
          "R",
          "P",
          "G",
          "Y",
          "R",
          "V",
          "K",
          "E",
          "I",
          "G",
          "S",
          "T",
          "M",
          "S",
          "G",
          "R",
          "K",
          "G",
          "T",
          "D",
          "D",
          "S",
          "M",
          "T",
          "L",
          "Q",
          "S",
          "Q",
          "K",
          "F",
          "Q",
          "I",
          "G",
          "D",
          "Y",
          "L",
          "D",
          "I",
          "A",
          "I",
          "T",
          "P",
          "P",
          "N",
          "R",
          "A",
          "P",
          "P",
          "P",
          "S",
          "G",
          "R",
          "M",
          "R",
          "P",
          "Y",
          "<eos>"
         ],
         "textposition": "top center",
         "type": "scatter3d",
         "x": [
          4.411383628845215,
          4.261977195739746,
          4.392644882202148,
          4.734272480010986,
          4.769791603088379,
          4.649461269378662,
          4.794907093048096,
          4.811542987823486,
          6.0583176612854,
          4.906784534454346,
          5.240900039672852,
          4.757221221923828,
          5.782926082611084,
          5.053470611572266,
          5.118185043334961,
          4.8761677742004395,
          4.938422679901123,
          5.616135120391846,
          5.312045574188232,
          5.884782791137695,
          6.374864101409912,
          6.771600723266602,
          6.016113758087158,
          6.910985469818115,
          6.715892314910889,
          7.171413898468018,
          7.1046037673950195,
          7.393794059753418,
          8.121615409851074,
          8.158243179321289,
          8.12824821472168,
          7.906938552856445,
          9.383389472961426,
          8.394587516784668,
          7.307471752166748,
          6.927516460418701,
          7.206298351287842,
          7.266345977783203,
          9.752724647521973,
          8.963190078735352,
          8.932022094726562,
          9.869566917419434,
          9.918144226074219,
          8.89867115020752,
          8.552406311035156,
          10.218825340270996,
          8.310997009277344,
          8.59717082977295,
          8.176279067993164,
          9.297839164733887,
          8.434951782226562,
          8.994851112365723,
          11.142617225646973,
          7.535225868225098,
          7.769137859344482,
          10.6376953125,
          10.314889907836914,
          9.752923011779785,
          13.731164932250977,
          10.517448425292969,
          9.975714683532715,
          9.216532707214355,
          9.174031257629395,
          10.378198623657227,
          9.035929679870605,
          12.577595710754395,
          9.296031951904297,
          10.379305839538574,
          11.023796081542969,
          11.077993392944336,
          11.386513710021973,
          13.780444145202637,
          11.131712913513184,
          14.142032623291016,
          13.783469200134277,
          10.732548713684082,
          13.890162467956543,
          10.830118179321289,
          12.339578628540039,
          11.53538703918457,
          11.033476829528809,
          12.104974746704102,
          12.192614555358887,
          11.934015274047852,
          11.723852157592773,
          12.370122909545898,
          14.22390365600586,
          12.703964233398438,
          13.840466499328613,
          14.073990821838379,
          13.547469139099121,
          14.220906257629395,
          13.044044494628906,
          12.568277359008789,
          12.59412956237793,
          13.325590133666992,
          13.329209327697754,
          12.287896156311035,
          13.233041763305664,
          11.697957038879395,
          13.596353530883789,
          14.157186508178711,
          13.694485664367676,
          13.946681022644043,
          12.964884757995605,
          13.618572235107422,
          14.113073348999023,
          14.149556159973145,
          13.644670486450195,
          13.093169212341309,
          12.713753700256348,
          12.624853134155273,
          13.034579277038574,
          12.61851692199707,
          11.478065490722656,
          12.85824966430664,
          11.680900573730469,
          11.927082061767578,
          14.362547874450684,
          13.06910228729248,
          12.147058486938477,
          12.747450828552246,
          12.439325332641602,
          12.990850448608398,
          11.920165061950684,
          11.773752212524414,
          12.241394996643066,
          12.10995864868164,
          12.735440254211426,
          13.230216979980469,
          13.085736274719238,
          12.099489212036133,
          12.940214157104492,
          12.366557121276855,
          14.201607704162598,
          13.247294425964355,
          12.2105073928833,
          13.010538101196289,
          11.093401908874512,
          10.856698036193848,
          11.171971321105957,
          7.61086368560791,
          7.251285076141357,
          6.497309684753418,
          6.683168888092041,
          6.90441370010376,
          6.9570393562316895,
          6.558549404144287,
          7.0260114669799805,
          6.571019172668457,
          6.56279182434082,
          6.279689788818359,
          6.353679180145264,
          6.572298526763916,
          5.331867694854736
         ],
         "y": [
          4.857485771179199,
          4.875953197479248,
          4.838873386383057,
          4.737726211547852,
          4.836273670196533,
          4.764159679412842,
          5.010009765625,
          4.836610317230225,
          4.41487979888916,
          4.836765766143799,
          4.578332424163818,
          4.835362434387207,
          4.39690637588501,
          4.628750801086426,
          4.693629264831543,
          4.75277042388916,
          4.690720081329346,
          4.4802775382995605,
          4.683549404144287,
          4.474151134490967,
          4.650174617767334,
          4.629171848297119,
          4.385787487030029,
          4.594334125518799,
          4.541998863220215,
          4.6776933670043945,
          4.518528461456299,
          4.537079811096191,
          5.040521621704102,
          5.003102779388428,
          5.12337064743042,
          4.8853678703308105,
          6.0625224113464355,
          5.198259353637695,
          4.597901821136475,
          4.530838966369629,
          4.594007968902588,
          4.570807933807373,
          6.273087978363037,
          5.904477596282959,
          6.0616960525512695,
          6.626455307006836,
          6.4462571144104,
          5.6113481521606445,
          5.579581260681152,
          6.912614822387695,
          5.12699031829834,
          5.5027546882629395,
          5.066412448883057,
          6.287569522857666,
          5.211466312408447,
          5.827439785003662,
          5.071749687194824,
          4.689990043640137,
          4.774523735046387,
          7.046632289886475,
          6.84635066986084,
          6.294205188751221,
          8.696049690246582,
          6.65125036239624,
          6.5472025871276855,
          6.3951096534729,
          6.131375312805176,
          6.90805196762085,
          5.833986759185791,
          6.475183963775635,
          6.175530910491943,
          6.733231067657471,
          7.146113395690918,
          7.298374176025391,
          7.153674125671387,
          8.750155448913574,
          6.78106164932251,
          8.741077423095703,
          8.51101016998291,
          6.772523403167725,
          8.373659133911133,
          7.01343297958374,
          6.549942970275879,
          6.985647201538086,
          6.652022361755371,
          6.755451202392578,
          6.696870803833008,
          5.087790012359619,
          5.12483549118042,
          5.5160136222839355,
          8.122221946716309,
          6.227904796600342,
          7.86792516708374,
          8.281354904174805,
          7.316521644592285,
          8.557917594909668,
          7.099841117858887,
          5.822708606719971,
          6.373116493225098,
          7.549694538116455,
          7.51945686340332,
          5.404947757720947,
          7.408537864685059,
          5.116483688354492,
          7.505741119384766,
          8.733698844909668,
          7.450501918792725,
          8.167959213256836,
          6.55832576751709,
          7.5551652908325195,
          8.719094276428223,
          8.15284252166748,
          8.044528007507324,
          6.525834083557129,
          6.074439525604248,
          5.886309623718262,
          6.913792133331299,
          6.723166465759277,
          5.176705360412598,
          6.364447116851807,
          5.427403926849365,
          5.8123369216918945,
          8.026925086975098,
          6.916174411773682,
          5.721227169036865,
          6.090010166168213,
          6.085327625274658,
          6.218930244445801,
          5.127193450927734,
          5.147997856140137,
          5.934356689453125,
          5.2655510902404785,
          6.297248840332031,
          6.838284492492676,
          6.345782279968262,
          5.2557220458984375,
          6.477237224578857,
          6.193104267120361,
          8.173452377319336,
          6.563352584838867,
          6.127187728881836,
          6.362199306488037,
          5.150081634521484,
          5.0088348388671875,
          5.074838161468506,
          4.65645694732666,
          4.590746879577637,
          4.549127101898193,
          4.676983833312988,
          4.549035549163818,
          4.5045247077941895,
          4.570952892303467,
          4.5863776206970215,
          4.645554542541504,
          4.581942558288574,
          4.556068420410156,
          4.5354132652282715,
          4.64667272567749,
          4.650947570800781
         ],
         "z": [
          6.187458515167236,
          5.925792217254639,
          6.150866985321045,
          6.284329414367676,
          6.300168514251709,
          6.379268169403076,
          6.37214994430542,
          6.348655700683594,
          5.228100776672363,
          6.167233467102051,
          5.496236801147461,
          5.891770362854004,
          5.4560441970825195,
          5.574088096618652,
          5.6066389083862305,
          5.751473426818848,
          6.004197120666504,
          5.465081214904785,
          5.600400447845459,
          5.504980564117432,
          5.253580570220947,
          4.873586654663086,
          5.12158203125,
          4.457869052886963,
          4.959831714630127,
          4.7253241539001465,
          4.1567254066467285,
          4.409948348999023,
          4.479203224182129,
          4.514943599700928,
          4.108669757843018,
          4.303625583648682,
          3.713106870651245,
          4.40137243270874,
          4.1962714195251465,
          4.380023956298828,
          4.1720404624938965,
          4.144286632537842,
          3.549551486968994,
          3.750911235809326,
          3.564957857131958,
          3.6322858333587646,
          3.5157692432403564,
          4.122339725494385,
          3.851323127746582,
          3.708047389984131,
          4.572625637054443,
          3.987550973892212,
          4.256307125091553,
          3.5552072525024414,
          4.284063816070557,
          3.858525276184082,
          5.350409984588623,
          4.4978461265563965,
          4.409011363983154,
          3.787820339202881,
          3.603074550628662,
          3.728699207305908,
          5.628878593444824,
          3.9957518577575684,
          3.613722085952759,
          3.4875881671905518,
          3.7019636631011963,
          3.74245023727417,
          3.8581249713897705,
          4.884286880493164,
          3.886329174041748,
          3.9390506744384766,
          4.004863262176514,
          4.070321083068848,
          4.247096538543701,
          5.537318229675293,
          4.129886627197266,
          5.784828186035156,
          5.761058330535889,
          4.089244842529297,
          5.606592178344727,
          3.892185926437378,
          4.748161792755127,
          4.382997512817383,
          4.213508605957031,
          4.835324287414551,
          4.78026008605957,
          5.1728692054748535,
          5.315349102020264,
          5.194177627563477,
          5.540780544281006,
          5.090002059936523,
          5.721306324005127,
          5.96365213394165,
          5.588197231292725,
          5.8383307456970215,
          5.537153244018555,
          5.128084182739258,
          4.947596549987793,
          5.671204090118408,
          5.461419105529785,
          5.225284099578857,
          5.570232391357422,
          5.377300262451172,
          5.6366448402404785,
          5.780428886413574,
          5.696938514709473,
          5.9363694190979,
          5.37377405166626,
          5.615688323974609,
          5.841353416442871,
          5.882073402404785,
          5.907803535461426,
          5.534685134887695,
          5.124826431274414,
          5.42155647277832,
          5.820542812347412,
          6.012542247772217,
          5.729637145996094,
          5.573471546173096,
          6.094079494476318,
          6.301614284515381,
          5.81210470199585,
          6.0657639503479,
          6.275187015533447,
          6.2236127853393555,
          6.327643871307373,
          5.911229610443115,
          5.614792346954346,
          5.73298978805542,
          6.424143314361572,
          5.569307804107666,
          6.108835220336914,
          5.876501083374023,
          5.99790096282959,
          5.685413837432861,
          6.030956745147705,
          6.42616081237793,
          5.9155120849609375,
          5.9519453048706055,
          6.514693260192871,
          6.325925827026367,
          5.9574737548828125,
          5.919246196746826,
          5.876471996307373,
          5.8481340408325195,
          5.964435577392578,
          6.237210273742676,
          5.859179496765137,
          5.938483238220215,
          5.97833776473999,
          5.693751811981201,
          5.909275531768799,
          6.133834362030029,
          6.132380485534668,
          5.95139741897583,
          5.930861473083496,
          5.8679633140563965,
          6.122185707092285
         ]
        }
       ],
       "layout": {
        "margin": {
         "b": 10,
         "l": 10,
         "r": 20,
         "t": 10
        },
        "scene": {
         "xaxis": {
          "title": {
           "text": "X AXIS"
          }
         },
         "yaxis": {
          "title": {
           "text": "Y AXIS"
          }
         },
         "zaxis": {
          "title": {
           "text": "Z AXIS"
          }
         }
        },
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "width": 700
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e96a6787c1946ad977c2bfcf1827f02",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(FloatSlider(value=0.0, description='eps:', max=1.6711198494853368, step=0.01), Output())…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<function __main__.update_plot(eps)>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from scipy.spatial import distance_matrix\n",
    "from transformers import AutoTokenizer, EsmModel\n",
    "from umap.umap_ import UMAP\n",
    "import plotly.graph_objects as go\n",
    "from ipywidgets import interact, FloatSlider\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"facebook/esm2_t6_8M_UR50D\")\n",
    "\n",
    "def compute_output(sentence, layer, head):\n",
    "    # Load pre-trained model\n",
    "    global tokenizer\n",
    "    model = EsmModel.from_pretrained(\"facebook/esm2_t6_8M_UR50D\")\n",
    "\n",
    "    # Set the output_attentions attribute of the configuration to True\n",
    "    model.config.output_attentions = True\n",
    "\n",
    "    # Tokenize input and convert to tensor\n",
    "    inputs = tokenizer(sentence, return_tensors=\"pt\")\n",
    "\n",
    "    # Forward pass\n",
    "    outputs = model(**inputs, output_hidden_states=True)\n",
    "\n",
    "    # Obtain the attention weights\n",
    "    attentions = outputs.attentions\n",
    "\n",
    "    # Obtain the attention weights for the specific layer and head\n",
    "    S = attentions[layer][0, head]\n",
    "\n",
    "    # Obtain the value vectors\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        hidden_states = outputs.hidden_states[layer]\n",
    "        all_W_v = model.encoder.layer[layer].attention.self.value.weight\n",
    "        num_heads = model.config.num_attention_heads\n",
    "        head_dim = model.config.hidden_size // num_heads\n",
    "        W_v_heads = all_W_v.view(num_heads, head_dim, model.config.hidden_size)\n",
    "        W_v = W_v_heads[head]\n",
    "        V = torch.matmul(hidden_states, W_v.t())\n",
    "\n",
    "    # Compute the output O\n",
    "    O = torch.matmul(S, V)\n",
    "\n",
    "    return O, inputs\n",
    "\n",
    "# Compute the output\n",
    "output, inputs = compute_output(\"MAVESRVTQEEIKKEPEKPIDREKTCPLLLRVFTTNNGRHHRMDEFSRGNVPSSELQIYTWMDATLKELTSLVKEVYPEARKKGTHFNFAIVFTDVKRPGYRVKEIGSTMSGRKGTDDSMTLQSQKFQIGDYLDIAITPPNRAPPPSGRMRPY\", 3, 2)\n",
    "\n",
    "# Convert the output tensor to numpy array\n",
    "output_np = output.detach().numpy()[0]\n",
    "\n",
    "# Compute the pairwise Euclidean distance matrix\n",
    "distances = distance_matrix(output_np, output_np)\n",
    "\n",
    "# Compute the number of tokens\n",
    "num_tokens = len(inputs[\"input_ids\"][0])\n",
    "\n",
    "# Decompose output to 3D for visualization using UMAP\n",
    "umap = UMAP(n_components=3)  # Set n_components to 3\n",
    "output_3d = umap.fit_transform(output_np)\n",
    "\n",
    "# Define the function to update the plot\n",
    "def update_plot(eps):\n",
    "    # Create a plotly graph object\n",
    "    fig = go.Figure()\n",
    "\n",
    "    # Get token labels\n",
    "    token_labels = [tokenizer.decode([token_id]) for token_id in inputs['input_ids'][0]]\n",
    "    \n",
    "    # Set the colors for the tokens\n",
    "    subset_positions = [45, 55, 56, 59, 63, 67, 72, 75, 80]\n",
    "    colors = ['red' if i in subset_positions else 'blue' for i in range(len(token_labels))]\n",
    "\n",
    "    # Add 3D scatter plot points\n",
    "    fig.add_trace(go.Scatter3d(x=output_3d[:, 0], y=output_3d[:, 1], z=output_3d[:, 2],\n",
    "                    mode='markers+text',   # Added 'text' to 'mode' to show the token labels\n",
    "                    text=token_labels,     # Specify the text(labels) to be displayed\n",
    "                    textposition='top center',   # Position the text to be top center wrt the marker\n",
    "                    marker=dict(\n",
    "                        size=6,\n",
    "                        color=colors,   # Specify the colors for the markers\n",
    "                        opacity=0.8\n",
    "                    ),\n",
    "                    hovertemplate = \"%{text}<br>x: %{x}<br>y: %{y}<br>z: %{z}<extra></extra>\",\n",
    "    ))\n",
    "\n",
    "    # Add edges\n",
    "    for i in range(distances.shape[0]):\n",
    "        for j in range(i+1, distances.shape[1]):\n",
    "            if distances[i, j] <= eps:\n",
    "                fig.add_trace(go.Scatter3d(x=[output_3d[i, 0], output_3d[j, 0]],\n",
    "                                        y=[output_3d[i, 1], output_3d[j, 1]], \n",
    "                                        z=[output_3d[i, 2], output_3d[j, 2]],\n",
    "                                        mode='lines',\n",
    "                                        line=dict(\n",
    "                                            color='darkblue',\n",
    "                                            width=2\n",
    "                                        )))\n",
    "\n",
    "    fig.update_layout(scene = dict(\n",
    "                    xaxis_title='X AXIS',\n",
    "                    yaxis_title='Y AXIS',\n",
    "                    zaxis_title='Z AXIS'),\n",
    "                    width=700,\n",
    "                    margin=dict(r=20, l=10, b=10, t=10))\n",
    "    fig.show()\n",
    "\n",
    "\n",
    "\n",
    "# Create the interactive widget for the threshold distance parameter\n",
    "eps_slider = FloatSlider(min=0, max=np.max(distances), step=0.01, value=0, description='eps:')\n",
    "interact(update_plot, eps=eps_slider)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}