01_mln-baselines/movielens/movielens-mln-ml-1m.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# MovieLens MLN Recommendation via PyTorch\n",
    "\n",
    "adapted from https://github.com/fastai/fastai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import math\n",
    "import copy\n",
    "import pickle\n",
    "from pathlib import Path\n",
    "from itertools import zip_longest\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "\n",
    "import time\n",
    "\n",
    "figure_path = '/home/weiss/git/thesis/doc/figures/'"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "def set_random_seed(state=1):\n",
    "    gens = (np.random.seed, torch.manual_seed, torch.cuda.manual_seed)\n",
    "    for set_state in gens:\n",
    "        set_state(state)\n",
    "\n",
    "RANDOM_STATE = 2021\n",
    "set_random_seed(RANDOM_STATE)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "def read_data(path):\n",
    "    files = {}\n",
    "    path = Path(path)\n",
    "    for filename in path.glob('*'):\n",
    "        if filename.suffix == '.csv':\n",
    "            files[filename.stem] = pd.read_csv(filename)\n",
    "        elif filename.suffix == '.dat':\n",
    "            if filename.stem == 'ratings':\n",
    "                columns = ['userId', 'movieId', 'rating', 'timestamp']\n",
    "            else:\n",
    "                columns = ['movieId', 'title', 'genres']\n",
    "            data = pd.read_csv(filename, sep='::', names=columns, engine='python')\n",
    "            files[filename.stem] = data\n",
    "        elif filename.suffix == '.data':  # ml-100k\n",
    "            if filename.stem == 'u':\n",
    "                columns = ['userId', 'movieId', 'rating', 'timestamp']\n",
    "                data = pd.read_csv(filename, sep='\\t', names=columns, engine='python')\n",
    "                files['ratings'] = data\n",
    "        elif filename.suffix == '.item':  # ml-100k\n",
    "            if filename.stem == 'u':\n",
    "                columns = ['movieId', 'title', 'genres']\n",
    "                data = pd.read_csv(filename, sep='|', names=columns, engine='python')\n",
    "                files['movies'] = data\n",
    "    return files['ratings'], files['movies']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "dataset = 'ml-1m'\n",
    "\n",
    "# pick one of the available folders\n",
    "ratings, movies = read_data('/home/weiss/rs_data/'+dataset)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "   userId  movieId  rating  timestamp\n0       1     1193       5  978300760\n1       1      661       3  978302109\n2       1      914       3  978301968\n3       1     3408       4  978300275\n4       1     2355       5  978824291",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>userId</th>\n      <th>movieId</th>\n      <th>rating</th>\n      <th>timestamp</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>1193</td>\n      <td>5</td>\n      <td>978300760</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>661</td>\n      <td>3</td>\n      <td>978302109</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>914</td>\n      <td>3</td>\n      <td>978301968</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>3408</td>\n      <td>4</td>\n      <td>978300275</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>2355</td>\n      <td>5</td>\n      <td>978824291</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "   movieId                               title                        genres\n0        1                    Toy Story (1995)   Animation|Children's|Comedy\n1        2                      Jumanji (1995)  Adventure|Children's|Fantasy\n2        3             Grumpier Old Men (1995)                Comedy|Romance\n3        4            Waiting to Exhale (1995)                  Comedy|Drama\n4        5  Father of the Bride Part II (1995)                        Comedy",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>movieId</th>\n      <th>title</th>\n      <th>genres</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>Toy Story (1995)</td>\n      <td>Animation|Children's|Comedy</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>Jumanji (1995)</td>\n      <td>Adventure|Children's|Fantasy</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>Grumpier Old Men (1995)</td>\n      <td>Comedy|Romance</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>4</td>\n      <td>Waiting to Exhale (1995)</td>\n      <td>Comedy|Drama</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>5</td>\n      <td>Father of the Bride Part II (1995)</td>\n      <td>Comedy</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embeddings: 6040 users, 3706 movies\n",
      "Dataset shape: (1000209, 2)\n",
      "Target shape: (1000209,)\n"
     ]
    }
   ],
   "source": [
    "def create_dataset(ratings, top=None):\n",
    "    if top is not None:\n",
    "        ratings.groupby('userId')['rating'].count()\n",
    "\n",
    "    unique_users = ratings.userId.unique()\n",
    "    user_to_index = {old: new for new, old in enumerate(unique_users)}\n",
    "    new_users = ratings.userId.map(user_to_index)\n",
    "\n",
    "    unique_movies = ratings.movieId.unique()\n",
    "    movie_to_index = {old: new for new, old in enumerate(unique_movies)}\n",
    "    new_movies = ratings.movieId.map(movie_to_index)\n",
    "\n",
    "    n_users = unique_users.shape[0]\n",
    "    n_movies = unique_movies.shape[0]\n",
    "\n",
    "    X = pd.DataFrame({'user_id': new_users, 'movie_id': new_movies})\n",
    "    y = ratings['rating'].astype(np.float32)\n",
    "    return (n_users, n_movies), (X, y), (user_to_index, movie_to_index)\n",
    "\n",
    "(n, m), (X, y), _ = create_dataset(ratings)\n",
    "print(f'Embeddings: {n} users, {m} movies')\n",
    "print(f'Dataset shape: {X.shape}')\n",
    "print(f'Target shape: {y.shape}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.4, random_state=RANDOM_STATE)\n",
    "X_valid, X_test, y_valid, y_test = train_test_split(X_valid, y_valid, test_size=0.5, random_state=RANDOM_STATE)\n",
    "datasets = {'train': (X_train, y_train), 'val': (X_valid, y_valid), 'test': (X_test, y_test)}\n",
    "dataset_sizes = {'train': len(X_train), 'val': len(X_valid), 'test': len(X_test)}\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "class RatingsIterator:\n",
    "\n",
    "    def __init__(self, X, y, batch_size=32, shuffle=True):\n",
    "        X, y = np.asarray(X), np.asarray(y)\n",
    "        if shuffle:\n",
    "            index = np.random.permutation(X.shape[0])\n",
    "            X, y = X[index], y[index]\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.batch_size = batch_size\n",
    "        self.shuffle = shuffle\n",
    "        self.n_batches = int(math.ceil(X.shape[0] // batch_size))\n",
    "        self._current = 0\n",
    "\n",
    "    def __iter__(self):\n",
    "        return self\n",
    "\n",
    "    def __next__(self):\n",
    "        return self.next()\n",
    "\n",
    "    def next(self):\n",
    "        if self._current >= self.n_batches:\n",
    "            raise StopIteration()\n",
    "        k = self._current\n",
    "        self._current += 1\n",
    "        bs = self.batch_size\n",
    "        return self.X[k*bs:(k + 1)*bs], self.y[k*bs:(k + 1)*bs]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "def batches(X, y, bs=32, shuffle=True):\n",
    "    for xb, yb in RatingsIterator(X, y, bs, shuffle):\n",
    "        xb = torch.LongTensor(xb)\n",
    "        yb = torch.FloatTensor(yb)\n",
    "        yield xb, yb.view(-1, 1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "class RecommenderNet(nn.Module):\n",
    "    \"\"\"\n",
    "    Creates dense MLN with embedding layers.\n",
    "\n",
    "    Args:\n",
    "        n_users:\n",
    "            Number of unique users in the dataset.\n",
    "\n",
    "        n_movies:\n",
    "            Number of unique movies in the dataset.\n",
    "\n",
    "        n_factors:\n",
    "            Number of columns in the embeddings matrix.\n",
    "\n",
    "        embedding_dropout:\n",
    "            Dropout rate to apply right after embeddings layer.\n",
    "\n",
    "        hidden:\n",
    "            A single integer or a list of integers defining the number of\n",
    "            units in hidden layer(s).\n",
    "\n",
    "        dropouts:\n",
    "            A single integer or a list of integers defining the dropout\n",
    "            layers rates applied right after each of hidden layers.\n",
    "\n",
    "    \"\"\"\n",
    "    def __init__(self, n_users, n_movies,\n",
    "                 n_factors=50, embedding_dropout=0.02,\n",
    "                 hidden=10, dropouts=0.2):\n",
    "\n",
    "        super().__init__()\n",
    "        hidden = get_list(hidden)\n",
    "        dropouts = get_list(dropouts)\n",
    "        n_last = hidden[-1]\n",
    "\n",
    "        def gen_layers(n_in):\n",
    "            \"\"\"\n",
    "            A generator that yields a sequence of hidden layers and\n",
    "            their activations/dropouts.\n",
    "\n",
    "            Note that the function captures `hidden` and `dropouts`\n",
    "            values from the outer scope.\n",
    "            \"\"\"\n",
    "            nonlocal hidden, dropouts\n",
    "            assert len(dropouts) <= len(hidden)\n",
    "\n",
    "            for n_out, rate in zip_longest(hidden, dropouts):\n",
    "                yield nn.Linear(n_in, n_out)\n",
    "                yield nn.ReLU()\n",
    "                if rate is not None and rate > 0.:\n",
    "                    yield nn.Dropout(rate)\n",
    "                n_in = n_out\n",
    "\n",
    "        self.u = nn.Embedding(n_users, n_factors)\n",
    "        self.m = nn.Embedding(n_movies, n_factors)\n",
    "        self.drop = nn.Dropout(embedding_dropout)\n",
    "        self.hidden = nn.Sequential(*list(gen_layers(n_factors * 2)))\n",
    "        self.fc = nn.Linear(n_last, 1)\n",
    "        self.relu = nn.ReLU()\n",
    "        self._init()\n",
    "\n",
    "    def forward(self, users, movies, minmax=None):\n",
    "        features = torch.cat([self.u(users), self.m(movies)], dim=1)\n",
    "        x = self.drop(features)\n",
    "        x = self.hidden(x)\n",
    "        out = torch.sigmoid(self.fc(x))\n",
    "        #out = self.relu(self.fc(x))  # relu delivers worse rsme\n",
    "        if minmax is not None:\n",
    "            min_rating, max_rating = minmax\n",
    "            out = out*(max_rating - min_rating + 1) + min_rating - 0.5\n",
    "        return out\n",
    "\n",
    "    def _init(self):\n",
    "        \"\"\"\n",
    "        Setup embeddings and hidden layers with reasonable initial values.\n",
    "        \"\"\"\n",
    "\n",
    "        def init(m):\n",
    "            if type(m) == nn.Linear:\n",
    "                torch.nn.init.xavier_uniform_(m.weight)\n",
    "                m.bias.data.fill_(0.01)\n",
    "\n",
    "        self.u.weight.data.uniform_(-0.05, 0.05)\n",
    "        self.m.weight.data.uniform_(-0.05, 0.05)\n",
    "        self.hidden.apply(init)\n",
    "        init(self.fc)\n",
    "\n",
    "\n",
    "def get_list(n):\n",
    "    if isinstance(n, (int, float)):\n",
    "        return [n]\n",
    "    elif hasattr(n, '__iter__'):\n",
    "        return list(n)\n",
    "    raise TypeError('layers configuraiton should be a single number or a list of numbers')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [],
   "source": [
    "def plot_lr(schedule):\n",
    "    ts = list(range(1000))\n",
    "    y = [schedule(t, 0.001) for t in ts]\n",
    "    plt.plot(ts, y)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "(1.0, 5.0)"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "minmax = float(ratings.rating.min()), float(ratings.rating.max())\n",
    "minmax"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [],
   "source": [
    "# small net\n",
    "\n",
    "start = time.time()\n",
    "net = RecommenderNet(\n",
    "    n_users=n, n_movies=m,\n",
    "    n_factors=10, hidden=[10],\n",
    "    embedding_dropout=0.05, dropouts=[0.3])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [],
   "source": [
    "lr = 1e-3\n",
    "wd = 1e-5\n",
    "bs = 4096\n",
    "n_epochs = 300\n",
    "patience = 300\n",
    "no_improvements = 0\n",
    "best_loss = np.inf\n",
    "best_weights = None\n",
    "history = []\n",
    "lr_history = []\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "net.to(device)\n",
    "criterion = nn.MSELoss(reduction='sum')\n",
    "optimizer = optim.Adam(net.parameters(), lr=lr, weight_decay=wd)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss improvement on epoch: 1\n",
      "[001/300] train: 1.1976 - val: 0.9426\n",
      "loss improvement on epoch: 2\n",
      "[002/300] train: 0.9012 - val: 0.8667\n",
      "loss improvement on epoch: 3\n",
      "[003/300] train: 0.8622 - val: 0.8519\n",
      "loss improvement on epoch: 4\n",
      "[004/300] train: 0.8468 - val: 0.8432\n",
      "loss improvement on epoch: 5\n",
      "[005/300] train: 0.8368 - val: 0.8390\n",
      "loss improvement on epoch: 6\n",
      "[006/300] train: 0.8290 - val: 0.8346\n",
      "loss improvement on epoch: 7\n",
      "[007/300] train: 0.8233 - val: 0.8299\n",
      "[008/300] train: 0.8203 - val: 0.8301\n",
      "loss improvement on epoch: 9\n",
      "[009/300] train: 0.8160 - val: 0.8289\n",
      "loss improvement on epoch: 10\n",
      "[010/300] train: 0.8138 - val: 0.8273\n",
      "loss improvement on epoch: 11\n",
      "[011/300] train: 0.8118 - val: 0.8265\n",
      "[012/300] train: 0.8099 - val: 0.8266\n",
      "loss improvement on epoch: 13\n",
      "[013/300] train: 0.8088 - val: 0.8258\n",
      "[014/300] train: 0.8074 - val: 0.8259\n",
      "loss improvement on epoch: 15\n",
      "[015/300] train: 0.8059 - val: 0.8238\n",
      "[016/300] train: 0.8056 - val: 0.8245\n",
      "[017/300] train: 0.8046 - val: 0.8250\n",
      "[018/300] train: 0.8028 - val: 0.8240\n",
      "[019/300] train: 0.8018 - val: 0.8239\n",
      "loss improvement on epoch: 20\n",
      "[020/300] train: 0.8001 - val: 0.8232\n",
      "[021/300] train: 0.7989 - val: 0.8242\n",
      "loss improvement on epoch: 22\n",
      "[022/300] train: 0.7990 - val: 0.8232\n",
      "loss improvement on epoch: 23\n",
      "[023/300] train: 0.7984 - val: 0.8230\n",
      "loss improvement on epoch: 24\n",
      "[024/300] train: 0.7969 - val: 0.8230\n",
      "[025/300] train: 0.7962 - val: 0.8234\n",
      "[026/300] train: 0.7947 - val: 0.8236\n",
      "[027/300] train: 0.7939 - val: 0.8237\n",
      "[028/300] train: 0.7932 - val: 0.8231\n",
      "loss improvement on epoch: 29\n",
      "[029/300] train: 0.7919 - val: 0.8223\n",
      "[030/300] train: 0.7919 - val: 0.8233\n",
      "[031/300] train: 0.7903 - val: 0.8241\n",
      "[032/300] train: 0.7891 - val: 0.8233\n",
      "[033/300] train: 0.7882 - val: 0.8228\n",
      "[034/300] train: 0.7879 - val: 0.8232\n",
      "loss improvement on epoch: 35\n",
      "[035/300] train: 0.7881 - val: 0.8217\n",
      "[036/300] train: 0.7873 - val: 0.8227\n",
      "loss improvement on epoch: 37\n",
      "[037/300] train: 0.7863 - val: 0.8214\n",
      "[038/300] train: 0.7863 - val: 0.8250\n",
      "[039/300] train: 0.7856 - val: 0.8225\n",
      "[040/300] train: 0.7844 - val: 0.8231\n",
      "[041/300] train: 0.7839 - val: 0.8233\n",
      "loss improvement on epoch: 42\n",
      "[042/300] train: 0.7839 - val: 0.8213\n",
      "[043/300] train: 0.7834 - val: 0.8225\n",
      "[044/300] train: 0.7826 - val: 0.8230\n",
      "[045/300] train: 0.7813 - val: 0.8230\n",
      "[046/300] train: 0.7811 - val: 0.8226\n",
      "[047/300] train: 0.7806 - val: 0.8235\n",
      "[048/300] train: 0.7801 - val: 0.8217\n",
      "[049/300] train: 0.7796 - val: 0.8233\n",
      "[050/300] train: 0.7790 - val: 0.8219\n",
      "[051/300] train: 0.7792 - val: 0.8222\n",
      "[052/300] train: 0.7785 - val: 0.8214\n",
      "[053/300] train: 0.7785 - val: 0.8215\n",
      "[054/300] train: 0.7771 - val: 0.8239\n",
      "[055/300] train: 0.7774 - val: 0.8236\n",
      "[056/300] train: 0.7774 - val: 0.8227\n",
      "[057/300] train: 0.7767 - val: 0.8232\n",
      "[058/300] train: 0.7773 - val: 0.8241\n",
      "[059/300] train: 0.7763 - val: 0.8219\n",
      "[060/300] train: 0.7754 - val: 0.8243\n",
      "[061/300] train: 0.7748 - val: 0.8235\n",
      "[062/300] train: 0.7753 - val: 0.8240\n",
      "[063/300] train: 0.7747 - val: 0.8256\n",
      "[064/300] train: 0.7738 - val: 0.8231\n",
      "[065/300] train: 0.7747 - val: 0.8238\n",
      "[066/300] train: 0.7734 - val: 0.8229\n",
      "[067/300] train: 0.7731 - val: 0.8233\n",
      "[068/300] train: 0.7741 - val: 0.8242\n",
      "[069/300] train: 0.7730 - val: 0.8239\n",
      "[070/300] train: 0.7730 - val: 0.8237\n",
      "[071/300] train: 0.7735 - val: 0.8253\n",
      "[072/300] train: 0.7717 - val: 0.8242\n",
      "[073/300] train: 0.7720 - val: 0.8236\n",
      "[074/300] train: 0.7714 - val: 0.8245\n",
      "[075/300] train: 0.7720 - val: 0.8255\n",
      "[076/300] train: 0.7709 - val: 0.8252\n",
      "[077/300] train: 0.7721 - val: 0.8249\n",
      "[078/300] train: 0.7718 - val: 0.8242\n",
      "[079/300] train: 0.7711 - val: 0.8223\n",
      "[080/300] train: 0.7705 - val: 0.8239\n",
      "[081/300] train: 0.7706 - val: 0.8246\n",
      "[082/300] train: 0.7705 - val: 0.8242\n",
      "[083/300] train: 0.7701 - val: 0.8253\n",
      "[084/300] train: 0.7699 - val: 0.8238\n",
      "[085/300] train: 0.7704 - val: 0.8248\n",
      "[086/300] train: 0.7703 - val: 0.8258\n",
      "[087/300] train: 0.7702 - val: 0.8255\n",
      "[088/300] train: 0.7703 - val: 0.8252\n",
      "[089/300] train: 0.7698 - val: 0.8253\n",
      "[090/300] train: 0.7687 - val: 0.8249\n",
      "[091/300] train: 0.7688 - val: 0.8283\n",
      "[092/300] train: 0.7702 - val: 0.8269\n",
      "[093/300] train: 0.7688 - val: 0.8258\n",
      "[094/300] train: 0.7691 - val: 0.8271\n",
      "[095/300] train: 0.7696 - val: 0.8255\n",
      "[096/300] train: 0.7690 - val: 0.8232\n",
      "[097/300] train: 0.7693 - val: 0.8264\n",
      "[098/300] train: 0.7691 - val: 0.8258\n",
      "[099/300] train: 0.7692 - val: 0.8256\n",
      "[100/300] train: 0.7690 - val: 0.8252\n",
      "[101/300] train: 0.7697 - val: 0.8269\n",
      "[102/300] train: 0.7678 - val: 0.8269\n",
      "[103/300] train: 0.7679 - val: 0.8262\n",
      "[104/300] train: 0.7686 - val: 0.8265\n",
      "[105/300] train: 0.7692 - val: 0.8257\n",
      "[106/300] train: 0.7677 - val: 0.8259\n",
      "[107/300] train: 0.7676 - val: 0.8270\n",
      "[108/300] train: 0.7690 - val: 0.8242\n",
      "[109/300] train: 0.7688 - val: 0.8254\n",
      "[110/300] train: 0.7677 - val: 0.8253\n",
      "[111/300] train: 0.7679 - val: 0.8252\n",
      "[112/300] train: 0.7669 - val: 0.8265\n",
      "[113/300] train: 0.7668 - val: 0.8264\n",
      "[114/300] train: 0.7681 - val: 0.8268\n",
      "[115/300] train: 0.7671 - val: 0.8260\n",
      "[116/300] train: 0.7680 - val: 0.8264\n",
      "[117/300] train: 0.7669 - val: 0.8276\n",
      "[118/300] train: 0.7672 - val: 0.8254\n",
      "[119/300] train: 0.7674 - val: 0.8278\n",
      "[120/300] train: 0.7683 - val: 0.8286\n",
      "[121/300] train: 0.7666 - val: 0.8265\n",
      "[122/300] train: 0.7671 - val: 0.8259\n",
      "[123/300] train: 0.7663 - val: 0.8254\n",
      "[124/300] train: 0.7667 - val: 0.8272\n",
      "[125/300] train: 0.7662 - val: 0.8275\n",
      "[126/300] train: 0.7681 - val: 0.8259\n",
      "[127/300] train: 0.7671 - val: 0.8259\n",
      "[128/300] train: 0.7663 - val: 0.8268\n",
      "[129/300] train: 0.7658 - val: 0.8262\n",
      "[130/300] train: 0.7658 - val: 0.8277\n",
      "[131/300] train: 0.7665 - val: 0.8260\n",
      "[132/300] train: 0.7656 - val: 0.8286\n",
      "[133/300] train: 0.7657 - val: 0.8274\n",
      "[134/300] train: 0.7657 - val: 0.8278\n",
      "[135/300] train: 0.7660 - val: 0.8272\n",
      "[136/300] train: 0.7661 - val: 0.8282\n",
      "[137/300] train: 0.7661 - val: 0.8280\n",
      "[138/300] train: 0.7654 - val: 0.8269\n",
      "[139/300] train: 0.7660 - val: 0.8288\n",
      "[140/300] train: 0.7650 - val: 0.8262\n",
      "[141/300] train: 0.7656 - val: 0.8283\n",
      "[142/300] train: 0.7651 - val: 0.8278\n",
      "[143/300] train: 0.7653 - val: 0.8295\n",
      "[144/300] train: 0.7655 - val: 0.8277\n",
      "[145/300] train: 0.7652 - val: 0.8266\n",
      "[146/300] train: 0.7657 - val: 0.8284\n",
      "[147/300] train: 0.7654 - val: 0.8270\n",
      "[148/300] train: 0.7648 - val: 0.8279\n",
      "[149/300] train: 0.7648 - val: 0.8251\n",
      "[150/300] train: 0.7644 - val: 0.8265\n",
      "[151/300] train: 0.7659 - val: 0.8269\n",
      "[152/300] train: 0.7633 - val: 0.8270\n",
      "[153/300] train: 0.7655 - val: 0.8277\n",
      "[154/300] train: 0.7643 - val: 0.8271\n",
      "[155/300] train: 0.7653 - val: 0.8284\n",
      "[156/300] train: 0.7649 - val: 0.8268\n",
      "[157/300] train: 0.7649 - val: 0.8276\n",
      "[158/300] train: 0.7650 - val: 0.8272\n",
      "[159/300] train: 0.7652 - val: 0.8313\n",
      "[160/300] train: 0.7644 - val: 0.8295\n",
      "[161/300] train: 0.7649 - val: 0.8293\n",
      "[162/300] train: 0.7632 - val: 0.8270\n",
      "[163/300] train: 0.7643 - val: 0.8272\n",
      "[164/300] train: 0.7632 - val: 0.8298\n",
      "[165/300] train: 0.7631 - val: 0.8301\n",
      "[166/300] train: 0.7644 - val: 0.8277\n",
      "[167/300] train: 0.7645 - val: 0.8282\n",
      "[168/300] train: 0.7639 - val: 0.8286\n",
      "[169/300] train: 0.7645 - val: 0.8288\n",
      "[170/300] train: 0.7636 - val: 0.8292\n",
      "[171/300] train: 0.7648 - val: 0.8290\n",
      "[172/300] train: 0.7640 - val: 0.8279\n",
      "[173/300] train: 0.7637 - val: 0.8287\n",
      "[174/300] train: 0.7642 - val: 0.8268\n",
      "[175/300] train: 0.7637 - val: 0.8295\n",
      "[176/300] train: 0.7642 - val: 0.8259\n",
      "[177/300] train: 0.7640 - val: 0.8264\n",
      "[178/300] train: 0.7640 - val: 0.8284\n",
      "[179/300] train: 0.7637 - val: 0.8275\n",
      "[180/300] train: 0.7626 - val: 0.8286\n",
      "[181/300] train: 0.7640 - val: 0.8275\n",
      "[182/300] train: 0.7638 - val: 0.8292\n",
      "[183/300] train: 0.7627 - val: 0.8290\n",
      "[184/300] train: 0.7641 - val: 0.8302\n",
      "[185/300] train: 0.7643 - val: 0.8293\n",
      "[186/300] train: 0.7635 - val: 0.8282\n",
      "[187/300] train: 0.7638 - val: 0.8267\n",
      "[188/300] train: 0.7636 - val: 0.8297\n",
      "[189/300] train: 0.7641 - val: 0.8288\n",
      "[190/300] train: 0.7642 - val: 0.8310\n",
      "[191/300] train: 0.7629 - val: 0.8271\n",
      "[192/300] train: 0.7635 - val: 0.8310\n",
      "[193/300] train: 0.7633 - val: 0.8279\n",
      "[194/300] train: 0.7634 - val: 0.8287\n",
      "[195/300] train: 0.7639 - val: 0.8279\n",
      "[196/300] train: 0.7623 - val: 0.8275\n",
      "[197/300] train: 0.7641 - val: 0.8283\n",
      "[198/300] train: 0.7626 - val: 0.8293\n",
      "[199/300] train: 0.7631 - val: 0.8298\n",
      "[200/300] train: 0.7634 - val: 0.8278\n",
      "[201/300] train: 0.7622 - val: 0.8287\n",
      "[202/300] train: 0.7629 - val: 0.8274\n",
      "[203/300] train: 0.7629 - val: 0.8271\n",
      "[204/300] train: 0.7634 - val: 0.8289\n",
      "[205/300] train: 0.7626 - val: 0.8285\n",
      "[206/300] train: 0.7614 - val: 0.8301\n",
      "[207/300] train: 0.7623 - val: 0.8278\n",
      "[208/300] train: 0.7615 - val: 0.8282\n",
      "[209/300] train: 0.7633 - val: 0.8265\n",
      "[210/300] train: 0.7629 - val: 0.8290\n",
      "[211/300] train: 0.7628 - val: 0.8280\n",
      "[212/300] train: 0.7625 - val: 0.8288\n",
      "[213/300] train: 0.7626 - val: 0.8294\n",
      "[214/300] train: 0.7630 - val: 0.8274\n",
      "[215/300] train: 0.7623 - val: 0.8290\n",
      "[216/300] train: 0.7617 - val: 0.8294\n",
      "[217/300] train: 0.7620 - val: 0.8275\n",
      "[218/300] train: 0.7622 - val: 0.8287\n",
      "[219/300] train: 0.7627 - val: 0.8299\n",
      "[220/300] train: 0.7625 - val: 0.8293\n",
      "[221/300] train: 0.7614 - val: 0.8272\n",
      "[222/300] train: 0.7623 - val: 0.8284\n",
      "[223/300] train: 0.7634 - val: 0.8277\n",
      "[224/300] train: 0.7629 - val: 0.8270\n",
      "[225/300] train: 0.7615 - val: 0.8285\n",
      "[226/300] train: 0.7614 - val: 0.8285\n",
      "[227/300] train: 0.7630 - val: 0.8310\n",
      "[228/300] train: 0.7623 - val: 0.8289\n",
      "[229/300] train: 0.7637 - val: 0.8295\n",
      "[230/300] train: 0.7609 - val: 0.8277\n",
      "[231/300] train: 0.7630 - val: 0.8297\n",
      "[232/300] train: 0.7621 - val: 0.8282\n",
      "[233/300] train: 0.7616 - val: 0.8311\n",
      "[234/300] train: 0.7622 - val: 0.8278\n",
      "[235/300] train: 0.7619 - val: 0.8294\n",
      "[236/300] train: 0.7625 - val: 0.8285\n",
      "[237/300] train: 0.7624 - val: 0.8295\n",
      "[238/300] train: 0.7623 - val: 0.8301\n",
      "[239/300] train: 0.7624 - val: 0.8282\n",
      "[240/300] train: 0.7621 - val: 0.8296\n",
      "[241/300] train: 0.7619 - val: 0.8316\n",
      "[242/300] train: 0.7614 - val: 0.8306\n",
      "[243/300] train: 0.7619 - val: 0.8286\n",
      "[244/300] train: 0.7614 - val: 0.8293\n",
      "[245/300] train: 0.7626 - val: 0.8289\n",
      "[246/300] train: 0.7629 - val: 0.8280\n",
      "[247/300] train: 0.7619 - val: 0.8277\n",
      "[248/300] train: 0.7625 - val: 0.8297\n",
      "[249/300] train: 0.7625 - val: 0.8291\n",
      "[250/300] train: 0.7608 - val: 0.8309\n",
      "[251/300] train: 0.7612 - val: 0.8325\n",
      "[252/300] train: 0.7605 - val: 0.8293\n",
      "[253/300] train: 0.7617 - val: 0.8281\n",
      "[254/300] train: 0.7614 - val: 0.8294\n",
      "[255/300] train: 0.7612 - val: 0.8305\n",
      "[256/300] train: 0.7603 - val: 0.8297\n",
      "[257/300] train: 0.7619 - val: 0.8298\n",
      "[258/300] train: 0.7609 - val: 0.8295\n",
      "[259/300] train: 0.7599 - val: 0.8294\n",
      "[260/300] train: 0.7612 - val: 0.8306\n",
      "[261/300] train: 0.7611 - val: 0.8298\n",
      "[262/300] train: 0.7612 - val: 0.8289\n",
      "[263/300] train: 0.7606 - val: 0.8301\n",
      "[264/300] train: 0.7623 - val: 0.8300\n",
      "[265/300] train: 0.7612 - val: 0.8309\n",
      "[266/300] train: 0.7609 - val: 0.8294\n",
      "[267/300] train: 0.7615 - val: 0.8289\n",
      "[268/300] train: 0.7602 - val: 0.8291\n",
      "[269/300] train: 0.7614 - val: 0.8298\n",
      "[270/300] train: 0.7611 - val: 0.8290\n",
      "[271/300] train: 0.7633 - val: 0.8286\n",
      "[272/300] train: 0.7616 - val: 0.8313\n",
      "[273/300] train: 0.7617 - val: 0.8300\n",
      "[274/300] train: 0.7604 - val: 0.8295\n",
      "[275/300] train: 0.7621 - val: 0.8298\n",
      "[276/300] train: 0.7593 - val: 0.8288\n",
      "[277/300] train: 0.7602 - val: 0.8276\n",
      "[278/300] train: 0.7613 - val: 0.8300\n",
      "[279/300] train: 0.7604 - val: 0.8285\n",
      "[280/300] train: 0.7616 - val: 0.8302\n",
      "[281/300] train: 0.7610 - val: 0.8298\n",
      "[282/300] train: 0.7603 - val: 0.8324\n",
      "[283/300] train: 0.7610 - val: 0.8286\n",
      "[284/300] train: 0.7607 - val: 0.8307\n",
      "[285/300] train: 0.7616 - val: 0.8303\n",
      "[286/300] train: 0.7606 - val: 0.8289\n",
      "[287/300] train: 0.7610 - val: 0.8322\n",
      "[288/300] train: 0.7604 - val: 0.8287\n",
      "[289/300] train: 0.7605 - val: 0.8316\n",
      "[290/300] train: 0.7615 - val: 0.8292\n",
      "[291/300] train: 0.7619 - val: 0.8294\n",
      "[292/300] train: 0.7600 - val: 0.8296\n",
      "[293/300] train: 0.7607 - val: 0.8286\n",
      "[294/300] train: 0.7597 - val: 0.8318\n",
      "[295/300] train: 0.7611 - val: 0.8273\n",
      "[296/300] train: 0.7611 - val: 0.8311\n",
      "[297/300] train: 0.7595 - val: 0.8305\n",
      "[298/300] train: 0.7612 - val: 0.8307\n",
      "[299/300] train: 0.7599 - val: 0.8306\n",
      "[300/300] train: 0.7598 - val: 0.8301\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(n_epochs):\n",
    "    stats = {'epoch': epoch + 1, 'total': n_epochs}\n",
    "\n",
    "    for phase in ('train', 'val'):\n",
    "        if phase == 'train':\n",
    "          training = True\n",
    "        else:\n",
    "          training = False\n",
    "        running_loss = 0\n",
    "        n_batches = 0\n",
    "        for batch in batches(*datasets[phase], shuffle=training, bs=bs):\n",
    "            x_batch, y_batch = [b.to(device) for b in batch]\n",
    "            optimizer.zero_grad()\n",
    "            # compute gradients only during 'train' phase\n",
    "            with torch.set_grad_enabled(training):\n",
    "                outputs = net(x_batch[:,0], x_batch[:,1], minmax)\n",
    "                loss = criterion(outputs, y_batch)\n",
    "                # don't update weights and rates when in 'val' phase\n",
    "                if training:\n",
    "                    #scheduler.step()\n",
    "                    loss.backward()\n",
    "                    optimizer.step()\n",
    "                    #lr_history.extend(scheduler.get_lr())\n",
    "            running_loss += loss.item()\n",
    "        epoch_loss = running_loss / dataset_sizes[phase]\n",
    "        stats[phase] = epoch_loss\n",
    "        # early stopping: save weights of the best model so far\n",
    "        if phase == 'val':\n",
    "            if epoch_loss < best_loss:\n",
    "                print('loss improvement on epoch: %d' % (epoch + 1))\n",
    "                best_loss = epoch_loss\n",
    "                best_weights = copy.deepcopy(net.state_dict())\n",
    "                no_improvements = 0\n",
    "            else:\n",
    "                no_improvements += 1\n",
    "    history.append(stats)\n",
    "    with open(f'logs/small.txt', 'a') as f:\n",
    "        print('[{epoch:03d}/{total:03d}] train: {train:.4f} - val: {val:.4f}'.format(**stats))\n",
    "        f.write('[{epoch:03d}/{total:03d}] train: {train:.4f} - val: {val:.4f}'.format(**stats))\n",
    "    if no_improvements >= patience:\n",
    "        print('early stopping after epoch {epoch:03d}'.format(**stats))\n",
    "        break"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "data": {
      "text/plain": "<All keys matched successfully>"
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.load_state_dict(best_weights)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "groud_truth, predictions = [], []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for batch in batches(*datasets['val'], shuffle=False, bs=bs):\n",
    "        x_batch, y_batch = [b.to(device) for b in batch]\n",
    "        outputs = net(x_batch[:, 0], x_batch[:, 1], minmax)\n",
    "        groud_truth.extend(y_batch.tolist())\n",
    "        predictions.extend(outputs.tolist())\n",
    "\n",
    "groud_truth = np.asarray(groud_truth).ravel()\n",
    "predictions = np.asarray(predictions).ravel()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation RMSE: 0.9145\n"
     ]
    }
   ],
   "source": [
    "valid_loss = np.sqrt(np.mean((predictions - groud_truth)**2))\n",
    "print(f'Validation RMSE: {valid_loss:.4f}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "groud_truth, predictions = [], []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for batch in batches(*datasets['test'], shuffle=False, bs=bs):\n",
    "        x_batch, y_batch = [b.to(device) for b in batch]\n",
    "        outputs = net(x_batch[:, 0], x_batch[:, 1], minmax)\n",
    "        groud_truth.extend(y_batch.tolist())\n",
    "        predictions.extend(outputs.tolist())\n",
    "\n",
    "groud_truth = np.asarray(groud_truth).ravel()\n",
    "predictions = np.asarray(predictions).ravel()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Small Test RMSE: 0.9168\n"
     ]
    }
   ],
   "source": [
    "final_loss = np.sqrt(np.mean((predictions - groud_truth)**2))\n",
    "print(f'Small Test RMSE: {final_loss:.4f}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Small duration: 92.2848\n"
     ]
    }
   ],
   "source": [
    "print(f'Small duration: {round(time.time() - start, 4)}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAacAAAEfCAYAAADldgmAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8m0lEQVR4nO3deXwV5aH/8c/ZsyeEhMWwSeREVheKuCtaBUFvFVu4KqggdcMfolbBl0ttbcvV1qW9XmyrlSsqKBYsVhSQxeUqqCAqQggQAkEIISH7cnK2+f0x5EBMAlnJAN/365VXkplnZp6Z55z5nme2YzMMw0BERMRC7B1dARERkR9TOImIiOUonERExHIUTiIiYjkKJxERsRyFk4iIWI6zoytwPFm/fn1HV0FE5Lg0dOjQZpVXODVTczcwQGZmJv3792+H2khrqF2sSe1iPa1tk5Z8sNdhPRERsRyFk4iIWI7CSURELEfhJCIilqNwEhERy1E4iYiI5SicRETEchROIiJiOQonERGxHIWTiIhYjsJJREQsR+EkIiKWo3ASERHLUTiJiIjlKJxERMRyFE4iImI5CicREbEchZOIiFiOwklERCxH4SQiIpajcBIREctROImIiOUonEREjkOXXXYZGRkZLFq0qKOr0i6cHV0BERFpvgsuuIADBw7QvXv3jq5Ku1DPSUSklf71r3+RkZHBF198ccyW+eSTTzJ79mzOO++8Y7bMY0nhJCLSSt9++21HV+GEo3ASEWklhVPbUziJiLTQzJkzycjIYNOmTQDcfPPNZGRkMHHiRAAyMjLIyMjgm2++4f3332fUqFEMHDiQ//3f/60zn6+//poHHniAyy67jMGDBzNo0CAuu+wyZs6cybZt2xpcdkMXRGzbto2MjAwGDx4MQFZWFvfddx8XXnghgwYN4vzzz2fatGls3769HbZG21I4iYi00IABA7j88ssj/5999tlcfvnlnH322XXKbdmyhQcffBCXy8WFF15Ily5dIuPeeOMNbrzxRt577z1qamo466yzOPvss6muruadd95h7NixrF27tkn18Xg8kb+/+uorxo8fz/r16zn99NM544wzqKioYNmyZYwfP549e/a0cu3bl67WExFpoZtvvjnSWwKYPn06w4cPr1duzpw5TJ06lbvvvrvO8MLCQv7rv/4LwzD4z//8Tx577DGcTnO37PP5eOihh1i2bBm//vWvWbZs2VHrY7eb/Y1wOMyvfvUr7rrrLn75y19Ghu/Zs4exY8dSUlLC66+/zowZM1q1/u3J8uGUm5vLQw89xIYNG0hLS2PVqlUtms+6deu46aabjlrO7XazcePGFi1D5ESxcP0PLFi3u0OWXVVVRcwnJW0+33E/6cn1Q3u0+XybIhwOc+edd9YbXlpayrhx4ygqKuLuu++OBBNAVFQU9957L8uWLWPnzp3s3LmTPn36NGl5wWCQvn37cscdd9QZnpaWxjXXXMNrr73Ghg0bWrVO7c3S4bRgwQJmzZpFVVVVq+dVVlYGQHR0NOeff36j5VwuV6uXJSJyuEsuuSTSezlceno6jz32WKPT9ezZM/J3YWFhk8MJYPz48Q0OP/XUUwEoLi5u8rw6giXDqbCwkEcffZTVq1eTmJjIVVddxQcffNCqedaGU8+ePZk9e3ZbVFPkhHX90B4d1svIzMykf//+HbLs9tK7d+8jjt+9ezerVq0iOzub0tJSAoFAvTKhUKhNlhkdHQ3Q4DKsxJLhtGjRIlavXs2wYcP44x//yJo1a1odTqWlpQDEx8e3RRVFRJosJiamweGGYfD0008zZ84cDMNo02Ue7/s6S4aT0+lk2rRp3HXXXQ12hVuivLwcOP4bTESOPzabrcHh8+bN45VXXgFg5MiRTJ48mfT0dGJjYyP7vtqLLU42lgynCRMm4Ha723Se6jmJiNW8+eabAJxzzjn8+c9/rhdiFRUVHVEtS7BkOLV1MMGhc04JCQls376d5cuXk5OTQyAQIDU1leHDhzNixAgcDkebL1tEpCE7d+4EzAsmGupdff3118e4RtZhyXBqD7XhtHz5cubNm1fv+O7cuXPxer385S9/iVzNIiLSHM29aMHpdOL3+/H7/fXGBQIBXnjhhRbP+3h30jwhojacCgsLGTduHIsXL2bjxo2sWbOGp556ipSUFLZu3cqkSZMoKirq4NqKyPEkKSkJoNn3SJ5xxhkALF68OHLqASA/P5+pU6cSHR0duepu69atbVPZ48RJ03O66667KC0tJS0tjTPPPDMyPDk5mWuvvZbBgwczduxY8vLyeOmllxq9czozM7PZy/b5fC2aTtqX2sWajsd28Xq9fPnllzz33HO8+eabVFdXM2fOnMj4vXv3NrhOY8aM4YsvvmDnzp1cfvnl9O3bl+rqarKzs+natStPPvkk//jHP9i1axfPPPMM77//Pj/72c84++yzI72tw+edn58fmff27dsjF4Idbu/evQD4/f4mb+eOaJOTJpwuvPDCI45PT09nzJgxLFy4kBUrVjQaTi25/+JEvG/jRKB2sabjsV2eeuopHnnkETZs2EBJSQl9+vSpsw6nnHJKg+vUv39/evTowezZs/n+++/Jysqie/fu3HbbbUyZMoXExER69OjBzJkz2bx5M/v27YvMu/bc/OHzPvyCr9NOO40ePerfq1YbMm63u8nbubVtsn79+mZPc9KEU1MMGjSIhQsXsmfPHsLhcJtdxi4iJ7ZTTjmlTk+pVlZW1lGnPe+88474hYF9+vSJXNV3uIYe5dajR4+jLnPs2LGMHTv2qPXqaNr7NsBmsymYREQ60EmxBy4rK+Pjjz9m/vz5+Hy+RsvVfsdJQ11hERE5dk6Kw3qlpaXcfvvtADgcDsaNG9dgmSVLlgBw6aWXHsvqiYjIj5xQPaf8/HxGjRrFqFGjWLduXWR4z549GTlyJACzZs3io48+qjPd/v37ueeeeygpKSEpKYnJkycfy2qLiMiPWLLn9OMv5MrLywPgwIED9cZNnz4dr9cLmDet5eTkANT7mo3f/va35OXl8d1333HHHXfQu3dvevbsSVVVFRs3biQQCJCcnMwLL7xA165d22vVRESkCSwZTitXrmxwuM/nqzfulltuadI8k5KSmD9/PosWLWLJkiVkZWWxdu1aoqKiyMjI4JJLLmHChAkkJye3uv4iItI6lgynplx+2ZCjXUbpdDoZN25cg+ecRETEOk6oc04iInJiUDiJiIjlKJxERMRyFE4iImI5CicREbEchZOIiFiOwklERCxH4SQiIpajcBIREctROImIiOUonEREjhMTJ04kIyOD//7v/+7oqrQ7hZOIiFiOwklERCxH4SQiIpajcBIREctROImItNCkSZPIyMjgrrvuOmK5u+++m4yMDG677bbIsK1bt/LII48wcuRIhgwZwqBBg7j44ouZNm0aGzZsaO+qW57CSUSkhf7jP/4DgP/7v/+joqKiwTLl5eV8+umnAPzsZz8DYMWKFYwdO5Z//vOfFBcXM2TIEIYNGwbAsmXLuPHGG1m8ePExWAPrUjiJiLTQFVdcQVRUFH6/n1WrVjVYZsWKFfj9fmJjY7niiivw+/08/vjjBAIBRowYwaeffsrrr7/OnDlzWLVqFRMnTiQcDvO73/2OqqqqY7xG1mHJr2kXkQ72zXzY8HqHLLpXVSWsjW37GZ81Ac68oU1nGRcXx4gRI/jggw9YunRppCd1uCVLlgAwcuRIoqOjKSgoYMyYMRQVFTFlyhQ8Hk+krNPp5P777+f111+nrKyMDRs2cMEFF7RpnY8XCicRkVa45ppr+OCDDyKH9uLi4iLjiouLWbNmDXDoEGBqaiqPPPJIo/OLiYkhJSWFgoICCgoK2rfyFqZwEpH6zryhzXsZTZWbmUn//v07ZNktcfHFF5OUlERJSQkfffQRV199dWTc8uXLCQaDdO/eneHDh9eZbv/+/axcuZKsrCxKS0vx+/0YhgGY56kAwuHwsVsRi1E4iYi0gsvlYuTIkbz11lssXbq0Tji9//77gNlrstsPneJ/5ZVXePbZZwkEAse8vscLXRAhItJKtYfsPvnkEyorKwEoLCzkq6++Ag5dpQfw0Ucf8dRTTxEIBBg+fDivvvoqX3zxBZs3byYrK4usrCzS0tKO/UpYjMJJRKSVhg4dSlpaGjU1NXz88ccALF26lFAoxKBBg0hPT4+UffPNNwHo3bs3L7/8Mueeey5JSUk4HI5ImdqAO5kpnEREWslms0UO5y1fvhwwwwng2muvrVN2586dAFxwwQW43e5688rOzqakpKTd6nq8UDiJiLSBa665BjBvyN2/fz/r16/H5XIxZsyYOuVcLhcAfr+/wfk8//zzkb9DoVD7VPY4oHASEWkD/fr14/TTT6e8vJw///nPhMNhLrzwQpKTk+uUGzJkCGDenLt3797I8NLSUh5++GF27NjBT37yEwCysrKO3QpYjK7WExFpI9dccw1btmxh0aJFQP1DegC33XYb77//PiUlJYwZM4YhQ4bg9/vZvHkzcXFxvPLKKyxevJh169bxxhtvsHXrVq699lrGjh17jNemY6nnJCLSRq6++mrsdjvhcJiEhAQuu+yyemX69u3LG2+8waWXXorD4eDrr7+moKCA66+/noULF0YeEHvhhRfi8XjYtm0bNputA9amY6nnJCLSRrp160ZmZuZRyw0YMIC//e1vjY7v3Lkz//jHP+oNf+2111pVv+OJek7HwJz1RTz0z287uhoiIscN9ZyOgdxSP6UHyjq6GiIixw31nI4Bl91GIHTyPiNLRKS5FE7HgNNhwx9UOImINJXC6RhQz0lEpHkUTseA0456TiIizaBwOgZcDht+9ZxERJpM4XQMuOw65yQi0hztEk6GYZCTk0NmZuZJ/eDCWk6dcxIRaZYWh1NOTg7Tpk3jvffeqzP8m2++4YorrmD06NGMHTuWSy65JPL9Jicrl8NG2ICgAkpEpElaFE779+/nlltu4cMPP4x8NwlAUVERt99+O3v27MEwDAzDoLCwkGnTprFr1662qvNxx2k3n4sVCBkdXBMRkeNDi8Jp/vz57N+/n1GjRnHTTTdFhs+bN4+ysjK6du3KwoUL+eKLL5g0aRI1NTUn1TOhfszlMMNJF0WIiDRNi8Jp9erVxMTE8Ic//IFOnTpFhi9duhSbzca9997LwIEDSUxM5IEHHiApKYk1a9a0WaWPN66DW1kXRYiINE2Lwik/P58BAwYQHR0dGZaXl8f27dtxOBxcccUVkeFOp5N+/frV+VKtk01tz0kXRYiINE2Lwqm8vJyYmJg6w9auXQvAGWecQVxcXJ1x0dHRjX4l8cmg9pyTek4iIk3TonCKj48nPz+/zrCPP/4Ym83G+eefX6/8gQMHiI2NbVkNTwDqOYmINE+Lwqlfv35s3bqV7777DoDt27ezcuVKgHrf/FhQUMCWLVvo06dP62p6HKvtOdWo5yQi0iQt+j6na665hi+//JKJEyfi9XrJzs4mGAwyZMgQ+vfvHymXm5vLww8/TDgc5uKLL26zSh9vXHb1nEREmqNFPafrr7+eyy+/nJqaGjZu3EhVVRWdO3fmqaeeqlPuD3/4A+vXr6d3795MnDixTSp8PHI6dM5JRKQ5WtRzstvt/M///A+ff/45mzZtIjExkZEjR5KYmFin3ODBg4mNjeXBBx+sN66pcnNzeeihh9iwYQNpaWmsWrWqRfOplZ+fz8svv8ynn37Kvn37cDgc9OzZk5EjR3LrrbfWuQKxrbh0E66ISLO06mvazz///AYvgKg1derU1syeBQsWMGvWLKqqqlo1n1rr16/n9ttvp6KiguTkZM466yx8Ph8bN24kMzOTxYsX8/rrr5OSktImy6sVuc9JzxkUEWkSSz74tbCwkDvvvJPHHnsMl8vFVVdd1eo6lZeXM3XqVCoqKrj11lv55JNPmDNnDvPnz2fFihWcfvrp5OTk8MADD7R6WT8WeUJEUD0nEZGmsOSDXxctWsTq1asZNmwYixcvbpOLKV599VWKi4s566yzmDlzJi6XKzKuW7duPPvss9hsNtauXcsXX3zR6uUdLnKfky6IEBFpEks++NXpdDJt2jTmzp1L9+7dW1LFepYuXQrAuHHjsNls9canp6czdOhQAN5///02WWatyH1OuiBCRKRJLPng1wkTJjB16lTs9rY56lhWVsa2bdsAIgHUkNpxX331VZsst5Z6TiIizWPJB7+63e6WVKtRO3bsAMyrDNPS0hot17NnT8C8QjAYDLbZ8vWECBGR5jkpHvxaWFgIQGJiIk5n4xcodu7cGYBAIEBpaWmbLd+lZ+uJiDTLSfHg1+rqagA8Hs8Ry0VFRUX+bqvL1wFcDvO3DuuJiDRNi+5zOlEf/GoYR7/UOzMzs9nzDfprANi7bz+ZmYFmTy/tw+fztag9pX2pXaynI9qkReHUr18/vvrqK7777juGDBnSpAe/Dhw4sPW1baHaXp7P5ztiucPHNxamhz87sKkyMzNx2m0kdkqmf//Tmz29tI/MzMwWtae0L7WL9bS2TdavX9/saU6KB7+mpqYCUFpait/vb/SCi4KCAsC8IKOlj1tqjMth1+OLRESa6KR48Gt6ejo2mw3DMNi9e3ej5XJycgDo27cvDoejTevgdtp1QYSISBNZ/sGvbSE2NpZBgwaxceNGvvzyS9LT0xssV/tkiPPOO6/N6+By2HVBhIhIE1n6wa9t6eqrr2bjxo0sWLCA8ePH17vB9+uvv46c8LvmmmvafPluh009JxGRJmqzB79WVFSwd+9e8vLyqKysbKvZNkt+fj6jRo1i1KhRrFu3rs64G264gbS0NDZv3syTTz5Z59L2nJwcZsyYAcDo0aPb/OINZ1U+fez5uglXRKSJWtVz2rx5M6+++iqff/555EbXWqeccgojRozglltuiTx5oanuvvvuOv/n5eUB5iXpPx43ffp0vF4vYN48W3ve6Mf3KXk8Hl588UVuvfVW5s2bx9KlSxkwYACVlZV89913hEIhzjzzTH73u981q65N0eXb/+Ex/07+Evxrm89bRORE1OJweuONN5g1axahUKjB+4P27NnDG2+8waJFi3jmmWcYMWJEk+dde1n6j/l8vnrjbrnllibPNyMjgyVLlvDSSy+xatUq1q1bh8vlYuDAgVxzzTXceOONR3yCREuFndF0C+9Xz0lEpIlatCf+/vvv+d3vfodhGHi9Xq688krS09NJTEzEMAxKS0vJyspi6dKl7Nq1i/vvv5933323yT2orKysllSLHj16HHXa5ORkZsyYETmMdywEo1PobJQSCOgGXBGRpmhROL322msYhsGUKVP41a9+1WCZ0aNHc++99/LEE0+wYMECXn31VR599NFWVfZ4FYzqjB2DmMCBjq6KiMhxoUUXRKxfv57OnTtz//33H3nmdjuPPvooiYmJfPbZZy2q4IkgGG1+7Xu8wklEpElaFE6FhYWcfvrpTfq+JbfbzYABA9i3b19LFnVCCEaZ4RSncBIRaZIWhZPNZiMcbvrJfbvd3qSHqp6oantO9or8k3o7iIg0VYvCqWvXrmzatKlJX4Ph9/v5/vvv6dq1a0sWdUIIRiVjYCMuUMC+siM/fFZERFoYTsOHD6e8vJzf/OY3hEKhRssFg0Eef/xxysrKuOCCC1pcyeOe3UkwKpkulLBpT1lH10ZExPJadLXe5MmTeeedd1i0aBFr165l5MiReL1eOnXqhGEYlJSUsGXLFpYtW8a+ffuIiopi0qRJbV3344o9oTtdKkv4fm8pPx1w8vYiRUSaokXh1Lt3b/70pz8xY8YM9uzZw5w5cxosZxgGsbGxPPvss81+SsSJxpF4ChkF3/PWnrb7+ncRkRNVi5+td+WVV/Luu+8yYcIEevbsiWEYkR+APn36MHnyZJYsWcIll1zSZhU+bg24lp7GXqJ2LKO0SjfjiogcSaue1dOzZ8/IjbU1NTWUlZVhs9lISEho9Av9TlpDxuNf/TTTS15n3idXcdeooR1dIxERy2qzp5J7PB5SU1NJSUlRMDXE4cR93Qv0se/n/DW3k5v5VUfXSETEso7ac2rN98YfzmazsXnz5jaZ13Hr1Isov/pv9HlvOnFvXYlv6C+JuuQ+iEoAd2xH105OZKV7AAMSexwaFg5DTSm448Dhav0ywmHwV5iv51qGAeEglO2FpF5Q+oNZB5ut/vShIDga2SUFqsEZ1fB0jQkFzd+NzbO6xJxvQvdDdTUMCFSCrwwS0+rOK+QHV7S5DsU55vok9a5fp3AYjDDkfw8HtsOpl0BcKoRDUJEPnniw127vg8sEc97+SnM5VQcgNgWiOx1cfgBsdrA7oHinWZ+kXuA8rCMQDh1ch4PtUHDwOaPdBpvLjU2FygLz7+R0KNsD3c88VP+qIijdDV0Hwr6N5rKcUeCKwRasbvp2byNHDSfdNNq2kn7yC9bHnMm2N2cybv3fYf3fzBEJaZDiNd+4To/5onTHQfczoKbMfBPFdzd3IjaH+cKpfbHafvS3w2lOX1EAFfvMN0NMCoQD5nJCfvPNseMjc3kxyeabIrEHONzmG6Om3HxDhPzmmzQcMpftObjjsdmgZDeU55n1dXjMN7Ur1ixX+oMZuEU5EKox18MwIDoJ/FWH3oT+CijJhYRTzHmX/mC+gdyx5k4uJgVS+kHRDqgsNNezqhASe5m/S3abdXFGmfUI1kDmv81pO/eD6mLo1Ad6DoPdX5rL8ySCO5au2esh0wl2p7ntojuZ8yjPM7d1Ui9zJ2V3QuV+c/m1644NfCXmb3cs7P3arFvaT8ydSqAaCrea06b0M7elrxQwzHaJSoR935k7DCNsLrey0Nx20cnmzq/LAHPbGCE4sMNsn5R+5rDYFHNd0842d1RVhVBTYa6fM8rchjHJ5ush6INdn5nLsTnM11nndMhda07nSTTn668w51FTbgaK022+HmK7gA1zW2RcZS6/ZLdZ55oys+2Lcw6+XmrM+qd4zXXPXWP+DtVAXDfz9ZhwMJw8CeZ2qC4yt7mvFDqfRrqvCtb0AU8c7M8061KeZ04X08l8DRduM7d3OAAV+81tE6w5WO4UcxscyDbHJ/eFmM7mdojtYq5nRb4Z2OEAuGLMdXHFmtvDZodA1cHX2AGzfasKzddvTGfz71pRieb298Sbrx+bHfZvNn8bB2+zsTvNOlfkm3VojDMafhwCdpfZBoFKc55RSeb2quVJMOsUk2y+R/xV5rY9fDk2x6G6/JjdZW7f2v1JyG+uk6/uhVv9XHGQsQ3cMY3Xv43ZjKOkz5dfftlmCzvnnHPabF4dYf369Qwd2vxzRZmZmfV6oJ9tL+QvbyxkcGAjZ3Rzc2lyMfHlO6B8n/nCqt2RtbXaF2pDL1iHx1xm6Ag3V9vs5pvIFV33TWolqf3NN1vhNjMMK/YDhll3hyeyAwg5Y3DEpZjBGw6anxzDAfPNXl1s7sxr2RzmDgAOftI1zJ1WsMYM2l7nme22b6PZdnYnpJ5uDivaYYZOVJK5zWNSzLBLG3qobKDaDKqQ/+DONg3yN0Hn08ydU6c+ZnAXZkFiTzMY7U7Y+425Y43rAu74g3XymTvKqgOHPrT0HG7Oo3wf7FlnBmFqBvQYBvmboXyv+WHIE3+oJxXymz9FOeZ62h2wZ7258+98mrkcT4JZ95R+Zj1jOptBVbjN3IZ9LzXrEJ0EO/8Pep1v9ihcMWYI+krM7Rrf3ZzX/s2UVgdI5GBQpvQzX2sJp5jzDFSZ9enUx9xR2l3mdivKNssl9Ta3txE2p3VGQ943Zhu5Y81t6/SYYZaQZoZ8WZ7Zg6ypgPhu5mshNtUMmfhTzGCI62bu9Et/MHsbKaeZ4bd/s7mMmjJzXYI15ocKu8P8ndQbti0z6xTfHZJPNesSDmImPuZ8DcNsr5hkMySjEs0w85WYoe+JN6epLDwUtmV7zGmqDpjD47qa6wPm+sV1MV9fJbmQkmGWt9nNdSzJNetWkGm+J8IB830Q0xl++AoG/Mxso6APAtX8cKCCHj+9s3k918O0ZN951J7T8R4oVnXBaSn0nX4zr/xfDg+tzaX6hxDpqbFc5E3lvPTO9OsSR093Ba6ireaOxuExX6zhoLmDC4cP/g6Zv43wwb/D5hukuth8g8V3Mz8pVhaab5iiHPMTaVUR9DrXPLQR8pufNPdvNsvEn2L2PBxucwd4YLv5hg75zU9mgSpzh5TcF7r0N4cHfYc+sQdrzB1fsMbcqRiGOQ+b3XwTu2MP7QBrd7wV+WZdat9U/iqzbNlec6eUcpr55guHzO1Rutv8P6mXuUGDNeZPOGiu8+Fvoqoi2PM1dDndnH8oADXlbM3ZS/+Bgw6VC4fN6Ww2KM83t1tMsjncHQuuqPoNWXsYxe44bD4H26EtDpUdTU2FuVM+fPlWdfGDTSq2NzOTxDY6ndCu0i9rWrmew9q3HsdAeWZmi4Oppdr+m/WkybonRvPImAFMuagv//52L59sK2T+l7n87+c7AXDabfTqHEPflBrSuzhJT+lDepc4+nePJ8Z9nDVd9yFHHn/4uRA4eKw9zQy/0y6vX75T77r/Oz2NzzsmGfr99ND/Dpc5zJ5ft9zhDzKO7wo04WZpm83sVdWZjwM4RmHhiTs2yxE5xo6zPdyJqWtCFFMu6suUi/riC4TYnFfGjoJKdhRUmL8LK/hkawH+g9+ka7NBr+QYYt1O4jxOMrrF0zc1lj6dY+nRKZpTkqKJ9ahpReT4pT2YxUS5HJzdqxNn9+pUZ3gobPBDcRVb8yvYtLeU7fsr8AXClFb7+deGPZTXBOuUT4x2cUpSNGlJUfRKjqVPSgyJ0S7iPE5iPWaoxXmcdIpxEx/lxG4/tl12EZEjUTgdJxx2G707x9K7cyxX/OjZfIZhUFjhZ9eBSvaUVLO3xMfekmr2llSzu6iaz7YfoDrQ+AN6HXYbnWLcdI51k/yjn85x7kPj4tw47Xai3Q5iXA7iopy4HG12q5yISITC6QRgs9lIjfeQGu/hJw2Mrw2v0uoAlTVBKmuCVNQEKfcFKakOUFRZQ1GlnwMVfoqr/GTuK6Oo0k/JUR6z5HbY6RTrIhAy8DjtOB027DYb3RKi6J4YRYzHiWFAl3gPbqcdfzBMj07RGAZU+oN0jvMQDIVxOez06xqH3WYjyunAHwqRFONmX6kPp8OG22HH43Ic/G03fzvt2I7xCVoROXYUTieBw8OrOYKh8MHwMoOrqNJPMBymyh+i2h8iv8xHSVUAp8OGLxAmFA4TMiC/1Mf63GKqasze2oFK/8F6HLrfsC24HWYgOu02XA47bqed7olR2G02couq6BznIcbtIBg2CIXDBEMGdpsNh92G3W6jqKyS3p+V0SnWTbkvQGl1gE4xbromRJFf5qPCF6RTrIuEKBexHiedY90cqPTjC4RwOeyEDYPEaBflviBlvgAuh53eyTGEDIP4KBfhsEGVP8SuokpS4zx0TYiic1zdp6fUbg8D49DfhtmbTY33UOUPYQMcDlttQYyDtxhEOR3EeJx4nHb2FFdzamosRZV+opwOqgMh/MEwdjs47XaC4TCxbid2m42UePfBbWcnMdrFgYoaiqv8keWbHyoMyn1BOsW4cTpsBEMGUS47gZCBgYHLbifG48Bpt3OgsoZYt1mP/PIaDMMgOdZNKGwQCBkED54rTY33YLPZCIUNfIEQIcMgIcpFaVWAhGgnNput3n2V4bBR75CzYRgt+mBSO+8fTxsOm1vUoUPblqJwkkY5HXZS4jykxHmadOFaYwIhMxhsNsgv82HDRozHQUmVH5fDTrkvSE5hJTYbVPvNHX9RpZ/uiVEYQE3Q3NH6g2FqDv7U/h0MhQmGDQIh8/+9JdWEwgYX9ksxwzRk4LAfDKSDO6VQ2Jwmzm6GzZ6SauI8ThKjXewuquK7H0pJiXOTFOMiu6CSyoO9zIqaILFuB9FuB4GD61PuC5IQ5SQh2kWFLxgJ4sN1S4iiqNIfuaDFSpx2G8Fwyz8xuB32yHq5HDYCocbn5T4Y6IcvLz7KSbkvSIzbgWFAdSBE52gHDuces5fvD9EpxkXP5Bgqa4J4nA5yi6rwBUJEuRxEuRy4DvbYbTaI8zhxOmzUBML4Q+brJM7jxGaDH4qr8QVCdE+MJi0pmv3lPoqrAoTCZlumJUWzt6SalDgP0W4z4J12G/6g+Xpx2m04HfbIhyGnw4bLbq7/7qIqgmGDLvEeAqEwIcMgPdU8GhDtcpBf5sPjspMQ5SLK5WDngUqSol2kdYomv6yGUNjA7bDjcNjIKajEboeUOA/+YJhOsW58/hD+UJjMvDLcDjtdEqLoHOumvCZIYrSLvimx7DpQRdgwCBvmh6LOcR7sNvAFQgRCBv5gmMQYF8FQmCiXg7AB+8t8GAb0TY0lGDaIcTvYW1KNx+kgJc5NXqmPHp1icPnL8GYYxzTAj3oTrhzSljfhSsdrTrsYhkEobL45D//kffineMMw8IfC2G02KmuCkTdyfJQLwzAorjJ7oXDolhEbhz7J2w4bHgwb7C+rIdbjiPz/4/K+QIgqf5Bqf5guCR625JWRGh8FGES7nWavMWQQDIdx2G1U1oQIGwZ5JdWEDLNnXFodIDnWTZeEqMjyy6rNi2uSYlwUV5kBX7vDdznM+8CCoTDlPvMQcVqnaCprQpRU++mdHIvDDkWVAVyOQzvxUNhgT3E1DrsNj9NBlMs8V7mrqIoenaLZX1aD22kert2yax+dkzsR43YSF+WksKKG3UVVxLqd+IIhenSKJiHKhS8QpjoQMnvsYQgbBhU1QUJh8zCz22keAi6pDmAY5hWu0W47uUXV7C/z0SnGTWq8uQMv9wXZV+YjPTWOoko/1YEQ0S4HwXAYj9MRCfFgOBzpDdZ+KLLbbPTuHIPTbmd/uQ+304FhGGQXVGK3QZU/RNcED4GQQdnBQ+s9k2Mo9wXZU1JNaryHKJd52DsQMuiZHIPDBgUVNbgddgor/MS4zSDulRxD2DAorQ6wv6yGKLeDvJJqSqoD9OwUjd1mBnVclJMDFTWRwHHYze1R7gvgstupCZpHNVLjzXrlFlXhcpivkR6doqkOhNhX6iOtUzT7Sn0kuG18NOOnRLladotEu9yEKyJmIDgd9T81Hh5UNpu54wVIinHXK1d7kUlTpac27x6mYX2Sm1XeqjIzw/ow1wzmVxXRrlfcZmZmtjiYWkrhJCJyHLMdPKR5otF1wCIiYjkKJxERsRyFk4iIWI7CSURELEfhJCIilqNwEhERy1E4iYiI5SicRETEchROIiJiOQonERGxHIWTiIhYjsJJREQsR+EkIiKWo3ASERHLUTiJiIjlKJxERMRyFE4iImI5CicREbEchZOIiFiOwklERCxH4SQiIpajcBIREctROImIiOUonERExHKcHV2BxoRCIRYsWMC///1vsrOzqaqqIjU1leHDhzNp0iS8Xm+z5rdu3Tpuuummo5Zzu91s3LixpdUWEZE2YMlwqq6uZsqUKaxbtw6n08mgQYOIi4sjKyuLRYsW8e9//5unn36a0aNHN3meZWVlAERHR3P++ec3Ws7lcrW6/iIi0jqWDKff//73rFu3Dq/Xy4svvkiPHj0ACAaDPPfcc7z88svMmDGDAQMG0KdPnybNszacevbsyezZs9ur6iIi0gYsd85p9+7dLFq0CJvNxvPPPx8JJgCn08mvfvUrzjrrLPx+P3/961+bPN/S0lIA4uPj27zOIiLStiwXTsuXLycUCjFs2DDS09PrjbfZbPz85z8H4MMPP8Tv9zdpvuXl5YDCSUTkeGC5cFq/fj0AZ599dqNlhg4dCkBFRQVbtmxp0nzVcxIROX5Y7pzTjh07AOjVq1ejZXr06IHdbiccDrNjxw6GDBly1PnWnnNKSEhg+/btLF++nJycHAKBQOQqwBEjRuBwONpmRUREpMUsF04HDhwAoHPnzo2WcblcJCQkUFJSQkFBQZPmWxtOy5cvZ968eRiGUWf83Llz8Xq9/OUvf+HUU09tYe1FRKQtWO6wXnV1NQAej+eI5WrHV1VVNWm+teFUWFjIuHHjWLx4MRs3bmTNmjU89dRTpKSksHXrViZNmkRRUVEr1kBERFrLcj2npqrt+dhstiaVv+uuuygtLSUtLY0zzzwzMjw5OZlrr72WwYMHM3bsWPLy8njppZeYMWNGg/PJzMxsdl19Pl+LppP2pXaxJrWL9XREm1gunGJiYigtLcXn8x2xXE1NDQCxsbFNmu+FF154xPHp6emMGTOGhQsXsmLFikbDqX///k1a3uEyMzNbNJ20L7WLNaldrKe1bVJ7oVtzWO6wXmpqKmAefmuM3++PHKarLd8WBg0aBMCePXsIh8NtNl8REWkey4VT7b1NO3fubLTMjh07Iof1+vXr1+Z1sNls2O2W2zQiIicNy+2Bhw8fDsBXX33VaJkvvvgCMM8XNeUBsGVlZXz88cfMnz//iIcLt2/fDlDnqRQiInLsWS6crrzySjweD99++y2bN2+uNz4YDPLWW28BMGbMmCbdl1RaWsrtt9/OE088wbvvvttomSVLlgBw6aWXtnwFRESk1SwXTqmpqdx8880APPDAA+Tm5kbG+f1+fv3rX5OdnU18fDx33nlnnWnz8/MZNWoUo0aNYt26dZHhPXv2ZOTIkQDMmjWLjz76qM50+/fv55577qGkpISkpCQmT57cTmsnIiJNYbmr9QCmT5/O9u3bWb16NVdddRWDBw8mNjaWTZs2UVxcTGxsLC+88AIpKSl1pgsEAuTk5AD173/67W9/S15eHt999x133HEHvXv3pmfPnlRVVbFx40YCgQDJycm88MILdO3a9Zitq4iI1GfJcHI6nbz44ossWrSIRYsWsW3bNqqrq+nSpQtXXXUVU6ZMIS0trVnzTEpKYv78+SxatIglS5aQlZXF2rVriYqKIiMjg0suuYQJEyaQnJzcTmslIiJNZclwAvOKueuvv57rr7++ydP06NGDrKysRsc7nU7GjRvHuHHj2qKKIiLSTix3zklEREThJCIilqNwEhERy1E4iYiI5SicRETEchROIiJiOQonERGxHIWTiIhYjsJJREQsR+EkIiKWo3ASERHLUTiJiIjlKJxERMRyFE4iImI5CicREbEchZOIiFiOwklERCxH4SQiIpajcBIREctROImIiOUonERExHIUTiIiYjkKJxERsRyFk4iIWI7CSURELEfhJCIilqNwEhERy1E4iYiI5SicRETEchROIiJiOTbDMIyOrsTxYv369R1dBRGR49LQoUObVV7hJCIilqPDeiIiYjkKJxERsRyFk4iIWI6zoytwIgqFQixYsIB///vfZGdnU1VVRWpqKsOHD2fSpEl4vd6OruJxLzc3l4ceeogNGzaQlpbGqlWrjjpNdnY2c+bMYc2aNRQUFBAVFcWpp57K1VdfzQ033IDT2fDbQe15dGVlZcydO5dVq1aRk5NDIBAgKSmJwYMHM378eC699NIGp1ObtK+ioiJeffVVPv74Y3bt2hVpl0GDBjF27FiuvPLKBqezQrvogog2Vl1dzZQpU1i3bh1Op5NBgwYRFxdHVlYWBQUFuFwunn76aUaPHt3RVT1uLViwgFmzZlFVVQXQpHBaunQpDz74IH6/n27dunHaaadRVlbG999/Tzgc5qyzzmLOnDlER0fXmU7teXRZWVlMmTKF/fv343K58Hq9xMfHk52dTUFBAQA33XQTjz/+eJ3p1Cbta8OGDdx5552UlJQQFRWF1+slJiamTrtcffXVPP300zgcjsh0lmkXQ9rUI488Yni9XuPqq682du/eHRkeCASMp59+2vB6vcagQYOMnJycjqvkcaqgoMC44447DK/XawwbNsy49957Da/Xa4wYMeKI0+3atcsYPHiw4fV6jTlz5hihUCgybuvWrcall15qeL1e4+GHH643rdrzyCorK40RI0YYXq/XuP76643c3NzIuEAgYDz77LOG1+s1vF6vsWzZssg4tUn7KikpMc4//3zD6/UakydPNgoKCiLjAoGA8ec//znSLvPmzYuMs1K7KJzaUG5urtG/f38jIyPD2L59e73x4XDYGD9+vOH1eo0ZM2Z0QA2Pb3/7298Mr9dr3HTTTcbevXuNhQsXNimcZs6caXi9XuP+++9vcPwnn3xieL1eo3///sauXbsiw9WeR7dgwQLD6/UaAwYMMPbu3VtvfDgcNq699lrD6/Uad911V2S42qR9vfrqq4bX6zWGDh1qlJWVNVjm5z//ueH1eo0bbrghMsxK7aILItrQ8uXLCYVCDBs2jPT09HrjbTYbP//5zwH48MMP8fv9x7qKxzWn08m0adOYO3cu3bt3b9I0wWCQFStWADBu3LgGy1x00UWccsophEIhli5dGhmu9jy62NhYRo8ezS9+8YsG28RmszFkyBAAdu7cCahNjoXOnTszduxYbr31VuLj4xssc9ZZZwGwb98+wHrtonBqQ7VPkDj77LMbLVN7l3RFRQVbtmw5JvU6UUyYMIGpU6ditzf9Zbtt2zbKyspwOByceeaZjZarbbOvvvoqMkzteXSjR4/mueee44knnmi0TDAYBMDtdgNqk2NhzJgxzJo1i3vuuafRMoFAAIBu3boB1msXhVMb2rFjBwC9evVqtEyPHj0iO9fa8tI0tTu35qjdxl27dsXj8TRarmfPnnXKH/632rPlgsEgn332GQA/+clPALWJFZSUlPDhhx8CcMUVVwDWaxeFUxs6cOAAYHapG+NyuUhISACIXDEj7aewsBCA5OTkI5arbbPa8qD2bAt//etfycvLw+12c/PNNwNqk44SDofJy8vjvffeY/z48RQUFHDFFVcwceJEwHrtovuc2lB1dTXAET91HD6+9lJoaT9NbZOoqCgAfD4f4XAYu92u9mylhQsX8sILLwDw4IMPRj5Vq02OvbvvvpuVK1dG/j/vvPN48MEH+elPfxoZZrV2UTh1AOPgrWU2m62DayK1jFbc7qf2rO/FF1/k+eefB2Dy5MmRXlNzqE3azhlnnIFhGFRUVLB9+3bWrl3LgQMH8Pv9zb4f7Fi1i8KpDcXExFBaWorP5ztiuZqaGsC80knaV0xMDMBR26R2fExMTOS4uNqz+QKBAL/5zW94++23sdls3Hfffdxxxx11yqhNjr3D2yAcDrN06VIee+wx7rvvPrKysrjvvvss1y4659SGUlNTgbrHYn/M7/dTVlZWp7y0n6a0CRw6Bn54m6g9m6e8vJxf/vKXvP3220RFRfHss8/WCyZQm3Q0u93O6NGjeeyxxwB46aWXyMvLs1y7KJzaUO31/bX3czRkx44dka5tv379jkW1TmqnnXYaYL6hKisrGy2Xk5NTpzyoPZujqqqKX/7yl6xZs4bU1FRef/31Rg8XqU2sofZ5h6FQiG+++cZy7aJwakPDhw8H6l7//2NffPEFYF4Ro4dTtr/09HRSUlIIh8ONfpNxMBhk3bp1gHmiuJbas2n8fj9Tp05lw4YN9O7dm7feeovBgwc3Wl5t0v7uvPNORo4cyezZsxstU9uLAXA4HJZrF4VTG7ryyivxeDx8++23bN68ud74YDDIW2+9BZg3yR3+sEVpH7WHMADefPPNBst88MEHFBcX43a7GTlyZGS42rNpnn76aT7//HO6du3Ka6+9Rlpa2hHLq03an9PpZOfOnbz77ruNPo3h888/j/zt9Xot1y4KpzaUmpoauSrpgQceIDc3NzLO7/fz61//muzsbOLj47nzzjs7qponnTvuuIP4+HhWrlzJ3/72N8LhcGTct99+y+9//3sAbrnlFrp06RIZp/Y8uu+//57XX38dgGeffZauXbs2aTq1SfuaMmUKdrudnJwcZs6cSXFxcZ3xn376Kc888wxg9oD69OkDWKtd9JUZbSwYDHLPPfewevVqnE4ngwcPJjY2lk2bNlFcXExsbCyzZ8/m3HPP7eiqHnfuvvvuOv/n5eWxefNmoqKiuOCCC+qMmz59ep1DB59//jlTp06lqqoq8jUAxcXFbNq0CYDLL7+cv/zlL/W+p0bteWTTp0/ngw8+ICYmps5hnsY8+eSTkRs11Sbt6+233+Y3v/kNgUCAmJgY0tPTSUhIYPfu3ZHw6NevH3PmzKlzgYJV2kXh1A4Mw2DRokUsWrSIbdu2UV1dTZcuXbj44ouZMmXKUQ97SMMyMjKaXHbu3LmR4+C1du/ezcsvv8ynn35KQUEBMTExeL1err/+en72s581eu+F2rNxEydO5Msvv2xy+ZUrV9KjR4/I/2qT9rVjxw7mzZvH2rVr2bNnD36/n/j4ePr168fIkSMZN25cg48Fs0K7KJxERMRydM5JREQsR+EkIiKWo3ASERHLUTiJiIjlKJxERMRyFE4iImI5CicREbEchZOIiFiOwklEGjVx4kQyMjKO+HRrkfagcBIREctROImIiOUonERExHIUTiIiYjnOoxcRkaOprq7mjTfeYPny5eTk5FBdXU2nTp0444wzGD9+PBdddFGd8mPHjmXTpk08+uijXHfddcyePZsVK1awb98+3G43AwYMYNKkSYwYMaLB5ZWVlTF37lxWrVpFbm4uPp+PpKQkBg4cyHXXXcfIkSMb/VqDzz77jDfeeINvv/2W0tJS4uLiOPPMM7nllluO+J1M4XCYuXPnsmjRInbv3o1hGPTt25cbbriBX/ziFy3feCIN0FdmiLTSvn37mDx5MtnZ2bjdbk477TQSExPJyclh3759gHnV26OPPhqZ5sYbb2T9+vXcc889fPjhh+Tk5DBw4EBiY2PJzMzkwIEDADz22GNMmDChzvKys7O57bbbyMvLw+VyMWjQIOLj48nNzWXnzp0AjB49mmeeeQa7ve7BkWeeeYa///3vgPnV3F26dCE3Nzfy5XPTp0/nrrvuipSv/b6me++9ly1btrB69WqGDBlCVFQUW7duZf/+/QA89NBD3HbbbW24VeWkZ4hIi4XDYWP8+PGG1+s1JkyYYOzdu7fO+HfeeccYOHCg4fV6jXfffTcyfMKECYbX6zXOOecc47rrrjP2798fGVdTU2NMmzbN8Hq9xpAhQ4x9+/ZFxgUCAePqq682vF6vcd111xl5eXl1lvfBBx9Eljdnzpw645YtW2Z4vV5j8ODBxqefflpn3Jw5cwyv12tkZGQY69atq1fPK664wrj++uvr1CUYDBoPPPCA4fV6jeHDhxuhUKj5G1CkETrnJNIKn3zyCRs2bCAuLo7nnnuO7t271xl/7bXXMnnyZABefvnletOXlpby1FNP1fmabLfbzRNPPIHH48Hn8/H+++9Hxq1evZqtW7dis9n405/+RLdu3erMb9SoUYwfPx6AV199FeOwAyMvvvgiYPbaLrzwwjrT3XrrrQwePBjDMFiwYEG9eu7Zs4c//vGPdO3aNTLM4XBwxx13AFBcXBzpfYm0BYWTSCusXLkSgDPPPJOUlJQGy1x11VUAbNmyhaKiojrjvF4v/fr1qzdN7fkqgA0bNkSGf/TRRwAMGDCAvn37Nri8kSNHArB3715ycnIAyM/PZ/PmzQCNnsf661//yueff84f/vCHeuOGDx/OqaeeWm/44V+5/uN1E2kNXRAh0gpZWVkA7Nixg7vvvrvBMsFgMPJ3Tk4OycnJkf9PP/30Rufdu3dvvvzyS3744YfIsG3btgGQkZHR6HTp6emRv3fs2EHfvn3ZunVrZFifPn0anK6xcK2tS0Oio6MjfwcCgUanF2kuhZNIK5SUlABmL2Xv3r1HLV9eXl7n/8TExEbLxsfHA1BZWRkZVlpaCkBSUlKj0yUkJET+LisrqzMdQFxc3FHr+WOxsbHNnkakNRROIq1QezXcpEmTmDlzZrOndzobfwuGw2GAOpeE1/5tHOEi28PH1ZY/fB5HmlbEKnTOSaQVanswhYWFLZr+xz2phsbV9qAOX15tj60hh/eSantmh/fQDh8vYlUKJ5FWqD1n9N1337Vo+tpzSA3ZtWsXUPd8j9frBcyLKxpTex7s8PK1v388/nA7duxg9erVrFu3rgk1F2lfCieRVrj88ssBM0jWrFnTYJmPP/6YcePGMXfu3HrjNm7cWOeCh1pFRUWRwBs2bFhk+GWXXQZAZmZmo8G2ZMkSAPr16xe5mq5Lly4MGDAAoM6l6YebNWsWd955J/PmzWtwvMixpHASaYULLriAs846C4AZM2bU69GsW7eOhx9+mG+//ZaKiop603s8Hh588MHIEyEAampqeOKJJ/D7/SQkJDBq1KjIuIsvvpiBAwcC5lMZ8vPz68zvn//8J//6178A6l09WHtP0nvvvcc777xTZ9xbb73FJ598AsANN9zQ5PUXaS+6IEKkFWw2G8899xy33XYb2dnZXHvttQwcOJBOnTqxd+9esrOzAfPeoylTptSbfvz48Xz00UeMGDGCIUOG4PF42LRpE8XFxdhsNh5//PE654vsdjvPP/88t956K5s3b+byyy/njDPOIDo6mh07drBnzx7ADKLRo0fXWdaoUaOYPHkyr7zyCjNnzuTvf/87aWlp5ObmRg4hTps2rU5PTaSjKJxEWql79+4sXLiQ+fPns3z5crKzs8nKyiIlJYWLLrqIsWPHctVVVzX4INbY2FjefvttZs+ezcqVK9m3bx8ej4cLLriA22+/nXPPPbfeNL169WLx4sXMnTuXFStWkJmZid/vJzk5mauuuoobb7yRc845p8G6zpgxg3PPPZf58+fz3XffkZubS1xcHCNGjDjqg19FjiU9+FWkA9Q+UPWee+7h//2//9fR1RGxHJ1zEhERy1E4iYiI5SicRETEchROIiJiObogQkRELEc9JxERsRyFk4iIWI7CSURELEfhJCIilqNwEhERy1E4iYiI5fx/BtUyWFRCjiEAAAAASUVORK5CYII=\n"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.size'] = 26\n",
    "plt.rcParams['axes.labelsize'] = 26\n",
    "plt.rcParams['xtick.labelsize'] = 26\n",
    "plt.rcParams['ytick.labelsize'] = 26\n",
    "plt.rcParams['legend.fontsize'] = 26\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "ax = pd.DataFrame(history).drop(columns='total').plot(x='epoch')\n",
    "plt.ylabel('loss')\n",
    "plt.ylim([0.2, 1.8])\n",
    "\n",
    "plt.savefig(figure_path + '/' + dataset +'-small-loss.png', dpi=300, bbox_inches='tight')\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [],
   "source": [
    "with open('best.weights.small', 'wb') as file:\n",
    "    pickle.dump(best_weights, file)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [],
   "source": [
    "# medium net\n",
    "\n",
    "start = time.time()\n",
    "net = RecommenderNet(\n",
    "    n_users=n, n_movies=m,\n",
    "    n_factors=20, hidden=[10 ,10],\n",
    "    embedding_dropout=0.05, dropouts=[0.3, 0.3])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [],
   "source": [
    "lr = 1e-3\n",
    "wd = 1e-5\n",
    "bs = 4096\n",
    "n_epochs = 300\n",
    "patience = 300\n",
    "no_improvements = 0\n",
    "best_loss = np.inf\n",
    "best_weights = None\n",
    "history = []\n",
    "lr_history = []\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "net.to(device)\n",
    "criterion = nn.MSELoss(reduction='sum')\n",
    "optimizer = optim.Adam(net.parameters(), lr=lr, weight_decay=wd)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss improvement on epoch: 1\n",
      "[001/300] train: 1.1910 - val: 0.9509\n",
      "loss improvement on epoch: 2\n",
      "[002/300] train: 0.9255 - val: 0.8859\n",
      "loss improvement on epoch: 3\n",
      "[003/300] train: 0.8761 - val: 0.8629\n",
      "loss improvement on epoch: 4\n",
      "[004/300] train: 0.8557 - val: 0.8562\n",
      "loss improvement on epoch: 5\n",
      "[005/300] train: 0.8459 - val: 0.8535\n",
      "loss improvement on epoch: 6\n",
      "[006/300] train: 0.8396 - val: 0.8493\n",
      "loss improvement on epoch: 7\n",
      "[007/300] train: 0.8350 - val: 0.8492\n",
      "loss improvement on epoch: 8\n",
      "[008/300] train: 0.8311 - val: 0.8488\n",
      "[009/300] train: 0.8265 - val: 0.8489\n",
      "loss improvement on epoch: 10\n",
      "[010/300] train: 0.8259 - val: 0.8446\n",
      "[011/300] train: 0.8229 - val: 0.8475\n",
      "[012/300] train: 0.8202 - val: 0.8485\n",
      "[013/300] train: 0.8188 - val: 0.8483\n",
      "[014/300] train: 0.8173 - val: 0.8476\n",
      "[015/300] train: 0.8146 - val: 0.8466\n",
      "[016/300] train: 0.8136 - val: 0.8466\n",
      "[017/300] train: 0.8124 - val: 0.8464\n",
      "[018/300] train: 0.8098 - val: 0.8449\n",
      "[019/300] train: 0.8091 - val: 0.8478\n",
      "[020/300] train: 0.8067 - val: 0.8475\n",
      "loss improvement on epoch: 21\n",
      "[021/300] train: 0.8054 - val: 0.8444\n",
      "[022/300] train: 0.8044 - val: 0.8466\n",
      "[023/300] train: 0.8031 - val: 0.8462\n",
      "[024/300] train: 0.8031 - val: 0.8466\n",
      "[025/300] train: 0.8009 - val: 0.8455\n",
      "[026/300] train: 0.8011 - val: 0.8453\n",
      "[027/300] train: 0.7994 - val: 0.8471\n",
      "[028/300] train: 0.7982 - val: 0.8455\n",
      "[029/300] train: 0.7974 - val: 0.8479\n",
      "[030/300] train: 0.7965 - val: 0.8477\n",
      "[031/300] train: 0.7951 - val: 0.8470\n",
      "[032/300] train: 0.7957 - val: 0.8483\n",
      "[033/300] train: 0.7949 - val: 0.8477\n",
      "[034/300] train: 0.7939 - val: 0.8464\n",
      "[035/300] train: 0.7938 - val: 0.8477\n",
      "[036/300] train: 0.7921 - val: 0.8505\n",
      "[037/300] train: 0.7913 - val: 0.8494\n",
      "[038/300] train: 0.7916 - val: 0.8484\n",
      "[039/300] train: 0.7910 - val: 0.8502\n",
      "[040/300] train: 0.7899 - val: 0.8488\n",
      "[041/300] train: 0.7904 - val: 0.8495\n",
      "[042/300] train: 0.7902 - val: 0.8498\n",
      "[043/300] train: 0.7882 - val: 0.8506\n",
      "[044/300] train: 0.7879 - val: 0.8497\n",
      "[045/300] train: 0.7873 - val: 0.8497\n",
      "[046/300] train: 0.7862 - val: 0.8496\n",
      "[047/300] train: 0.7875 - val: 0.8483\n",
      "[048/300] train: 0.7878 - val: 0.8502\n",
      "[049/300] train: 0.7855 - val: 0.8511\n",
      "[050/300] train: 0.7841 - val: 0.8509\n",
      "[051/300] train: 0.7855 - val: 0.8519\n",
      "[052/300] train: 0.7851 - val: 0.8489\n",
      "[053/300] train: 0.7836 - val: 0.8519\n",
      "[054/300] train: 0.7838 - val: 0.8509\n",
      "[055/300] train: 0.7841 - val: 0.8502\n",
      "[056/300] train: 0.7837 - val: 0.8511\n",
      "[057/300] train: 0.7829 - val: 0.8509\n",
      "[058/300] train: 0.7839 - val: 0.8501\n",
      "[059/300] train: 0.7825 - val: 0.8504\n",
      "[060/300] train: 0.7829 - val: 0.8520\n",
      "[061/300] train: 0.7822 - val: 0.8504\n",
      "[062/300] train: 0.7824 - val: 0.8495\n",
      "[063/300] train: 0.7812 - val: 0.8506\n",
      "[064/300] train: 0.7800 - val: 0.8527\n",
      "[065/300] train: 0.7812 - val: 0.8519\n",
      "[066/300] train: 0.7804 - val: 0.8490\n",
      "[067/300] train: 0.7802 - val: 0.8529\n",
      "[068/300] train: 0.7806 - val: 0.8509\n",
      "[069/300] train: 0.7800 - val: 0.8494\n",
      "[070/300] train: 0.7788 - val: 0.8519\n",
      "[071/300] train: 0.7798 - val: 0.8517\n",
      "[072/300] train: 0.7795 - val: 0.8499\n",
      "[073/300] train: 0.7798 - val: 0.8522\n",
      "[074/300] train: 0.7785 - val: 0.8508\n",
      "[075/300] train: 0.7782 - val: 0.8540\n",
      "[076/300] train: 0.7797 - val: 0.8522\n",
      "[077/300] train: 0.7797 - val: 0.8527\n",
      "[078/300] train: 0.7784 - val: 0.8521\n",
      "[079/300] train: 0.7789 - val: 0.8539\n",
      "[080/300] train: 0.7778 - val: 0.8525\n",
      "[081/300] train: 0.7790 - val: 0.8528\n",
      "[082/300] train: 0.7773 - val: 0.8519\n",
      "[083/300] train: 0.7780 - val: 0.8531\n",
      "[084/300] train: 0.7770 - val: 0.8529\n",
      "[085/300] train: 0.7764 - val: 0.8527\n",
      "[086/300] train: 0.7773 - val: 0.8540\n",
      "[087/300] train: 0.7770 - val: 0.8526\n",
      "[088/300] train: 0.7760 - val: 0.8500\n",
      "[089/300] train: 0.7763 - val: 0.8503\n",
      "[090/300] train: 0.7759 - val: 0.8525\n",
      "[091/300] train: 0.7747 - val: 0.8521\n",
      "[092/300] train: 0.7756 - val: 0.8538\n",
      "[093/300] train: 0.7751 - val: 0.8520\n",
      "[094/300] train: 0.7743 - val: 0.8504\n",
      "[095/300] train: 0.7743 - val: 0.8547\n",
      "[096/300] train: 0.7749 - val: 0.8523\n",
      "[097/300] train: 0.7748 - val: 0.8519\n",
      "[098/300] train: 0.7740 - val: 0.8558\n",
      "[099/300] train: 0.7751 - val: 0.8519\n",
      "[100/300] train: 0.7740 - val: 0.8516\n",
      "[101/300] train: 0.7742 - val: 0.8531\n",
      "[102/300] train: 0.7737 - val: 0.8527\n",
      "[103/300] train: 0.7740 - val: 0.8540\n",
      "[104/300] train: 0.7737 - val: 0.8539\n",
      "[105/300] train: 0.7725 - val: 0.8551\n",
      "[106/300] train: 0.7726 - val: 0.8533\n",
      "[107/300] train: 0.7729 - val: 0.8557\n",
      "[108/300] train: 0.7734 - val: 0.8527\n",
      "[109/300] train: 0.7731 - val: 0.8516\n",
      "[110/300] train: 0.7724 - val: 0.8523\n",
      "[111/300] train: 0.7731 - val: 0.8514\n",
      "[112/300] train: 0.7717 - val: 0.8519\n",
      "[113/300] train: 0.7733 - val: 0.8517\n",
      "[114/300] train: 0.7718 - val: 0.8522\n",
      "[115/300] train: 0.7726 - val: 0.8497\n",
      "[116/300] train: 0.7729 - val: 0.8531\n",
      "[117/300] train: 0.7720 - val: 0.8520\n",
      "[118/300] train: 0.7711 - val: 0.8528\n",
      "[119/300] train: 0.7727 - val: 0.8543\n",
      "[120/300] train: 0.7719 - val: 0.8534\n",
      "[121/300] train: 0.7727 - val: 0.8525\n",
      "[122/300] train: 0.7707 - val: 0.8521\n",
      "[123/300] train: 0.7717 - val: 0.8532\n",
      "[124/300] train: 0.7708 - val: 0.8541\n",
      "[125/300] train: 0.7713 - val: 0.8529\n",
      "[126/300] train: 0.7709 - val: 0.8535\n",
      "[127/300] train: 0.7710 - val: 0.8545\n",
      "[128/300] train: 0.7707 - val: 0.8536\n",
      "[129/300] train: 0.7702 - val: 0.8521\n",
      "[130/300] train: 0.7691 - val: 0.8542\n",
      "[131/300] train: 0.7696 - val: 0.8518\n",
      "[132/300] train: 0.7702 - val: 0.8564\n",
      "[133/300] train: 0.7702 - val: 0.8533\n",
      "[134/300] train: 0.7698 - val: 0.8525\n",
      "[135/300] train: 0.7705 - val: 0.8535\n",
      "[136/300] train: 0.7702 - val: 0.8532\n",
      "[137/300] train: 0.7707 - val: 0.8555\n",
      "[138/300] train: 0.7695 - val: 0.8535\n",
      "[139/300] train: 0.7700 - val: 0.8528\n",
      "[140/300] train: 0.7710 - val: 0.8510\n",
      "[141/300] train: 0.7697 - val: 0.8538\n",
      "[142/300] train: 0.7692 - val: 0.8526\n",
      "[143/300] train: 0.7698 - val: 0.8523\n",
      "[144/300] train: 0.7687 - val: 0.8563\n",
      "[145/300] train: 0.7694 - val: 0.8510\n",
      "[146/300] train: 0.7694 - val: 0.8544\n",
      "[147/300] train: 0.7688 - val: 0.8525\n",
      "[148/300] train: 0.7690 - val: 0.8518\n",
      "[149/300] train: 0.7692 - val: 0.8538\n",
      "[150/300] train: 0.7690 - val: 0.8528\n",
      "[151/300] train: 0.7689 - val: 0.8550\n",
      "[152/300] train: 0.7699 - val: 0.8552\n",
      "[153/300] train: 0.7690 - val: 0.8561\n",
      "[154/300] train: 0.7685 - val: 0.8539\n",
      "[155/300] train: 0.7687 - val: 0.8546\n",
      "[156/300] train: 0.7684 - val: 0.8544\n",
      "[157/300] train: 0.7683 - val: 0.8532\n",
      "[158/300] train: 0.7672 - val: 0.8527\n",
      "[159/300] train: 0.7686 - val: 0.8557\n",
      "[160/300] train: 0.7676 - val: 0.8520\n",
      "[161/300] train: 0.7685 - val: 0.8533\n",
      "[162/300] train: 0.7693 - val: 0.8536\n",
      "[163/300] train: 0.7669 - val: 0.8535\n",
      "[164/300] train: 0.7671 - val: 0.8553\n",
      "[165/300] train: 0.7691 - val: 0.8561\n",
      "[166/300] train: 0.7675 - val: 0.8553\n",
      "[167/300] train: 0.7669 - val: 0.8586\n",
      "[168/300] train: 0.7684 - val: 0.8538\n",
      "[169/300] train: 0.7671 - val: 0.8546\n",
      "[170/300] train: 0.7684 - val: 0.8548\n",
      "[171/300] train: 0.7671 - val: 0.8555\n",
      "[172/300] train: 0.7673 - val: 0.8556\n",
      "[173/300] train: 0.7669 - val: 0.8560\n",
      "[174/300] train: 0.7675 - val: 0.8535\n",
      "[175/300] train: 0.7674 - val: 0.8559\n",
      "[176/300] train: 0.7670 - val: 0.8556\n",
      "[177/300] train: 0.7676 - val: 0.8563\n",
      "[178/300] train: 0.7668 - val: 0.8542\n",
      "[179/300] train: 0.7667 - val: 0.8565\n",
      "[180/300] train: 0.7672 - val: 0.8536\n",
      "[181/300] train: 0.7679 - val: 0.8527\n",
      "[182/300] train: 0.7661 - val: 0.8553\n",
      "[183/300] train: 0.7664 - val: 0.8549\n",
      "[184/300] train: 0.7665 - val: 0.8554\n",
      "[185/300] train: 0.7661 - val: 0.8544\n",
      "[186/300] train: 0.7670 - val: 0.8548\n",
      "[187/300] train: 0.7669 - val: 0.8521\n",
      "[188/300] train: 0.7669 - val: 0.8551\n",
      "[189/300] train: 0.7663 - val: 0.8568\n",
      "[190/300] train: 0.7669 - val: 0.8549\n",
      "[191/300] train: 0.7666 - val: 0.8547\n",
      "[192/300] train: 0.7669 - val: 0.8558\n",
      "[193/300] train: 0.7656 - val: 0.8561\n",
      "[194/300] train: 0.7656 - val: 0.8591\n",
      "[195/300] train: 0.7660 - val: 0.8525\n",
      "[196/300] train: 0.7664 - val: 0.8549\n",
      "[197/300] train: 0.7657 - val: 0.8540\n",
      "[198/300] train: 0.7669 - val: 0.8549\n",
      "[199/300] train: 0.7662 - val: 0.8541\n",
      "[200/300] train: 0.7654 - val: 0.8555\n",
      "[201/300] train: 0.7660 - val: 0.8579\n",
      "[202/300] train: 0.7661 - val: 0.8552\n",
      "[203/300] train: 0.7651 - val: 0.8561\n",
      "[204/300] train: 0.7665 - val: 0.8547\n",
      "[205/300] train: 0.7668 - val: 0.8550\n",
      "[206/300] train: 0.7659 - val: 0.8564\n",
      "[207/300] train: 0.7654 - val: 0.8572\n",
      "[208/300] train: 0.7657 - val: 0.8569\n",
      "[209/300] train: 0.7652 - val: 0.8535\n",
      "[210/300] train: 0.7656 - val: 0.8566\n",
      "[211/300] train: 0.7657 - val: 0.8554\n",
      "[212/300] train: 0.7652 - val: 0.8571\n",
      "[213/300] train: 0.7654 - val: 0.8559\n",
      "[214/300] train: 0.7651 - val: 0.8550\n",
      "[215/300] train: 0.7665 - val: 0.8551\n",
      "[216/300] train: 0.7656 - val: 0.8561\n",
      "[217/300] train: 0.7652 - val: 0.8556\n",
      "[218/300] train: 0.7649 - val: 0.8546\n",
      "[219/300] train: 0.7662 - val: 0.8572\n",
      "[220/300] train: 0.7643 - val: 0.8572\n",
      "[221/300] train: 0.7640 - val: 0.8560\n",
      "[222/300] train: 0.7651 - val: 0.8561\n",
      "[223/300] train: 0.7656 - val: 0.8570\n",
      "[224/300] train: 0.7644 - val: 0.8562\n",
      "[225/300] train: 0.7635 - val: 0.8571\n",
      "[226/300] train: 0.7648 - val: 0.8538\n",
      "[227/300] train: 0.7639 - val: 0.8566\n",
      "[228/300] train: 0.7646 - val: 0.8548\n",
      "[229/300] train: 0.7631 - val: 0.8559\n",
      "[230/300] train: 0.7638 - val: 0.8546\n",
      "[231/300] train: 0.7643 - val: 0.8561\n",
      "[232/300] train: 0.7656 - val: 0.8560\n",
      "[233/300] train: 0.7648 - val: 0.8568\n",
      "[234/300] train: 0.7635 - val: 0.8553\n",
      "[235/300] train: 0.7647 - val: 0.8582\n",
      "[236/300] train: 0.7636 - val: 0.8577\n",
      "[237/300] train: 0.7641 - val: 0.8575\n",
      "[238/300] train: 0.7637 - val: 0.8569\n",
      "[239/300] train: 0.7642 - val: 0.8564\n",
      "[240/300] train: 0.7642 - val: 0.8571\n",
      "[241/300] train: 0.7645 - val: 0.8553\n",
      "[242/300] train: 0.7642 - val: 0.8554\n",
      "[243/300] train: 0.7637 - val: 0.8599\n",
      "[244/300] train: 0.7633 - val: 0.8582\n",
      "[245/300] train: 0.7632 - val: 0.8547\n",
      "[246/300] train: 0.7643 - val: 0.8569\n",
      "[247/300] train: 0.7639 - val: 0.8576\n",
      "[248/300] train: 0.7627 - val: 0.8594\n",
      "[249/300] train: 0.7629 - val: 0.8598\n",
      "[250/300] train: 0.7650 - val: 0.8561\n",
      "[251/300] train: 0.7631 - val: 0.8581\n",
      "[252/300] train: 0.7632 - val: 0.8570\n",
      "[253/300] train: 0.7632 - val: 0.8560\n",
      "[254/300] train: 0.7635 - val: 0.8576\n",
      "[255/300] train: 0.7636 - val: 0.8563\n",
      "[256/300] train: 0.7622 - val: 0.8578\n",
      "[257/300] train: 0.7635 - val: 0.8590\n",
      "[258/300] train: 0.7634 - val: 0.8556\n",
      "[259/300] train: 0.7628 - val: 0.8574\n",
      "[260/300] train: 0.7628 - val: 0.8560\n",
      "[261/300] train: 0.7632 - val: 0.8578\n",
      "[262/300] train: 0.7618 - val: 0.8582\n",
      "[263/300] train: 0.7636 - val: 0.8582\n",
      "[264/300] train: 0.7639 - val: 0.8586\n",
      "[265/300] train: 0.7616 - val: 0.8578\n",
      "[266/300] train: 0.7625 - val: 0.8598\n",
      "[267/300] train: 0.7638 - val: 0.8580\n",
      "[268/300] train: 0.7629 - val: 0.8563\n",
      "[269/300] train: 0.7620 - val: 0.8574\n",
      "[270/300] train: 0.7617 - val: 0.8571\n",
      "[271/300] train: 0.7620 - val: 0.8594\n",
      "[272/300] train: 0.7619 - val: 0.8579\n",
      "[273/300] train: 0.7634 - val: 0.8574\n",
      "[274/300] train: 0.7633 - val: 0.8580\n",
      "[275/300] train: 0.7625 - val: 0.8585\n",
      "[276/300] train: 0.7611 - val: 0.8557\n",
      "[277/300] train: 0.7634 - val: 0.8566\n",
      "[278/300] train: 0.7631 - val: 0.8576\n",
      "[279/300] train: 0.7628 - val: 0.8580\n",
      "[280/300] train: 0.7623 - val: 0.8564\n",
      "[281/300] train: 0.7637 - val: 0.8574\n",
      "[282/300] train: 0.7629 - val: 0.8576\n",
      "[283/300] train: 0.7619 - val: 0.8574\n",
      "[284/300] train: 0.7629 - val: 0.8560\n",
      "[285/300] train: 0.7621 - val: 0.8576\n",
      "[286/300] train: 0.7628 - val: 0.8581\n",
      "[287/300] train: 0.7639 - val: 0.8557\n",
      "[288/300] train: 0.7614 - val: 0.8579\n",
      "[289/300] train: 0.7621 - val: 0.8606\n",
      "[290/300] train: 0.7628 - val: 0.8572\n",
      "[291/300] train: 0.7613 - val: 0.8595\n",
      "[292/300] train: 0.7624 - val: 0.8603\n",
      "[293/300] train: 0.7616 - val: 0.8598\n",
      "[294/300] train: 0.7630 - val: 0.8572\n",
      "[295/300] train: 0.7618 - val: 0.8574\n",
      "[296/300] train: 0.7612 - val: 0.8569\n",
      "[297/300] train: 0.7618 - val: 0.8579\n",
      "[298/300] train: 0.7609 - val: 0.8565\n",
      "[299/300] train: 0.7615 - val: 0.8559\n",
      "[300/300] train: 0.7617 - val: 0.8605\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(n_epochs):\n",
    "    stats = {'epoch': epoch + 1, 'total': n_epochs}\n",
    "\n",
    "    for phase in ('train', 'val'):\n",
    "        if phase == 'train':\n",
    "          training = True\n",
    "        else:\n",
    "          training = False\n",
    "        running_loss = 0\n",
    "        n_batches = 0\n",
    "        for batch in batches(*datasets[phase], shuffle=training, bs=bs):\n",
    "            x_batch, y_batch = [b.to(device) for b in batch]\n",
    "            optimizer.zero_grad()\n",
    "            # compute gradients only during 'train' phase\n",
    "            with torch.set_grad_enabled(training):\n",
    "                outputs = net(x_batch[:,0], x_batch[:,1], minmax)\n",
    "                loss = criterion(outputs, y_batch)\n",
    "                # don't update weights and rates when in 'val' phase\n",
    "                if training:\n",
    "                    #scheduler.step()\n",
    "                    loss.backward()\n",
    "                    optimizer.step()\n",
    "                    #lr_history.extend(scheduler.get_lr())\n",
    "            running_loss += loss.item()\n",
    "        epoch_loss = running_loss / dataset_sizes[phase]\n",
    "        stats[phase] = epoch_loss\n",
    "        # early stopping: save weights of the best model so far\n",
    "        if phase == 'val':\n",
    "            if epoch_loss < best_loss:\n",
    "                print('loss improvement on epoch: %d' % (epoch + 1))\n",
    "                best_loss = epoch_loss\n",
    "                best_weights = copy.deepcopy(net.state_dict())\n",
    "                no_improvements = 0\n",
    "            else:\n",
    "                no_improvements += 1\n",
    "    history.append(stats)\n",
    "    with open(f'logs/medium.txt', 'a') as f:\n",
    "        print('[{epoch:03d}/{total:03d}] train: {train:.4f} - val: {val:.4f}'.format(**stats))\n",
    "        f.write('[{epoch:03d}/{total:03d}] train: {train:.4f} - val: {val:.4f}'.format(**stats))\n",
    "    if no_improvements >= patience:\n",
    "        print('early stopping after epoch {epoch:03d}'.format(**stats))\n",
    "        break"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "outputs": [
    {
     "data": {
      "text/plain": "<All keys matched successfully>"
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.load_state_dict(best_weights)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [],
   "source": [
    "groud_truth, predictions = [], []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for batch in batches(*datasets['val'], shuffle=False, bs=bs):\n",
    "        x_batch, y_batch = [b.to(device) for b in batch]\n",
    "        outputs = net(x_batch[:, 0], x_batch[:, 1], minmax)\n",
    "        groud_truth.extend(y_batch.tolist())\n",
    "        predictions.extend(outputs.tolist())\n",
    "\n",
    "groud_truth = np.asarray(groud_truth).ravel()\n",
    "predictions = np.asarray(predictions).ravel()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation RMSE: 0.9281\n"
     ]
    }
   ],
   "source": [
    "valid_loss = np.sqrt(np.mean((predictions - groud_truth)**2))\n",
    "print(f'Validation RMSE: {valid_loss:.4f}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [],
   "source": [
    "groud_truth, predictions = [], []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for batch in batches(*datasets['test'], shuffle=False, bs=bs):\n",
    "        x_batch, y_batch = [b.to(device) for b in batch]\n",
    "        outputs = net(x_batch[:, 0], x_batch[:, 1], minmax)\n",
    "        groud_truth.extend(y_batch.tolist())\n",
    "        predictions.extend(outputs.tolist())\n",
    "\n",
    "groud_truth = np.asarray(groud_truth).ravel()\n",
    "predictions = np.asarray(predictions).ravel()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Medium test RMSE: 0.9312\n"
     ]
    }
   ],
   "source": [
    "final_loss = np.sqrt(np.mean((predictions - groud_truth)**2))\n",
    "print(f'Medium test RMSE: {final_loss:.4f}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Medium duration: 104.2433\n"
     ]
    }
   ],
   "source": [
    "print(f'Medium duration: {round(time.time() - start, 4)}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [],
   "source": [
    "with open('best.weights.medium', 'wb') as file:\n",
    "    pickle.dump(best_weights, file)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAacAAAEfCAYAAADldgmAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA9P0lEQVR4nO3deXwV9b3/8dfZT/YQEraAIJETWRWp4q7oVTa5VeyFq4IKUvcfblXwoba2tuVq69JeL7bVygUVFAsWKwrIongV1CAKQggQAsEQIIHs29nm98eQAzEJhCxkgPfz8eDByXy/853vzPfMfM535jszNsMwDERERCzE3t4VEBER+TEFJxERsRwFJxERsRwFJxERsRwFJxERsRwFJxERsRxne1fgZLJu3br2roKIyElpyJAhx5Vfwek4He8GBsjMzKRv375tUBtpCbWLNaldrKelbdKcH/Y6rSciIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQichK66qqrSE9PZ+HChe1dlTbhbO8KiIjI8bvkkks4cOAAXbt2be+qtAn1nEREWuif//wn6enpfPnllydsmc888wwzZ87koosuOmHLPJEUnEREWui7775r7yqcchScRERaSMGp9Sk4iYg00/Tp00lPT2fTpk0A3HrrraSnpzNx4kQA0tPTSU9P59tvv+XDDz9kxIgR9O/fn//93/+tU84333zDI488wlVXXcXAgQMZMGAAV111FdOnT2fbtm0NLruhARHbtm0jPT2dgQMHApCVlcVDDz3EpZdeyoABA7j44ouZOnUq27dvb4Ot0boUnEREmqlfv35cffXVkb/PO+88rr76as4777w6+bZs2cKjjz6Ky+Xi0ksvpVOnTpG0t956i5tvvpkPPviAmpoaBg8ezHnnnUdVVRXvvfceY8eOZe3atU2qj8fjiXz++uuvGT9+POvWrePss8/mnHPOoby8nKVLlzJ+/Hjy8vJauPZtS6P1RESa6dZbb430lgAefPBBhg4dWi/frFmzuO+++7j33nvrTC8sLOS//uu/MAyD//zP/+Spp57C6TQPy9XV1Tz22GMsXbqUX/3qVyxduvSY9bHbzf5GOBzmF7/4Bffccw8///nPI9Pz8vIYO3YsxcXFvPnmm0ybNq1F69+WLB+ccnNzeeyxx1i/fj2pqamsXLmyWeVkZGRwyy23HDOf2+1m48aNzVqGyKliwbofmJ+xu12WXVlZSfTq4lYvd9xPenDjkO6tXm5ThMNh7r777nrTS0pKGDduHAcPHuTee++NBCYAr9fLAw88wNKlS9m5cyc7d+6kV69eTVpeMBikd+/e3HXXXXWmp6amMmbMGN544w3Wr1/fonVqa5YOTvPnz2fGjBlUVla2uKzS0lIAoqKiuPjiixvN53K5WrwsEZEjXXHFFZHey5HS0tJ46qmnGp2vR48ekc+FhYVNDk4A48ePb3D6mWeeCUBRUVGTy2oPlgxOhYWFPPnkk6xatYqEhARGjhzJRx991KIya4NTjx49mDlzZmtUU+SUdeOQ7u3Wy8jMzKRv377tsuy20rNnz6Om7969m5UrV5KdnU1JSQmBQKBenlAo1CrLjIqKAmhwGVZiyeC0cOFCVq1axfnnn88f/vAH1qxZ0+LgVFJSAkBcXFxrVFFEpMmio6MbnG4YBs899xyzZs3CMIxWXebJfqyzZHByOp1MnTqVe+65p8GucHOUlZUBJ3+DicjJx2azNTh97ty5vP766wAMHz6cyZMnk5aWRkxMTOTYVzvY4nRjyeA0YcIE3G53q5apnpOIWM3bb78NwAUXXMCf/vSnekGsvLy8PaplCZYMTq0dmODwNaf4+Hi2b9/OsmXLyMnJIRAIkJKSwtChQxk2bBgOh6PVly0i0pCdO3cC5oCJhnpX33zzzQmukXVYMji1hdrgtGzZMubOnVvv/O6cOXPw+Xz8+c9/joxmERE5Hsc7aMHpdOL3+/H7/fXSAoEAL7/8crPLPtmdNk+IqA1OhYWFjBs3jkWLFrFx40bWrFnDs88+S3JyMlu3bmXSpEkcPHiwnWsrIieTxMREgOO+R/Kcc84BYNGiRZFLDwD79u3jvvvuIyoqKjLqbuvWra1T2ZPEadNzuueeeygpKSE1NZVzzz03Mj0pKYnrr7+egQMHMnbsWPLz83n11VcbvXM6MzPzuJddXV3drPmkbaldrOlkbBefz8dXX33Fiy++yNtvv01VVRWzZs2KpO/Zs6fBdRo9ejRffvklO3fu5Oqrr6Z3795UVVWRnZ1N586deeaZZ/j73//Orl27eP755/nwww/56U9/ynnnnRfpbR1Z9r59+yJlb9++PTIQ7Eh79uwBwO/3N3k7t0ebnDbB6dJLLz1qelpaGqNHj2bBggUsX7680eDUnPsvTsX7Nk4FahdrOhnb5dlnn+WJJ55g/fr1FBcX06tXrzrr0K1btwbXqW/fvnTv3p2ZM2fy/fffk5WVRdeuXbnjjjuYMmUKCQkJdO/enenTp7N582b27t0bKbv22vyRZR854Ouss86ie/f696rVBhm3293k7dzSNlm3bt1xz3PaBKemGDBgAAsWLCAvL49wONxqw9hF5NTWrVu3Oj2lWllZWcec96KLLjrqCwN79eoVGdV3pIYe5da9e/djLnPs2LGMHTv2mPVqbzr6NsBmsykwiYi0o9PiCFxaWsqnn37KvHnzqK6ubjRf7TtOGuoKi4jIiXNanNYrKSnhzjvvBMDhcDBu3LgG8yxevBiAK6+88kRWT0REfuSU6jnt27ePESNGMGLECDIyMiLTe/TowfDhwwGYMWMGn3zySZ359u/fz/33309xcTGJiYlMnjz5RFZbRER+xJI9px+/kCs/Px+AAwcO1Et78MEH8fl8gHnTWk5ODkC912z85je/IT8/nw0bNnDXXXfRs2dPevToQWVlJRs3biQQCJCUlMTLL79M586d22rVRESkCSwZnFasWNHg9Orq6nppt912W5PKTExMZN68eSxcuJDFixeTlZXF2rVr8Xq9pKenc8UVVzBhwgSSkpJaXH8REWkZSwanpgy/bMixhlE6nU7GjRvX4DUnERGxjlPqmpOIiJwaFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERE4SEydOJD09nf/+7/9u76q0OQUnERGxHAUnERGxHAUnERGxHAUnERGxHAUnEZFmmjRpEunp6dxzzz1HzXfvvfeSnp7OHXfcEZm2detWnnjiCYYPH86gQYMYMGAAl19+OVOnTmX9+vVtXXXLU3ASEWmmf//3fwfg//7v/ygvL28wT1lZGZ999hkAP/3pTwFYvnw5Y8eO5R//+AdFRUUMGjSI888/H4ClS5dy8803s2jRohOwBtal4CQi0kzXXHMNXq8Xv9/PypUrG8yzfPly/H4/MTExXHPNNfj9fn75y18SCAQYNmwYn332GW+++SazZs1i5cqVTJw4kXA4zG9/+1sqKytP8BpZhyVf0y4i7ezbebD+zXZZ9BmVFbA2pvULHjwBzr2pVYuMjY1l2LBhfPTRRyxZsiTSkzrS4sWLARg+fDhRUVEUFBQwevRoDh48yJQpU/B4PJG8TqeThx9+mDfffJPS0lLWr1/PJZdc0qp1PlkoOImItMCYMWP46KOPIqf2YmNjI2lFRUWsWbMGOHwKMCUlhSeeeKLR8qKjo0lOTqagoICCgoK2rbyFKTiJSH3n3tTqvYymys3MpG/fvu2y7Oa4/PLLSUxMpLi4mE8++YTrrrsukrZs2TKCwSBdu3Zl6NChdebbv38/K1asICsri5KSEvx+P4ZhAOZ1KoBwOHziVsRiFJxERFrA5XIxfPhw3nnnHZYsWVInOH344YeA2Wuy2w9f4n/99dd54YUXCAQCJ7y+JwsNiBARaaHaU3arV6+moqICgMLCQr7++mvg8Cg9gE8++YRnn32WQCDA0KFDmT17Nl9++SWbN28mKyuLrKwsUlNTT/xKWIyCk4hICw0ZMoTU1FRqamr49NNPAViyZAmhUIgBAwaQlpYWyfv2228D0LNnT1577TUuvPBCEhMTcTgckTy1Ae50puAkItJCNpstcjpv2bJlgBmcAK6//vo6eXfu3AnAJZdcgtvtrldWdnY2xcXFbVbXk4WCk4hIKxgzZgxg3pC7f/9+1q1bh8vlYvTo0XXyuVwuAPx+f4PlvPTSS5HPoVCobSp7ElBwEhFpBX369OHss8+mrKyMP/3pT4TDYS699FKSkpLq5Bs0aBBg3py7Z8+eyPSSkhIef/xxduzYwU9+8hMAsrKyTtwKWIxG64mItJIxY8awZcsWFi5cCNQ/pQdwxx138OGHH1JcXMzo0aMZNGgQfr+fzZs3Exsby+uvv86iRYvIyMjgrbfeYuvWrVx//fWMHTv2BK9N+1LPSUSklVx33XXY7XbC4TDx8fFcddVV9fL07t2bt956iyuvvBKHw8E333xDQUEBN954IwsWLIg8IPbSSy/F4/Gwbds2bDZbO6xN+1LPSUSklXTp0oXMzMxj5uvXrx9//etfG03v2LEjf//73+tNf+ONN1pUv5OJek4nQNgwCIWN9q6GiMhJQ8HpBPiftYXc+9a69q6GiMhJQ6f1ToCiqhBFpafvo+9FRI6Xek4ngMthwx88fR/gKCJyvBScTgCXw0aNgpOISJMpOJ0ALrsNf0jBSUSkqRScTgCd1hMROT4KTieAy67gJCJyPNokOBmGQU5ODpmZmaf1gwtruR02aoLaDiIiTdXs4JSTk8PUqVP54IMP6kz/9ttvueaaaxg1ahRjx47liiuuiLzf5HTlctgIGxDUdScRkSZpVnDav38/t912Gx9//HHk3SQABw8e5M477yQvLw/DMDAMg8LCQqZOncquXbtaq84nHZfDfC6WBkWIiDRNs4LTvHnz2L9/PyNGjOCWW26JTJ87dy6lpaV07tyZBQsW8OWXXzJp0iRqampOq2dC/ZjLfig46bqTiEiTNCs4rVq1iujoaH7/+9/ToUOHyPQlS5Zgs9l44IEH6N+/PwkJCTzyyCMkJiayZs2aVqv0ycbtUHASETkezQpO+/bto1+/fkRFRUWm5efns337dhwOB9dcc01kutPppE+fPnVeqnW6qT2tpxtxRUSaplnBqaysjOjo6DrT1q5dC8A555xDbGxsnbSoqKhGX0l8Oqg9rafgJCLSNM0KTnFxcezbt6/OtE8//RSbzcbFF19cL/+BAweIiYlpXg1PAS6d1hMROS7NCk59+vRh69atbNiwAYDt27ezYsUKgHpvfiwoKGDLli306tWrZTU9iWm0nojI8WnWKzPGjBnDV199xcSJE/H5fGRnZxMMBhk0aBB9+/aN5MvNzeXxxx8nHA5z+eWXt1qlTzaRa04B3YgrItIUzeo53XjjjVx99dXU1NSwceNGKisr6dixI88++2ydfL///e9Zt24dPXv2ZOLEia1S4ZNRZCi5ek4iIk3SrJ6T3W7nf/7nf/jiiy/YtGkTCQkJDB8+nISEhDr5Bg4cSExMDI8++mi9tKbKzc3lscceY/369aSmprJy5cpmlVNr3759vPbaa3z22Wfs3bsXh8NBjx49GD58OLfffnudEYitxeUw/9c1JxGRpmnRm3AvvvjiBgdA1LrvvvtaUjzz589nxowZVFa2zltk161bx5133kl5eTlJSUkMHjyY6upqNm7cSGZmJosWLeLNN98kOTm5VZZXy+0wO6gKTiIiTWPJB78WFhZy991389RTT+FyuRg5cmSL61RWVsZ9991HeXk5t99+O6tXr2bWrFnMmzeP5cuXc/bZZ5OTk8MjjzzS4mX9mOvQVtZpPRGRprHkg18XLlzIqlWrOP/881m0aFGrDKaYPXs2RUVFDB48mOnTp+NyuSJpXbp04YUXXsBms7F27Vq+/PLLFi/vSIcHRCg4iYg0hSUf/Op0Opk6dSpz5syha9euzaliPUuWLAFg3Lhx2Gy2eulpaWkMGTIEgA8//LBVllkrEpzUcxIRaRJLPvh1woQJ3HfffdjtrXPWsbS0lG3btgFEAlBDatO+/vrrVlluLT34VUTk+Fjywa9ut7s51WrUjh07AHOUYWpqaqP5evToAZgjBIPBYKstXw9+FRE5PqfFg18LCwsBSEhIwOlsfIBix44dAQgEApSUlLTa8g8/+FU34YqINMVp8eDXqqoqADwez1Hzeb3eyOfWGr4OYLfZcNpt6jmJiDRRs+5zOlUf/GoYxjHzZGZmHne51dXVOO2wd39hs+aXtlFdXa32sCC1i/W0R5s0Kzj16dOHr7/+mg0bNjBo0KAmPfi1f//+La9tM9X28qqrq4+a78j0xoLpkc8ObKrMzEyi3E5iExKbNb+0jczMTLWHBaldrKelbbJu3brjnue0ePBrSkoKACUlJfj9/kYHXBQUFADmgIzmPm6pMW6nXaf1RESa6LR48GtaWho2mw3DMNi9e3ej+XJycgDo3bs3DoejVevgdtr1skERkSay/INfW0NMTAwDBgxg48aNfPXVV6SlpTWYr/bJEBdddFGr18HtUM9JRKSpLP3g19Z03XXXsXHjRubPn8/48ePr3eD7zTffRC74jRkzplWX7SnexiD2UxY8t1XLFRE5VbXag1/Ly8vZs2cP+fn5VFRUtFaxx2Xfvn2MGDGCESNGkJGRUSftpptuIjU1lc2bN/PMM8/UGdqek5PDtGnTABg1alSrD97ouHk2D1T+tx78KiLSRC3qOW3evJnZs2fzxRdfRG50rdWtWzeGDRvGbbfdFnnyQlPde++9df7Oz88HzCHpP0578MEH8fl8gHnzbO11ox/fp+TxeHjllVe4/fbbmTt3LkuWLKFfv35UVFSwYcMGQqEQ5557Lr/97W+Pq65NEfJ2oFN4P37dhCsi0iTNDk5vvfUWM2bMIBQKNXh/UF5eHm+99RYLFy7k+eefZ9iwYU0uu3ZY+o9VV1fXS7vtttuaXG56ejqLFy/m1VdfZeXKlWRkZOByuejfvz9jxozh5ptvPuoTJJorEN2ZJKMSh7+s1csWETkVNetI/P333/Pb3/4WwzDw+Xxce+21pKWlkZCQgGEYlJSUkJWVxZIlS9i1axcPP/ww77//fpN7UFlZWc2pFt27dz/mvElJSUybNi1yGu9ECER3BiDOv+8YOUVEBJoZnN544w0Mw2DKlCn84he/aDDPqFGjeOCBB3j66aeZP38+s2fP5sknn2xRZU9WwUPByV7yA4ZhNPjKDhEROaxZAyLWrVtHx44defjhh49euN3Ok08+SUJCAp9//nmzKngqCER3ASAxWEBhefs9Y1BE5GTRrOBUWFjI2Wef3aT3Lbndbvr168fevXubs6hTQtDbEcPmoJutkB0F5e1dHRERy2tWcLLZbITDTR8Wbbfbm/RQ1VOW3UEotgtdbQfYUdg+w+xFRE4mzQpOnTt3ZtOmTU16DYbf7+f777+nc+fOzVnUKcOReAbd7QfUcxIRaYJmBaehQ4dSVlbGr3/9a0Khxu/dCQaD/PKXv6S0tJRLLrmk2ZU8Fdg692OQLYcf9h9s76qIiFhes0brTZ48mffee4+FCxeydu1ahg8fjs/no0OHDhiGQXFxMVu2bGHp0qXs3bsXr9fLpEmTWrvuJ5e+1xGV8XdifviUUPgSHHaN2BMRaUyzglPPnj354x//yLRp08jLy2PWrFkN5jMMg5iYGF544YXjfkrEKafXZfjdCVxRtZrPt9/J5b6U9q6RSPsxDDjZb6kwDKguBm9i3XWpLoFQEGI6tqz86lJwRYHDdXhaKAg2OzRhMNrRyy4xy0/oDqV7oKYMYjtBdFL9vOEwjuoDLVteMzT7cQjXXnstffv2Zfbs2axevZrc3NxIms1mo1evXgwbNoxbb72VLl26tEplT2oOF/bBNzPmy1f4fPFDkPQrSO7T3rU6fuGQuXOAuXM2tJOEw2CEDu9UgSqoKIC4rubftZ9tNgjWgN1lfg5UQagG8tZBUhpgADY4uAPiU8HuMA8EZXsO1cEGpXmQ2BNS0s0yygsg5IeD2eaOl3AGbP0IYrtAQips+id0PAt6X4G7dCfsDUH5PvCXQ/l+8MRD5QHwxIJvBDg9YHOAKxqC1eY/d4z5f9k+M9+B7ea8cV3M9XK4oGgn1JSbO3zZXrP8pDQzf9FOM8+2jyHtakg8AyoLwV8J+d9B5/7QMQ0SesAPX4M7FmzAhvnQ/wazfvGp5vrt32xuv33fQ8kP5sEyOd1cjhGG6GSI7WyWE98Vkn1QVWS244FtULjNbJO8b8wD4eCJZl1rSs0DVk2Zuc1jkqFwq1mmzWEexCoKIX2kOW3PerA7IRyE6I7gcJvf761LzbboPBBcXnOegizI+RSufNxcpjvWPFi6oqBwK722LIesn5j547qa5Rbnmt8DIwxnXgal+eb3o3y/mZ50ptkuFYXmdyO2E+zbZM4XlQRRHSBQabZboNLc5rFdzO9ibCcIBczll+ZBoNr8/oYCsOdbs1xvvPndcLjNdezU11znnE/NdkofaW7Lop1QZD5CjR5DzfrEpJjfjZLdZnuU5sGBbOh5MYQD5vYt2mm2S6Dq0Hcs1vxeuqIgdYhZXrAa9mea3/vEHmb9qorMdenUD8ryzX9Or7lOlQchWHWoPTxmO7hjzbbMWW3uJzEp5jYAs9zOA8zvWvDQeILEHlBRyFn7NkP/nea2OEFsRisNo6upqaG0tBSbzUZ8fHyjL/Q7ma1bt44hQ4Yc93yRt0iGgnz113s4b98/cNjBlnY1uKPNnSvxDOh9JeSuNb9U3gTzSxXbyTyQuWPAGQW5a8y0XpeZO6vdaX7J4ruZCyvcan758r8z05xe85eRKxo69ISDObB1iVleQndzp4hKMg/sFQXmzuSNP3RAtJk7cPleKNhqBo6aMvPAGPKbeeO7mjHEX24uy+kxD57+CvMA64mDfZvNncQTb/5dmmceML0JZn2dXnNdS/MOH+COV3x3cDjN9TmSM8pcdmtyHdpBAy0deWnD3HitxO4yt2/VcVzXdMea/8enmu1/5Lw2O7jjwF9mBgVPvPkDIeg3170p29buhLhuUJJ7uI7x3cz23vNNg+tQ2eFsost2mj98qkvM6d5ESOptBo3878zvs8NlHlwNA4p3md8bd6z5d6DC/F53GWD+YKkpMw/0rijz+7Zv06HvZJyZZnOY88d2Mr/DBmb6mZebP6BqSs2eRshvll+YZZZ/3q3meuz41FxWhzOh22AzyGxZbAbNqiLzu53Qw9w3Erqb23v7CjPI15Sa+39cNzMgO73mshJ7mPPu/tIMik6vuQ3sDvMHj8Nl7kN53xz+wRfX9VBdi81t7Iwyf8wF/eYPuLL8wz8qPPHmepxxkRm8C7fCDxmHluU2f8AU50Kgirw+E0kd8UDTv1c/0pxjZ6sFp9NBi4MTUB0IMenlxYwsnssNidnEuWzmTlG009zxsB3eYVrrwGVzHCr7kO7nmwee4t3mjlJdbB5EYpLNHa6m1Ny5jLD5azw6CbqcY+7Ynljz16TDbf56LN1jlumNP9SzqDG/9FGJ5i/kmjLzV11KuhlYy/eZvYW9G83eQo+hUFVs7kCd+plB7swrzHIdLvNgkNTb7KUYYXNnje9qbqdw0DzQFW41d3QM81emO9YMxEW7YO8G8wADZhk9LzIDbUkue8qhW88+5k4clWj+X3nQ3P4VhfDDV+avUyNk/pp2ec2dvWCLOe2Mi8x1rj2wVOw3DxrBaujQy6xHbQB3Rpk9rKqD5oGpNA/SR5m/vquKzG1vc5i9psKt5o+Iwixz+4CZp8dQyHzf/HVbU2quT3Ifc7sk+8xl2mxmu2KY5VUUmNuy41nmL2d/hXkAttnNetTOA+YBseQHc1t44s0fMDabuQ7VJeaPDTgcnOwuyMswTzX1vNjMa7Ob27DqoBkAzvo38/vjrzDbyxNv5gsFYedqs21DQXOZ/nKI60rmth2H36hdU3Zon4g9/P09mGO2lTf+8LRwyCzf4TbLrykzt7mjkZNDNWXmekR1OLzfhfzmj8XmCAUbX9YpoDVe097qwaklFaqzIJuNzZs3t0pZ7aU1ghPA/rJqfj47g+9+KOHf+nbiydH96BXeDQWZ5s7siTNPjVUdNE9bhINmAKkoNHtXZXth1+fQZeCh888OKMkzD1Id08z0boPNg7u/8tBpC7+5U8d1afi88mmopTuctA21i/W0R3A6ZqhXx6r1dYrzMv/ui/jfz3fy5xXbuPbF1Vw/uBvpXc5lTI2LTh7MUxoxyea/H4tOgs796k5LPaLhuww8/Nl76A3E9qj684iIWNQxg9OcOXNORD1OOx6ng7uuSOOGwak8uySLJd/vZX7GD/z+w0wuTuvIuT0SGXxGIpeclYzH6Wjv6oqInFDHDE4XXHDBiajHaatTvJfnx50DnEN2QTnvZvzAqi37+Z9V2wkb4HXZ6Z0cy9ld4rgiPYW0lFj6dY3HrvukROQUdupewTsJpaXEMn3k2UwfeTZV/hBrcw6wemsBOwoqWJW1n4Xr8wBIinFzdpc40lJi6ZUcQ5zHyTk9Eon1Ouka71XgEpGTnoKTRUW5HQxL78Sw9E4ABENhsvaVsW1fOf+3vZDt+8v55/o8ymrqDruOcTtI7xJHt8QoUuI8dIrz0inOQ8qhf1EuB6kdonA5WngTn4hIG1JwOkk4HXb6d0ugf7cErh+cCpiDVYorAxys9LPhh2Iq/SG27i1jy94yNu0pZX9pNRX++s8+9LrsdEuMAsxrX3FeJ3EeJ163A1+nOHolRxPnddIlPooKf5AD5X66Jnjp3y0+8tglvTBRRNqSgtNJzGaz0SHGTYcYN2kpsQ3mqagJUlBWw/6yGgrLa6ioCZK1t4w9JVXYsFETDFNWHSC/pJqqQIgPN+bT2ADNWI/5danwB+mWEEWXBG9kNGe020kobGCzQac4DzabjRiPg2i3k7SUGGI8TooqA8R7nXicdsIGdEuMItbjpKjST7TbQZzHRYzHQdiAXQcqcDnspHaIImwYJES5KKkMkHKobBE5tSk4neJiPE5iPE56JTftsSOl1QEKymoorQqQV1xFjMdJSqyHnMIKvsw5gN1mI97rYueBCoorA4B5z2NZdRC7DUJhg3VFlYTDUOkPUuEP4Q82/d1fxxLlchAMh0mKcZMU46Gk0o/LaSfK5SDG4yTa7aBbgtnjcznsxHmdxHqcVPpDGIaBy2HH6bDjctgoPlhE133bcTvsOO02OsV7ife6CBkG4bBBMGwQChuEDYPEKJf5IIxQmOQYD0WVfmw2M8CGwwYHKvx4XQ5sQK+OMVT4gyREuagKhIhyOfC6HE162K9hGNhsNgzDwB8Ka6SmnLYUnKSOeK+LeK/5TLzBZ3SITB+QmsCYc7odd3mhsEFeURXlNUE6xropqw5QEwxjw8ae4irKagJ0jPFQ6Q9RUROkvCZIKGxwZkoMgWCYvOIqHHYbRRUB4qOc5B6sxON0cLCihgPlfvp2jSMcNqjwh6jyhyitCrB5TymxXrMnV1oVoLwmSIzbid1uIxgKEwiZB35TUWtstiZxHwqiUS4HLqeN6kAYfzCM026jrDqIw27DHwrTKc5DSVWASn+IlDgPHaLNIJcc6yFsQE0gRIzHicNmI2yYwROgsNyPx2knOdaD12XHbrNhs9lIjnVHyg8ZBvnFVXSK83JWp1i27isjFDY4q1MspdXBSH0CoTBdE72EwhA2zCDtsNvMhy8EwsR6nDjsNkqqAhiGOT82G6FQmOARgR3A63JQWRPE63IQ5XYQ43HgcTqo9Ieo8geJdjvZX1ZDfJT5Q2hnbjmbK38gIcrF3tJqyqqDxEc56RjjAcDjsoMBVYEQNsDlsON1OcgvqSJsGHSM8RAf5eJAeQ3RHrPnXh0Isa+0BrvNRq/kaMKGeVbB63TgcNjoEu+lvCZIWbX5fXHYbCREuyirDpIU7aYmGMbttFMTDFETCBMMh/G6zDMD0W4HNpu5/aNd5mnykGGQvb+CM5Nj8LrMMwXBcBjDMLdHMBQmbIDDbsNuO3ya3B8MU3noh43NZqM6EMJus+F21r1GXHvGonY+wzAOPerS/Dt0aPv/eL5AKEwobOB1Wf9Hj4KTtCmH3cYZHQ8/EqZzvDfyuV+3+IZmaXW1vZEfT9u0OZOzfOn4Q2ECwTD5JdVU+kM47OCw23HYbNjt5joUlNUA4HbYKakK0CHGTThskFdchWFAp3gPNYEw1cEQew71OMuqg0S5HNQEQ1T5w1QFzINxVcDsTUa5HbgddgJhwzyghQycDjv7y6rpEO0m3uvih6JKSqsDeF0OCspqcDrseOM8lNcECRsGLrsdG+a6ndMjmkAwbJ6+9ZvpwZDB+twiYjxODAycdjtd4r18u7uYZZv30r1DNG6nnf/bXojX5cDttBMOGzgdNgrL/Ye3gc12qEcJHqf9UPkQ53VGfhy0rv2tXN6J5bDbIsH5x5Ji3BRX+jky2WaDbglR7C+rJhAyDp0ah32l5veuQ7SLlDgPbqedYMjgh6IqqgIhYj1OuiVGcaDcPHUf43YQ7XFGfgR2ivPgtNupDoTM73F5DS6HnfTOcRRV+gmHDVLiPJQeOvNRUFZDlwQvHaLdBEJhOkS7qQqEMPxVzO6TXi/YtSUFJznlNXSNymaz4bDb8B465QbQMdbTaBlnn+IP1m8ogB8rfyhsBtNQ2KCgrAa7HZx2O06HDafd3L6GYT5PMtrtPBSkQ1T4Q9QEQ0S7nHjddsqrg6TEeaioCVFQVsPu3J30851FUaWfzvHmgbL2dLPNBtWBMHYbkXbzB8NUB0Ikx3nwuhwUltVQVm32PkqqAuwprsLjstM53kswZLDzgPnA3jivk5pgmJpAiMJyvzkwyOskzuuiJhiipDJAfJSLosoAXpcdf9A8zep1maeGq/0hKv1BKgMhQiGD5DgPVf4QZdVBaoIh+nSOZdeBykPB3n5oexjsPlhljpx1OyKnjQOhMLsOVJKaGEVyrIeNeSW4HHZ6dTR7eQXl1RSU1RAIGdhtNs7vlUR8lPkDaNeBSnonx5CWEkPFoTpFuZzERznJK6oiEAoT5XYSCofpmhBFSVWA7IJy0lJiMIAD5X66J5k/bC5OS2ZfaTXFlQGi3A7yS6qJ8ThI9Jz40b0KTiJy3INMbDYbToc5j8Nuo0uCt9G8tUHE7bQT53XVS+8UZ/4f53XRJcGLs8xNr+QYenH4OmmU21Gn1300qYdGojbm0j4NPBJMjiozM/OE9pqgma9pFxERaUsKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjkKTiIiYjnO9q5AY0KhEPPnz+df//oX2dnZVFZWkpKSwtChQ5k0aRI+n++4ysvIyOCWW245Zj63283GjRubW20REWkFlgxOVVVVTJkyhYyMDJxOJwMGDCA2NpasrCwWLlzIv/71L5577jlGjRrV5DJLS0sBiIqK4uKLL240n8vlanH9RUSkZSwZnH73u9+RkZGBz+fjlVdeoXv37gAEg0FefPFFXnvtNaZNm0a/fv3o1atXk8qsDU49evRg5syZbVV1ERFpBZa75rR7924WLlyIzWbjpZdeigQmAKfTyS9+8QsGDx6M3+/nL3/5S5PLLSkpASAuLq7V6ywiIq3LcsFp2bJlhEIhzj//fNLS0uql22w2fvaznwHw8ccf4/f7m1RuWVkZoOAkInIysFxwWrduHQDnnXdeo3mGDBkCQHl5OVu2bGlSueo5iYicPCx3zWnHjh0AnHHGGY3m6d69O3a7nXA4zI4dOxg0aNAxy6295hQfH8/27dtZtmwZOTk5BAKByCjAYcOG4XA4WmdFRESk2SwXnA4cOABAx44dG83jcrmIj4+nuLiYgoKCJpVbG5yWLVvG3LlzMQyjTvqcOXPw+Xz8+c9/5swzz2xm7UVEpDVY7rReVVUVAB6P56j5atMrKyubVG5tcCosLGTcuHEsWrSIjRs3smbNGp599lmSk5PZunUrkyZN4uDBgy1YAxERaSnL9ZyaqrbnY7PZmpT/nnvuoaSkhNTUVM4999zI9KSkJK6//noGDhzI2LFjyc/P59VXX2XatGkNlpOZmXncda2urm7WfNK21C7WpHaxnvZoE8sFp+joaEpKSqiurj5qvpqaGgBiYmKaVO6ll1561PS0tDRGjx7NggULWL58eaPBqW/fvk1a3pEyMzObNZ+0LbWLNaldrKelbVI70O14WO60XkpKCmCefmuM3++PnKarzd8aBgwYAEBeXh7hcLjVyhURkeNjueBUe2/Tzp07G82zY8eOyGm9Pn36tHodbDYbdrvlNo2IyGnDckfgoUOHAvD11183mufLL78EzOtFTXkAbGlpKZ9++inz5s076unC7du3A9R5KoWIiJx4lgtO1157LR6Ph++++47NmzfXSw8Gg7zzzjsAjB49ukn3JZWUlHDnnXfy9NNP8/777zeaZ/HixQBceeWVzV8BERFpMcsFp5SUFG699VYAHnnkEXJzcyNpfr+fX/3qV2RnZxMXF8fdd99dZ959+/YxYsQIRowYQUZGRmR6jx49GD58OAAzZszgk08+qTPf/v37uf/++ykuLiYxMZHJkye30dqJiEhTWG60HsCDDz7I9u3bWbVqFSNHjmTgwIHExMSwadMmioqKiImJ4eWXXyY5ObnOfIFAgJycHKD+/U+/+c1vyM/PZ8OGDdx111307NmTHj16UFlZycaNGwkEAiQlJfHyyy/TuXPnE7auIiJSnyWDk9Pp5JVXXmHhwoUsXLiQbdu2UVVVRadOnRg5ciRTpkwhNTX1uMpMTExk3rx5LFy4kMWLF5OVlcXatWvxer2kp6dzxRVXMGHCBJKSktporUREpKksGZzAHDF34403cuONNzZ5nu7du5OVldVoutPpZNy4cYwbN641qigiIm3EctecREREFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRyFJxERMRybIZhGO1diZPFunXr2rsKIiInpSFDhhxXfgUnERGxHJ3WExERy1FwEhERy1FwEhERy3G2dwVORaFQiPnz5/Ovf/2L7OxsKisrSUlJYejQoUyaNAmfz9feVTzp5ebm8thjj7F+/XpSU1NZuXLlMefJzs5m1qxZrFmzhoKCArxeL2eeeSbXXXcdN910E05nw7uD2vPYSktLmTNnDitXriQnJ4dAIEBiYiIDBw5k/PjxXHnllQ3OpzZpWwcPHmT27Nl8+umn7Nq1K9IuAwYMYOzYsVx77bUNzmeFdtGAiFZWVVXFlClTyMjIwOl0MmDAAGJjY8nKyqKgoACXy8Vzzz3HqFGj2ruqJ6358+czY8YMKisrAZoUnJYsWcKjjz6K3++nS5cunHXWWZSWlvL9998TDocZPHgws2bNIioqqs58as9jy8rKYsqUKezfvx+Xy4XP5yMuLo7s7GwKCgoAuOWWW/jlL39ZZz61Sdtav349d999N8XFxXi9Xnw+H9HR0XXa5brrruO5557D4XBE5rNMuxjSqp544gnD5/MZ1113nbF79+7I9EAgYDz33HOGz+czBgwYYOTk5LRfJU9SBQUFxl133WX4fD7j/PPPNx544AHD5/MZw4YNO+p8u3btMgYOHGj4fD5j1qxZRigUiqRt3brVuPLKKw2fz2c8/vjj9eZVex5dRUWFMWzYMMPn8xk33nijkZubG0kLBALGCy+8YPh8PsPn8xlLly6NpKlN2lZxcbFx8cUXGz6fz5g8ebJRUFAQSQsEAsaf/vSnSLvMnTs3kmaldlFwakW5ublG3759jfT0dGP79u310sPhsDF+/HjD5/MZ06ZNa4cantz++te/Gj6fz7jllluMPXv2GAsWLGhScJo+fbrh8/mMhx9+uMH01atXGz6fz+jbt6+xa9euyHS157HNnz/f8Pl8Rr9+/Yw9e/bUSw+Hw8b1119v+Hw+45577olMV5u0rdmzZxs+n88YMmSIUVpa2mCen/3sZ4bP5zNuuummyDQrtYsGRLSiZcuWEQqFOP/880lLS6uXbrPZ+NnPfgbAxx9/jN/vP9FVPKk5nU6mTp3KnDlz6Nq1a5PmCQaDLF++HIBx48Y1mOeyyy6jW7duhEIhlixZEpmu9jy2mJgYRo0axX/8x3802CY2m41BgwYBsHPnTkBtciJ07NiRsWPHcvvttxMXF9dgnsGDBwOwd+9ewHrtouDUimqfIHHeeec1mqf2Luny8nK2bNlyQup1qpgwYQL33XcfdnvTv7bbtm2jtLQUh8PBueee22i+2jb7+uuvI9PUnsc2atQoXnzxRZ5++ulG8wSDQQDcbjegNjkRRo8ezYwZM7j//vsbzRMIBADo0qULYL12UXBqRTt27ADgjDPOaDRP9+7dIwfX2vzSNLUHt+NRu407d+6Mx+NpNF+PHj3q5D/ys9qz+YLBIJ9//jkAP/nJTwC1iRUUFxfz8ccfA3DNNdcA1msXBadWdODAAcDsUjfG5XIRHx8PEBkxI22nsLAQgKSkpKPmq22z2vyg9mwNf/nLX8jPz8ftdnPrrbcCapP2Eg6Hyc/P54MPPmD8+PEUFBRwzTXXMHHiRMB67aL7nFpRVVUVwFF/dRyZXjsUWtpOU9vE6/UCUF1dTTgcxm63qz1baMGCBbz88ssAPProo5Ff1WqTE+/ee+9lxYoVkb8vuugiHn30Uf7t3/4tMs1q7aLg1A6MQ7eW2Wy2dq6J1DJacLuf2rO+V155hZdeegmAyZMnR3pNx0Nt0nrOOeccDMOgvLyc7du3s3btWg4cOIDf7z/u+8FOVLsoOLWi6OhoSkpKqK6uPmq+mpoawBzpJG0rOjoa4JhtUpseHR0dOS+u9jx+gUCAX//617z77rvYbDYeeugh7rrrrjp51CYn3pFtEA6HWbJkCU899RQPPfQQWVlZPPTQQ5ZrF11zakUpKSlA3XOxP+b3+yktLa2TX9pOU9oEDp8DP7JN1J7Hp6ysjJ///Oe8++67eL1eXnjhhXqBCdQm7c1utzNq1CieeuopAF599VXy8/Mt1y4KTq2odnx/7f0cDdmxY0eka9unT58TUa3T2llnnQWYO1RFRUWj+XJycurkB7Xn8aisrOTnP/85a9asISUlhTfffLPR00VqE2uofd5hKBTi22+/tVy7KDi1oqFDhwJ1x///2JdffgmYI2L0cMq2l5aWRnJyMuFwuNE3GQeDQTIyMgDzQnEttWfT+P1+7rvvPtavX0/Pnj155513GDhwYKP51SZt7+6772b48OHMnDmz0Ty1vRgAh8NhuXZRcGpF1157LR6Ph++++47NmzfXSw8Gg7zzzjuAeZPckQ9blLZRewoD4O23324wz0cffURRURFut5vhw4dHpqs9m+a5557jiy++oHPnzrzxxhukpqYeNb/apO05nU527tzJ+++/3+jTGL744ovIZ5/PZ7l2UXBqRSkpKZFRSY888gi5ubmRNL/fz69+9Suys7OJi4vj7rvvbq9qnnbuuusu4uLiWLFiBX/9618Jh8ORtO+++47f/e53ANx222106tQpkqb2PLbvv/+eN998E4AXXniBzp07N2k+tUnbmjJlCna7nZycHKZPn05RUVGd9M8++4znn38eMHtAvXr1AqzVLnplRisLBoPcf//9rFq1CqfTycCBA4mJiWHTpk0UFRURExPDzJkzufDCC9u7qiede++9t87f+fn5bN68Ga/XyyWXXFIn7cEHH6xz6uCLL77gvvvuo7KyMvIagKKiIjZt2gTA1VdfzZ///Od676lRex7dgw8+yEcffUR0dHSd0zyNeeaZZyI3aqpN2ta7777Lr3/9awKBANHR0aSlpREfH8/u3bsjwaNPnz7MmjWrzgAFq7SLglMbMAyDhQsXsnDhQrZt20ZVVRWdOnXi8ssvZ8qUKcc87SENS09Pb3LeOXPmRM6D19q9ezevvfYan332GQUFBURHR+Pz+bjxxhv56U9/2ui9F2rPxk2cOJGvvvqqyflXrFhB9+7dI3+rTdrWjh07mDt3LmvXriUvLw+/309cXBx9+vRh+PDhjBs3rsHHglmhXRScRETEcnTNSURELEfBSURELEfBSURELEfBSURELEfBSURELEfBSURELEfBSURELEfBSURELEfBSUQaNXHiRNLT04/6dGuRtqDgJCIilqPgJCIilqPgJCIilqPgJCIiluM8dhYROZaqqireeustli1bRk5ODlVVVXTo0IFzzjmH8ePHc9lll9XJP3bsWDZt2sSTTz7JDTfcwMyZM1m+fDl79+7F7XbTr18/Jk2axLBhwxpcXmlpKXPmzGHlypXk5uZSXV1NYmIi/fv354YbbmD48OGNvtbg888/56233uK7776jpKSE2NhYzj33XG677bajvpMpHA4zZ84cFi5cyO7duzEMg969e3PTTTfxH//xH83feCIN0CszRFpo7969TJ48mezsbNxuN2eddRYJCQnk5OSwd+9ewBz19uSTT0bmufnmm1m3bh33338/H3/8MTk5OfTv35+YmBgyMzM5cOAAAE899RQTJkyos7zs7GzuuOMO8vPzcblcDBgwgLi4OHJzc9m5cycAo0aN4vnnn8dur3ty5Pnnn+dvf/sbYL6au1OnTuTm5kZePvfggw9yzz33RPLXvq/pgQceYMuWLaxatYpBgwbh9XrZunUr+/fvB+Cxxx7jjjvuaMWtKqc9Q0SaLRwOG+PHjzd8Pp8xYcIEY8+ePXXS33vvPaN///6Gz+cz3n///cj0CRMmGD6fz7jggguMG264wdi/f38kraamxpg6darh8/mMQYMGGXv37o2kBQIB47rrrjN8Pp9xww03GPn5+XWW99FHH0WWN2vWrDppS5cuNXw+nzFw4EDjs88+q5M2a9Ysw+fzGenp6UZGRka9el5zzTXGjTfeWKcuwWDQeOSRRwyfz2cMHTrUCIVCx78BRRqha04iLbB69WrWr19PbGwsL774Il27dq2Tfv311zN58mQAXnvttXrzl5SU8Oyzz9Z5Tbbb7ebpp5/G4/FQXV3Nhx9+GElbtWoVW7duxWaz8cc//pEuXbrUKW/EiBGMHz8egNmzZ2MccWLklVdeAcxe26WXXlpnvttvv52BAwdiGAbz58+vV8+8vDz+8Ic/0Llz58g0h8PBXXfdBUBRUVGk9yXSGhScRFpgxYoVAJx77rkkJyc3mGfkyJEAbNmyhYMHD9ZJ8/l89OnTp948tderANavXx+Z/sknnwDQr18/evfu3eDyhg8fDsCePXvIyckBYN++fWzevBmg0etYf/nLX/jiiy/4/e9/Xy9t6NChnHnmmfWmH/nK9R+vm0hLaECESAtkZWUBsGPHDu69994G8wSDwcjnnJwckpKSIn+fffbZjZbds2dPvvrqK3744YfItG3btgGQnp7e6HxpaWmRzzt27KB3795s3bo1Mq1Xr14NztdYcK2tS0OioqIinwOBQKPzixwvBSeRFiguLgbMXsqePXuOmb+srKzO3wkJCY3mjYuLA6CioiIyraSkBIDExMRG54uPj498Li0trTMfQGxs7DHr+WMxMTHHPY9ISyg4ibRA7Wi4SZMmMX369OOe3+lsfBcMh8MAdYaE1342jjLI9si02vxHlnG0eUWsQtecRFqgtgdTWFjYrPl/3JNqKK22B3Xk8mp7bA05spdU2zM7sod2ZLqIVSk4ibRA7TWjDRs2NGv+2mtIDdm1axdQ93qPz+cDzMEVjam9DnZk/tr/f5x+pB07drBq1SoyMjKaUHORtqXgJNICV199NWAGkjVr1jSY59NPP2XcuHHMmTOnXtrGjRvrDHiodfDgwUjAO//88yPTr7rqKgAyMzMbDWyLFy8GoE+fPpHRdJ06daJfv34AdYamH2nGjBncfffdzJ07t8F0kRNJwUmkBS655BIGDx4MwLRp0+r1aDIyMnj88cf57rvvKC8vrze/x+Ph0UcfjTwRAqCmpoann34av99PfHw8I0aMiKRdfvnl9O/fHzCfyrBv37465f3jH//gn//8J0C90YO19yR98MEHvPfee3XS3nnnHVavXg3ATTfd1OT1F2krGhAh0gI2m40XX3yRO+64g+zsbK6//nr69+9Phw4d2LNnD9nZ2YB579GUKVPqzT9+/Hg++eQThg0bxqBBg/B4PGzatImioiJsNhu//OUv61wvstvtvPTSS9x+++1s3ryZq6++mnPOOYeoqCh27NhBXl4eYAaiUaNG1VnWiBEjmDx5Mq+//jrTp0/nb3/7G6mpqeTm5kZOIU6dOrVOT02kvSg4ibRQ165dWbBgAfPmzWPZsmVkZ2eTlZVFcnIyl112GWPHjmXkyJENPog1JiaGd999l5kzZ7JixQr27t2Lx+Phkksu4c477+TCCy+sN88ZZ5zBokWLmDNnDsuXLyczMxO/309SUhIjR47k5ptv5oILLmiwrtOmTePCCy9k3rx5bNiwgdzcXGJjYxk2bNgxH/wqciLpwa8i7aD2gar3338//+///b/2ro6I5eiak4iIWI6Ck4iIWI6Ck4iIWI6Ck4iIWI4GRIiIiOWo5yQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpaj4CQiIpbz/wEB0mIfoedLXAAAAABJRU5ErkJggg==\n"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.size'] = 26\n",
    "plt.rcParams['axes.labelsize'] = 26\n",
    "plt.rcParams['xtick.labelsize'] = 26\n",
    "plt.rcParams['ytick.labelsize'] = 26\n",
    "plt.rcParams['legend.fontsize'] = 26\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "ax = pd.DataFrame(history).drop(columns='total').plot(x='epoch')\n",
    "plt.ylabel('loss')\n",
    "plt.ylim([0.2, 1.8])\n",
    "\n",
    "plt.savefig(figure_path + '/' + dataset +'-medium-loss.png', dpi=300, bbox_inches='tight')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "outputs": [],
   "source": [
    "# large net\n",
    "\n",
    "start = time.time()\n",
    "net = RecommenderNet(\n",
    "    n_users=n, n_movies=m,\n",
    "    n_factors=50, hidden=[100, 100, 100],\n",
    "    embedding_dropout=0.05, dropouts=[0.3, 0.3, 0.3])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "outputs": [],
   "source": [
    "lr = 1e-3\n",
    "wd = 1e-5\n",
    "bs = 4096\n",
    "n_epochs = 300\n",
    "patience = 300\n",
    "no_improvements = 0\n",
    "best_loss = np.inf\n",
    "best_weights = None\n",
    "history = []\n",
    "lr_history = []\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "net.to(device)\n",
    "criterion = nn.MSELoss(reduction='sum')\n",
    "optimizer = optim.Adam(net.parameters(), lr=lr, weight_decay=wd)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss improvement on epoch: 1\n",
      "[001/300] train: 0.9961 - val: 0.8413\n",
      "loss improvement on epoch: 2\n",
      "[002/300] train: 0.8310 - val: 0.8156\n",
      "loss improvement on epoch: 3\n",
      "[003/300] train: 0.8079 - val: 0.8084\n",
      "loss improvement on epoch: 4\n",
      "[004/300] train: 0.7930 - val: 0.8048\n",
      "loss improvement on epoch: 5\n",
      "[005/300] train: 0.7785 - val: 0.7975\n",
      "loss improvement on epoch: 6\n",
      "[006/300] train: 0.7621 - val: 0.7943\n",
      "[007/300] train: 0.7451 - val: 0.7964\n",
      "[008/300] train: 0.7292 - val: 0.7986\n",
      "[009/300] train: 0.7148 - val: 0.7975\n",
      "[010/300] train: 0.7004 - val: 0.8052\n",
      "[011/300] train: 0.6878 - val: 0.8071\n",
      "[012/300] train: 0.6751 - val: 0.8159\n",
      "[013/300] train: 0.6650 - val: 0.8217\n",
      "[014/300] train: 0.6547 - val: 0.8263\n",
      "[015/300] train: 0.6454 - val: 0.8258\n",
      "[016/300] train: 0.6367 - val: 0.8326\n",
      "[017/300] train: 0.6300 - val: 0.8318\n",
      "[018/300] train: 0.6227 - val: 0.8328\n",
      "[019/300] train: 0.6159 - val: 0.8363\n",
      "[020/300] train: 0.6099 - val: 0.8436\n",
      "[021/300] train: 0.6035 - val: 0.8423\n",
      "[022/300] train: 0.5989 - val: 0.8463\n",
      "[023/300] train: 0.5944 - val: 0.8501\n",
      "[024/300] train: 0.5904 - val: 0.8467\n",
      "[025/300] train: 0.5860 - val: 0.8512\n",
      "[026/300] train: 0.5820 - val: 0.8573\n",
      "[027/300] train: 0.5780 - val: 0.8535\n",
      "[028/300] train: 0.5748 - val: 0.8595\n",
      "[029/300] train: 0.5719 - val: 0.8542\n",
      "[030/300] train: 0.5696 - val: 0.8494\n",
      "[031/300] train: 0.5663 - val: 0.8479\n",
      "[032/300] train: 0.5630 - val: 0.8620\n",
      "[033/300] train: 0.5604 - val: 0.8599\n",
      "[034/300] train: 0.5584 - val: 0.8677\n",
      "[035/300] train: 0.5555 - val: 0.8586\n",
      "[036/300] train: 0.5540 - val: 0.8578\n",
      "[037/300] train: 0.5510 - val: 0.8595\n",
      "[038/300] train: 0.5482 - val: 0.8641\n",
      "[039/300] train: 0.5467 - val: 0.8691\n",
      "[040/300] train: 0.5456 - val: 0.8613\n",
      "[041/300] train: 0.5439 - val: 0.8691\n",
      "[042/300] train: 0.5411 - val: 0.8630\n",
      "[043/300] train: 0.5388 - val: 0.8717\n",
      "[044/300] train: 0.5377 - val: 0.8661\n",
      "[045/300] train: 0.5367 - val: 0.8729\n",
      "[046/300] train: 0.5343 - val: 0.8689\n",
      "[047/300] train: 0.5346 - val: 0.8744\n",
      "[048/300] train: 0.5312 - val: 0.8702\n",
      "[049/300] train: 0.5302 - val: 0.8771\n",
      "[050/300] train: 0.5293 - val: 0.8746\n",
      "[051/300] train: 0.5283 - val: 0.8751\n",
      "[052/300] train: 0.5264 - val: 0.8702\n",
      "[053/300] train: 0.5267 - val: 0.8686\n",
      "[054/300] train: 0.5244 - val: 0.8695\n",
      "[055/300] train: 0.5248 - val: 0.8801\n",
      "[056/300] train: 0.5219 - val: 0.8700\n",
      "[057/300] train: 0.5211 - val: 0.8782\n",
      "[058/300] train: 0.5194 - val: 0.8755\n",
      "[059/300] train: 0.5189 - val: 0.8772\n",
      "[060/300] train: 0.5186 - val: 0.8740\n",
      "[061/300] train: 0.5176 - val: 0.8795\n",
      "[062/300] train: 0.5158 - val: 0.8753\n",
      "[063/300] train: 0.5137 - val: 0.8797\n",
      "[064/300] train: 0.5146 - val: 0.8824\n",
      "[065/300] train: 0.5125 - val: 0.8807\n",
      "[066/300] train: 0.5142 - val: 0.8872\n",
      "[067/300] train: 0.5112 - val: 0.8799\n",
      "[068/300] train: 0.5111 - val: 0.8846\n",
      "[069/300] train: 0.5104 - val: 0.8829\n",
      "[070/300] train: 0.5100 - val: 0.8771\n",
      "[071/300] train: 0.5083 - val: 0.8857\n",
      "[072/300] train: 0.5077 - val: 0.8848\n",
      "[073/300] train: 0.5057 - val: 0.8850\n",
      "[074/300] train: 0.5051 - val: 0.8887\n",
      "[075/300] train: 0.5058 - val: 0.8798\n",
      "[076/300] train: 0.5047 - val: 0.8873\n",
      "[077/300] train: 0.5056 - val: 0.8847\n",
      "[078/300] train: 0.5035 - val: 0.8863\n",
      "[079/300] train: 0.5028 - val: 0.8822\n",
      "[080/300] train: 0.5020 - val: 0.8975\n",
      "[081/300] train: 0.5012 - val: 0.8899\n",
      "[082/300] train: 0.5019 - val: 0.8855\n",
      "[083/300] train: 0.5006 - val: 0.8869\n",
      "[084/300] train: 0.4994 - val: 0.8855\n",
      "[085/300] train: 0.4996 - val: 0.8878\n",
      "[086/300] train: 0.4990 - val: 0.8856\n",
      "[087/300] train: 0.4983 - val: 0.8865\n",
      "[088/300] train: 0.4969 - val: 0.8806\n",
      "[089/300] train: 0.4954 - val: 0.8863\n",
      "[090/300] train: 0.4970 - val: 0.8886\n",
      "[091/300] train: 0.4957 - val: 0.8850\n",
      "[092/300] train: 0.4953 - val: 0.8963\n",
      "[093/300] train: 0.4941 - val: 0.8905\n",
      "[094/300] train: 0.4935 - val: 0.8940\n",
      "[095/300] train: 0.4940 - val: 0.8905\n",
      "[096/300] train: 0.4936 - val: 0.8920\n",
      "[097/300] train: 0.4923 - val: 0.8881\n",
      "[098/300] train: 0.4919 - val: 0.8916\n",
      "[099/300] train: 0.4924 - val: 0.8906\n",
      "[100/300] train: 0.4903 - val: 0.8881\n",
      "[101/300] train: 0.4905 - val: 0.8969\n",
      "[102/300] train: 0.4905 - val: 0.8913\n",
      "[103/300] train: 0.4901 - val: 0.8904\n",
      "[104/300] train: 0.4908 - val: 0.8914\n",
      "[105/300] train: 0.4892 - val: 0.8889\n",
      "[106/300] train: 0.4888 - val: 0.8950\n",
      "[107/300] train: 0.4877 - val: 0.8888\n",
      "[108/300] train: 0.4879 - val: 0.8905\n",
      "[109/300] train: 0.4875 - val: 0.8913\n",
      "[110/300] train: 0.4875 - val: 0.8926\n",
      "[111/300] train: 0.4877 - val: 0.8912\n",
      "[112/300] train: 0.4861 - val: 0.8928\n",
      "[113/300] train: 0.4854 - val: 0.8988\n",
      "[114/300] train: 0.4866 - val: 0.8889\n",
      "[115/300] train: 0.4845 - val: 0.9011\n",
      "[116/300] train: 0.4846 - val: 0.8950\n",
      "[117/300] train: 0.4849 - val: 0.8984\n",
      "[118/300] train: 0.4847 - val: 0.8906\n",
      "[119/300] train: 0.4846 - val: 0.8954\n",
      "[120/300] train: 0.4830 - val: 0.8974\n",
      "[121/300] train: 0.4831 - val: 0.8941\n",
      "[122/300] train: 0.4824 - val: 0.8950\n",
      "[123/300] train: 0.4821 - val: 0.8948\n",
      "[124/300] train: 0.4812 - val: 0.8996\n",
      "[125/300] train: 0.4812 - val: 0.8919\n",
      "[126/300] train: 0.4813 - val: 0.9019\n",
      "[127/300] train: 0.4806 - val: 0.9003\n",
      "[128/300] train: 0.4803 - val: 0.8947\n",
      "[129/300] train: 0.4806 - val: 0.8969\n",
      "[130/300] train: 0.4797 - val: 0.8953\n",
      "[131/300] train: 0.4798 - val: 0.9019\n",
      "[132/300] train: 0.4795 - val: 0.8964\n",
      "[133/300] train: 0.4793 - val: 0.8941\n",
      "[134/300] train: 0.4787 - val: 0.8921\n",
      "[135/300] train: 0.4781 - val: 0.8921\n",
      "[136/300] train: 0.4769 - val: 0.8986\n",
      "[137/300] train: 0.4786 - val: 0.8996\n",
      "[138/300] train: 0.4768 - val: 0.9009\n",
      "[139/300] train: 0.4767 - val: 0.8960\n",
      "[140/300] train: 0.4760 - val: 0.8918\n",
      "[141/300] train: 0.4758 - val: 0.9027\n",
      "[142/300] train: 0.4758 - val: 0.8992\n",
      "[143/300] train: 0.4760 - val: 0.8951\n",
      "[144/300] train: 0.4762 - val: 0.8958\n",
      "[145/300] train: 0.4754 - val: 0.8950\n",
      "[146/300] train: 0.4743 - val: 0.8904\n",
      "[147/300] train: 0.4752 - val: 0.8985\n",
      "[148/300] train: 0.4744 - val: 0.9029\n",
      "[149/300] train: 0.4744 - val: 0.8972\n",
      "[150/300] train: 0.4729 - val: 0.8984\n",
      "[151/300] train: 0.4737 - val: 0.8951\n",
      "[152/300] train: 0.4738 - val: 0.9018\n",
      "[153/300] train: 0.4732 - val: 0.8978\n",
      "[154/300] train: 0.4733 - val: 0.8928\n",
      "[155/300] train: 0.4720 - val: 0.8952\n",
      "[156/300] train: 0.4721 - val: 0.9048\n",
      "[157/300] train: 0.4716 - val: 0.9005\n",
      "[158/300] train: 0.4727 - val: 0.9023\n",
      "[159/300] train: 0.4708 - val: 0.9001\n",
      "[160/300] train: 0.4715 - val: 0.8965\n",
      "[161/300] train: 0.4715 - val: 0.9025\n",
      "[162/300] train: 0.4709 - val: 0.9020\n",
      "[163/300] train: 0.4695 - val: 0.9045\n",
      "[164/300] train: 0.4708 - val: 0.9008\n",
      "[165/300] train: 0.4689 - val: 0.9065\n",
      "[166/300] train: 0.4699 - val: 0.9078\n",
      "[167/300] train: 0.4704 - val: 0.9040\n",
      "[168/300] train: 0.4695 - val: 0.9002\n",
      "[169/300] train: 0.4683 - val: 0.9001\n",
      "[170/300] train: 0.4691 - val: 0.9035\n",
      "[171/300] train: 0.4683 - val: 0.8993\n",
      "[172/300] train: 0.4690 - val: 0.9007\n",
      "[173/300] train: 0.4682 - val: 0.9073\n",
      "[174/300] train: 0.4675 - val: 0.9015\n",
      "[175/300] train: 0.4694 - val: 0.8969\n",
      "[176/300] train: 0.4676 - val: 0.9048\n",
      "[177/300] train: 0.4674 - val: 0.9036\n",
      "[178/300] train: 0.4675 - val: 0.9023\n",
      "[179/300] train: 0.4669 - val: 0.9025\n",
      "[180/300] train: 0.4662 - val: 0.9034\n",
      "[181/300] train: 0.4650 - val: 0.9126\n",
      "[182/300] train: 0.4663 - val: 0.9097\n",
      "[183/300] train: 0.4664 - val: 0.8993\n",
      "[184/300] train: 0.4663 - val: 0.9005\n",
      "[185/300] train: 0.4654 - val: 0.9005\n",
      "[186/300] train: 0.4644 - val: 0.9039\n",
      "[187/300] train: 0.4652 - val: 0.9045\n",
      "[188/300] train: 0.4643 - val: 0.9011\n",
      "[189/300] train: 0.4646 - val: 0.8982\n",
      "[190/300] train: 0.4649 - val: 0.9064\n",
      "[191/300] train: 0.4638 - val: 0.9117\n",
      "[192/300] train: 0.4646 - val: 0.9043\n",
      "[193/300] train: 0.4648 - val: 0.9012\n",
      "[194/300] train: 0.4640 - val: 0.9021\n",
      "[195/300] train: 0.4640 - val: 0.9085\n",
      "[196/300] train: 0.4640 - val: 0.9014\n",
      "[197/300] train: 0.4629 - val: 0.9110\n",
      "[198/300] train: 0.4635 - val: 0.8959\n",
      "[199/300] train: 0.4624 - val: 0.9073\n",
      "[200/300] train: 0.4626 - val: 0.9121\n",
      "[201/300] train: 0.4632 - val: 0.9033\n",
      "[202/300] train: 0.4624 - val: 0.9056\n",
      "[203/300] train: 0.4623 - val: 0.9066\n",
      "[204/300] train: 0.4616 - val: 0.9031\n",
      "[205/300] train: 0.4619 - val: 0.9039\n",
      "[206/300] train: 0.4619 - val: 0.9049\n",
      "[207/300] train: 0.4616 - val: 0.9046\n",
      "[208/300] train: 0.4615 - val: 0.9014\n",
      "[209/300] train: 0.4614 - val: 0.9056\n",
      "[210/300] train: 0.4611 - val: 0.9103\n",
      "[211/300] train: 0.4607 - val: 0.9066\n",
      "[212/300] train: 0.4595 - val: 0.9072\n",
      "[213/300] train: 0.4599 - val: 0.9059\n",
      "[214/300] train: 0.4602 - val: 0.9078\n",
      "[215/300] train: 0.4607 - val: 0.9055\n",
      "[216/300] train: 0.4600 - val: 0.9025\n",
      "[217/300] train: 0.4597 - val: 0.9009\n",
      "[218/300] train: 0.4590 - val: 0.9084\n",
      "[219/300] train: 0.4588 - val: 0.9038\n",
      "[220/300] train: 0.4585 - val: 0.9120\n",
      "[221/300] train: 0.4581 - val: 0.9019\n",
      "[222/300] train: 0.4577 - val: 0.9043\n",
      "[223/300] train: 0.4596 - val: 0.9054\n",
      "[224/300] train: 0.4585 - val: 0.9226\n",
      "[225/300] train: 0.4586 - val: 0.9073\n",
      "[226/300] train: 0.4584 - val: 0.9053\n",
      "[227/300] train: 0.4582 - val: 0.9122\n",
      "[228/300] train: 0.4577 - val: 0.9062\n",
      "[229/300] train: 0.4582 - val: 0.9038\n",
      "[230/300] train: 0.4568 - val: 0.9032\n",
      "[231/300] train: 0.4569 - val: 0.9155\n",
      "[232/300] train: 0.4563 - val: 0.9177\n",
      "[233/300] train: 0.4583 - val: 0.9081\n",
      "[234/300] train: 0.4584 - val: 0.9070\n",
      "[235/300] train: 0.4562 - val: 0.9101\n",
      "[236/300] train: 0.4568 - val: 0.9064\n",
      "[237/300] train: 0.4560 - val: 0.9111\n",
      "[238/300] train: 0.4573 - val: 0.9047\n",
      "[239/300] train: 0.4562 - val: 0.9055\n",
      "[240/300] train: 0.4545 - val: 0.9088\n",
      "[241/300] train: 0.4569 - val: 0.9132\n",
      "[242/300] train: 0.4556 - val: 0.9091\n",
      "[243/300] train: 0.4564 - val: 0.9093\n",
      "[244/300] train: 0.4542 - val: 0.9037\n",
      "[245/300] train: 0.4560 - val: 0.9060\n",
      "[246/300] train: 0.4534 - val: 0.9068\n",
      "[247/300] train: 0.4553 - val: 0.9014\n",
      "[248/300] train: 0.4553 - val: 0.9069\n",
      "[249/300] train: 0.4549 - val: 0.9111\n",
      "[250/300] train: 0.4544 - val: 0.9087\n",
      "[251/300] train: 0.4540 - val: 0.9199\n",
      "[252/300] train: 0.4549 - val: 0.9106\n",
      "[253/300] train: 0.4541 - val: 0.9045\n",
      "[254/300] train: 0.4541 - val: 0.9079\n",
      "[255/300] train: 0.4545 - val: 0.9113\n",
      "[256/300] train: 0.4523 - val: 0.9170\n",
      "[257/300] train: 0.4536 - val: 0.9137\n",
      "[258/300] train: 0.4544 - val: 0.9098\n",
      "[259/300] train: 0.4532 - val: 0.9135\n",
      "[260/300] train: 0.4535 - val: 0.9143\n",
      "[261/300] train: 0.4536 - val: 0.9087\n",
      "[262/300] train: 0.4526 - val: 0.9107\n",
      "[263/300] train: 0.4540 - val: 0.9114\n",
      "[264/300] train: 0.4523 - val: 0.9139\n",
      "[265/300] train: 0.4520 - val: 0.9088\n",
      "[266/300] train: 0.4526 - val: 0.9118\n",
      "[267/300] train: 0.4508 - val: 0.9031\n",
      "[268/300] train: 0.4516 - val: 0.9083\n",
      "[269/300] train: 0.4514 - val: 0.9120\n",
      "[270/300] train: 0.4520 - val: 0.9152\n",
      "[271/300] train: 0.4509 - val: 0.9140\n",
      "[272/300] train: 0.4509 - val: 0.9120\n",
      "[273/300] train: 0.4501 - val: 0.9104\n",
      "[274/300] train: 0.4523 - val: 0.9120\n",
      "[275/300] train: 0.4517 - val: 0.9168\n",
      "[276/300] train: 0.4512 - val: 0.9098\n",
      "[277/300] train: 0.4499 - val: 0.9072\n",
      "[278/300] train: 0.4513 - val: 0.9153\n",
      "[279/300] train: 0.4502 - val: 0.9211\n",
      "[280/300] train: 0.4505 - val: 0.9112\n",
      "[281/300] train: 0.4503 - val: 0.9048\n",
      "[282/300] train: 0.4510 - val: 0.9173\n",
      "[283/300] train: 0.4504 - val: 0.9112\n",
      "[284/300] train: 0.4493 - val: 0.9168\n",
      "[285/300] train: 0.4497 - val: 0.9112\n",
      "[286/300] train: 0.4494 - val: 0.9107\n",
      "[287/300] train: 0.4501 - val: 0.9159\n",
      "[288/300] train: 0.4500 - val: 0.9111\n",
      "[289/300] train: 0.4490 - val: 0.9130\n",
      "[290/300] train: 0.4494 - val: 0.9105\n",
      "[291/300] train: 0.4491 - val: 0.9172\n",
      "[292/300] train: 0.4488 - val: 0.9111\n",
      "[293/300] train: 0.4497 - val: 0.9106\n",
      "[294/300] train: 0.4486 - val: 0.9173\n",
      "[295/300] train: 0.4492 - val: 0.9152\n",
      "[296/300] train: 0.4492 - val: 0.9152\n",
      "[297/300] train: 0.4485 - val: 0.9117\n",
      "[298/300] train: 0.4485 - val: 0.9104\n",
      "[299/300] train: 0.4488 - val: 0.9111\n",
      "[300/300] train: 0.4487 - val: 0.9162\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(n_epochs):\n",
    "    stats = {'epoch': epoch + 1, 'total': n_epochs}\n",
    "\n",
    "    for phase in ('train', 'val'):\n",
    "        if phase == 'train':\n",
    "          training = True\n",
    "        else:\n",
    "          training = False\n",
    "        running_loss = 0\n",
    "        n_batches = 0\n",
    "        for batch in batches(*datasets[phase], shuffle=training, bs=bs):\n",
    "            x_batch, y_batch = [b.to(device) for b in batch]\n",
    "            optimizer.zero_grad()\n",
    "            # compute gradients only during 'train' phase\n",
    "            with torch.set_grad_enabled(training):\n",
    "                outputs = net(x_batch[:,0], x_batch[:,1], minmax)\n",
    "                loss = criterion(outputs, y_batch)\n",
    "                # don't update weights and rates when in 'val' phase\n",
    "                if training:\n",
    "                    #scheduler.step()\n",
    "                    loss.backward()\n",
    "                    optimizer.step()\n",
    "                    #lr_history.extend(scheduler.get_lr())\n",
    "            running_loss += loss.item()\n",
    "        epoch_loss = running_loss / dataset_sizes[phase]\n",
    "        stats[phase] = epoch_loss\n",
    "        # early stopping: save weights of the best model so far\n",
    "        if phase == 'val':\n",
    "            if epoch_loss < best_loss:\n",
    "                print('loss improvement on epoch: %d' % (epoch + 1))\n",
    "                best_loss = epoch_loss\n",
    "                best_weights = copy.deepcopy(net.state_dict())\n",
    "                no_improvements = 0\n",
    "            else:\n",
    "                no_improvements += 1\n",
    "    history.append(stats)\n",
    "    with open(f'logs/large.txt', 'a') as f:\n",
    "        print('[{epoch:03d}/{total:03d}] train: {train:.4f} - val: {val:.4f}'.format(**stats))\n",
    "        f.write('[{epoch:03d}/{total:03d}] train: {train:.4f} - val: {val:.4f}'.format(**stats))\n",
    "    if no_improvements >= patience:\n",
    "        print('early stopping after epoch {epoch:03d}'.format(**stats))\n",
    "        break"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "outputs": [
    {
     "data": {
      "text/plain": "<All keys matched successfully>"
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.load_state_dict(best_weights)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "outputs": [],
   "source": [
    "groud_truth, predictions = [], []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for batch in batches(*datasets['val'], shuffle=False, bs=bs):\n",
    "        x_batch, y_batch = [b.to(device) for b in batch]\n",
    "        outputs = net(x_batch[:, 0], x_batch[:, 1], minmax)\n",
    "        groud_truth.extend(y_batch.tolist())\n",
    "        predictions.extend(outputs.tolist())\n",
    "\n",
    "groud_truth = np.asarray(groud_truth).ravel()\n",
    "predictions = np.asarray(predictions).ravel()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation RMSE: 0.8991\n"
     ]
    }
   ],
   "source": [
    "valid_loss = np.sqrt(np.mean((predictions - groud_truth)**2))\n",
    "print(f'Validation RMSE: {valid_loss:.4f}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "outputs": [],
   "source": [
    "groud_truth, predictions = [], []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for batch in batches(*datasets['test'], shuffle=False, bs=bs):\n",
    "        x_batch, y_batch = [b.to(device) for b in batch]\n",
    "        outputs = net(x_batch[:, 0], x_batch[:, 1], minmax)\n",
    "        groud_truth.extend(y_batch.tolist())\n",
    "        predictions.extend(outputs.tolist())\n",
    "\n",
    "groud_truth = np.asarray(groud_truth).ravel()\n",
    "predictions = np.asarray(predictions).ravel()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Large test RMSE: 0.9022\n"
     ]
    }
   ],
   "source": [
    "final_loss = np.sqrt(np.mean((predictions - groud_truth)**2))\n",
    "print(f'Large test RMSE: {final_loss:.4f}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Large duration: 117.7924\n"
     ]
    }
   ],
   "source": [
    "print(f'Large duration: {round(time.time() - start, 4)}')\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "outputs": [],
   "source": [
    "with open('best.weights.big', 'wb') as file:\n",
    "    pickle.dump(best_weights, file)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAacAAAEfCAYAAADldgmAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABDTUlEQVR4nO3dd3wVVf7/8dftNz0EQktCC9wIJEhHAQsihuZXBIVVQQSxgYu4roI/y9pWFndF1/WL6+rKFxRQlLioSJEi4FIkMRQhBEgCwfRAerttfn8MuRCSQEghA3yej4cPwsyZmXNzZN5zzpyZq1MURUEIIYTQEH1zV0AIIYQ4n4STEEIIzZFwEkIIoTkSTkIIITRHwkkIIYTmSDgJIYTQHGNzV+BKEhcX19xVEEKIK1K/fv0uqbyE0yW61F8wQEJCAt27d2+C2oiGkHbRJmkX7Wlom9Tnwl6G9YQQQmiOhJMQQgjNkXASQgihORJOQgghNEfCSQghhOZIOAkhhNAcCSchhBCaI+EkhBBCcySchBBCaI6EkxBCCM2RcBJCCKE5Ek5CCCE0R8JJCCGE5kg4CSGE0BwJJyGEEJoj4SSEEEJzJJyEEEJojoSTEEIIzZFwEkIIoTkSTkIIITRHwkkIIYTmSDgJIYTQHAknIYS4At12221EREQQExPT3FVpEsbmroAQQohLN2TIEE6dOkW7du2auypNQnpOQgjRQP/5z3+IiIhg9+7dl+2Yr7/+OosWLeLGG2+8bMe8nCSchBCigfbt29fcVbjqSDgJIUQDSTg1PgknIYSop3nz5hEREcHBgwcBePDBB4mIiGDKlCkAREREEBERwd69e/n+++8ZOXIkPXv25P/+7/+q7OeXX37hmWee4bbbbiMqKorIyEhuu+025s2bx9GjR2s8dk0TIo4ePUpERARRUVEAJCYm8vTTTzN06FAiIyMZPHgws2fP5tixY03w22hcEk5CCFFPPXr0YPjw4Z6/9+3bl+HDh9O3b98q5Q4fPsyzzz6LyWRi6NChtG7d2rNu2bJl3H///Xz33XdUVFTQp08f+vbtS1lZGV9//TXjx49n165ddaqPxWLx/Lxnzx4mTZpEXFwc1113Hddffz3FxcWsX7+eSZMmkZaW1sBP37Rktp4QQtTTgw8+6OktAcyZM4dBgwZVK7d48WJmzZrFzJkzqyzPzc3lL3/5C4qi8Lvf/Y6XXnoJo1E9LZeXl/Pcc8+xfv16/vSnP7F+/fqL1kevV/sbbrebP/7xjzzxxBM88sgjnuVpaWmMHz+e/Px8PvvsM+bOndugz9+UNB9OqampPPfcc8THxxMSEsLmzZvrtZ/Y2FgeeOCBi5Yzm80cOHCgXscQ4mqxKu43VsaebJZjl5aW4r0tv9H3O7F/GBP6hTb6fuvC7Xbz+OOPV1teUFDAxIkTOX36NDNnzvQEE4DVauWpp55i/fr1HD9+nOPHj9OpU6c6Hc/pdNKlSxcee+yxKstDQkK48847+fTTT4mPj2/QZ2pqmg6nlStXMn/+fEpLSxu8r8LCQgC8vLwYPHhwreVMJlODjyWEEOe65ZZbPL2Xc4WHh/PSSy/Vul1YWJjn59zc3DqHE8CkSZNqXN65c2cA8vLy6ryv5qDJcMrNzeXFF19ky5YtBAQEMGrUKNauXdugfVaGU1hYGIsWLWqMagpx1ZrQL7TZehkJCQl07969WY7dVDp27HjB9SdPnmTz5s0kJSVRUFCAw+GoVsblcjXKMb28vABqPIaWaDKcYmJi2LJlCwMGDOCvf/0rO3fubHA4FRQUAODn59cYVRRCiDrz9vaucbmiKLz11lssXrwYRVEa9ZhX+rlOk+FkNBqZPXs2TzzxRI1d4fooKioCrvwGE0JceXQ6XY3Lly9fzieffAJAdHQ006dPJzw8HB8fH8+5r3KyxbVGk+E0efJkzGZzo+5Tek5CCK35/PPPARg4cCB///vfq4VYcXFxc1RLEzQZTo0dTHD2npO/vz/Hjh1jw4YNpKSk4HA4CA4OZtCgQQwbNgyDwdDoxxZCiJocP34cUCdM1NS7+uWXXy5zjbRDk+HUFCrDacOGDSxfvrza+O7SpUux2Wy89957ntksQghxKS510oLRaMRut2O326utczgcvP/++/Xe95XumnlDRGU45ebmMnHiRFavXs2BAwfYuXMnCxYsoFWrVhw5coRp06Zx+vTpZq6tEOJKEhgYCHDJz0hef/31AKxevdpz6wEgKyuLWbNm4eXl5Zl1d+TIkcap7BXimuk5PfHEExQUFBASEkLv3r09y4OCghg3bhxRUVGMHz+ejIwMPvroo1qfnE5ISLjkY5eXl9drO9G0pF206UpsF5vNxs8//8w777zD559/TllZGYsXL/asT09Pr/EzjRkzht27d3P8+HGGDx9Oly5dKCsrIykpiTZt2vD666/z73//mxMnTvD222/z/fffc9ddd9G3b19Pb+vcfWdlZXn2fezYMc9EsHOlp6cDYLfb6/x7bo42uWbCaejQoRdcHx4ezpgxY1i1ahUbN26sNZzq8/zF1fjcxtVA2kWbrsR2WbBgAS+88ALx8fHk5+fTqVOnKp+hffv2NX6m7t27ExoayqJFi/j1119JTEykXbt2PPzww8yYMYOAgABCQ0OZN28ehw4dIjMz07Pvynvz5+773AlfXbt2JTS0+rNqlSFjNpvr/HtuaJvExcVd8jbXTDjVRWRkJKtWrSItLQ23291o09iFEFe39u3bV+kpVUpMTLzotjfeeOMFvzCwU6dOnll956rpVW6hoaEXPeb48eMZP378RevV3OTsWwOdTifBJIQQzeiaOAMXFhaydetWVqxYQXl5ea3lKr/jpKausBBCiMvnmhjWKygo4NFHHwXAYDAwceLEGsusWbMGgFtvvfVyVk8IIcR5rqqeU1ZWFiNHjmTkyJHExsZ6loeFhREdHQ3A/Pnz+fHHH6tsl52dzZNPPkl+fj6BgYFMnz79clZbCCHEeTTZczr/C7kyMjIAOHXqVLV1c+bMwWazAepDaykpKQDVvmbjtddeIyMjg/379/PYY4/RsWNHwsLCKC0t5cCBAzgcDoKCgnj//fdp06ZNU300IYQQdaDJcNq0aVONy8vLy6utmzp1ap32GRgYyIoVK4iJiWHNmjUkJiaya9curFYrERER3HLLLUyePJmgoKAG118IIUTDaDKc6jL9siYXm0ZpNBqZOHFijfechBBCaMdVdc9JCCHE1UHCSQghhOZIOAkhhNAcCSchhBCaI+EkhBBCcySchBBCaI6EkxBCCM2RcBJCCKE5Ek5CCCE0R8JJCCGE5kg4CSHEFWLKlClERETwj3/8o7mr0uQknIQQQmiOhJMQQgjNkXASQgihORJOQgghNEfCSQgh6mnatGlERETwxBNPXLDczJkziYiI4OGHH/YsO3LkCC+88ALR0dH06tWLyMhIbr75ZmbPnk18fHxTV13zJJyEEKKe/ud//geAn376ieLi4hrLFBUVsX37dgDuuusuADZu3Mj48eP56quvyMvLo1evXgwYMACA9evXc//997N69erL8Am0S8JJCCHqacSIEVitVux2O5s3b66xzMaNG7Hb7fj4+DBixAjsdjsvv/wyDoeDYcOGsX37dj777DMWL17M5s2bmTJlCm63mzfeeIPS0tLL/Im0Q5Nf0y6EaGZ7V0D8Z81y6A6lJbDLp/F33Gcy9L6vUXfp6+vLsGHDWLt2LevWrfP0pM61Zs0aAKKjo/Hy8iInJ4cxY8Zw+vRpZsyYgcVi8ZQ1Go384Q9/4LPPPqOwsJD4+HiGDBnSqHW+Ukg4CSFEA9x5552sXbvWM7Tn6+vrWZeXl8fOnTuBs0OAwcHBvPDCC7Xuz9vbm1atWpGTk0NOTk7TVl7DJJyEENX1vq/Rexl1lZqQQPfu3Zvl2PVx8803ExgYSH5+Pj/++CNjx471rNuwYQNOp5N27doxaNCgKttlZ2ezadMmEhMTKSgowG63oygKoN6nAnC73Zfvg2iMhJMQQjSAyWQiOjqaL774gnXr1lUJp++//x5Qe016/dlb/J988gkLFy7E4XBc9vpeKWRChBBCNFDlkN22bdsoKSkBIDc3lz179gBnZ+kB/PjjjyxYsACHw8GgQYNYsmQJu3fv5tChQyQmJpKYmEhISMjl/xAaI+EkhBAN1K9fP0JCQqioqGDr1q0ArFu3DpfLRWRkJOHh4Z6yn3/+OQAdO3bk448/5oYbbiAwMBCDweApUxlw1zIJJyGEaCCdTucZztuwYQOghhPAuHHjqpQ9fvw4AEOGDMFsNlfbV1JSEvn5+U1W1yuFhJMQQjSCO++8E1AfyM3OziYuLg6TycSYMWOqlDOZTADY7fYa9/Puu+96fna5XE1T2SuAhJMQQjSCbt26cd1111FUVMTf//533G43Q4cOJSgoqEq5Xr16AerDuenp6Z7lBQUFPP/88yQnJ9O/f38AEhMTL98H0BiZrSeEEI3kzjvv5PDhw8TExADVh/QAHn74Yb7//nvy8/MZM2YMvXr1wm63c+jQIXx9ffnkk09YvXo1sbGxLFu2jCNHjjBu3DjGjx9/mT9N85KekxBCNJKxY8ei1+txu934+/tz2223VSvTpUsXli1bxq233orBYOCXX34hJyeHCRMmsGrVKs8LYocOHYrFYuHo0aPodLpm+DTNS3pOQgjRSNq2bUtCQsJFy/Xo0YMPP/yw1vUtW7bk3//+d7Xln376aYPqdyWRnpMQQgjNkXASQgihORJOQgghNEfCSQghhOZIOAkhhNAcCSchhBCaI+EkhBBCcySchBBCaE6ThJOiKKSkpJCQkHBNv7hQCCFE/dQ7nFJSUpg9ezbfffddleV79+5lxIgRjB49mvHjx3PLLbd4vt9ECCGEqIt6hVN2djZTp07lhx9+8Hw3CcDp06d59NFHSUtLQ1EUFEUhNzeX2bNnc+LEicaqsxBCiKtcvcJpxYoVZGdnM3LkSB544AHP8uXLl1NYWEibNm1YtWoVu3fvZtq0aVRUVFxT74QSQgjRMPUKpy1btuDt7c2bb75JixYtPMvXrVuHTqfjqaeeomfPngQEBPDMM88QGBjIzp07G63SQgghrm71CqesrCx69OiBl5eXZ1lGRgbHjh3DYDAwYsQIz3Kj0Ui3bt2qfKmWEEIIcSH1CqeioiK8vb2rLNu1axcA119/Pb6+vlXWeXl51fqVxEIIIcT56hVOfn5+ZGVlVVm2detWdDodgwcPrlb+1KlT+Pj41K+GQgghrjn1Cqdu3bpx5MgR9u/fD8CxY8fYtGkTQLVvfszJyeHw4cN06tSpYTUVQghxzajXN+Heeeed/Pzzz0yZMgWbzUZSUhJOp5NevXrRvXt3T7nU1FSef/553G43N998c6NVWgghxNWtXj2nCRMmMHz4cCoqKjhw4AClpaW0bNmSBQsWVCn35ptvEhcXR8eOHZkyZUqjVFgIIcTVr149J71ez//+7/+yY8cODh48SEBAANHR0QQEBFQpFxUVhY+PD88++2y1dXWVmprKc889R3x8PCEhIWzevLle+6mUlZXFxx9/zPbt28nMzMRgMBAWFkZ0dDQPPfRQlRmIQgghmke9wqnS4MGDa5wAUWnWrFkN2T0rV65k/vz5lJaWNmg/leLi4nj00UcpLi4mKCiIPn36UF5ezoEDB0hISGD16tV89tlntGrVqlGOJ4QQon40+eLX3NxcHn/8cV566SVMJhOjRo1qcJ2KioqYNWsWxcXFPPTQQ2zbto3FixezYsUKNm7cyHXXXUdKSgrPPPNMg48lhBCiYTT54teYmBi2bNnCgAEDWL16daNMpliyZAl5eXn06dOHefPmYTKZPOvatm3LwoUL0el07Nq1i927dzf4eEIIIepPky9+NRqNzJ49m6VLl9KuXbv6VLGadevWATBx4kR0Ol219eHh4fTr1w+A77//vlGOKYQQon40+eLXyZMnM2vWLPT6xhl1LCws5OjRowCeAKpJ5bo9e/Y0ynGFEELUjyZf/Go2m+tTrVolJycD6izDkJCQWsuFhYUB6gxBp9PZqHUQQghRd9fEi19zc3MBCAgIwGisfYJiy5YtAXA4HBQUFFyWugkhhKjumnjxa1lZGQAWi+WC5axWq+fnxpq+DvBLeinrfs1stP0JIcTVrl7POV2tL35VFOWiZRISEi55v6sP5ZMbe5qOhrz6VEs0kfLy8nq1p2ha0i7a0xxtUq9w6tatG3v27GH//v306tWrTi9+7dmzZ8NrW0+Vvbzy8vILljt3fW1heu67A+t8/G1ZuMv19dpWNJ2EhARpEw2SdtGehrZJXFzcJW9zTbz4NTg4GICCggLsdnutEy5ycnIAdUJGfV+3VBOrUU+JfJ+VEELU2TXx4tfw8HB0Oh2KonDy5Mlay6WkpADQpUsXDAZDox3fatRRZr/0N2UIIcS1SvMvfm0MPj4+REZGcuDAAX7++WfCw8NrLFf5Zogbb7yxUY/vZdJTYneiKEqNDwALIYSoStMvfm1MY8eO5cCBA6xcuZJJkyZVe8D3l19+8dzwu/POOxv12FajHkWBcocbL3Pj9ciEEOJq1Wgvfi0uLiY9PZ2MjAxKSkoaa7eXJCsri5EjRzJy5EhiY2OrrLvvvvsICQnh0KFDvP7661WmtqekpDB37lwARo8e3eiTN6xGtbdUapcHe4UQoi4a1HM6dOgQS5YsYceOHZ4HXSu1b9+eYcOGMXXqVM+bF+pq5syZVf6ekZEBqFPSz183Z84cbDYboD48W3nf6PznlCwWCx988AEPPfQQy5cvZ926dfTo0YOSkhL279+Py+Wid+/evPHGG5dU17rwMqnXAKV2Fy0bfe9CCHH1qXc4LVu2jPnz5+NyuWp8PigtLY1ly5YRExPD22+/zbBhw+q878pp6ecrLy+vtm7q1Kl13m9ERARr1qzho48+YvPmzcTGxmIymejZsyd33nkn999//wXfIFFfZ3tOMilCCCHqol5n4l9//ZU33ngDRVGw2WzccccdhIeHExAQgKIoFBQUkJiYyLp16zhx4gR/+MMf+Oabb+rcg0pMTKxPtQgNDb3otkFBQcydO9czjHc5WI1qz6lEhvWEaF4J30JFEfS+//IcL+cItOoGl2siVFocKEBo7S+4vlLUK5w+/fRTFEVhxowZ/PGPf6yxzOjRo3nqqad45ZVXWLlyJUuWLOHFF19sUGWvVF6mMz2nCuk5iauMvRSyD0H7vnChbxFQFCg9DT41DGy7XaA/M1HodIpatrGUF4DOABZfcJTBN7PBXgxdbgWzL9hLwP/M1/Ic2wQbXgIUeGgNeAedrbvLDsYzrz8rPQ2Z+6HjUDDUcgp1OWDvcvh2Ntz0Rxj+kro8ZRus+3/Q+SYY+Aic3AMFJ6HHOAjsoP6cugvCBoI1ALxagMEEBb9B7GJQXBDSH0L7q59t02vg2wZGvQV7Pob1z4PRCo9shgNfQno8BIWr+x35FziyHrIOQO8H1Dqm7oQj62Dsu5C8BeI/g9Y91H32vk/9+ch62v+8HLouBZO15s/bBHRKXd7Zc57bb7+dsrIytm/fftGvtbDb7dx0000EBQWxdu3aeldUC+Li4i74lRu1+fanvfz+uzT+NaUfd/Rs2wQ1E/Vx1b+JoCgTLH5gruFtJyW5YDBDxl44vAb82kL/h8HqX7VcyjY4dQz6PgRuJ5xOhhYdweQFLicsvxeSNkPwdTBtrXpCVxQoTFNPfkGd1Z7K6ifh0H8g+k24YSbkHIZtf1NPwlv+DOM+gINfw4EvyQu/mxYD7oX8k+pJ3Bqo7k+nhzaRkPg9/DgfAsKgbZR6Ai/OVo8dMVqty9YFan33LgOLPzz4H4hfBlvO3FO2jVJDtSQXRs5Xj/PxCDXECtKgbSR4t4S8E+DTCjJ/VcsVZ8LO/4WyPPBrpwbKkDnqcmcF2EbCljfhwEo1FI1WcJTA0Kch+zAcWatuV5ytBk0lo5f6d9eZiVp6k1r/rsPBP0T9HIqi9sDc54zAGK3gLIdWEZCbqH6utFjQG9X2R6laDtTfo+Kuemyn+v5R2veFikIoTAfH2fv2pS0j8Z65tfYwvoj6nDvrdaTc3Fz69etXp+9bMpvN9OjRg71799bnUFcFuefUCAp+g4DQupd3OS/6D0nvKKlTOY/sBPUEdOoYZB2EyAnqiaJ9b/Uk5yiDwDND1+l71atrawD89C50HAw3PaNeqaf/Aj9/BLe/ol4BAzjK1RNa9mEoyYEBM9QT5RdTwBatngB9W6snJ0WBX5bAtreh510QdoP6u9nyplqXIXPUK+WPbgOTt3oV7N0Cet6tDmtlH4aiDHWdo+TMn6Xw379DxBg48ZP6ewm/FfauUE+aCd+qV+ppcYBO/VwVhepJbuCj6lX9snvVMMo9Ahn71M8V0l89GabFqSe+9f9PLVuYrh7716/Ucl8+pJ6YOw6lRdLXkPR1zW1QeSJt3UP9/aZsVdug8uS7+XXwCoKy02p5s596kl7Y/Wx9Ot8MP72jhlpwhNq7AbWeU75W23nLG1ByCvzawKkkCAiBb55Uy3W9Hbr/jxrKaXHw+X1n67dunhr6faeqAXb7K2rv5qd31HrdMg8G/x5Kc2H/l2o4B3WGHxeov9PgCGjdHfavVH+/+79QQ27ADBj8JPgEQ8Z+9f8ho0UNw+QfYecitU4T/g3ZB2H5JPWC474V6v+zPsFw9Afwb68GeuZ+NcBcDuhwAxz6Rr0wiZqo9oDL8tReVsFJCAjjhLEn3esZTPVVr55Tnz596N27N4sXL65T+Ycffpi4uLgrPqDq23P6Ke4Ak79M5c27o7h/UIcmqNlVxO2uPjy0fyXEPKL+w7513sXH73OPwce3QZdhatnWPdXeQ0g/9QpccYNvG5wr7scYGAY3PwNOu3qFvOff6p/9pqlX93kp6lWsowR2faD+g1aUs1eaeiN0ukkdEgF1CMVeAiXZ6nF0evBtC8VZZ65WlbPbuZ3qiaLzLXByN/y2Rz35Gs1qEHi3VE8sjjPHQlFPcKH91aGYll3VoKxkMKsn+IAwdTsUaHe9OmxWmKaGhslbPYm16KT2grxbwu1/Uk/IW99Sh4FadFLrlnMYut0BYYPUE6yzHIa/rP5ZkqOe3NtGQY+7YPe/4IeXzgRhS4i6R61T7GK1jnd/qIb5L0vO9NTawfW/g30r1J7BtrfUoab/+QfpaxfSvudgtVeSuFYd8gsIU497ZD2E9IV+D6m/W1BPpGZfNZCOrFOH58IGqSd+i7/akziVpPaOQgeqoV96Wj25G63qMFrGPvVzXze65v+nHGWQ9ot6EdCi49nlZflq27frrQb+nn/D0Dnq76WSoqjDZ+16g7nqtzlc1MH/qOHV7vpL264sT/1/2q/NpW1Xi8Z4t96lnjvrFU4jR47k9OnT/PTTTxf9YsDKYb3AwEDWr19/qYfSlPqGU+y+g9yz4jgvjunOjJu6NEHNNM5eovYsDv1HPREM/r06TNLxRkAHcf+nhkfmfvgtTv3HnfANtO2l3h/4LVa9+nVVqFe+BpN6lZl9CIK7qyf6yPHqGHvyVnXIyVmunlCsAVCeX2O1nJZAjAYDlJ46u7DyBF9Jpz87nNItWj3R2Ush+s/qyTDhO7UeN84Cg0X9DF6BaiA5y9Uex7S1as8v4Rv1hOuyQ9S96on56A9qMJm8Yew70HO82ovZ9pZ6Yp/wb2gZDvGfqnXJTlCHtXr9Th0Ky9yvHidpM0SMgopidZjM4gfD/h+076N+DkVRr7ADQtUb9HVR+blB7emU5EK7XpfS8uo+SnLUXl9t3C44/B2EDweL79U/3HoFao5wqlc/bdCgQaxcuZJXX32V1157rdb30DmdTl5++WUKCwsZM2ZMfQ51Vagc1iu5EidEOMrVMEjdqV7NB4WrJzjLme/s2vCSerLtcAPkJKonX4NZ/fnUUfVK+pelVcavSfz+zA869STqLFdPYj7B6pXlplfV7XKPqscuy4N7l6i9j9jF6sn/VBK0iVKPUZYPiWvAEqBenadsgzFvq1fOPq2hKF0dGjm6Xr069m0LKVtJVULp0n+E2oMwWtTjteyqhk5hmnqlWxkmoAboqSR1uKXypH/jk2og1HSjH+CON9QTfEAIdBhUdd3Qp9X/zmfxhRGvwe2vng2H2185uz7vOAR0UHuY7XuryzrccHZ953XV96nTQXjdH+fwbFPJv73636XS6S4cTKD2jHrcden7Fle1eoXT9OnT+frrr4mJiWHXrl1ER0djs9lo0aIFiqKQn5/P4cOHWb9+PZmZmVitVqZNm9bYdb9iGPQ6LEY9pY5mnkpeckq9aVx50lEUdQgkpK96Rd6ik3rfoWW4OvTiHQQp29WeSME5L8z1aQ1DZqs3une8py7b/c/qxzNa1eGl6+9Tj2H2UYdwkjZBnynw33fh2GaYsQmC1QepyUlUw+zmZ9UQspeqvYOwQWq9Bz1W/TjlhRD7iXqCC+pcfX2LTuqfA2acXdY2koqEBDUM20aqy2rrURhMZ39ued57GY2Ws7O4atKQKcS1bVv5eYS4itUrnDp27Mjf/vY35s6dS1paWq33nhRFwcfHh4ULF17yWyKuNt5mw+WbSu5ynD2hKoo6tFRwEta/oA4/Rf9Z7QF8+xQc3w5+7dXehd4Eboe6nclbvUcS1EXdR9REdWppURb8/CFsOPNYQKeb1Hs0ZfkwdqE6zl2cpZ5Y20SqPabzJzJ0uUX9867/rTp0BOoN4eg/n/272btqr6AmVn91KFAIcdWo9/SLO+64g+7du7NkyRK2bdtGamqqZ51Op6NTp04MGzaMBx98kLZtZfq0t9nYtLP1Tv6sTs0N7KDew+k/TR0KOrIOvpp+phKtYOf7cOArdeqrJUAdVtq5SJ08UJqr9j782qn3ewI7qL2dc3sOoIZUYYZ6Dygg7OwzKpVaX3fOX4IuXG95S7sQogYNmhsYFhbmebC2oqKCwsJCdDod/v7+F50oca3xsRga98WvTjvEL1VnM7XopA6FgXp/pF1vdarykfVqL6p1D7U3EjpAHf7KPgwtu0DvyeoDiINnq8+S1OHRAI/KBxeFEKIJNNrEdYvF4vnGWVGVV3Y8NyuZHLEPbfjO9q9Up/zai9Wpqz6t4dhG9RmW0W+rU5iDr4MTO2Djn9Qguut9CL9N3X7IU9X36X2R3o0QQlxmFw2nxprSqdPpOHToUKPs60oTcGIds4t/5BGvAZe+cckpdWjOYILjP6k9pDY91Zv3/d9XnyAvyzsbMJWzxjoNgRkbG+9DCCHEZXTRcKrHY1DiPKVt+hOSvJqggoPATXXb6OTP6rMfh75RHwQF9TmaATPgjtfVGXSVpOcjhLjKXDScli5dejnqcVUrbjMQN3quK97FsewH6Nrar/bCLgdsfEWduKAzqM/7TPka/EPVZ2Vqek+aEEJcZS4aTgMHDrwc9biquS0BONsPYGLaVjbsiaPrmFtrKeiGmEfhYIz6Es4Rr6lhJDPahBDXmEb7mnZxYeaxC/DV27lnz+9wb3u7+tcCJK6Df92sBtPwP6nPDFl8JZiEENeky/ua2WtZ+z7sHPYFyg9/Inrza+pLOMOHqe9EK/hNfb9aULj6vSwDH23u2gohRLOScLqMbhsymOG7XiBP+ZxJv36Fbv/n6hucW3ZVv4fmrvfVF5UKIcQ1TsLpMjIa9Mwa1o3nVt1F6KTfM7StW33Fz/lvWBBCiGuc3HO6zO7uG0JIoBd/21GI0raXBJMQQtRAwukyMxn0zBwWzt6T+Ww/mtvc1RFCCE2ScGoG9/QLpV2AlX9sPtrcVRFCCE2ScGoGFqOBR27qwp7jecSdyGvu6gghhOZIODWTSQPCCPAy8a9tSc1dFSGE0BwJp2biYzEy+YYObDiURUpuSXNXRwghNEXCqRlNHdwJk17Px9uTm7sqQgihKRJOzai1n5UJ/UL4Mu43MgrKmrs6QgihGRJOzWzmrV1RFIV/bD7W3FURQgjNkHBqZmFB3tw3sAMr95zkxCm59ySEECDhpAlPDuuK0aDj3Y3y3JMQQoCEkya09rcydXAn/rM3jf2/5Td3dYQQotlJOGnEk8O60tLHwsurD+J2KxffQAghrmISThrhZzXx/Kjr2Hsyn69++a25qyOEEM1KwklD7u4TQt8OgSxYe5jsovLmro4QQjQbCScN0et1LJjQi1K7i9kr4nHJ8J4Q4hol4aQx3dr48fq4SHYln+bdjUeauzpCCNEsJJw06J5+oUzsH8o/Nh/jx8Ts5q6OEEJcdhJOGvXq/0RyXVs/nv5irzycK4S45kg4aZSX2cAHk/uhAA8t3kN6vrx7Twhx7ZBw0rDOrXz499T+5BZVMH7RDg5nFjZ3lYQQ4rKQcNK4fh2DWPn4jSgo3PvBTnYk5TZ3lYQQoslJOF0BurfzJ2bmENoGWJn6yc98sy+9uaskhBBNSsLpChES6MVXjw+mT1gLZq+I58OtSSiKPAclhLg6SThdQQK8TSx9eCBjotoxf+1hJn64k1/TCpq7WkII0egknK4wVpOBf9zXh7+MjyI5p4Q73/+J52MOcKq4ormrJoQQjUbC6Qqk1+v43cAObP7jrUwb3JkvY09y699+5L1NRykqdzR39YQQosEknK5gAV4mXr6zB+vm3MSgzi1Z+MMRhi7YwnubjpIrPSkhxBXM2NwVEA3XtbUfH0/tz4HfCvj7pqMs/OEI7206yu3d2zBxQCg3dwvGaJDrECHElUPC6SoSFRrAx1P7czSriJWxJ4n5JY11BzNp7WdhQr9QRvRoQ+/QQPR6XXNXVQghLkiz4eRyuVi5ciXffvstSUlJlJaWEhwczKBBg5g2bRo2m+2S9hcbG8sDDzxw0XJms5kDBw7Ut9qa0K2NHy+M6cFzI69j8+Fsvow9yb+2JfPBj0mEBHoxpGtLonu25Rab9KiEENqkyXAqKytjxowZxMbGYjQaiYyMxNfXl8TERGJiYvj222956623GD16dJ33WViovvrHy8uLwYMH11rOZDI1uP5aYTLoie7ZluiebckrsbPtaA7f7E1nw6EsVsb+Rlt/K4O7tqRHO396tg+gT4dArCZDc1dbCCG0GU5//vOfiY2NxWaz8cEHHxAaGgqA0+nknXfe4eOPP2bu3Ln06NGDTp061WmfleEUFhbGokWLmqrqmtXCx8xdvUO4q3cIDpebTQnZxPzyGz8dzSXmlzQAfC1G7ujRhv6dgujU0puo0AD8rFdPWAshrhyaC6eTJ08SExODTqfj3Xff9QQTgNFo5I9//CNxcXHEx8fzz3/+k7/85S912m9Bgfqwqp+fX5PU+0piMugZGdmWkZFtAcguKmf/yQI2HMpk3a+ZxMSrYWU26PH3MhIZEkB0z7aEBHoR0sKLkEAv6WEJIZqU5sJpw4YNuFwuBg4cSHh4eLX1Op2Oe+65h/j4eH744Qdee+01zGbzRfdbVFQESDjVpLWfldt7WLm9Rxvmj+9FZmE5yTnF/HQsl7wSO5sP5/BjYk6VbVr6mOkZEkCXVj60D7Ry23WtCQvyxmKU0BJCNJzmwikuLg6Avn371lqmX79+ABQXF3P48GF69ep10f1Kz6luDHqd2kMK9OKmbsEAuNwKmYXlpOWVkZZfSlpeGb/llbHn+GniT+RRVOHkze8PYzbo6dzKhyAfM0E+ZkKDvBgc3opOLb3R63QE+1kod7gI9L74xYQQ4tqmuXBKTk4GoEOHDrWWCQ0NRa/X43a7SU5OrlM4Vd5z8vf359ixY2zYsIGUlBQcDodnFuCwYcMwGOTK/3znBhYEVVuflFPM/t/yOZxRREpuCXmldhIyC9lwKJMPtyZ7ylXOYO/boQUtfMz4WYwE+1no3s6fkBZe9AoNwGI0oCgKOp1MdxfiWqa5cDp16hQALVu2rLWMyWTC39+f/Px8cnJyai13rspw2rBhA8uXL6/2Ru+lS5dis9l477336Ny5cz1rf20KD/YlPNgX+lRdXmp3sud4HlmF5ZTZXeQWV+BwKcQeP83J06UUlTvJKa7A7nQDagh6mQye3lWFw0XvDoGEBXnTxs9KCx8T5Q4XYS286RLsi0GvI8DLRJCPGYM8uyXEVUVz4VRWpn4ducViuWC5yvWlpaV12m9lOOXm5jJx4kTuv/9+unTpQnFxMdu2beOvf/0rR44cYdq0acTExBAUVL2HIC6Nt9nILbbgC5axO92cOFVCSm4JB9IKKK5wYjEayCuxo9fr2Hcyn0PphZwqsde6D4NeR9dgX7q29iWjoAxfq4mQQCvtArxoG2DF7VYwGvR0bKmGnN3lol2AV2N/XCFEI9JcONVVZc+nrsM/TzzxBAUFBYSEhNC7d2/P8qCgIMaNG0dUVBTjx48nIyODjz76iLlz59a4n4SEhEuua3l5eb22u5aE6SEsDNTXPSrAmSns3VsB4HApFNtdmAw60gudpBc5QIHCChenSl0cz7cTl5JDsI+RzJIy9qeeJr/cVevxjHrwMelROI5Rr6Pc6cbfYqCdnwmrUYeXSY/VqMNq1GPQg7/FQIdAM3aXG6NOR+cg9b5ZZQfcx6zHzyJDwo1B/r1oT3O0iebCydvbm4KCAsrLyy9YrqJCfbGpj49PnfY7dOjQC64PDw9nzJgxrFq1io0bN9YaTt27d6/T8c6VkJBQr+1Ew5Q7XGQXVmAw6LA73RzPLeFUiR2jXkdiVhEn0rNpGRSEw+XGajKQVVhOVmE5eXYXaSUuSu1OiiucuBU8Q48X0sLbhMmgJ8jHTEtfM4FeZsxGPWaDHrNRj7fZgNVkwNtsOOdnI95mA75WI0E+ZtLyyrC18SPAy4TZqL8mhyvl34v2NLRNKie6XQrNhVNwcDAFBQXk5ubWWsZut3uG6YKDLzxsdCkiIyNZtWoVaWlpuN1u9Hp5tc+VzGoy0KGlt+fvnVtVvZBJSFDq/A/uVHEFqadLMRn0FFc4ST1VCjrQofbec4sr+C2vFIdT4XSpnVPFFWQWFGJ3ubE73VQ43ZTZXVTUIeTOZdDrMBl0ZwLOgMWop7W/heJyJ75WI239rbTxt+JwuSmucGIy6PGzGvGzmvC3GvG1GMkrddDK10wrXwtlDhcmg57QFuqwprfZQEtfC9mF5Rj0OtoHemE8E4g6nc4zOUVRFFxnhkeFuBw0F07h4eEcO3aM48eP11omOTnZM6zXrVu3Rq+DTqeTYBJVtPS10NL37H3QG7rUPmHnQtxuhTKHi1K7i/Izf5banRSUOcguqqB9gBdJOcWU2l04zgSbw6WGm93lptzuIrOwnGBfCyV2J0eyivjpWC5mgx5fqxGH001hudrjqw+DXodRr8OtKBj1elxuxfMIQFG5ky7BPvhZjXiZjfiYDQR6m/D3MpFVUK4Oxhr0tA/0oqWPmRK7k5Y+ZuwuBZfLTbCflVK7k7IzE14MOh0GPQT7WXG63LTxt2Ix6ckpcRJYUIZRr/Y6TUY1nA16nczivIZoLpwGDRrE+vXr2bNnT61ldu/eDaj3i+ryAtjCwkLi4+NJT0/n7rvvxmq11lju2LFjAFXeSiFEY9LrdfhYjPhYav+nN7RbqwYfx+VWKK5wUlTuwN/LRGZBOSUVTrzMBhxOheTcYvQ6HQVlDoornLT2s+B0KZw4XYLd6cag1+NwuTEadOQUVqDX6wj0MnH8VAmldhcFZQ4y8ss4VWKnsMxBu0ArBp2OcoebrKJyzpsMWw+p1ZbodGr4WQx6TEY9JoOOtv5WLEYDRRVOiiscmA16WnibCfQ242814m1Rh069TAbS8ssotTsJ9rWg1+sw6HRYTHoMej0mvY5OrXywGPXkFFeQlF2Cv5eR7u38cbsVTAY9p0vtRLTxw9dq5EhmEa3O/M66tvYlt7iCrq19ySwox2hQZ51aTQbMBj0FZQ7MRj1eJgMuRd0XqBcq8g0BtdNcON1xxx0sWLCAffv2cejQIXr06FFlvdPp5IsvvgBgzJgxdXouqaCggEcffRQAg8HAxIkTayyzZs0aAG699dYGfgohmlflNPsAL3Viif9570iMCg1otGOdf5K1O90UlDnwNhvIK7VjMRrQ6SCnqAJfixGLSU9+qQNFAafbTVZhOSaDnqzCCpwuNxkZGbRt1w6nW8FxpsfoONODtLuUc3qTLtLyy3C5FUICvfC1+GJ3uckrcfBbnvqogtpLdVLucNPC20Sgt5kdpadwuRVPL9ZdQ5B6m9VHGmpaVxujXoezDht0a+1Lqd1FRkEZQT5mTAb1QiDQ20xucQVt/a0E+1mwGPWYDGf/K3M48bWoQ7Y+FiO/nS6lxO5Uh3YDrBSXO3G5FQK8TZQ73FQ41AlBvmcuhnwtRkxG9QJCr4OWPhYU1OcPFQVKzgR3xZnft9eZ+6Jut0JBgZ3LfRdQc+EUHBzMgw8+yEcffcQzzzzDhx9+6Hkg12638+qrr5KUlISfnx+PP/54lW2zsrKYOnUqAG+88Qb9+/cH1Je9RkdHs379eubPn0/r1q2rBFB2djbPPPMM+fn5BAYGMn369MvzYYW4Cpx/9W826gn2U4dAz+0htjpnWLS139nRi57tqwZlQkIJ3bvX/hB+fbjcCnpd9dm9breCApQ5XGTkl1HhdONjMdKppTeFZU6OnyrBaNB5wu1IVhG5xXYiQwLIL7WjKHAsu5hAbxMJGUV0be2LXgeldhdlDhcVZ4YwHS43pXYXbkXh17QCArxMtA9sz+kSO25FwaDXc6q4ggGdWpBRUE5BmYPTJWoIO84EstWkp7DcSWmFk1KHixbeZlr6mNmRdIqicidGvQ69XueZvGMx6lGo22Sei9Hr4ECfnhfs8Tc2zYUTwJw5czh27Bhbtmxh1KhRREVF4ePjw8GDB8nLy8PHx4f333+fVq2qDn84HA5SUlKA6s8/vfbaa2RkZLB//34ee+wxOnbsSFhYGKWlpRw4cACHw0FQUBDvv/8+bdq0uWyfVQjR9Gqb9VgZrL4WI93aVH21WYC3ieu9A6ss6xLsW20fw65r3TiVvAQOlxuDTuepf6ndeeZ+oR67043FqPesc7jclFQ4KSp34nQreJkMOFxu8krt6HU6XGcC2sdsIKe4Ai+T4UxPTb0vqtfpKM5Ju6zBBBoNJ6PRyAcffEBMTAwxMTEcPXqUsrIyWrduzahRo5gxYwYhISGXtM/AwEBWrFhBTEwMa9asITExkV27dmG1WomIiOCWW25h8uTJ8vCtEELzTOfNmvQ2nz2Ve5kN1coGnrkPd66wIG/Od35AV0pw1O1NPI1Jk+EEavd7woQJTJgwoc7bhIaGkpiYWOt6o9HIxIkTa7znJIQQQjtkvrQQQgjNkXASQgihORJOQgghNEfCSQghhOZIOAkhhNAcCSchhBCaI+EkhBBCcySchBBCaI6EkxBCCM2RcBJCCKE5Ek5CCCE0R8JJCCGE5kg4CSGE0BwJJyGEEJoj4SSEEEJzJJyEEEJojoSTEEIIzZFwEkIIoTkSTkIIITRHwkkIIYTmSDgJIYTQHAknIYQQmiPhJIQQQnMknIQQQmiOhJMQQgjNkXASQgihORJOQgghNEfCSQghhOZIOAkhhNAcCSchhBCao1MURWnuSlwp4uLimrsKQghxRerXr98llZdwEkIIoTkyrCeEEEJzJJyEEEJojoSTEEIIzTE2dwWuRi6Xi5UrV/Ltt9+SlJREaWkpwcHBDBo0iGnTpmGz2Zq7ile81NRUnnvuOeLj4wkJCWHz5s0X3SYpKYnFixezc+dOcnJysFqtdO7cmbFjx3LfffdhNNb8z0Ha8+IKCwtZunQpmzdvJiUlBYfDQWBgIFFRUUyaNIlbb721xu2kTZrW6dOnWbJkCVu3buXEiROedomMjGT8+PHccccdNW6nhXaRCRGNrKysjBkzZhAbG4vRaCQyMhJfX18SExPJycnBZDLx1ltvMXr06Oau6hVr5cqVzJ8/n9LSUoA6hdO6det49tlnsdvttG3blq5du1JYWMivv/6K2+2mT58+LF68GC8vryrbSXteXGJiIjNmzCA7OxuTyYTNZsPPz4+kpCRycnIAeOCBB3j55ZerbCdt0rTi4+N5/PHHyc/Px2q1YrPZ8Pb2rtIuY8eO5a233sJgMHi200y7KKJRvfDCC4rNZlPGjh2rnDx50rPc4XAob731lmKz2ZTIyEglJSWl+Sp5hcrJyVEee+wxxWazKQMGDFCeeuopxWazKcOGDbvgdidOnFCioqIUm82mLF68WHG5XJ51R44cUW699VbFZrMpzz//fLVtpT0vrKSkRBk2bJhis9mUCRMmKKmpqZ51DodDWbhwoWKz2RSbzaasX7/es07apGnl5+crgwcPVmw2mzJ9+nQlJyfHs87hcCh///vfPe2yfPlyzzottYuEUyNKTU1VunfvrkRERCjHjh2rtt7tdiuTJk1SbDabMnfu3Gao4ZXtww8/VGw2m/LAAw8o6enpyqpVq+oUTvPmzVNsNpvyhz/8ocb127ZtU2w2m9K9e3flxIkTnuXSnhe3cuVKxWazKT169FDS09OrrXe73cq4ceMUm82mPPHEE57l0iZNa8mSJYrNZlP69eunFBYW1ljmnnvuUWw2m3Lfffd5lmmpXWRCRCPasGEDLpeLAQMGEB4eXm29TqfjnnvuAeCHH37Abrdf7ipe0YxGI7Nnz2bp0qW0a9euTts4nU42btwIwMSJE2ssc9NNN9G+fXtcLhfr1q3zLJf2vDgfHx9Gjx7NvffeW2Ob6HQ6evXqBcDx48cBaZPLoWXLlowfP56HHnoIPz+/Gsv06dMHgMzMTEB77SLh1Igq3yDRt2/fWstUPiVdXFzM4cOHL0u9rhaTJ09m1qxZ6PV1/9/26NGjFBYWYjAY6N27d63lKttsz549nmXSnhc3evRo3nnnHV555ZVayzidTgDMZjMgbXI5jBkzhvnz5/Pkk0/WWsbhcADQtm1bQHvtIuHUiJKTkwHo0KFDrWVCQ0M9J9fK8qJuKk9ul6Lyd9ymTRssFkut5cLCwqqUP/dnac/6czqd/Pe//wWgf//+gLSJFuTn5/PDDz8AMGLECEB77SLh1IhOnToFqF3q2phMJvz9/QE8M2ZE08nNzQUgKCjoguUq26yyPEh7NoZ//vOfZGRkYDabefDBBwFpk+bidrvJyMjgu+++Y9KkSeTk5DBixAimTJkCaK9d5DmnRlRWVgZwwauOc9dXToUWTaeubWK1WgEoLy/H7Xaj1+ulPRto1apVvP/++wA8++yznqtqaZPLb+bMmWzatMnz9xtvvJFnn32W22+/3bNMa+0i4dQMlDOPlul0umauiaikNOBxP2nP6j744APeffddAKZPn+7pNV0KaZPGc/3116MoCsXFxRw7doxdu3Zx6tQp7Hb7JT8PdrnaRcKpEXl7e1NQUEB5efkFy1VUVADqTCfRtLy9vQEu2iaV6729vT3j4tKel87hcPDqq6/y5ZdfotPpePrpp3nssceqlJE2ufzObQO32826det46aWXePrpp0lMTOTpp5/WXLvIPadGFBwcDFQdiz2f3W6nsLCwSnnRdOrSJnB2DPzcNpH2vDRFRUU88sgjfPnll1itVhYuXFgtmEDapLnp9XpGjx7NSy+9BMBHH31ERkaG5tpFwqkRVc7vr3yeoybJycmerm23bt0uR7WuaV27dgXUf1AlJSW1lktJSalSHqQ9L0VpaSmPPPIIO3fuJDg4mM8++6zW4SJpE22ofN+hy+Vi7969mmsXCadGNGjQIKDq/P/z7d69G1BnxMjLKZteeHg4rVq1wu121/pNxk6nk9jYWEC9UVxJ2rNu7HY7s2bNIj4+no4dO/LFF18QFRVVa3lpk6b3+OOPEx0dzaJFi2otU9mLATAYDJprFwmnRnTHHXdgsVjYt28fhw4dqrbe6XTyxRdfAOpDcue+bFE0jcohDIDPP/+8xjJr164lLy8Ps9lMdHS0Z7m0Z9289dZb7NixgzZt2vDpp58SEhJywfLSJk3PaDRy/Phxvvnmm1rfxrBjxw7PzzabTXPtIuHUiIKDgz2zkp555hlSU1M96+x2O3/6059ISkrCz8+Pxx9/vLmqec157LHH8PPzY9OmTXz44Ye43W7Pun379vHnP/8ZgKlTp9K6dWvPOmnPi/v111/57LPPAFi4cCFt2rSp03bSJk1rxowZ6PV6UlJSmDdvHnl5eVXWb9++nbfffhtQe0CdOnUCtNUu8pUZjczpdPLkk0+yZcsWjEYjUVFR+Pj4cPDgQfLy8vDx8WHRokXccMMNzV3VK87MmTOr/D0jI4NDhw5htVoZMmRIlXVz5sypMnSwY8cOZs2aRWlpqedrAPLy8jh48CAAw4cP57333qv2PTXSnhc2Z84c1q5di7e3d5Vhntq8/vrrngc1pU2a1pdffsmrr76Kw+HA29ub8PBw/P39OXnypCc8unXrxuLFi6tMUNBKu0g4NQFFUYiJiSEmJoajR49SVlZG69atufnmm5kxY8ZFhz1EzSIiIupcdunSpZ5x8EonT57k448/Zvv27eTk5ODt7Y3NZmPChAncddddtT57Ie1ZuylTpvDzzz/XufymTZsIDQ31/F3apGklJyezfPlydu3aRVpaGna7HT8/P7p160Z0dDQTJ06s8bVgWmgXCSchhBCaI/echBBCaI6EkxBCCM2RcBJCCKE5Ek5CCCE0R8JJCCGE5kg4CSGE0BwJJyGEEJoj4SSEEEJzJJyEELWaMmUKERERF3y7tRBNQcJJCCGE5kg4CSGE0BwJJyGEEJoj4SSEEEJzjBcvIoS4mLKyMpYtW8aGDRtISUmhrKyMFi1acP311zNp0iRuuummKuXHjx/PwYMHefHFF7n77rtZtGgRGzduJDMzE7PZTI8ePZg2bRrDhg2r8XiFhYUsXbqUzZs3k5qaSnl5OYGBgfTs2ZO7776b6OjoWr/W4L///S/Lli1j3759FBQU4OvrS+/evZk6deoFv5PJ7XazdOlSYmJiOHnyJIqi0KVLF+677z7uvffe+v/yhKiBfGWGEA2UmZnJ9OnTSUpKwmw207VrVwICAkhJSSEzMxNQZ729+OKLnm3uv/9+4uLiePLJJ/nhhx9ISUmhZ8+e+Pj4kJCQwKlTpwB46aWXmDx5cpXjJSUl8fDDD5ORkYHJZCIyMhI/Pz9SU1M5fvw4AKNHj+btt99Gr686OPL222/zr3/9C1C/mrt169akpqZ6vnxuzpw5PPHEE57yld/X9NRTT3H48GG2bNlCr169sFqtHDlyhOzsbACee+45Hn744Ub8rYprniKEqDe3261MmjRJsdlsyuTJk5X09PQq67/++mulZ8+eis1mU7755hvP8smTJys2m00ZOHCgcvfddyvZ2dmedRUVFcrs2bMVm82m9OrVS8nMzPSsczgcytixYxWbzabcfffdSkZGRpXjrV271nO8xYsXV1m3fv16xWazKVFRUcr27durrFu8eLFis9mUiIgIJTY2tlo9R4wYoUyYMKFKXZxOp/LMM88oNptNGTRokOJyuS79FyhELeSekxANsG3bNuLj4/H19eWdd96hXbt2VdaPGzeO6dOnA/Dxxx9X276goIAFCxZU+Zpss9nMK6+8gsVioby8nO+//96zbsuWLRw5cgSdTsff/vY32rZtW2V/I0eOZNKkSQAsWbIE5ZyBkQ8++ABQe21Dhw6tst1DDz1EVFQUiqKwcuXKavVMS0vjr3/9K23atPEsMxgMPPbYYwDk5eV5el9CNAYJJyEaYNOmTQD07t2bVq1a1Vhm1KhRABw+fJjTp09XWWez2ejWrVu1bSrvVwHEx8d7lv/4448A9OjRgy5dutR4vOjoaADS09NJSUkBICsri0OHDgHUeh/rn//8Jzt27ODNN9+stm7QoEF07ty52vJzv3L9/M8mREPIhAghGiAxMRGA5ORkZs6cWWMZp9Pp+TklJYWgoCDP36+77rpa992xY0d+/vlnfvvtN8+yo0ePAhAREVHrduHh4Z6fk5OT6dKlC0eOHPEs69SpU43b1RaulXWpiZeXl+dnh8NR6/ZCXCoJJyEaID8/H1B7Kenp6RctX1RUVOXvAQEBtZb18/MDoKSkxLOsoKAAgMDAwFq38/f39/xcWFhYZTsAX1/fi9bzfD4+Ppe8jRANIeEkRANUzoabNm0a8+bNu+Ttjcba/wm63W6AKlPCK39WLjDJ9tx1leXP3ceFthVCK+SekxANUNmDyc3Nrdf25/ekalpX2YM693iVPbaanNtLquyZndtDO3e9EFol4SREA1TeM9q/f3+9tq+8h1STEydOAFXv99hsNkCdXFGbyvtg55av/PP89edKTk5my5YtxMbG1qHmQjQtCSchGmD48OGAGiQ7d+6ssczWrVuZOHEiS5curbbuwIEDVSY8VDp9+rQn8AYMGOBZfttttwGQkJBQa7CtWbMGgG7dunlm07Vu3ZoePXoAVJmafq758+fz+OOPs3z58hrXC3E5STgJ0QBDhgyhT58+AMydO7dajyY2Npbnn3+effv2UVxcXG17i8XCs88+63kjBEBFRQWvvPIKdrsdf39/Ro4c6Vl3880307NnT0B9K0NWVlaV/X311Vf85z//Aag2e7DymaTvvvuOr7/+usq6L774gm3btgFw33331fnzC9FUZEKEEA2g0+l45513ePjhh0lKSmLcuHH07NmTFi1akJ6eTlJSEqA+ezRjxoxq20+aNIkff/yRYcOG0atXLywWCwcPHiQvLw+dTsfLL79c5X6RXq/n3Xff5aGHHuLQoUMMHz6c66+/Hi8vL5KTk0lLSwPUIBo9enSVY40cOZLp06fzySefMG/ePP71r38REhJCamqqZwhx9uzZVXpqQjQXCSchGqhdu3asWrWKFStWsGHDBpKSkkhMTKRVq1bcdNNNjB8/nlGjRtX4IlYfHx++/PJLFi1axKZNm8jMzMRisTBkyBAeffRRbrjhhmrbdOjQgdWrV7N06VI2btxIQkICdrudoKAgRo0axf3338/AgQNrrOvcuXO54YYbWLFiBfv37yc1NRVfX1+GDRt20Re/CnE5yYtfhWgGlS9UffLJJ/n973/f3NURQnPknpMQQgjNkXASQgihORJOQgghNEfCSQghhObIhAghhBCaIz0nIYQQmiPhJIQQQnMknIQQQmiOhJMQQgjNkXASQgihORJOQgghNOf/A2RN1l0+DjT3AAAAAElFTkSuQmCC\n"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.size'] = 26\n",
    "plt.rcParams['axes.labelsize'] = 26\n",
    "plt.rcParams['xtick.labelsize'] = 26\n",
    "plt.rcParams['ytick.labelsize'] = 26\n",
    "plt.rcParams['legend.fontsize'] = 26\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "ax = pd.DataFrame(history).drop(columns='total').plot(x='epoch')\n",
    "plt.ylabel('loss')\n",
    "plt.ylim([0.2, 1.8])\n",
    "\n",
    "plt.savefig(figure_path + '/' + dataset +'-large-loss.png', dpi=300, bbox_inches='tight')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "pycharm-f57757a1",
   "language": "python",
   "display_name": "PyCharm (rs-via-gnn)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}