Adam_problem.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adam convergence problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from scipy.signal import savgol_filter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Synthetic example 1: online learning\n",
    "\n",
    "We will solve the online optimization problem for function  $f_t(x) = \\begin{cases} 1010x, \\text{for t mod 101} = 1 \\\\ \n",
    "-10x, otherwise \\end{cases}$\n",
    "\n",
    "Online optimization task: $ F(x) = \\sum_t f_t(x)  \\to min_x$\n",
    "\n",
    "with the constraint set $\\mathcal{F} =[-1, 1]$\n",
    "\n",
    "The analitical solution for this task is $x^*=-1$\n",
    "\n",
    "In order to understand the problems of Adam, we denote the concept of \"regret\": $R_T = \\sum\\limits_{t=1}^T f_t(x_t) - \\min\\limits_{x \\in \\mathcal{F}} \\sum\\limits_{t=1}^T f_t(x)$\n",
    "\n",
    "The regret should tend to zero during optimization, it implies that on average,\n",
    "the model’s performance converges to the optimal one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_buffer = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import adashift.optimizers as ad_opt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def online_f_t(x, t):\n",
    "    if t % 101 == 1:\n",
    "        return 1010.0*x\n",
    "    else:\n",
    "        return -10.0*x\n",
    "    \n",
    "def f_min(t):\n",
    "    if t % 101 == 1:\n",
    "        return -1010.0\n",
    "    else:\n",
    "        return 10.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def OnlineLearning(optimizer_type='Adam', alpha=1, lr_decay=True, eps=0.,\n",
    "                   x_init=0.0, betas=(0.9, 0.99), amsgrad=False, max_step = 1000001):\n",
    "    x = torch.tensor([x_init], requires_grad=True)\n",
    "    lr = alpha\n",
    "    if optimizer_type == \"Adam\":\n",
    "        optimizer = torch.optim.Adam([x], amsgrad=amsgrad, lr=lr, betas=betas, eps=eps)\n",
    "    else:\n",
    "        optimizer = ad_opt.AdaShift([x], lr=lr, betas=betas, eps=eps)\n",
    "    \n",
    "    \n",
    "    regret_sum = 0\n",
    "    time_steps = []\n",
    "    avg_regret_history = []\n",
    "    x_history = []\n",
    "    \n",
    "    if lr_decay:\n",
    "        lambda_lr_decay = lambda step: alpha / np.sqrt(step + 1) \n",
    "        scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_lr_decay)\n",
    "\n",
    "    for step in range(1, max_step):\n",
    "        # constraint\n",
    "        x.data = x.data.clamp(-1, 1)\n",
    "        \n",
    "        loss = online_f_t(x, step)\n",
    "\n",
    "        regret_sum += (loss.item() - f_min(step))\n",
    "        regret_avg = regret_sum / step\n",
    "    \n",
    "        loss.backward()\n",
    "        \n",
    "        x_history.append(x.item())\n",
    "\n",
    "        if step % 10000 == 0:\n",
    "            time_steps.append(step)\n",
    "            avg_regret_history.append(regret_avg)\n",
    "        if step % 50000 == 0:\n",
    "            print('step : {} loss: {} regret_avg : {} x: {}'.format(step, loss.item() , regret_avg, x.item()))\n",
    "\n",
    "        optimizer.step()\n",
    "                                \n",
    "        if lr_decay:\n",
    "            scheduler.step()\n",
    "        optimizer.zero_grad()\n",
    "                                                      \n",
    "    return time_steps, avg_regret_history, x_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step : 50000 loss: -9.889703750610352 regret_avg : 0.5064160318643972 x: 0.9889703989028931\n",
      "step : 100000 loss: -9.864534378051758 regret_avg : 0.4067958696881495 x: 0.9864534735679626\n",
      "step : 150000 loss: -9.865385055541992 regret_avg : 0.36496170430763314 x: 0.9865384697914124\n",
      "step : 200000 loss: -9.87474250793457 regret_avg : 0.34069641541677526 x: 0.9874742031097412\n",
      "step : 250000 loss: -9.886850357055664 regret_avg : 0.3244257141243443 x: 0.9886850118637085\n",
      "step : 300000 loss: -9.899446487426758 regret_avg : 0.31256548273217555 x: 0.9899446368217468\n",
      "step : 350000 loss: -9.911629676818848 regret_avg : 0.30343590262934034 x: 0.9911629557609558\n",
      "step : 400000 loss: -9.92305850982666 regret_avg : 0.296132920313864 x: 0.9923058152198792\n",
      "step : 450000 loss: -9.933679580688477 regret_avg : 0.29012169665317983 x: 0.9933679699897766\n",
      "step : 500000 loss: -9.943526268005371 regret_avg : 0.28506324361689017 x: 0.9943526387214661\n",
      "step : 550000 loss: -9.952676773071289 regret_avg : 0.28073099849078803 x: 0.9952676892280579\n",
      "step : 600000 loss: -9.96122932434082 regret_avg : 0.27696713664756156 x: 0.9961228966712952\n",
      "step : 650000 loss: -9.969263076782227 regret_avg : 0.27365787758814314 x: 0.9969263076782227\n",
      "step : 700000 loss: -9.976862907409668 regret_avg : 0.27071892833493383 x: 0.9976863265037537\n",
      "step : 750000 loss: -9.984089851379395 regret_avg : 0.26808631337917593 x: 0.9984089732170105\n",
      "step : 800000 loss: -9.991005897521973 regret_avg : 0.2657104797034408 x: 0.9991006255149841\n",
      "step : 850000 loss: -9.997658729553223 regret_avg : 0.2635523970429051 x: 0.9997658729553223\n",
      "step : 900000 loss: -10.0 regret_avg : 0.2615808684745493 x: 1.0\n",
      "step : 950000 loss: -10.0 regret_avg : 0.2597706107866509 x: 1.0\n",
      "step : 1000000 loss: -10.0 regret_avg : 0.2581008625005837 x: 1.0\n"
     ]
    }
   ],
   "source": [
    "time_steps, avg_regret_history, x_history = OnlineLearning()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step : 50000 loss: 9.866371154785156 regret_avg : 0.31438388655662536 x: -0.9866371154785156\n",
      "step : 100000 loss: 9.961386680603027 regret_avg : 0.21818185344219207 x: -0.9961386919021606\n",
      "step : 150000 loss: 9.991243362426758 regret_avg : 0.17665850755373638 x: -0.9991243481636047\n",
      "step : 200000 loss: 10.0 regret_avg : 0.15222457282304763 x: -1.0\n",
      "step : 250000 loss: 9.999848365783691 regret_avg : 0.13568644660758972 x: -0.999984860420227\n",
      "step : 300000 loss: 9.997462272644043 regret_avg : 0.12354939424355825 x: -0.9997462034225464\n",
      "step : 350000 loss: 9.993610382080078 regret_avg : 0.11415800917625427 x: -0.9993610382080078\n",
      "step : 400000 loss: 9.989237785339355 regret_avg : 0.10661424065232276 x: -0.9989237785339355\n",
      "step : 450000 loss: 9.984785079956055 regret_avg : 0.1003834655518002 x: -0.9984785318374634\n",
      "step : 500000 loss: 9.980443954467773 regret_avg : 0.09512502759456634 x: -0.9980443716049194\n",
      "step : 550000 loss: 9.976297378540039 regret_avg : 0.09061028428251093 x: -0.997629702091217\n",
      "step : 600000 loss: 9.97236156463623 regret_avg : 0.08667933124303817 x: -0.9972361326217651\n",
      "step : 650000 loss: 9.96863842010498 regret_avg : 0.08321655632972717 x: -0.996863842010498\n",
      "step : 700000 loss: 9.965115547180176 regret_avg : 0.08013602013519833 x: -0.9965115189552307\n",
      "step : 750000 loss: 9.961771011352539 regret_avg : 0.07737233556048076 x: -0.996177077293396\n",
      "step : 800000 loss: 9.95859432220459 regret_avg : 0.07487476393163205 x: -0.9958594441413879\n",
      "step : 850000 loss: 9.955567359924316 regret_avg : 0.07260324488191043 x: -0.9955567121505737\n",
      "step : 900000 loss: 9.952668190002441 regret_avg : 0.07052567510763805 x: -0.9952667951583862\n",
      "step : 950000 loss: 9.949888229370117 regret_avg : 0.06861598586082458 x: -0.9949888586997986\n",
      "step : 1000000 loss: 9.947218894958496 regret_avg : 0.06685275888586044 x: -0.9947218894958496\n"
     ]
    }
   ],
   "source": [
    "amsgrad_time_steps, amsgrad_avg_regret_history, amsgrad_x_history = OnlineLearning(amsgrad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step : 50000 loss: 9.873220443725586 regret_avg : 0.46133322309844194 x: -0.9873220920562744\n",
      "step : 100000 loss: 10.0 regret_avg : 0.32524554844077674 x: -1.0\n",
      "step : 150000 loss: 9.991798400878906 regret_avg : 0.2652160040167719 x: -0.9991798400878906\n",
      "step : 200000 loss: 9.98381519317627 regret_avg : 0.22950586252776906 x: -0.9983815550804138\n",
      "step : 250000 loss: 9.977197647094727 regret_avg : 0.2051678520629257 x: -0.9977197647094727\n",
      "step : 300000 loss: 9.971391677856445 regret_avg : 0.18721873679219433 x: -0.9971391558647156\n",
      "step : 350000 loss: 9.966115951538086 regret_avg : 0.173278321160408 x: -0.9966115951538086\n",
      "step : 400000 loss: 9.961211204528809 regret_avg : 0.16204738431974314 x: -0.9961211085319519\n",
      "step : 450000 loss: 9.956570625305176 regret_avg : 0.1527487501317842 x: -0.9956570863723755\n",
      "step : 500000 loss: 9.952126502990723 regret_avg : 0.14488528773724288 x: -0.9952126741409302\n",
      "step : 550000 loss: 9.947837829589844 regret_avg : 0.1381222433925691 x: -0.9947838187217712\n",
      "step : 600000 loss: 9.943656921386719 regret_avg : 0.13222486033627764 x: -0.9943656921386719\n",
      "step : 650000 loss: 9.939566612243652 regret_avg : 0.12702299467407166 x: -0.9939566254615784\n",
      "step : 700000 loss: 9.935537338256836 regret_avg : 0.12238990945841052 x: -0.9935536980628967\n",
      "step : 750000 loss: 9.93155574798584 regret_avg : 0.11822900174863636 x: -0.9931555986404419\n",
      "step : 800000 loss: 9.927606582641602 regret_avg : 0.11446517678997944 x: -0.9927606582641602\n",
      "step : 850000 loss: 9.92368221282959 regret_avg : 0.11103906178326729 x: -0.992368221282959\n",
      "step : 900000 loss: 9.919768333435059 regret_avg : 0.10790301073305723 x: -0.991976797580719\n",
      "step : 950000 loss: 9.915854454040527 regret_avg : 0.10501828575052322 x: -0.9915854334831238\n",
      "step : 1000000 loss: 9.91193962097168 regret_avg : 0.10235302256887778 x: -0.991193950176239\n"
     ]
    }
   ],
   "source": [
    "adashift_time_steps, adashift_avg_regret_history, adashift_x_history = OnlineLearning(optimizer_type='Adashift') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Regret should tend to zero. As we can see it is done only for amsgrad."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEWCAYAAACqitpwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmYXFWd//H3t2vtrt637El3yB4SErKQEMQEUQIomyAiKuOIy4wwKuMgzM9BxMFRwVEZGREFwQUUBDUsTgTZFAkkYUnIvpClk3TS+1Z79fn9cW9VqpteqjvVXdWd7+t57lP3nnvq3lNdnf7k3OVcMcaglFJKpVNOphuglFJq9NFwUUoplXYaLkoppdJOw0UppVTaabgopZRKOw0XpZRSaafhopRSKu00XJQaJBHZJyIBEWkXkVoReUBE8jPQjpUiUjPc+1WqLxouSp2YDxlj8oEFwELg5nTvQEQc6d6mUkNNw0WpNDDG1AJrsUIGEfGIyJ0ickBEjorIPSKSG68vIjeKyBEROSwi14qIEZFp9roHROTHIvK0iHQAq3rbnoj4gD8B4+0eVLuIjM/Aj0CpLjRclEoDEZkInA/stou+DczACptpwATgFrvuauAG4Fx73coeNvkx4HagAPhbb9szxnTY+z1sjMm3p8ND8BGVGhDRscWUGhwR2QeUAwbIB54DPgy0AO3AfGPMHrvucuAhY0y1iNwPHDXG3GyvmwbsAqYbY3aLyANAjjHmk/Z66Wd7K4FfGWMmDs8nV6p/zkw3QKkR7hJjzLMi8l7gIaywcQN5wEYrFwAQIH7uZDywIWkbB3vYbnJZRT/bUyrraLgolQbGmBftHsedwGVAAJhrjDnUQ/UjQHIvY1JPm0yar+9ne3r4QWUdPeeiVPr8AHg/MA/4KfB9EakEEJEJInKeXe8R4FMiMltE8oD/6GujxpjOfrZ3FCgTkaK0fyKlBknDRak0McbUAb/AOnH/VayT++tEpBV4Fphp1/sTcBfwfLyOvYlQH5vva3vbgYeBvSLSrFeLqWygJ/SVyjARmQ28DXiMMdFMt0epdNCei1IZICKX2veulADfAZ7QYFGjiYaLUpnxOeAYsAeIAf+U2eYolV56WEwppVTaac9FKaVU2p1U97mUl5ebqqqqTDdDKaVGlI0bN9YbYyoG8p6TKlyqqqrYsGFD/xWVUkoliMj+gb5HD4sppZRKOw0XpZRSaafhopRSKu1OqnMuSqnRLxKJUFNTQzAYzHRTRhyv18vEiRNxuVwnvC0NF6XUqFJTU0NBQQFVVVUkPaJA9cMYQ0NDAzU1NVRXV5/w9vSwmFJqVAkGg5SVlWmwDJCIUFZWlrYen4aLUmrU0WAZnHT+3DRcUvDSzjp++OyuTDdDKaVGDA2XFKzb28D/PLcLHYdNKZUODzzwANddd92A3/fCCy/wwQ9+sMd11157LVu3bgXg0UcfZfbs2axatYo333yTp59++oTaOxgaLinweZxEOw3hWGemm6KUUj362c9+xpw5cwC47777+OlPf8rzzz+v4ZLNfG4HAB2hWIZbopQaCS655BIWLVrE3LlzuffeewH4+c9/zowZM1i6dCkvv/xyou4TTzzBGWecwcKFCzn33HM5evQoAC+++CILFixgwYIFLFy4kLa2NgDa29u5/PLLmTVrFldffXXiiMrKlSvZsGEDt912G3/729/49Kc/zZe//GVuueUWfvvb37JgwQJ++9vfDtvPQC9FToHPY/2YOkJRSn3uDLdGKZWqbzyxha2HW9O6zTnjC/n6h+b2Wef++++ntLSUQCDAkiVLuPDCC/n617/Oxo0bKSoqYtWqVSxcuBCAs846i3Xr1iEi/OxnP+O73/0u3/ve97jzzju5++67WbFiBe3t7Xi9XgDeeOMNtmzZwvjx41mxYgUvv/wyZ511VmLft9xyC8899xx33nknixcv5rTTTmPDhg386Ec/SuvPoT/ac0lBIlzC+qBApVT/7rrrLk477TSWLVvGwYMH+eUvf8nKlSupqKjA7XZz5ZVXJurW1NRw3nnnMW/ePO644w62bNkCwIoVK7jhhhu46667aG5uxum0/g4tXbqUiRMnkpOTw4IFC9i3b18mPmK/tOeSguM9Fz0sptRI0l8PYyi88MILPPvss7zyyivk5eWxcuVKZs2alTjZ3t3111/PDTfcwEUXXcQLL7zArbfeCsBNN93EhRdeyNNPP82KFStYu3YtAB6PJ/Feh8NBNJqd/+nNyp6LiNwvIsdE5O1e1ouI3CUiu0Vkk4icPpTtOX7OJTu/RKVU9mhpaaGkpIS8vDy2b9/OunXrCAQCvPjiizQ0NBCJRHj00Ue71J8wYQIADz74YKJ8z549zJs3j69+9assWbKE7du3D6o9BQUFifM1wykrwwV4AFjdx/rzgen29Fngx0PZmMpjL/Mvjsfx62ExpVQ/Vq9eTTQaZfbs2dx0000sW7aMcePGceutt7J8+XJWrFjB7NmzE/VvvfVWrrjiChYtWkR5eXmi/Ac/+AGnnnoq8+fPx+Vycf755w+qPatWrWLr1q3DfkJfsvXeDRGpAp40xpzaw7qfAC8YYx62l3cAK40xR/ra5uLFi81gHhbW8uR/kLf+bv74obe4fPGkAb9fKTV8tm3b1uWPtxqYnn5+IrLRGLN4INvJ1p5LfyYAB5OWa+yydxGRz4rIBhHZUFdXN6idOXMLcUmMULBjUO9XSqmTzUgNl5QZY+41xiw2xiyuqBjQI6AT3L5iAML+lnQ2TSmlRq2RGi6HgOTjUxPtsiHhzC0EIOZP7/XySik1Wo3UcFkDfNK+amwZ0NLf+ZYTIZ4CAGJBDRellEpFVt7nIiIPAyuBchGpAb4OuACMMfcATwMXALsBP/CpIW2QHS5ouCilVEqyMlyMMVf1s94AXxim5iTCxYTah22XSik1ko3Uw2LDyw4XCQ//jUhKqdFnsEPujyQaLqnwWCf0cyLac1FKqVRouKTC7rk4NFyUUilIx5D7t956K9dccw3vec97mDJlCo8//jg33ngj8+bNY/Xq1UQikYx8tlRl5TmXrOP0EMWJO6rhotSI8qeboHZzerc5dh6c/+0+q6RjyH2wxhd7/vnn2bp1K8uXL+exxx7ju9/9LpdeeilPPfUUl1xySXo/WxppuKRChJDDhzuqd+grpfp311138fvf/x7gXUPuA1x55ZXs3LkTsIbcv/LKKzly5AjhcJjq6urEds4//3xcLhfz5s0jFouxerU15OK8efOydqj9OA2XFIWdPjwBDRelRpR+ehhDIV1D7sPx4fVzcnJwuVyISGI5W4faj9NzLimKOn3kGj+xzuwc6FMplR3SNeT+SKfhkqKYM598AjrsvlKqT+kacn+ky9oh94fCYIfcBzh09wdpOFpD5b+uY2yRN80tU0qliw65f2JO9iH3h51xF+AjSIf2XJRSql8aLqnyFFAgAX3UsVJKpUDDJUXiLSCfAB2hWKabopRSWU/DJUWO3ELyJIQ/EMx0U5RSKutpuKTI4S0CIKRPo1RKqX5puKTI7bPCJaxPo1RKqX5puKTIlWeFSyygPRel1MhWVVVFfX39kO5DwyVFHl8xANGA9lyUUtkn24aD0XBJkcNrP41SH3WslOpHT0Pu5+fn82//9m/MnTuXc889l9dee42VK1cydepU1qxZA8CWLVtYunQpCxYsYP78+ezatQuAb37zm8ycOZOzzjqLq666ijvvvBOAlStX8qUvfYnFixfzwx/+sNfh+xsaGvjABz7A3LlzufbaaxmOm+d14MpUxR91HNSnUSo1Unznte+wvXF7Wrc5q3QWX1361T7rdB9y/8Mf/jAdHR2cc8453HHHHVx66aV87Wtf45lnnmHr1q1cc801XHTRRdxzzz188Ytf5OqrryYcDhOLxVi/fj2PPfYYb731FpFIhNNPP51FixYl9hUOh4mPPNLU1NTj8P3f+MY3OOuss7jlllt46qmnuO+++9L6M+mJhkuq9FHHSqkUdR9yf9euXbjd7i5D5ns8nsRw+vHh85cvX87tt99OTU0Nl112GdOnT+fll1/m4osvxuv14vV6+dCHPtRlX1deeWVivrfh+1966SUef/xxAC688EJKSkqG+keg4ZKyRLjoA8OUGin662EMhZ6G3A8Gg+8aMj95OP34+ZKPfexjnHHGGTz11FNccMEF/OQnP+l3fz6fLzHf1/D9w03PuaTKnQ+AUx91rJTqQ09D7qdq7969TJ06lX/5l3/h4osvZtOmTaxYsYInnniCYDBIe3s7Tz75ZJ/77mn4/rPPPpuHHnoIgD/96U80NTUN8tOlTnsuqcrJISB5Gi5KqT6tXr2ae+65h9mzZzNz5kyWLVuW8nsfeeQRfvnLX+JyuRg7diz//u//TmlpKRdddBHz589nzJgxzJs3j6Kioh7fHx++v6SkhHPOOYd33nkHgK9//etcddVVzJ07lzPPPJPJkyen5bP2RYfcH4Cm/zyFV2QhF/y/36WxVUqpdBqNQ+63t7eTn5+P3+/n7LPP5t577+X0008fkn2la8h97bkMQNjhwxPRRx0rpYbXZz/7WbZu3UowGOSaa64ZsmBJJw2XAYg48vGGNFyUUsMrfr5kJNET+gMQdfnI7fQPyw1ISqnB03+jg5POn5uGywDEXPn4CBCKdma6KUqpXni9XhoaGjRgBsgYQ0NDA15veh7jrofFBsC488m3n0bpdTky3RylVA8mTpxITU0NdXV1mW7KiOP1epk4cWJatqXhMgDGbT2NsjUcoyzTjVFK9cjlciXuTFeZo4fFBkC8heQToD0YyXRTlFIqq2m4DIB4C3GIIejX8cWUUqovWRsuIrJaRHaIyG4RuamH9ZNF5HkReUNENonIBUPdJmduIQDBDn1gmFJK9SUrw0VEHMDdwPnAHOAqEZnTrdrXgEeMMQuBjwL/O9TtcuZZ4RLpaB7qXSml1IiWleECLAV2G2P2GmPCwG+Ai7vVMUChPV8EHB7qRrnzrKdRhv3ac1FKqb5ka7hMAA4mLdfYZcluBT4uIjXA08D1PW1IRD4rIhtEZMOJXpro9lmDxcUCes5FKaX6kq3hkoqrgAeMMROBC4Bfisi7Po8x5l5jzGJjzOKKiooT2qE33+q5xILac1FKqb5ka7gcAiYlLU+0y5J9GngEwBjzCuAFyoeyUW77nAv6qGOllOpTtobLemC6iFSLiBvrhP2abnUOAO8DEJHZWOEypLfkiscOl5CGi1JK9SUrw8UYEwWuA9YC27CuCtsiIreJyEV2tX8FPiMibwEPA/9ghnowIY/1NEp91LFSSvUta4d/McY8jXWiPrnslqT5rcCKYW2U00MYFzkR7bkopVRfsrLnks38kodTHximlFJ90nAZoGBOHq6oHhZTSqm+aLgMUMjhwx3TnotSSvVFw2WAIg4fHg0XpZTqk4bLAEVc+eR2+jPdDKWUymoaLilYs2cNX3nxK4D1qONco+GilFJ90XBJwf7W/fx535/pNJ10ugrwESAa68x0s5RSKmtpuKSg0F2IwdAWbkO8+RQQoDmgT6NUSqneaLikoMhjjYbcGm7F4yvGIxGONurglUop1RsNlxQUuq0xxVrDreSWWSP/Nx7Zn8kmKaVUVtNwSUEiXEKt5I+dBkDw2O5MNkkppbKahksKCj3Hey5F42cAEGt4J5NNUkqprKbhkoLkw2I5heMJ4cLVqofFlFKqNxouKYiHS0uoBXJyqHOMxeevyXCrlFIqe2m4pCDXmYszx0lruBWAFu8ESkPdH4yplFIqTsMlBSJCobuQ1pAVLoH8SYzrrKVTb6RUSqkeabikqNBdmOi5dBZXUSABGutrM9wqpZTKThouKSryFCXCxVk+FYDmQ7sy2SSllMpaGi4pSj4sFr/XpeOohotSSvVEwyVFhZ7jh8VKJkwH9F4XpZTqjYZLipLPuZQVF3PUlOBo2ZfZRimlVJbScElRobuQ9nA7naaTnByh1jEWX/vBTDdLKaWykoZLipKH3Qdo9kygWO91UUqpHmm4pCh5fDEAv28SpZ0NEAlmsllKKZWVNFxSlDwyMkC0aAo5GDqbdIwxpZTqTsMlRYnxxcLWQ8Ic9r0urUd2ZqxNSimVrTRcUtT9sFhepX2vS60+10UppbrTcElRkdt+1LF9WKx8zAQ6jIdI3d5MNksppbKShkuKuvdcxhXncsCMIadZz7kopVR3Gi4p8jq8uHJciXAp9bmpYQy5HQcy3DKllMo+Gi4p6j7svojQ4B5PUfAQGJPh1imlVHbJ2nARkdUiskNEdovITb3U+YiIbBWRLSLy0FC3KXl8MYD2vEm4TRha9WZKpZRK5sx0A3oiIg7gbuD9QA2wXkTWGGO2JtWZDtwMrDDGNIlI5VC3K3l8MYDW0lOhFTj4GhRNHOrdK6XUiJGtPZelwG5jzF5jTBj4DXBxtzqfAe42xjQBGGOODXWjkg+LAUTHzKPDeDD7/z7Uu1ZKqRElW8NlApA8KmSNXZZsBjBDRF4WkXUisrqnDYnIZ0Vkg4hsqKurO6FGdT8sNq44n9c7pxPdp+GilFLJsjVcUuEEpgMrgauAn4pIcfdKxph7jTGLjTGLKyoqTmiH3Xsu0yoLWN85C2fdVgg0n9C2lVJqNMnWcDkETEpanmiXJasB1hhjIsaYd4CdWGEzZIo8RbRF2oh1xgCYM66Q9WYmgrHOuyillAKyN1zWA9NFpFpE3MBHgTXd6vwBq9eCiJRjHSYb0tvl4+OLtUfaASjKc3G0YB4xHHBAD40ppVRcVoaLMSYKXAesBbYBjxhjtojIbSJykV1tLdAgIluB54F/M8Y0DGW7uo+MDHDKhAp25JwC+18Zyl0rpdSIktZLkUUkZoxxpGNbxpingae7ld2SNG+AG+xpWCTCJemk/pxxhfxt53RmH34GiQTB5R2u5iilVNZKd89F0ry9rBIfXyw+7D7AnPGFvNY5C4mF4dDGTDVNKaWyyqDCRUT+NWl+ZtKqUT0OSm89lw2dM6wFPe+ilFLAAA+L2Zf6fh+YJSIBYBPwaeBTPdStMsbsS0cjs0VP51wmluTS6S3hqLeaMXreRSmlgAH2XIwxzcaYTwG3Aq8C04DHe6n+rnIRWTbQBmaT7sPugzWA5Zzxhbwhs63Lke3LlJVS6mTWb7iIyCdEpE5EakTkGru4BbgUuN4Y80TX6nKFiHwbKBCR2SKSvI9709f04dd92P24OeOKeKZjGoTb4NDrGWqdUkplj1R6LrcAFwALgGoReQZ4FHABX+qh/stYlw+XAP8N7BaR10XkSSCQllZniIhQ5CnqclgMrJP6z4Tn0enwwOZHM9Q6pZTKHqmcc2k3xqwHEJFvAEeBGcaYnsY7McaYw8CDIrLbGPOy/b4yoArYnp5mZ073kZHBOqnfio8jY1cx4e3H4LzbweHKUAuVUirzUum5jLUHf3wvMAao6SVYuogHiz3fYIzZaIzpOIG2ZoXu44sBTKvMx+UQXvG9D/z1sOe5DLVOKaWyQyrh8nVgHnAbsBWYJyLPisgdIvKxIW1dFuo+MjKA25nD9MoCngrMhdxSeOs3GWqdUkplh34PixljupyEF5GJWGEzHzgfSH4C5Ki+iRKsnsue5j3vKp8zvpAXdtTBwsvgjV9BsBW8hRlooVJKZd6Ab6I0xtQYY/5kjPmOMeYT3dZl5Vhl6dTTYTGAueMLqW8P0XjKJRANwrYneni3UkqdHEZ9GKRboaewy7D7cadNsh4l8/fQVCiphk16aEwpdfLScBmgilzrgWN1ga5PtZw/oYhCr5OXdtXD/Cvhnb9CS00mmqiUUhmn4TJAVYVVALzT8k6XcqcjhxXTyvnrrnrMaR8FyYFX7s5AC5VSKvM0XAaoqqgKgH2t+9617uwZFRxpCbI7Ug6nfRQ23A9ttcPbQKWUygIaLgNUkVuBz+V7V88FrHABeHFnHbznXyEWgZd/ONxNVEqpjNNwGSARoaqwin0t+961bkJxLqdU+KzzLmWnaO9FKXXS0nAZhOqiat5pfXfPBeA90yt4dW8DwUhMey9KqZOWhssgVBVWUdtRiz/if9e6986oIBTtZP2+xq69l9YjGWipUkplhobLIFQXVQOwv3X/u9adMbUUtyOHl3balyqf/RUwnbD25uFsolJKZZSGyyD0dcVYntvJ4qoSXtpZbxWUToX3fhW2/B62rhm+RiqlVAZpuAzClMIpCNLjFWNgXTW242gbtS1Bq2DFF2HsfHjqX8HfOIwtVUqpzNBwGQSPw8P4/PE9XjEGcM6sSgD+7237PIvDBRffDYFGWPvvw9RKpZTKHA2XQerrirEZYwo4dUIhv3s9afiXcfPhrBvgrYdh25PD1EqllMoMDZdBqiqsYn/rfjpNZ4/rLz99Im8famXbkaQRlM/+CoxfCL//HBzdOkwtVUqp4afhMkjVRdUEogGOdhztcf1FCybgcgiPbUzqvTg98NGHwJ0PD18JHfXD1FqllBpeGi6DFL8cubdDY6U+N++bNYY/vHmISCypd1M43gqYtqPwyCchGh6O5iql1LDScBmkRLj0csUYwOWLJlLfHubFHV2H52fiIusE//6XrUNksehQNlUppYadhssglXnLyHfl93rFGMB7Z1ZQnu/mdxt7eK7L/Cvg/bfBlsfhsU9bw8QopdQo4cx0A0YqEenzijEAlyOHSxZM4MFX9tHYEabU5+5aYcUXree+/Plr1l38l99vXbaslFIjnPZcTkBvoyMnu3zxRCIxw8OvHei5wpnXw3nfgm1r4KErIdiS/oYqpdQwy9pwEZHVIrJDRHaLyE191PuwiBgRWTyc7QNrGJij/qM9DmAZN2tsIe+dUcF9f3sHf7iXcyvLvwAfugveeRF+di407BmiFiul1PDIynAREQdwN3A+MAe4SkTm9FCvAPgi8OrwttAyq3QWAJvqN/VZ77pzptHYEebh1w72XmnRNfDJP1qXJ//0HNj9bDqbqpRSwyorwwVYCuw2xuw1xoSB3wAX91Dvm8B3gOBwNi5u0ZhFOMTBa0de67PekqpSzqgu5d6X9hCKxnqvWHUWfOY563LlX30Y/u9miGTkoyml1AnJ1nCZACT/N7/GLksQkdOBScaYp4azYcl8Lh+nlp/Kq0f67zhdf850jraGer5yLFlpNVz7F1jyGVj3v1YvpvbtNLVYKaWGR7aGS59EJAf4b+BfU6j7WRHZICIb6urq+qs+YGeMO4O3G96mLdzWZ70V08pYMKmYH7+wp+tNlT1x58GFd8LHHoWOOvjJ2dYVZaH2NLZcKaWGTraGyyFgUtLyRLssrgA4FXhBRPYBy4A1PZ3UN8bca4xZbIxZXFFRkfaGLhu3jE7TycajG/usJyJct2oaNU2BrkPC9GXGB+ALr8LCj8Pf/wfuXgpvPw7GpKHlSik1dLI1XNYD00WkWkTcwEeBxJO2jDEtxphyY0yVMaYKWAdcZIzZMNwNnV8xH4/Dk9KhsffNrmTRlBK+u3YHLf4Ub5rMK4WL7oJPPwO5pfC7T8FPV8HeF06s4UopNYSyMlyMMVHgOmAtsA14xBizRURuE5GLMtu6rjwODwsrF/Jqbf/hIiLcdvFcmv1hvvfMjoHtaNJS+NyLcPH/Qnsd/OJiePAieOcl7ckopbJOVoYLgDHmaWPMDGPMKcaY2+2yW4wx73pWsDFmZSZ6LXFnjDuDXU27aAg09Ft37vgiPrm8il+t28/bhwZ4w2SOAxZeDddvhA/cDse2wYMfsu6N2fYkdPZxJZpSSg2jrA2XkeSMsWcAsL52fUr1v/z+GZT63PzHH9+ms3MQvQ6XF868Dr60GS78HnQcg99eDT9cAH/7gT5KWSmVcRouaTC7bDb5rnzWHVmXUv2iXBc3nz+bNw4085v1fdxY2R+XF5ZcC9e/AR/5BZRMgWe/Dt+bBY9+Cnb/RXszSqmM0IEr08CZ42Tx2MW8Vtv3zZTJLjt9Ao+/UcM3n9zK0upSplXmD74BDifMudiajm6BjQ/Cpt9aIy4XToBTL4NTPwzjFoDI4PejlFIp0p5Lmpwx9gwOth2kpi21y4xFhP/+yAJy3Q6uf/gNgpE09TDGzIULvgtf2QlXPABjToV1P4Z7V8L/nA7P3AIH10NnP/faKKXUCdBwSZNVk1cB8NTe1AcMGFPo5Y7L57PtSCvf+b/t6W2Q0wNzL4WrH4Gv7LIGxiyeAq/cDfedC/89G/54HWxdA8HW9O5bKXXSE3MSXca6ePFis2HD0F1Udu3aaznUfoinLnuKHEk9t29ds4UH/r6Pn31yMefOGTNk7QMg0AS7noHtT8Ge5yHUAjlOmLgEpq60pvGng9Pd93aUUicNEdlojBnQyPMaLmn05N4nufmvN3P/efezZOySlN8XjMS4/J6/805dB49+/kzmjC8csjZ2EYvAwddg9zOw90U4/AZgwJlr3VczZQVMWQ4TFoHbNzxtUkplHQ2Xfgx1uASjQc555BxWTlrJt97zrQG9t7YlyCV3v4wI/OELKxhT6B2iVvbB3wj7/gb7X4Z9L8PRtwED4oCx86zezYRF1lQ2DXL0qKpSJwMNl34MdbgAfPOVb7Jmzxqe+8hzFLgLBvTeLYdbuOKeV5ha4eORzy0nz53hi/kCTVCzAQ6sg4OvWj2bsD14prsAxs23rkAbdxqMPRXKZ+hjmpUahTRc+jEc4fJ2/dtc9dRV3LL8Fq6YccWA3//c9qNc++AGzppewb2fWITX5RiCVg5SZwzqd8GhDVbQHHkLajdD1H7mjMMNFTOhci6MmQOVc6zlwonay1FqBNNw6cdwhIsxhsvWXEaeM49fX/jrQW3jt+sP8NXHNrNyZgX3fDzLAqa7WBQadlshU7sJjm2Fo1uh7fDxOi4flE+3ejbl061DamXToOwUPZej1AgwmHDRmyjTTES4dNql3LHhDnY07mBm6cwBb+PKJZMxBm56fDOf/9XG7A4YhxMqZ1nT/KSemr8R6nZA3fbjrwdegc2PdH1/wTgonWo9JK2kGkqqrKl4CvjK9aZPpUYo7bkMgZZQC+c9dh7Lxy3n+6u+P+jt/Oa1A9z0+GbOmlbOjz9+OgXeUXA+I9wBDXugcY/V42nYC03vQOM70F7bta5vT1pfAAAdIElEQVQrD4onW1PRJCieZL0WTbRGHigYZ4WbUmpIac8lSxR5irhm7jX875v/y9v1b3Nq+amD2s5Hl07G6cjhpsc2ccU9r/DzTy1hXFFumls7zNw++0KA+e9eF+6A5gPQtM+amg9C835rqllvXWCQTHIgfywUjofCcVCQ9Fow1gqfgjHgKdQekFLDTHsuQ6Qj0sH5j53PrNJZ3PuBe09oWy/trOOff/06+R4n9//DkuG7DybbhNqgpQZaDkFr/PUwtNqvbUcg1MNoA85cK2Tyx0B+Jfgq7fkKe77SOgTnqwB3vgaRUt3oCf1+DGe4APxiyy+4Y8Md3PeB+1g6bukJbWvbkVY+9fP1tAQi/Ndl87hk4YQ0tXKUCbVbIdNWa09HoP2oNbXVQkedNd+9FxTn9EJeOfjK7NdyyCuzngiaV2ZNufZ8bolV7vQM72dUaphpuPRjuMMlFAtx4eMXMsY3hl+d/yvkBP9HfKw1yHUPvcFr+xr5+LLJ/McH5+BxZumJ/mwXDYO/HtqPQUe99UycjnorfDrqrXXxV3/j8ft7euLyWSHjLYbcYit04q/JZd4iaznxWqj3BakRQc+5ZBmPw8PnT/s833jlG6zdv5bVVatPaHuVhV5+/ZkzuGPtDu59aS9vHWzh+1eexrTKgd2sqbDGTiscb02piAQh0GgFjb/BmgJNdlmTNR9sttbX77TXNUEs3Pd2XT47bAqtV0+hNZ94LbDmPfH5gqSyAvDkWxc+6KE8lWW05zLEop1Rrn76ag63H+b3F/+e8tzytGx37ZZabnpsE/5wjBtXz+JTZ1aRk6N/YLKKMRAJWKETaIZgiz0lz3ebQq3WKNXx11io//1IjjVigiffOmeUeC2wXt2+42VuX7fXvOPzrvi8z7ohVgNL2fSwWD8yES4Ae5r38JEnPsKZE87krlV3nfDhsbhjbUFufmwzf9l+jKXVpXzr0nkn9tAxlX2iIetChmCLdWgu1GaFTrjdCqBQm3WeKdR2fH243SoLd0C47fh8KkEVJw4rZFx5VgC5fODKPT7vzrOWu5THy/Ksc1fJy65c68mpzlx7PlcDbATRcOlHpsIF4MEtD3Lnhjv5zxX/ycXTLk7bdo0xPLqhhv98aiuBSIzPnX0K150zLXtvulSZE4tYwRP2Hw+esB8ifiuUIn57XXu3+YC93GG9RgL2fOD4us7IIBokVsgkgigePl6rzOntuSxRnjx5epj3JE1J5Q6PDkc0QBou/chkuMQ6Y/zj2n9kZ9NOHrvoMcbnp3isP0X17SG+9dQ2Hn/jEJNKc/nq6llcOG9c2npJSvUpFukaQtHg8RCKz8eDKRq054PH1/X6GoJo4Hjd+HSiclzHg8eRFEIOd1KZ2w4jd8/rHB7rgoz4uuR6iXmXXc9tv6f75Oo6n6X/XjVc+pHJcAE42HaQK564gkkFk3hw9YPkufLSvo9X9jTwjSe2sL22jYWTi/nahbNZNKU07ftRKmOMsS6UiAat0ImF7PAJWFcBJkIo1HU+/p5oKGmy3x9/Xyx8fF28PBZfjnStO6jeWj+6h06O690BlHhNms/poTzH2bVs9kXWQLKDoOHSj0yHC8Bfa/7Kdc9dx9kTzuYHq36AIyf9h69inYbHNtZw5593cKwtxKqZFXz5/TOYP7E47ftS6qTV2WkFTDy44sGUmLeDKRY5HkzvKrff0xlJWg7by93mY0l14suJ9/VRHg/BKx6EuZcM6qNquPQjG8IF4OHtD/OtV7/FJ+Z8ghuX3Dhk+/GHozzw933c+9Jemv0Rzp1dyT+tnMaiKSVDtk+lVJYxBjqj1lWFg/zPrN7nMkJcNesqDrQe4Jdbf8mYvDFcM/eaIdlPntvJP6+cxieWTeGBl/fxs7+9w7M//jtLq0r5/MqprJxRqZcvKzXaiWTkZl3tuWRIrDPGjS/dyJ/3/5kvnv5Frp137ZDvsyMU5TfrD3LfX/dyuCVIdbmPTy6fwuWLJo6OEZeVUkNCD4v1I5vCBawbLL/28td4au9TfG7+5/jCgi8My9VdkVgnT28+wgN/38cbB5rxuR1ctGACH10yifkTi/QKM6VUF3pYbIRx5ji5fcXteBwefrLpJ7SEWrhx6Y24coa2F+Fy5HDxgglcvGACm2qa+cUr+/n9GzU8/NoBZo8r5IpFE7lowXjK83VARqXU4GjPJQt0mk6+v/H7PLDlAZaOXcqd772TEu/wnnRvDUb445uHeWT9QTYfasGRI5w9vZyLF0zg3DljyPfo/0OUOlnpYbF+ZGu4xK3Zs4Zv/P0bVORV8INVP2BW6ayMtGPX0TYef+MQf3jjEEdagnicOayaWckF88examaFnp9R6iSj4dKPbA8XgM11m/nS81+iKdTE9Quv55NzPjkk98KkorPTsPFAE0++dZinNtdS3x7C7cjhzGllfGDOWM6ZVcnYIm9G2qaUGj6jKlxEZDXwQ8AB/MwY8+1u628ArgWiQB3wj8aY/X1tcySEC0BjsJHbXrmNvxz4C4vGLOL2s25nQn5mHw4W6zS8fqCJtW/XsnZrLQcbAwDMHV/IqpmVvHdmBQsnFeN06JhNSo02oyZcRMQB7ATeD9QA64GrjDFbk+qsAl41xvhF5J+AlcaYK/va7kgJF7AGpPzjnj/y7de+TawzxudO+xzXzLkGVxY8XMoYw65j7fxl2zGe236U1w80E+s0FHicnDmtjBXTylkxrZyp5T698kypUWA0hcty4FZjzHn28s0Axpj/6qX+QuBHxpgVfW13JIVL3OH2w3znte/w3MHnqC6q5qalN3Hm+DMz3awuWgIR/r67nhd31vHXXfUcarZ6NWMKPZxRXcayqWUsrS7llAoNG6VGotEULpcDq40x19rLnwDOMMZc10v9HwG1xpj/7GHdZ4HPAkyePHnR/v19HjnLWi/VvMR/vfpf1LTXsGzcMr54+hc5tfzUTDfrXYwxHGj087fd9azb28i6vQ3UtVnPESnzuVlcVcLiKaWcPqWYueOL9NEASo0AJ2W4iMjHgeuA9xpj+nwa0kjsuSQLxUI8suMRfrrppzSFmnjf5PfxmXmfYW753Ew3rVfGGPbWd7D+nUbW72tiw/5G9jf4AXA5hDnji1g4qZjTJhVx2sRiqsp8OiSNUllmNIVLSofFRORc4H+wguVYf9sd6eES1x5u5xdbf8Gvtv6Ktkgby8Yt49PzPs0ZY88YEYedjrUFeeNAM68faOKN/c1sPtRCIBIDoMDjZO6EQuZNKGLu+CLmji9kakU+Dg0cpTJmNIWLE+uE/vuAQ1gn9D9mjNmSVGch8DusHs6uVLY7WsIlrj3cziM7H+EXW35BQ7CBU4pO4apZV/GhUz40JM+KGSrRWCe769p566AVNJsPtbLtSCvhaCcAXlcOM8cUMHtcIbPHFTJrbAEzxxZQnOfOcMuVOjmMmnABEJELgB9gXYp8vzHmdhG5DdhgjFkjIs8C84Aj9lsOGGMu6muboy1c4kKxEH965088tO0htjVuw+fysbpqNZdOv5T55fNHRG+mu0isk93H2tlyuJWth62w2VbbSrP/+AOaxhR6mDGmgOmVBUwfk8/0ynymVeZr6CiVZqMqXIbCaA2XOGMMb9W9xe92/o4/7/8zgWiA6qJqLqi+gAuqL2By4eRMN/GEGGOobQ2yo7aNnUfb2F7bxq6j7ew+1p44rAbWhQOnVORzSqWPqeX5TK3wUV3uY1JpHi69D0epAdNw6cdoD5dkHZEO1u5byx93/5HXj70OwJyyObx/yvs5Z/I5TC2amuEWpk9np6GmKcDuujZ2H7PCZm9dB3vrO2jsCCfqOXKESSW5TCnzUVWWx+QyH1NK85hSlsek0jy9ck2pXmi49ONkCpdktR21rN23lrX71rK5fjMA1UXVvHfiezl74tksqFww5CMxZ0pTR5i99R3sq+9gb307+xr87G/oYH+9n7ZQtEvdygIPk0utoJlYksukEut1Qkku44pycTu116NOThou/ThZwyVZbUctzx14jucPPs+GoxuIdkbJd+WzZOwSlo9fzrJxy6gqrBqR52kGwhhDY0eY/Y1+DjT4OdDo52Cj9VrTFOBIS4DOpH8aIlb4TCjOZXxxLhOKcxlX5GVccS7ji3IZV+ylNM+tl1GrUUnDpR8aLl11RDpYd2Qdf635K+uOrONQ+yEAKnMrWTR2EUvGLmFR5SKqi6pHfdh0F4l1UtsS5GCTn0NNAWrs6UhLgMPNAQ63BBNXs8W5HTmMKfIwrjCXMUVexhZ6GFPoZUyhl7FFXsYUeKks9OjhNzXiaLj0Q8Old8YYatpqeOXIK2yo3cD6o+upD9QDUOQpYmHFQk6rPI155fM4tfxUfC5fhlucWcYYGjrCHGkOcqg5QG1LgCOtQY40B6ltDXK0NUhtS5BQtwACKPQ6qSz0UlngoaLAk3itKPBQke+lvMBNRb6HEu0JqSyh4dIPDZfUGWPY17qPN4+9yevHXufNY2+yr3UfAIIwtWgqc8rmMLd8LrNLZzOzdOZJHzjdGWNoDUSpbbUC51hrkGNtIY62BqlrC3GsLcSxNms+GHl3CDlyhFKfmzKfm4oCD2U+N2X5Hsry3ZT7PJT63JTmW+tLfW7yPc6TroephoeGSz80XE5MS6iFzfWb2Vy3mS0NW9jSsCXRuwGYXDCZmaUzmV4ynRklM5hRPIMJBRPIET0R3hdjDO2hKMfaQtS3hahvD1PXFqS+PUxDR4i6tjD17SEaO8I0tIfoCMd63I7bkUOJz0VJnpuyfDfFeW5K89yU+NyU5Lko9VllJXlWnaI8FwUaSCoFGi790HBJv2P+Y2xr2Mb2xu1sb9zOjqYd1LTVYLB+r7wOL9VF1ZxSfApTi6ZSXVRNdVE1kwom4XbozY6DEQjHaOiIh02Yho4wjR0hGjrCNHWEaeyI0NgRotkfodEfpiUQobd/5o4coTjXRXGei+I8N8W5LoryXBTluijOdVNszxfmOinKjc+7KPS69NzRSUTDpR8aLsPDH/Gzu3l3YtrTvIc9zXs46j+aqJMjOUzIn8CUwilMKZzC5ILJTC6czKSCSYz3jc+K59aMFrFOQ0sgQmNHmGZ/mCZ/hGZ/mGZ/hObA8eWWQMQq80doCURo73apdnceZ04ibIpyXRR4nRR67Vc7gAq8zqRlJwWJMhd5LoeeUxohBhMuzqFqjDp55bnymF8xn/kV87uUd0Q62Neyj70te9nfuj8xvX70dfxRf6JejuQwJm8ME/InWFOB9TreN57x+eOpzKvEmaO/uqmKn7sp9Q2spxiJddIasIImeYqXtQajtAYitAYjifDaV99BSyBCWzBKtLPv/7iKQL7HCqR8j5N8O4jyPcdffZ7jyz57uSCpPF5H70HKPtpzURlnjKEh2MCB1gMcbDtITXsNB9sOcqjtEIfbD3Ms0HXA6xzJoTKvknG+cYzNG8tY31jG+MYwNm8slXmVVOZVUpZbpgGUQcYYgpFOWoMR2oIRWgJR2oJWb6gtaM8Ho7QGreX20PF17aEo7XZ58rA+fXE5xAoftxOfx5EInzy3wy5zkudxkO92kudx4nM7jr+67XoeB7luJ3kuB3keB25Hjp6PsulhsX5ouIxMoViI2o5aDrVbYXOk4wi1HbUcbj/MUf9RjnYcJdwZ7vKeHMmh3FtORV6FNeVaU3leufWaW055bjll3jI9BJfFYp3WxQ7toSgd9mtb8Ph8R6I8Zs2H42UxOsJR/KEY7aEo/nCUjnDsXfcm9cWRI+S5HfbkJNdlzecml7kdVhi5HXjt+Vy3A6/r+Hty3Y6ury4HXnfOiAovPSymRiWPw5M4N9MTYwyNwUaO+Y9xzH/MChz/Uer8dRzzH+Nw+2E21W2iMdjY4/sL3YWU5ZZR5i2j1FtqTbmllHqs1xJPCaXeUkq8JRR5ivTqt2HkyJHEhQTpEIl14g/F8EfsAApF8YdjBCJWQAXC1nK83JrsOvZyWzDKsdYQ/kiUQLw8Euv1oone5AhW0NhTPHy8rhxrObHu+LLH1bWO15WD12nV87hy8Di7r3fgcVrzw/1MJA0XNeKJiBUOuWXMLpvda71ILEJDsIH6QD11/joagg00BKzlhmADjcFGdjbtpCHYQFu4rcdt5EgORe4iir3FFHu6TkWeosRrYnIXUegpxOvwjpj/pY5mLkcORXk5FJHe3qoxhlC0MxE0idAJRwna5YFIlEC4k0DECrFgpJNgxKofTEyd+MNW78y6/8kqC0at7fV0U26qfvSxhXxw/vg0fuq+abiok4bL4WKszzpH059IZ4TmYDONwUYag400BZtoCjXRFGyiOWSVt4RaqGmvYUv9FppDze86NJfMneOmwF1AoaeQQrc9eQopcB0vK3AXdJ1cBeS78ylwFeihuywnIomeQskQ7qez0wqxYCRGMGoFjxU6x0MoFOm0l2OJAAtGOpk5pmAIW/ZuGi5K9cCV40qcr0mFMYZANEBLqIWWcAstoRaaQ820hltpDbXSEm6hNdSaWK4P1LO3ZS+t4Vbaw+2J+4J643V48bl8FLgLyHfl43P7yHflW5M7H5/Lh89lleW58vA5feS7j8/H1+c6c7UHNYLl5Ih1+Myd/fcYabgolQYiQp4rjzxXHuMYN6D3dppO/BE/reFW2sJtiak90t5lvj3STnu4nbZIG/6In/2B/bRH2umIdNAR6aDT9H/IRLDb6cxLhE2ey5rPc+YllpPnc525ieUukys3sc7j8GhoqS40XJTKsBzJId9t9UAGK95z8kf9tIePB048fPwRP/6oP1EeiAas8qgff8RPnb8u8X5/xE8gGiBmUrsMGKzQ8jq9ieDxOqx5r9ObKE8u8zg85Dpz8Tg8ifXxea/Di8fpsV7tsuRXvaBiZNBwUWoUSO45leeWn/D2jDFEOiOJIApEA4nQSZ78UT/BaDCxHJ8PxoL4o34CkQBNwSaORI8kyoPRIMFYMKWeVk/cOW48Tg8eR9fJ6/TidrjxOrq+xtfH53srczvc1raTlhPzOdayK8elPbQUabgopd5FRBJ/YEuG4BS1MYZoZ5RAzAqkUDREIBYgFA11CaBgNEgoFiIUCxGMBgnHwgRjwcRyfF0oFiIUtabWUGuiLF4/HAsTioXS0nZXjisRPPH5RPg4XMfLc6zl7uHkcri6rs9xJ7bjzHEer2eXxefj70uum7zOKdk1CKmGi1Jq2ImI9cfS4aLQXTgs+4z3xuJhEw+cLvOdXcsjnZEudeLruy9HOiNd3heMBhNl8XqRWOR4WR9XFp6IeNgkgicpkG5cciMrJqwYkv32RMNFKXVSSO6NZZoxhqiJJgInHjrJr/H5aGfUWu5WN14e7kyaT67fGeny3uF+3pKGi1JKDTMRwSVWj2K00ssulFJKpZ2Gi1JKqbTTcFFKKZV2Gi5KKaXSTsNFKaVU2mm4KKWUSjsNF6WUUmmn4aKUUirtxAz02ZwjmIjUAfsH+fZyoD6NzRkJ9DOfHPQznxxO5DNPMcak9nAj20kVLidCRDYYYxZnuh3DST/zyUE/88lhuD+zHhZTSimVdhouSiml0k7DJXX3ZroBGaCf+eSgn/nkMKyfWc+5KKWUSjvtuSillEo7DRellFJpp+GSAhFZLSI7RGS3iNyU6fb0R0QmicjzIrJVRLaIyBft8lIReUZEdtmvJXa5iMhd9ufbJCKnJ23rGrv+LhG5Jql8kYhstt9zl9gP7+5tH8P42R0i8oaIPGkvV4vIq3Y7fysibrvcYy/vttdXJW3jZrt8h4icl1Te4+9Bb/sYps9bLCK/E5HtIrJNRJaP9u9ZRL5s/16/LSIPi4h3tH3PInK/iBwTkbeTyjL2vfa1j14ZY3TqYwIcwB5gKuAG3gLmZLpd/bR5HHC6PV8A7ATmAN8FbrLLbwK+Y89fAPwJEGAZ8KpdXgrstV9L7PkSe91rdl2x33u+Xd7jPobxs98APAQ8aS8/AnzUnr8H+Cd7/p+Be+z5jwK/tefn2N+xB6i2v3tHX78Hve1jmD7vg8C19rwbKB7N3zMwAXgHyE362f/DaPuegbOB04G3k8oy9r32to8+P8Nw/SMYqROwHFibtHwzcHOm2zXAz/BH4P3ADmCcXTYO2GHP/wS4Kqn+Dnv9VcBPksp/YpeNA7YnlSfq9baPYfqcE4G/AOcAT9r/EOoBZ/fvElgLLLfnnXY96f79xuv19nvQ1z6G4fMWYf2hlW7lo/Z7xgqXg/YfTKf9PZ83Gr9noIqu4ZKx77W3ffTVfj0s1r/4L3NcjV02ItiHARYCrwJjjDFH7FW1wBh7vrfP2Fd5TQ/l9LGP4fAD4Eag014uA5qNMVF7Obmdic9mr2+x6w/0Z9HXPoZaNVAH/FysQ4E/ExEfo/h7NsYcAu4EDgBHsL63jYzu7zkuk9/rgP8OariMYiKSDzwGfMkY05q8zlj//RjS69CHYx9xIvJB4JgxZuNw7C9LOLEOnfzYGLMQ6MA6lJEwCr/nEuBirGAdD/iA1cOx72wyEr5XDZf+HQImJS1PtMuymoi4sILl18aYx+3ioyIyzl4/Djhml/f2Gfsqn9hDeV/7GGorgItEZB/wG6xDYz8EikXE2UM7E5/NXl8ENDDwn0VDH/sYajVAjTHmVXv5d1hhM5q/53OBd4wxdcaYCPA41nc/mr/nuEx+rwP+O6jh0r/1wHT7ShE31knBNRluU5/sKz/uA7YZY/47adUaIH7FyDVY52Li5Z+0rwhZBrTYXeO1wAdEpMT+H+MHsI4zHwFaRWSZva9PdttWT/sYUsaYm40xE40xVVjf0XPGmKuB54HLe2hPcjsvt+sbu/yj9lVG1cB0rJOfPf4e2O/pbR9DyhhTCxwUkZl20fuArYzi7xnrcNgyEcmz2xT/zKP2e06Sye+1t330bihPSI2WCetKiZ1YV5H8v0y3J4X2noXVnd0EvGlPF2AdN/4LsAt4Fii16wtwt/35NgOLk7b1j8Bue/pUUvli4G37PT/i+GgPPe5jmD//So5fLTYV64/GbuBRwGOXe+3l3fb6qUnv/3/259qBfRVNX78Hve1jmD7rAmCD/V3/AeuqoFH9PQPfALbb7fol1hVfo+p7Bh7GOqcUweqhfjqT32tf++ht0uFflFJKpZ0eFlNKKZV2Gi5KKaXSTsNFKaVU2mm4KKWUSjsNF6WUUmmn4aLUAIjI3+3XKhH5WJq3/e897UupkUgvRVZqEERkJfAVY8wHB/Aepzk+NlVP69uNMfnpaJ9SmaY9F6UGQETa7dlvA+8RkTfFer6IQ0TuEJH19vMuPmfXXykifxWRNVh3kiMifxCRjWI9k+Szdtm3gVx7e79O3pd9V/QdYj2/ZLOIXJm07Rfk+PNcfm3fcY2IfFus5/lsEpE7h/NnpBRYA98ppQbuJpJ6LnZItBhjloiIB3hZRP5s1z0dONUY8469/I/GmEYRyQXWi8hjxpibROQ6Y8yCHvZ1Gdad+KcB5fZ7XrLXLQTmAoeBl4EVIrINuBSYZYwxIlKc9k+vVD+056JUenwAa+ylN7Eeb1CGNV4VwGtJwQLwLyLyFrAOazDA6fTtLOBhY0zMGHMUeBFYkrTtGmNMJ9YwP1VYw8oHgftE5DLAf8KfTqkB0nBRKj0EuN4Ys8Ceqo0x8Z5LR6KSda7mXKwHWJ0GvIE1/tVghZLmY1gPs4oCS7FGSf4g8H8nsH2lBkXDRanBacN6hHTcWuCfxHrUASIyQ6wHd3VXBDQZY/wiMgvrkbFxkfj7u/krcKV9XqcC6xG4r/XWMLGe41NkjHka+DLW4TSlhpWec1FqcDYBMfvw1gNYz46pAl63T6rXAZf08L7/Az5vnxfZgXVoLO5eYJOIvG6sxwXE/R7rsbpvYY12faMxptYOp54UAH8UES9Wj+qGwX1EpQZPL0VWSimVdnpYTCmlVNppuCillEo7DRellFJpp+GilFIq7TRclFJKpZ2Gi1JKqbTTcFFKKZV2/x+norCsG6NDCAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"Regret\")\n",
    "plt.plot(np.linspace(1, 1000001, 100), adashift_avg_regret_history, label='adashift')\n",
    "plt.plot(np.linspace(1, 1000001, 100), avg_regret_history,label='adam')\n",
    "plt.plot(np.linspace(1, 1000001, 100), amsgrad_avg_regret_history,label='amsgrad')\n",
    "plt.xlabel(\"iterations\")\n",
    "plt.ylabel(r\"$\\dfrac{R_t}{t}$\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAEWCAYAAAC0Q+rDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xu8VWW97/HPd11ggSD3oygmWKRIKMgSUciNiIpaoGaB2Y7aeji7fdTKU4bZC8lyb1PblSdPSnjbVt51i4mRN2xnXliUoeAFvOxchLpEIRGR2+/8McaiyWJd5prMseZai+/79ZqvOcYznjGe35hjMX+My3weRQRmZmZZKCt1AGZm1nk5yZiZWWacZMzMLDNOMmZmlhknGTMzy4yTjJmZZcZJxqyNSfqIpPWSyjPY9hxJvyj2ds0K5SRj1gJJX5L0rKQNkt6Q9DNJvVux/muSJtXPR8RfIqJHRGzNJmKz9sNJxqwZkv4P8APgm0AvYCywP/CgpC6ljM2sI3CSMWuCpD2B7wLnRsRvImJzRLwGfA4YDHwhrTdH0p2SbpP0nqQ/Sjo0XXYz8BHgvvQS2QWSBksKSRVpnUWSvi/pD2md+yT1k/RLSX+TtFjS4Jy4fiLp9XTZEkmfzHN/viXpqZx2vyJpmaSqIn1kZjtxkjFr2lFAFXB3bmFErAcWAMflFE8F7gD6Ar8C/lNSZUT8I/AX4NPpJbLLm2hrOvCPwL7AR4EngBvS7T0PXJxTdzEwMqetO/JMFFcAHwLfkTQU+FfgCxGxMY91zQriJGPWtP7A2xGxpZFlq9Pl9ZZExJ0RsRn4d5LkNLYVbd0QES9HxDrgAeDliHgobfsOYFR9xYj4RUSsiYgtEfFDoCtwYEsNRMQ24IvAecB84PKI+FMrYjRrNScZs6a9DfSvv7zUwMB0eb3X6yfSL/NaYJ9WtPVmzvQHjcz3qJ+R9A1Jz0taJ2ktyb2i3ITXpPRy36Mkl/uubkV8ZgVxkjFr2hMkl5dOyy2U1AM4EXg4p3i/nOVlwCDgr2lR0bo6T++/XEByX6hPRPQG1gHKc/2TgSNJYr+iWHGZNcVJxqwJ6aWr7wL/V9JkSZXpDfjbSc5Ubs6pPlrSaelZz9dIktOT6bI3gQOKFFZPYAtQB1RImg3smc+KkvoD84CzgRnApyWdVKS4zBrlJGPWjPRG/beBK4G/AU+RXBo7NiI+zKl6LzANeJfkBv5p6f0ZgH8judm+VtI3djGkhcBvgJeA/wY2knOprgVzgXsjYkFErAHOAuZJ6reLMZk1SR60zGzXSJoDfCwivlDqWMzaG5/JmJlZZpxkzMwsM75cZmZmmfGZjJmZZaaxH5l1ev3794/BgweXOgwzsw5jyZIlb0fEgNaut1smmcGDB1NTU1PqMMzMOgxJ/13Ier5cZmZmmXGSMTOzzDjJmJlZZpxkzMwsM04yZmaWmXaRZCRdL+ktSc81sVySrpK0UtJSSYflLJshaUX6mtF2UZuZWUvaRZIBbgQmN7P8RGBo+poJ/AxAUl+SYWmPAMYAF0vqk2mkZmaWt3bxO5mI+F06TkdTpgL/EUkfOE9K6i1pIDABeDAi3gGQ9CBJsrolizj/sLKOo37xsSw2bWbWNma/A2XlbdZcezmTacm+7DhmRm1a1lT5TiTNlFQjqaaurq6gIGpu3NWhQMzMSqwNEwx0nCSzyyJibkRUR0T1gAGt7hkBgL15t8hRmZl1bu3iclkeVpEzhjrJ+Omr0teEBuWLsgqiqza3XKkz694f9hgA3XpD935Q1RuqekH3Psl0157QdU/osgd07QFdekJlFVR2h4quoDJQefqu5B0l0/WU11D1ZtZBdJQkMx84R9KtJDf510XEakkLgX/Nudl/PHBhVkGcVPZUcTc4+JNw+Fnw0WOTL2h/wZpZJ9MukoykW0jOSPpLqiV5YqwSICKuARYAJwErgQ3Al9Nl70j6HrA43dQl9Q8BZCHvkXc+fRUc9kUnDTPb7bWLJBMRZ7SwPID/3cSy64Hrs4irobX05H+wducFR50Lx3+/LUIwM+tQ2kWS6Sgq2bJz4Zx1bR+ImVkHsds8XVYMOyUZJxgzs2Y5ybTCDknm68tKF4iZWQfhJNMKXZWTZHoNKl0gZmYdhJOMmZllxkmmFd4I971pZtYaTjKt8Oy2IcnEXp8obSBmZh2Ek0wrVLA1mSjvUtpAzMw6CCeZVtieZMr88yIzs3w4ybTCJ8vTgTvfe6O0gZiZdRBOMoVY95dSR2Bm1iE4yZiZWWacZMzMLDNOMmZmlhknGTMzy4yTjJmZZaZdJBlJkyW9KGmlpFmNLP+RpGfS10uS1uYs25qzbH7bBNwuPjYzs3av5L8qlFQOXA0cB9QCiyXNj4jl9XUi4us59c8FRuVs4oOIGNlW8QLw0Ylt2pyZWUfVHv5LPgZYGRGvRMQm4FZgajP1zwBuaZPIGrh6y5Rk4qhzS9G8mVmH0x6SzL7A6znztWnZTiTtDwwBHskprpJUI+lJSac01YikmWm9mrq6uoICXbEtHUOm134FrW9mtrtpD0mmNaYDd0bE1pyy/SOiGvg88GNJH21sxYiYGxHVEVE9YMCAtojVzGy31x6SzCog99RgUFrWmOk0uFQWEavS91eARex4v6aoqrQpq02bmXVK7SHJLAaGShoiqQtJItnpKTFJBwF9gCdyyvpI6ppO9wfGAcsbrlssl1XOSybeW51VE2ZmnUrJny6LiC2SzgEWAuXA9RGxTNIlQE1E1Cec6cCtERE5qw8DrpW0jSRhXpb7VFpmNr2feRNmZp1ByZMMQEQsABY0KJvdYH5OI+v9ARiRaXCNUXmbN2lm1hG1h8tlHU+ZPzYzs3z427IQ/sW/mVle/G1ZCCcZM7O8+NuyEO8X9mNOM7PdjZNMIT5cX+oIzMw6BCeZQpT56TIzs3w4yRSirF08+W1m1u45yRTCv5MxM8uLk0whfLnMzCwvTjKFcJIxM8uLk0whfLnMzCwvTjKF6DO41BGYmXUITjKF6N6v1BGYmXUITjKF8D0ZM7O8OMkUwvdkzMzy4iRTCHf1b2aWl3bxbSlpsqQXJa2UNKuR5V+SVCfpmfR1ds6yGZJWpK8ZbROwz2TMzPJR8v5RJJUDVwPHAbXAYknzGxlG+baIOKfBun2Bi4FqIIAl6brvZht0u8jNZmbtXnv4thwDrIyIVyJiE3ArMDXPdU8AHoyId9LE8iAwOaM4/843/s3M8tIeksy+wOs587VpWUOfkbRU0p2S9mvlukiaKalGUk1d3S6OB+MzGTOzvHSUb8v7gMERcQjJ2cpNrd1ARMyNiOqIqB4wYMCuReMkY2aWl/bwbbkK2C9nflBatl1ErImID9PZecDofNfNhJOMmVle2sO35WJgqKQhkroA04H5uRUkDcyZnQI8n04vBI6X1EdSH+D4tCxjyr4JM7NOoORPl0XEFknnkCSHcuD6iFgm6RKgJiLmA+dJmgJsAd4BvpSu+46k75EkKoBLIuKdzIOWk4yZWT4UEaWOoc1VV1dHTU1N61ec0yt9X1fcgMzM2jlJSyKiurXrtYfLZWZm1kk5yZiZWWacZMzMLDNOMmZmlhknGTMzy4yTjJmZZcZJxszMMuMk0wort+1T6hDMzDoUJ5lWeHrbgbwZvUsdhplZh+Ek0wruTMbMrHWcZFopnGrMzPLmJNMKB5StZm9lO7KzmVln4iTTCkeUvVDqEMzMOhQnGTMzy4yTjJmZZaZdJBlJkyW9KGmlpFmNLD9f0nJJSyU9LGn/nGVbJT2TvuY3XNfMzEqn5CNjSioHrgaOA2qBxZLmR8TynGp/AqojYoOkrwCXA9PSZR9ExMg2DdrMzPLSHs5kxgArI+KViNgE3ApMza0QEY9GxIZ09klgUBvHaGZmBWgPSWZf4PWc+dq0rClnAQ/kzFdJqpH0pKRTmlpJ0sy0Xk1dXd2uRWxmZnkp+eWy1pD0BaAa+Iec4v0jYpWkA4BHJD0bES83XDci5gJzAaqrq6NNAjazkti8eTO1tbVs3Lix1KF0OFVVVQwaNIjKysqibK89JJlVwH4584PSsh1ImgRcBPxDRHxYXx4Rq9L3VyQtAkYBOyUZM9t91NbW0rNnTwYPHozkXjryFRGsWbOG2tpahgwZUpRttofLZYuBoZKGSOoCTAd2eEpM0ijgWmBKRLyVU95HUtd0uj8wDsh9YMDMdkMbN26kX79+TjCtJIl+/foV9Qyw5GcyEbFF0jnAQqAcuD4ilkm6BKiJiPnAFUAP4I70j+YvETEFGAZcK2kbScK8rMFTaWa2m3KCKUyxP7eSJxmAiFgALGhQNjtnelIT6/0BGJFtdGZmVqj2cLnMzGy3dOONN3LOOee0er1FixbxqU99qtFlZ599NsuXJxd07rjjDoYNG8YxxxzDM888w4IFCxpdJ0tOMmZmnci8efM4+OCDAbjuuuv4+c9/zqOPPuokY2bW2ZxyyimMHj2a4cOHM3fuXABuuOEGPv7xjzNmzBgef/zx7XXvu+8+jjjiCEaNGsWkSZN48803AXjssccYOXIkI0eOZNSoUbz33nsArF+/ntNPP52DDjqIM888k4jklxkTJkygpqaGSy65hN///vecddZZfP3rX2f27NncdtttjBw5kttuu63NPoN2cU/GzCwr371vGcv/+reibvPgffbk4k8Pb7He9ddfT9++ffnggw84/PDDOfnkk7n44otZsmQJvXr14phjjmHUqFEAjB8/nieffBJJzJs3j8svv5wf/vCHXHnllVx99dWMGzeO9evXU1VVBcCf/vQnli1bxj777MO4ceN4/PHHGT9+/Pa2Z8+ezSOPPMKVV15JdXU1hx56KDU1Nfz0pz8t6mfREp/JmJll5KqrruLQQw9l7NixvP7669x8881MmDCBAQMG0KVLF6ZNm7a9bm1tLSeccAIjRozgiiuuYNmyZQCMGzeO888/n6uuuoq1a9dSUZGcG4wZM4ZBgwZRVlbGyJEjee2110qxiy3ymYyZdWr5nHFkYdGiRTz00EM88cQTdO/enQkTJnDQQQdtvynf0Lnnnsv555/PlClTWLRoEXPmzAFg1qxZnHzyySxYsIBx48axcOFCALp27bp93fLycrZs2ZL5PhXCScbMLAPr1q2jT58+dO/enRdeeIEnn3ySDz74gMcee4w1a9aw5557cscdd3DooYdur7/vvkm3jTfddNP27bz88suMGDGCESNGsHjxYl544QV69+7d6nh69uy5/X5OW/LlMjOzDEyePJktW7YwbNgwZs2axdixYxk4cCBz5szhyCOPZNy4cQwbNmx7/Tlz5vDZz36W0aNH079//+3lP/7xj/nEJz7BIYccQmVlJSeeeGJB8RxzzDEsX768zW/8q/6JhN1JdXV11NTUtH7FOb3S93XFDcjMiur555/f4QvcWqexz0/Skoiobu22fCZjZmaZcZIxM7PMOMmYmVlmnGTMzCwzTjJmZpYZJxkzM8uMk4yZWYkU2tV/R9KqJCPp/+RMH1isICRNlvSipJWSZjWyvKuk29LlT0kanLPswrT8RUknFCsmMzPbdXklGUm9Jd0AfEbSv0gaD+yUDAohqRy4GjgROBg4Q9LBDaqdBbwbER8DfgT8IF33YGA6MByYDPy/dHtmZiVXjK7+58yZw4wZM/jkJz/J/vvvz913380FF1zAiBEjmDx5Mps3by7JvuUrr77LImIt8OX0TOFt4BDg7iLFMAZYGRGvAEi6FZgK5PYiNxWYk07fCfxUyUDUU4FbI+JD4FVJK9PtPVGk2HZwcf++DP9wE5/LYuNmlo0HZsEbzxZ3m3uPgBMva7FaMbr6h6T/skcffZTly5dz5JFHctddd3H55Zdz6qmncv/993PKKacUd/+KqMUkI+knwNcisTAtXlLEGPYFXs+ZrwWOaKpORGyRtA7ol5Y/2WDdfRtrRNJMYCbARz7ykYICfaxbN8p3w254zKwwV111Fffccw/ATl39A0ybNo2XXnoJSLr6nzZtGqtXr2bTpk0MGTJk+3ZOPPFEKisrGTFiBFu3bmXy5MkAjBgxot128V8vnzOZ94D5kqZHxPvp2czsiBiXcWxFFRFzgbmQ9F1W4nDMrK3kccaRhWJ19Q9/79a/rKyMyspKkgs5yXx77eK/Xov3ZCLiO8AtwCJJjwPnU6T7MalVwH4584PSskbrSKoAegFr8lzXzKzNtdTV/+bNm7njjjt2qN9YV/8dXYtJRtKxwP8E3gf6A+dFxH8VMYbFwFBJQyR1IbmRP79BnfnAjHT6dOCRSLqPng9MT58+GwIMBZ4uYmxmZgUpVlf/HV2LXf1LeoTk8tjvJY0AbgbOj4hHihaEdBLwY6AcuD4iLpV0CVATEfMlVaXtjgLeAabnPChwEfBPwBaSe0cPtNReoV39T7juYCZu2MDsc19r9bpm1nbc1f+uKWZX/y3ek4mIiTnTz0o6EbgLOKq1jTXTxgJgQYOy2TnTG4HPNrHupcClxYrFzMyKp9W/+I+I1cCxGcRiZmadTEHdykTEB8UOxMzMOh/3XWZmZplxkjEzs8w4yZiZWWacZMzMdlODBw/m7bffzrQNJxkzs06kvXUz4yRjZpaRxrr679GjB9/85jcZPnw4kyZN4umnn2bChAkccMABzJ+fdHaybNkyxowZw8iRIznkkENYsWIFAN/73vc48MADGT9+PGeccQZXXnklABMmTOBrX/sa1dXV/OQnP2ly2IA1a9Zw/PHHM3z4cM4++2xa+jF+MeTV1b+ZWUf1g6d/wAvvvFDUbR7U9yC+NeZbLdZr2NX/Zz7zGd5//30mTpzIFVdcwamnnsp3vvMdHnzwQZYvX86MGTOYMmUK11xzDV/96lc588wz2bRpE1u3bmXx4sXcdddd/PnPf2bz5s0cdthhjB49entbmzZtor4nk3fffbfRYQO++93vMn78eGbPns3999/PddddV9TPpTFOMmZmGWnY1f+KFSvo0qXLDl31d+3adXs3/vXd9h955JFceuml1NbWctpppzF06FAef/xxpk6dSlVVFVVVVXz605/eoa1p06Ztn25q2IDf/e533H13MhTYySefTJ8+fbL+CJxkzKxzy+eMIwuNdfW/cePGnbrqz+3Gv/5+yuc//3mOOOII7r//fk466SSuvfbaFtvbY489tk83N2xAW/M9GTOzDDTW1X++XnnlFQ444ADOO+88pk6dytKlSxk3bhz33XcfGzduZP369fz6179utu3Ghg04+uij+dWvfgXAAw88wLvvvlvg3uXPZzJmZhmYPHky11xzDcOGDePAAw9k7Nixea97++23c/PNN1NZWcnee+/Nt7/9bfr27cuUKVM45JBD2GuvvRgxYgS9evVqdP36YQP69OnDxIkTefXVVwG4+OKLOeOMMxg+fDhHHXVUwaMEt0aLXf13Ru7q36xz66xd/a9fv54ePXqwYcMGjj76aObOncthhx1W9HbatKt/MzNrH2bOnMny5cvZuHEjM2bMyCTBFJuTjJlZB1F/P6UjKemNf0l9JT0oaUX6vtPzdJJGSnpC0jJJSyVNy1l2o6RXJT2Tvka27R6YWXu1O94KKIZif26lfrpsFvBwRAwFHk7nG9oAfDEihgOTgR9L6p2z/JsRMTJ9PZN9yGbW3lVVVbFmzRonmlaKCNasWUNVVVXRtlnqy2VTgQnp9E3AImCHh9oj4qWc6b9KegsYAKxtmxDNrKMZNGgQtbW11NXVlTqUDqeqqopBgwYVbXulTjJ7pcM5A7wB7NVcZUljgC7AyznFl0qaTXomFBEfNrHuTGAm0CaP7ZlZ6VRWVm7/lbuVVuaXyyQ9JOm5Rl5Tc+tFcl7b5LmtpIHAzcCXI2JbWnwhcBBwONCXBmdBDbY/NyKqI6J6wIABu7pbZmaWh8zPZCJiUlPLJL0paWBErE6TyFtN1NsTuB+4KCK2/2w25yzoQ0k3AN8oYuhmZraLSn3jfz4wI52eAdzbsIKkLsA9wH9ExJ0Nlg1M3wWcAjyXabRmZtYqpU4ylwHHSVoBTErnkVQtaV5a53PA0cCXGnlU+ZeSngWeBfoD32/b8M3MrDklvfEfEWuAYxsprwHOTqd/AfyiifUnZhqgmZntklKfyZiZWSfmJGNmZplxkjEzs8w4yZiZWWacZMzMLDNOMmZmlhknGTMzy4yTjJmZZcZJxszMMuMkY2ZmmXGSMTOzzDjJmJlZZpxkzMwsM04yZmaWGScZMzPLTEmTjKS+kh6UtCJ979NEva05A5bNzykfIukpSSsl3ZaOomlmZu1Eqc9kZgEPR8RQ4OF0vjEfRMTI9DUlp/wHwI8i4mPAu8BZ2YZrZmatUeokMxW4KZ2+CTgl3xUlCZgI3FnI+mZmlr1SJ5m9ImJ1Ov0GsFcT9aok1Uh6UlJ9IukHrI2ILel8LbBvUw1Jmpluo6aurq7ggKPgNc3Mdj8VWTcg6SFg70YWXZQ7ExEhqanv8P0jYpWkA4BHJD0LrGtNHBExF5gLUF1dXVCuUCErmZntxjJPMhExqallkt6UNDAiVksaCLzVxDZWpe+vSFoEjALuAnpLqkjPZgYBq4q+A2ZmVrBSXy6bD8xIp2cA9zasIKmPpK7pdH9gHLA8IgJ4FDi9ufWL6e2Kch7ao3uWTZiZdSqlTjKXAcdJWgFMSueRVC1pXlpnGFAj6c8kSeWyiFieLvsWcL6klST3aK7LOuC15eVZN2Fm1mlkfrmsORGxBji2kfIa4Ox0+g/AiCbWfwUYk2WMZmZWuFKfyZiZWSfmJGNmZplxkjEzs8w4yZiZWWacZMzMLDNOMmZmlhknGTMzy4yTjJmZZcZJpgAbt2wsdQhmZh2Ck0wB6j4ofKgAM7PdiZNMAX5X+7tSh2Bm1iE4yRTgzpfubLmSmZk5yRRi5dqVpQ7BzKxDcJIxM7PMOMmYmVlmnGTMzCwzJU0ykvpKelDSivS9TyN1jpH0TM5ro6RT0mU3Sno1Z9nItoo9Gf3ZzMyaU+ozmVnAwxExFHg4nd9BRDwaESMjYiQwEdgA/Danyjfrl0fEM20SNbD4jcVt1ZSZWYdV6iQzFbgpnb4JOKWF+qcDD0TEhkyjysNZvz2r1CGYmbV7pU4ye0XE6nT6DWCvFupPB25pUHappKWSfiSpa1MrSpopqUZSTV2df7FvZtYWMk8ykh6S9Fwjr6m59SK5ydHkjQ5JA4ERwMKc4guBg4DDgb7At5paPyLmRkR1RFQPGDBgV3bJzMzyVJF1AxExqallkt6UNDAiVqdJ5K1mNvU54J6I2Jyz7fqzoA8l3QB8oyhB56nmjRqq965uyybNzDqUUl8umw/MSKdnAPc2U/cMGlwqSxMTkkRyP+e5DGJs0pcXfrktmzMz63BKnWQuA46TtAKYlM4jqVrSvPpKkgYD+wGPNVj/l5KeBZ4F+gPfb4OYzcwsT5lfLmtORKwBjm2kvAY4O2f+NWDfRupNzDK+fPiSmZlZ00p9JtPh+ZKZmVnTnGSKwL/+NzNrnJNMERzyH4eUOgQzs3bJSaYA61+6qNQhmJl1CE4yBYitPXcqm/7r6SWIxMysfXOSKdCNJ9y4w/yyNctKE4iZWTvmJFOgB2q67VQ2+ubRJYjEzKz9cpIp0Lzfv8o9U+7ZoWzTtk0lisbMrH1yktkFH+l5wE5lI24aUYJIzMzaJyeZXfDx7zzAH7/wx53K33j/jRJEY2bW/jjJ7KKhF/12p7Lj7jyuBJGYmbU/TjJF8N7zl+1U5stmZmZOMkXz/ivn7VS25oM1JYjEzKz9cJIpkm0f7rNT2YTbJzB41v2s3eCnzsxs91TSrv47m/eev4yew2btUNZz2CxGXtL8epOH780Fkw9kSP89SMZfMzPrHJxkiuy9F2fT88Ads0p94tn6wSA+fOtEtm4YQu5J5G+WvcFvlhX3ibRuleUctn9vDh3Um0/s24sD9+7JoD7d6FJe5kRmZm2mpElG0meBOcAwYEw6WFlj9SYDPwHKgXkRUT+C5hDgVqAfsAT4x4go7bWpbd3ZvHY0lb2X7LSovFst3ff/eZuFshRY+ibEqu5s29yH2NKTbVt6EVv2JDbvybYtPYmtPYit3WBbV2JbF4hykgToRFSvokx0qyyne9dy9uhSQdfKcnp0Ladblwq6VpSxR5dyunUpp2tF8t6tspyqyjKqKsupqiyna0Uy3aWijG7pfGV5GV0qyuhSXkZFuagsL6O8TFSUibL6dwkJypROA7n/P/B/FqwjKPWZzHPAacC1TVWQVA5cDRwH1AKLJc2PiOXAD4AfRcStkq4BzgJ+ln3Yzdu4+rONJplSUcUGyis2lDqMDm9D+tpBAB+mL7N2bsv7H2PpP99OZVllm7VZ0hv/EfF8RLzYQrUxwMqIeCU9S7kVmKrkv3ETgTvTejcBp2QXbes09lizmVkpVeyxErXxVYqO8HTZvsDrOfO1aVk/YG1EbGlQ3ihJMyXVSKqpq6srKJAzK45k0Gufyrv+e8//W0HtmJllpaKsbS9gZd6apIeAvRtZdFFE3Jt1+/UiYi4wF6C6urqg8ZJnnTmXWS1XayD/pGRm1tlknmQiYtIubmIVsF/O/KC0bA3QW1JFejZTX25mZu1ER7hcthgYKmmIpC7AdGB+RATwKHB6Wm8G0GZnRmZm1rKSJhlJp0qqBY4E7pe0MC3fR9ICgPQs5RxgIfA8cHtE1A9D+S3gfEkrSe7RXNfW+2BmZk1TckKwe6muro6amkZ/kmNmZo2QtCQiqlu7Xke4XGZmZh2Uk4yZmWXGScbMzDLjJGNmZpnZLW/8S6oD/rvA1fsDbxcxnI7A+9z57W77C97n1to/Iga0dqXdMsnsCkk1hTxh0ZF5nzu/3W1/wfvcVny5zMzMMuMkY2ZmmXGSab25pQ6gBLzPnd/utr/gfW4TvidjZmaZ8ZmMmZllxknGzMwy4ySTJ0mTJb0oaaWk1o9dVgKS9pP0qKTlkpZJ+mpa3lfSg5JWpO990nJJuirdx6WSDsvZ1oy0/gpJM3LKR0t6Nl3nqnRY7CbbaKP9Lpf0J0m/TueHSHoqjfG2dMgIJHVN51emywfnbOPCtPxFSSfklDf6d9BUG220v70l3SnpBUnPSzpyNzjBlnWqAAAGNElEQVTGX0//pp+TdIukqs52nCVdL+ktSc/llJXsuDbXRrMiwq8WXkA58DJwANAF+DNwcKnjyiPugcBh6XRP4CXgYOByYFZaPgv4QTp9EvAAIGAs8FRa3hd4JX3vk073SZc9ndZVuu6JaXmjbbTRfp8P/Ar4dTp/OzA9nb4G+Eo6/S/ANen0dOC2dPrg9Bh3BYakx768ub+Dptpoo/29CTg7ne4C9O7Mx5hkmPVXgW45n/2XOttxBo4GDgOeyykr2XFtqo0W96Ot/iF05BfJeDcLc+YvBC4sdVwF7Me9wHHAi8DAtGwg8GI6fS1wRk79F9PlZwDX5pRfm5YNBF7IKd9er6k22mAfBwEPAxOBX6f/IN4GKhoeS5Ixio5MpyvSemp4fOvrNfV30FwbbbC/vUi+cNWgvDMf432B19Mvzor0OJ/QGY8zMJgdk0zJjmtTbbS0D75clp/6P+p6tWlZh5FeIhgFPAXsFRGr00VvAHul003tZ3PltY2U00wbWfsxcAGwLZ3vB6yNZPC7hjFu3690+bq0fms/h+bayNoQoA64QcklwnmS9qATH+OIWAVcCfwFWE1y3JbQuY9zvVIe14K+B51kdgOSegB3AV+LiL/lLovkvySZPsfeFm0ASPoU8FZELMm6rXakguSSys8iYhTwPsklju060zEGSO8RTCVJsPsAewCT26Lt9qSjHFcnmfysAvbLmR+UlrV7kipJEswvI+LutPhNSQPT5QOBt9LypvazufJBjZQ310aWxgFTJL0G3EpyyewnQG9JFY3EuH2/0uW9gDW0/nNY00wbWasFaiPiqXT+TpKk01mPMcAk4NWIqIuIzcDdJMe+Mx/neqU8rgV9DzrJ5GcxMDR9sqQLyc3D+SWOqUXp0yLXAc9HxL/nLJoP1D9lMoPkXk19+RfTp0jGAuvS0+aFwPGS+qT/izye5Fr0auBvksambX2xwbYaayMzEXFhRAyKiMEkx+iRiDgTeBQ4vZFYcmM8Pa0fafn09KmkIcBQkpukjf4dpOs01UamIuIN4HVJB6ZFxwLL6aTHOPUXYKyk7mlM9fvcaY9zjlIe16baaF6WN60604vkyYqXSJ46uajU8eQZ83iSU92lwDPp6ySSa8sPAyuAh4C+aX0BV6f7+CxQnbOtfwJWpq8v55RXA8+l6/yUv/ci0WgbbbjvE/j702UHkHx5rATuALqm5VXp/Mp0+QE561+U7tOLpE/dNPd30FQbbbSvI4Ga9Dj/J8lTRJ36GAPfBV5I47qZ5AmxTnWcgVtI7jltJjljPauUx7W5Npp7uVsZMzPLjC+XmZlZZpxkzMwsM04yZmaWGScZMzPLjJOMmZllxknGrJUk/SF9Hyzp80Xe9rcba8uso/IjzGYFkjQB+EZEfKoV61TE3/u+amz5+ojoUYz4zNoDn8mYtZKk9enkZcAnJT2jZHyTcklXSFqcjrfxv9L6EyT9l6T5JL9MR9J/SlqiZEyUmWnZZUC3dHu/zG0r/ZX1FUrGT3lW0rScbS/S38eT+WX6C24kXaZkLKGlkq5sy8/IrF5Fy1XMrAmzyDmTSZPFuog4XFJX4HFJv03rHgZ8IiJeTef/KSLekdQNWCzproiYJemciBjZSFunkfyy/1Cgf7rO79Jlo4DhwF+Bx4Fxkp4HTgUOioiQ1Lvoe2+WB5/JmBXP8SR9Oz1DMqRCP5L+sACezkkwAOdJ+jPwJEmng0Np3njglojYGhFvAo8Bh+dsuzYitpF0HTSYpDv7jcB1kk4DNuzy3pkVwEnGrHgEnBsRI9PXkIioP5N5f3ul5F7OJJKBtA4F/kTSv1ahPsyZ3koyqNYWYAxJr8yfAn6zC9s3K5iTjFnh3iMZ1rreQuArSoZXQNLHlQwg1lAv4N2I2CDpIJKhbOttrl+/gf8CpqX3fQaQDM37dFOBKRlDqFdELAC+TnKZzazN+Z6MWeGWAlvTy143koxdMxj4Y3rzvQ44pZH1fgP8c3rf5EWSS2b15gJLJf0xkmEK6t1DMtzvn0l61r4gIt5Ik1RjegL3SqoiOcM6v7BdNNs1foTZzMwy48tlZmaWGScZMzPLjJOMmZllxknGzMwy4yRjZmaZcZIxM7PMOMmYmVlm/j96WEhvrhxzVQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"Optimal x\")\n",
    "plt.plot(adashift_x_history, label='adashift')\n",
    "plt.plot(x_history,label='adam')\n",
    "plt.plot(amsgrad_x_history,label='amsgrad')\n",
    "plt.legend()\n",
    "plt.xlabel(\"iterations\")\n",
    "plt.ylabel(r\"$x_t$\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Synthetic example 2: stochastic case\n",
    "\n",
    "Let's see on the similar task, but with stochasticity.\n",
    "\n",
    "$\\delta = 0.002$\n",
    "C = 101\n",
    "\n",
    "$f_t(x) = \\begin{cases} Cx, \\text{with p } =\\frac{1+\\delta}{C+1} \\\\ \n",
    "-x, otherwise \\end{cases}$\n",
    "\n",
    "Cost function is $\\delta x$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "delta = 0.02\n",
    "C = 101\n",
    "p = (1+delta)/(C + 1)\n",
    "\n",
    "def stoch_f_t(x, t):\n",
    "    if np.random.binomial(1,p):\n",
    "        return C*x\n",
    "    else:\n",
    "        return -x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def StochLearning(optimizer_type='Adam', alpha=1e-3, lr_decay=True,\\\n",
    "                  eps=1e-10, betas=(0.0, 0.999), amsgrad=False, max_step = 1000001):\n",
    "    \n",
    "    x = torch.tensor([0.0], requires_grad=True)\n",
    "    if optimizer_type == \"Adam\":\n",
    "        optimizer = torch.optim.Adam([x], lr=alpha, betas=betas, eps=eps, amsgrad=amsgrad)\n",
    "    else:\n",
    "        optimizer = ad_opt.AdaShift([x], lr=alpha, keep_num=1, betas=betas, eps=eps)\n",
    "\n",
    "    time_steps = []\n",
    "    avg_regret_history = []\n",
    "    x_history = []\n",
    "    loss_history = []\n",
    "    print(\"Optimization with {}\".format(optimizer_type))\n",
    "    \n",
    "    if lr_decay:\n",
    "        lambda_lr_decay = lambda step: alpha / np.sqrt(step + 1) \n",
    "        scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_lr_decay)\n",
    "\n",
    "    for step in range(1, max_step):\n",
    "        #x.data = x.data.clamp(-1.0, 1.0)\n",
    "\n",
    "        loss = stoch_f_t(x, step)\n",
    "\n",
    "        loss.backward()\n",
    "        if step % 10000 == 0:\n",
    "            x_history.append(x.item())\n",
    "            loss_history.append(loss.item())\n",
    "            time_steps.append(step)\n",
    "\n",
    "        if step % 100000 == 0:\n",
    "            print ('step : ', step, '  loss : ', loss.item(), ' x:', x.item())\n",
    "                                 \n",
    "        optimizer.step()   \n",
    "\n",
    "        if lr_decay:\n",
    "            scheduler.step()\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "            \n",
    "    return time_steps, loss_history, x_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization with Adam\n",
      "step :  100000   loss :  -1.1847261190414429  x: 1.1847261190414429\n",
      "step :  200000   loss :  -1.4070461988449097  x: 1.4070461988449097\n",
      "step :  300000   loss :  -1.7301219701766968  x: 1.7301219701766968\n",
      "step :  400000   loss :  -2.104252576828003  x: 2.104252576828003\n",
      "step :  500000   loss :  -2.7960855960845947  x: 2.7960855960845947\n",
      "step :  600000   loss :  -3.5971755981445312  x: 3.5971755981445312\n",
      "step :  700000   loss :  -4.240509033203125  x: 4.240509033203125\n",
      "step :  800000   loss :  -4.502536773681641  x: 4.502536773681641\n",
      "step :  900000   loss :  -4.662587642669678  x: 4.662587642669678\n",
      "step :  1000000   loss :  -5.070318698883057  x: 5.070318698883057\n",
      "step :  1100000   loss :  -5.384920597076416  x: 5.384920597076416\n",
      "step :  1200000   loss :  -5.2494916915893555  x: 5.2494916915893555\n",
      "step :  1300000   loss :  -5.3970627784729  x: 5.3970627784729\n",
      "step :  1400000   loss :  -5.596741676330566  x: 5.596741676330566\n",
      "step :  1500000   loss :  -6.008414268493652  x: 6.008414268493652\n",
      "step :  1600000   loss :  -6.78537654876709  x: 6.78537654876709\n",
      "step :  1700000   loss :  -7.100484848022461  x: 7.100484848022461\n",
      "step :  1800000   loss :  -7.642403602600098  x: 7.642403602600098\n",
      "step :  1900000   loss :  -8.169049263000488  x: 8.169049263000488\n",
      "step :  2000000   loss :  -8.562658309936523  x: 8.562658309936523\n",
      "step :  2100000   loss :  -8.710331916809082  x: 8.710331916809082\n",
      "step :  2200000   loss :  -9.028823852539062  x: 9.028823852539062\n",
      "step :  2300000   loss :  -10.531248092651367  x: 10.531248092651367\n",
      "step :  2400000   loss :  -10.376737594604492  x: 10.376737594604492\n",
      "step :  2500000   loss :  -10.805197715759277  x: 10.805197715759277\n",
      "step :  2600000   loss :  -11.040071487426758  x: 11.040071487426758\n",
      "step :  2700000   loss :  -11.600111961364746  x: 11.600111961364746\n",
      "step :  2800000   loss :  -12.283075332641602  x: 12.283075332641602\n",
      "step :  2900000   loss :  -12.854783058166504  x: 12.854783058166504\n",
      "step :  3000000   loss :  -12.953192710876465  x: 12.953192710876465\n",
      "step :  3100000   loss :  -13.579699516296387  x: 13.579699516296387\n",
      "step :  3200000   loss :  -13.71220588684082  x: 13.71220588684082\n",
      "step :  3300000   loss :  -14.549880027770996  x: 14.549880027770996\n",
      "step :  3400000   loss :  1484.45361328125  x: 14.69756031036377\n",
      "step :  3500000   loss :  -15.440301895141602  x: 15.440301895141602\n",
      "step :  3600000   loss :  -16.264249801635742  x: 16.264249801635742\n",
      "step :  3700000   loss :  -16.224853515625  x: 16.224853515625\n",
      "step :  3800000   loss :  -16.450313568115234  x: 16.450313568115234\n",
      "step :  3900000   loss :  -16.738100051879883  x: 16.738100051879883\n",
      "step :  4000000   loss :  -17.249980926513672  x: 17.249980926513672\n",
      "step :  4100000   loss :  -17.73457145690918  x: 17.73457145690918\n",
      "step :  4200000   loss :  -17.885164260864258  x: 17.885164260864258\n",
      "step :  4300000   loss :  -17.767845153808594  x: 17.767845153808594\n",
      "step :  4400000   loss :  -18.3480224609375  x: 18.3480224609375\n",
      "step :  4500000   loss :  -18.76740074157715  x: 18.76740074157715\n",
      "step :  4600000   loss :  -18.919906616210938  x: 18.919906616210938\n",
      "step :  4700000   loss :  -19.093120574951172  x: 19.093120574951172\n",
      "step :  4800000   loss :  -19.3359432220459  x: 19.3359432220459\n",
      "step :  4900000   loss :  -20.57540512084961  x: 20.57540512084961\n",
      "step :  5000000   loss :  -20.485252380371094  x: 20.485252380371094\n",
      "step :  5100000   loss :  -21.198232650756836  x: 21.198232650756836\n",
      "step :  5200000   loss :  -21.11430549621582  x: 21.11430549621582\n",
      "step :  5300000   loss :  -21.168184280395508  x: 21.168184280395508\n",
      "step :  5400000   loss :  -21.6210880279541  x: 21.6210880279541\n",
      "step :  5500000   loss :  -21.58517837524414  x: 21.58517837524414\n",
      "step :  5600000   loss :  -22.072921752929688  x: 22.072921752929688\n",
      "step :  5700000   loss :  -21.90089225769043  x: 21.90089225769043\n",
      "step :  5800000   loss :  -22.030771255493164  x: 22.030771255493164\n",
      "step :  5900000   loss :  -21.79073715209961  x: 21.79073715209961\n",
      "step :  6000000   loss :  -22.259828567504883  x: 22.259828567504883\n",
      "step :  6100000   loss :  -22.62893295288086  x: 22.62893295288086\n",
      "step :  6200000   loss :  -22.803356170654297  x: 22.803356170654297\n",
      "step :  6300000   loss :  -22.95709991455078  x: 22.95709991455078\n",
      "step :  6400000   loss :  -23.144319534301758  x: 23.144319534301758\n",
      "step :  6500000   loss :  -23.75922203063965  x: 23.75922203063965\n",
      "step :  6600000   loss :  -24.125959396362305  x: 24.125959396362305\n",
      "step :  6700000   loss :  -24.044891357421875  x: 24.044891357421875\n",
      "step :  6800000   loss :  -24.088661193847656  x: 24.088661193847656\n",
      "step :  6900000   loss :  -24.466503143310547  x: 24.466503143310547\n",
      "step :  7000000   loss :  -24.47447967529297  x: 24.47447967529297\n",
      "step :  7100000   loss :  -24.238370895385742  x: 24.238370895385742\n",
      "step :  7200000   loss :  -24.870508193969727  x: 24.870508193969727\n",
      "step :  7300000   loss :  -24.71416664123535  x: 24.71416664123535\n",
      "step :  7400000   loss :  -25.309829711914062  x: 25.309829711914062\n",
      "step :  7500000   loss :  -25.46268081665039  x: 25.46268081665039\n",
      "step :  7600000   loss :  -25.6578369140625  x: 25.6578369140625\n",
      "step :  7700000   loss :  -26.08003044128418  x: 26.08003044128418\n",
      "step :  7800000   loss :  -26.451635360717773  x: 26.451635360717773\n",
      "step :  7900000   loss :  -26.98204231262207  x: 26.98204231262207\n",
      "step :  8000000   loss :  -27.759675979614258  x: 27.759675979614258\n",
      "step :  8100000   loss :  -28.23503303527832  x: 28.23503303527832\n",
      "step :  8200000   loss :  -28.490901947021484  x: 28.490901947021484\n",
      "step :  8300000   loss :  -28.518051147460938  x: 28.518051147460938\n",
      "step :  8400000   loss :  -28.524202346801758  x: 28.524202346801758\n",
      "step :  8500000   loss :  -29.125585556030273  x: 29.125585556030273\n",
      "step :  8600000   loss :  -29.441280364990234  x: 29.441280364990234\n",
      "step :  8700000   loss :  -29.931488037109375  x: 29.931488037109375\n",
      "step :  8800000   loss :  -30.53805160522461  x: 30.53805160522461\n",
      "step :  8900000   loss :  -30.43130111694336  x: 30.43130111694336\n",
      "step :  9000000   loss :  -31.046688079833984  x: 31.046688079833984\n",
      "step :  9100000   loss :  -31.44439697265625  x: 31.44439697265625\n",
      "step :  9200000   loss :  -31.704538345336914  x: 31.704538345336914\n",
      "step :  9300000   loss :  -32.09241485595703  x: 32.09241485595703\n",
      "step :  9400000   loss :  -32.447166442871094  x: 32.447166442871094\n",
      "step :  9500000   loss :  -33.18728256225586  x: 33.18728256225586\n",
      "step :  9600000   loss :  -33.6731071472168  x: 33.6731071472168\n",
      "step :  9700000   loss :  -34.055484771728516  x: 34.055484771728516\n",
      "step :  9800000   loss :  -34.332462310791016  x: 34.332462310791016\n",
      "step :  9900000   loss :  -35.09705352783203  x: 35.09705352783203\n"
     ]
    }
   ],
   "source": [
    "lr = 1e-3\n",
    "time_steps, loss_history, x_history = StochLearning(alpha=lr, lr_decay=False, max_step=10000000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization with Adashift\n",
      "step :  100000   loss :  0.31325533986091614  x: -0.31325533986091614\n",
      "step :  200000   loss :  0.28532645106315613  x: -0.28532645106315613\n",
      "step :  300000   loss :  0.8368154168128967  x: -0.8368154168128967\n",
      "step :  400000   loss :  1.0624598264694214  x: -1.0624598264694214\n",
      "step :  500000   loss :  1.8906928300857544  x: -1.8906928300857544\n",
      "step :  600000   loss :  2.344602108001709  x: -2.344602108001709\n",
      "step :  700000   loss :  2.448505163192749  x: -2.448505163192749\n",
      "step :  800000   loss :  2.5806972980499268  x: -2.5806972980499268\n",
      "step :  900000   loss :  2.9236128330230713  x: -2.9236128330230713\n",
      "step :  1000000   loss :  2.4929535388946533  x: -2.4929535388946533\n",
      "step :  1100000   loss :  2.7226791381835938  x: -2.7226791381835938\n",
      "step :  1200000   loss :  2.6651761531829834  x: -2.6651761531829834\n",
      "step :  1300000   loss :  2.685107469558716  x: -2.685107469558716\n",
      "step :  1400000   loss :  2.215848684310913  x: -2.215848684310913\n",
      "step :  1500000   loss :  2.36421537399292  x: -2.36421537399292\n",
      "step :  1600000   loss :  2.2782225608825684  x: -2.2782225608825684\n",
      "step :  1700000   loss :  1.5842700004577637  x: -1.5842700004577637\n",
      "step :  1800000   loss :  1.69608736038208  x: -1.69608736038208\n",
      "step :  1900000   loss :  2.1388800144195557  x: -2.1388800144195557\n",
      "step :  2000000   loss :  2.4918770790100098  x: -2.4918770790100098\n",
      "step :  2100000   loss :  2.4855902194976807  x: -2.4855902194976807\n",
      "step :  2200000   loss :  2.48868989944458  x: -2.48868989944458\n",
      "step :  2300000   loss :  3.0740535259246826  x: -3.0740535259246826\n",
      "step :  2400000   loss :  3.3882386684417725  x: -3.3882386684417725\n",
      "step :  2500000   loss :  2.9736666679382324  x: -2.9736666679382324\n",
      "step :  2600000   loss :  3.432202100753784  x: -3.432202100753784\n",
      "step :  2700000   loss :  4.136037349700928  x: -4.136037349700928\n",
      "step :  2800000   loss :  4.427706241607666  x: -4.427706241607666\n",
      "step :  2900000   loss :  4.10300874710083  x: -4.10300874710083\n",
      "step :  3000000   loss :  4.190589427947998  x: -4.190589427947998\n",
      "step :  3100000   loss :  4.393651008605957  x: -4.393651008605957\n",
      "step :  3200000   loss :  4.594647407531738  x: -4.594647407531738\n",
      "step :  3300000   loss :  5.173344135284424  x: -5.173344135284424\n",
      "step :  3400000   loss :  5.416220188140869  x: -5.416220188140869\n",
      "step :  3500000   loss :  5.165672779083252  x: -5.165672779083252\n",
      "step :  3600000   loss :  5.9280476570129395  x: -5.9280476570129395\n",
      "step :  3700000   loss :  6.340357303619385  x: -6.340357303619385\n",
      "step :  3800000   loss :  6.415966987609863  x: -6.415966987609863\n",
      "step :  3900000   loss :  7.021544456481934  x: -7.021544456481934\n",
      "step :  4000000   loss :  7.241345405578613  x: -7.241345405578613\n",
      "step :  4100000   loss :  -710.30517578125  x: -7.032724380493164\n",
      "step :  4200000   loss :  7.147594451904297  x: -7.147594451904297\n",
      "step :  4300000   loss :  7.534843921661377  x: -7.534843921661377\n",
      "step :  4400000   loss :  7.39988374710083  x: -7.39988374710083\n",
      "step :  4500000   loss :  7.4088544845581055  x: -7.4088544845581055\n",
      "step :  4600000   loss :  7.626873970031738  x: -7.626873970031738\n",
      "step :  4700000   loss :  7.617283344268799  x: -7.617283344268799\n",
      "step :  4800000   loss :  7.9205322265625  x: -7.9205322265625\n",
      "step :  4900000   loss :  8.395398139953613  x: -8.395398139953613\n",
      "step :  5000000   loss :  8.0447998046875  x: -8.0447998046875\n",
      "step :  5100000   loss :  8.56939697265625  x: -8.56939697265625\n",
      "step :  5200000   loss :  8.791878700256348  x: -8.791878700256348\n",
      "step :  5300000   loss :  9.025267601013184  x: -9.025267601013184\n",
      "step :  5400000   loss :  9.052425384521484  x: -9.052425384521484\n",
      "step :  5500000   loss :  8.897454261779785  x: -8.897454261779785\n",
      "step :  5600000   loss :  8.217730522155762  x: -8.217730522155762\n",
      "step :  5700000   loss :  8.340407371520996  x: -8.340407371520996\n",
      "step :  5800000   loss :  8.280197143554688  x: -8.280197143554688\n",
      "step :  5900000   loss :  8.236777305603027  x: -8.236777305603027\n",
      "step :  6000000   loss :  8.052545547485352  x: -8.052545547485352\n",
      "step :  6100000   loss :  7.998349666595459  x: -7.998349666595459\n",
      "step :  6200000   loss :  8.614864349365234  x: -8.614864349365234\n",
      "step :  6300000   loss :  8.182519912719727  x: -8.182519912719727\n",
      "step :  6400000   loss :  8.396403312683105  x: -8.396403312683105\n",
      "step :  6500000   loss :  8.336465835571289  x: -8.336465835571289\n",
      "step :  6600000   loss :  8.933448791503906  x: -8.933448791503906\n",
      "step :  6700000   loss :  8.948176383972168  x: -8.948176383972168\n",
      "step :  6800000   loss :  9.225473403930664  x: -9.225473403930664\n",
      "step :  6900000   loss :  10.242803573608398  x: -10.242803573608398\n",
      "step :  7000000   loss :  10.632209777832031  x: -10.632209777832031\n",
      "step :  7100000   loss :  10.717737197875977  x: -10.717737197875977\n",
      "step :  7200000   loss :  10.950407981872559  x: -10.950407981872559\n",
      "step :  7300000   loss :  10.776394844055176  x: -10.776394844055176\n",
      "step :  7400000   loss :  10.952515602111816  x: -10.952515602111816\n",
      "step :  7500000   loss :  11.826210021972656  x: -11.826210021972656\n",
      "step :  7600000   loss :  12.273371696472168  x: -12.273371696472168\n",
      "step :  7700000   loss :  12.95554256439209  x: -12.95554256439209\n",
      "step :  7800000   loss :  12.145513534545898  x: -12.145513534545898\n",
      "step :  7900000   loss :  11.88361930847168  x: -11.88361930847168\n",
      "step :  8000000   loss :  12.281220436096191  x: -12.281220436096191\n",
      "step :  8100000   loss :  12.825567245483398  x: -12.825567245483398\n",
      "step :  8200000   loss :  12.678954124450684  x: -12.678954124450684\n",
      "step :  8300000   loss :  13.025280952453613  x: -13.025280952453613\n",
      "step :  8400000   loss :  13.25296401977539  x: -13.25296401977539\n",
      "step :  8500000   loss :  13.352694511413574  x: -13.352694511413574\n",
      "step :  8600000   loss :  13.931696891784668  x: -13.931696891784668\n",
      "step :  8700000   loss :  -1466.5198974609375  x: -14.519998550415039\n",
      "step :  8800000   loss :  15.219633102416992  x: -15.219633102416992\n",
      "step :  8900000   loss :  15.406670570373535  x: -15.406670570373535\n",
      "step :  9000000   loss :  16.272245407104492  x: -16.272245407104492\n",
      "step :  9100000   loss :  16.691661834716797  x: -16.691661834716797\n",
      "step :  9200000   loss :  16.458457946777344  x: -16.458457946777344\n",
      "step :  9300000   loss :  16.486913681030273  x: -16.486913681030273\n",
      "step :  9400000   loss :  16.584880828857422  x: -16.584880828857422\n",
      "step :  9500000   loss :  16.454532623291016  x: -16.454532623291016\n",
      "step :  9600000   loss :  16.8990478515625  x: -16.8990478515625\n",
      "step :  9700000   loss :  17.176965713500977  x: -17.176965713500977\n",
      "step :  9800000   loss :  16.944353103637695  x: -16.944353103637695\n",
      "step :  9900000   loss :  17.069684982299805  x: -17.069684982299805\n"
     ]
    }
   ],
   "source": [
    "adashift_time_steps, adashift_loss_history, adashift_x_history = \\\n",
    "                    StochLearning(optimizer_type='Adashift',lr_decay=False, max_step=10000000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization with Adam\n",
      "step :  100000   loss :  0.06807340681552887  x: -0.06807340681552887\n",
      "step :  200000   loss :  0.1619931012392044  x: -0.1619931012392044\n",
      "step :  300000   loss :  0.20775218307971954  x: -0.20775218307971954\n",
      "step :  400000   loss :  0.2950662672519684  x: -0.2950662672519684\n",
      "step :  500000   loss :  0.49955493211746216  x: -0.49955493211746216\n",
      "step :  600000   loss :  0.35960322618484497  x: -0.35960322618484497\n",
      "step :  700000   loss :  0.13698606193065643  x: -0.13698606193065643\n",
      "step :  800000   loss :  0.09306208044290543  x: -0.09306208044290543\n",
      "step :  900000   loss :  0.31118008494377136  x: -0.31118008494377136\n",
      "step :  1000000   loss :  0.3847825825214386  x: -0.3847825825214386\n",
      "step :  1100000   loss :  0.8786011338233948  x: -0.8786011338233948\n",
      "step :  1200000   loss :  1.1681876182556152  x: -1.1681876182556152\n",
      "step :  1300000   loss :  0.8730359673500061  x: -0.8730359673500061\n",
      "step :  1400000   loss :  0.822638750076294  x: -0.822638750076294\n",
      "step :  1500000   loss :  0.9281461238861084  x: -0.9281461238861084\n",
      "step :  1600000   loss :  1.3296902179718018  x: -1.3296902179718018\n",
      "step :  1700000   loss :  1.5351674556732178  x: -1.5351674556732178\n",
      "step :  1800000   loss :  1.7130892276763916  x: -1.7130892276763916\n",
      "step :  1900000   loss :  1.4914567470550537  x: -1.4914567470550537\n",
      "step :  2000000   loss :  1.44893479347229  x: -1.44893479347229\n",
      "step :  2100000   loss :  2.014101266860962  x: -2.014101266860962\n",
      "step :  2200000   loss :  1.9071861505508423  x: -1.9071861505508423\n",
      "step :  2300000   loss :  1.8100183010101318  x: -1.8100183010101318\n",
      "step :  2400000   loss :  1.7950518131256104  x: -1.7950518131256104\n",
      "step :  2500000   loss :  2.3157715797424316  x: -2.3157715797424316\n",
      "step :  2600000   loss :  2.3540592193603516  x: -2.3540592193603516\n",
      "step :  2700000   loss :  2.2407917976379395  x: -2.2407917976379395\n",
      "step :  2800000   loss :  1.930902123451233  x: -1.930902123451233\n",
      "step :  2900000   loss :  1.9305468797683716  x: -1.9305468797683716\n",
      "step :  3000000   loss :  2.236417531967163  x: -2.236417531967163\n",
      "step :  3100000   loss :  2.2402608394622803  x: -2.2402608394622803\n",
      "step :  3200000   loss :  2.492103338241577  x: -2.492103338241577\n",
      "step :  3300000   loss :  2.530390977859497  x: -2.530390977859497\n",
      "step :  3400000   loss :  2.8786776065826416  x: -2.8786776065826416\n",
      "step :  3500000   loss :  2.9307429790496826  x: -2.9307429790496826\n",
      "step :  3600000   loss :  3.037919282913208  x: -3.037919282913208\n",
      "step :  3700000   loss :  3.0348737239837646  x: -3.0348737239837646\n",
      "step :  3800000   loss :  2.962939500808716  x: -2.962939500808716\n",
      "step :  3900000   loss :  3.1734488010406494  x: -3.1734488010406494\n",
      "step :  4000000   loss :  3.2255141735076904  x: -3.2255141735076904\n",
      "step :  4100000   loss :  3.2913572788238525  x: -3.2913572788238525\n",
      "step :  4200000   loss :  3.536310911178589  x: -3.536310911178589\n",
      "step :  4300000   loss :  3.4712655544281006  x: -3.4712655544281006\n",
      "step :  4400000   loss :  3.7093303203582764  x: -3.7093303203582764\n",
      "step :  4500000   loss :  3.5753962993621826  x: -3.5753962993621826\n",
      "step :  4600000   loss :  3.7927944660186768  x: -3.7927944660186768\n",
      "step :  4700000   loss :  4.107938289642334  x: -4.107938289642334\n",
      "step :  4800000   loss :  3.9416916370391846  x: -3.9416916370391846\n",
      "step :  4900000   loss :  3.9370853900909424  x: -3.9370853900909424\n",
      "step :  5000000   loss :  4.218910217285156  x: -4.218910217285156\n",
      "step :  5100000   loss :  4.322911262512207  x: -4.322911262512207\n",
      "step :  5200000   loss :  4.0204691886901855  x: -4.0204691886901855\n",
      "step :  5300000   loss :  4.019057750701904  x: -4.019057750701904\n",
      "step :  5400000   loss :  3.8707454204559326  x: -3.8707454204559326\n",
      "step :  5500000   loss :  4.114096641540527  x: -4.114096641540527\n",
      "step :  5600000   loss :  3.8653604984283447  x: -3.8653604984283447\n",
      "step :  5700000   loss :  4.320676803588867  x: -4.320676803588867\n",
      "step :  5800000   loss :  4.162900924682617  x: -4.162900924682617\n",
      "step :  5900000   loss :  3.96732497215271  x: -3.96732497215271\n",
      "step :  6000000   loss :  3.9719762802124023  x: -3.9719762802124023\n",
      "step :  6100000   loss :  4.231784343719482  x: -4.231784343719482\n",
      "step :  6200000   loss :  4.8180060386657715  x: -4.8180060386657715\n",
      "step :  6300000   loss :  4.839340686798096  x: -4.839340686798096\n",
      "step :  6400000   loss :  5.039785861968994  x: -5.039785861968994\n",
      "step :  6500000   loss :  5.185120105743408  x: -5.185120105743408\n",
      "step :  6600000   loss :  5.17889928817749  x: -5.17889928817749\n",
      "step :  6700000   loss :  4.931568145751953  x: -4.931568145751953\n",
      "step :  6800000   loss :  5.228457450866699  x: -5.228457450866699\n",
      "step :  6900000   loss :  5.525346755981445  x: -5.525346755981445\n",
      "step :  7000000   loss :  5.560459136962891  x: -5.560459136962891\n",
      "step :  7100000   loss :  5.657571315765381  x: -5.657571315765381\n",
      "step :  7200000   loss :  6.292015075683594  x: -6.292015075683594\n",
      "step :  7300000   loss :  6.34090518951416  x: -6.34090518951416\n",
      "step :  7400000   loss :  6.582683563232422  x: -6.582683563232422\n",
      "step :  7500000   loss :  6.865795135498047  x: -6.865795135498047\n",
      "step :  7600000   loss :  7.0937957763671875  x: -7.0937957763671875\n",
      "step :  7700000   loss :  7.4526848793029785  x: -7.4526848793029785\n",
      "step :  7800000   loss :  7.728907585144043  x: -7.728907585144043\n",
      "step :  7900000   loss :  7.970766067504883  x: -7.970766067504883\n",
      "step :  8000000   loss :  8.357670783996582  x: -8.357670783996582\n",
      "step :  8100000   loss :  8.200352668762207  x: -8.200352668762207\n",
      "step :  8200000   loss :  7.967144012451172  x: -7.967144012451172\n",
      "step :  8300000   loss :  7.699153423309326  x: -7.699153423309326\n",
      "step :  8400000   loss :  8.17545223236084  x: -8.17545223236084\n",
      "step :  8500000   loss :  8.258461952209473  x: -8.258461952209473\n",
      "step :  8600000   loss :  8.251773834228516  x: -8.251773834228516\n",
      "step :  8700000   loss :  8.174808502197266  x: -8.174808502197266\n",
      "step :  8800000   loss :  8.273990631103516  x: -8.273990631103516\n",
      "step :  8900000   loss :  8.379947662353516  x: -8.379947662353516\n",
      "step :  9000000   loss :  8.519779205322266  x: -8.519779205322266\n",
      "step :  9100000   loss :  8.903507232666016  x: -8.903507232666016\n",
      "step :  9200000   loss :  8.941715240478516  x: -8.941715240478516\n",
      "step :  9300000   loss :  9.013797760009766  x: -9.013797760009766\n",
      "step :  9400000   loss :  8.909732818603516  x: -8.909732818603516\n",
      "step :  9500000   loss :  8.954715728759766  x: -8.954715728759766\n",
      "step :  9600000   loss :  9.365543365478516  x: -9.365543365478516\n",
      "step :  9700000   loss :  9.803470611572266  x: -9.803470611572266\n",
      "step :  9800000   loss :  9.699405670166016  x: -9.699405670166016\n",
      "step :  9900000   loss :  9.554691314697266  x: -9.554691314697266\n"
     ]
    }
   ],
   "source": [
    "ams_time_steps, ams_loss_history, ams_x_history = StochLearning(alpha=lr, amsgrad=True, \n",
    "                                                                lr_decay=False, max_step=10000000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEWCAYAAAB1xKBvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xd8FVX6+PHPuclN7wklhRB6ILQAhirSVFCWYkdUrKxd19W1fpW1/FZde0UUBQuKCCqILIIURelSEyCUBJKQhPR+k9x7z++PCSnUAEluyvN+vXjlzsyZmWcC5MmZmfMcpbVGCCGEOBuTowMQQgjRNEjCEEIIUSuSMIQQQtSKJAwhhBC1IglDCCFErUjCEEIIUSuSMIQ4D0qpi5VS+85z33ClVKFSyqmxxCREbSgZhyHE2SmlNNBFa33A0bEc1xhjEs2b9DCEEELUiiQM0WQppToppbKVUv0qlkOUUhlKqRGnad9dKbVGKZWrlIpVSk2otm2OUmqmUmqFUqpAKbVWKdW+YttvFc12VNxKul4pNUIplVxt/0Sl1GNKqZ1KqSKl1GylVBul1LKK461USvlXtI1QSmmllLNSanDFMY//sSilEivaxSil1lfEm6qUek8p5XIOMZ3tet9XSi2tiG+jUqrThf+tiOZMEoZosrTWB4HHgS+VUh7AZ8BcrfWaE9sqpczAEuAXoDXwAPCVUqpbtWZTgReAIGA78FXFeYZXbO+jtfbSWs8/TUhXA5cCXYG/AcuAp4BWGP/XHjzFNayvOKYX4A9sBL6u2GwD/lERz2BgNHBvbWKq5fXeAPy74rwHgJdOc11CAJIwRBOntf4Y44fdRiAYePo0TQcBXsDLWusyrfUq4CdgSrU2S7XWv2mtSyuOM1gp1e4cwnlXa52utU4Bfgc2aq23aa0twPdA9Fn2fwcoOH4NWuutWusNWmur1joR+Ai4pJax1OZ6v9dab9JaWzGSY99aHlu0UM6ODkCIOvAxsBiYXvHD/lRCgCSttb3ausNAaLXlpOMftNaFSqns4/vVMo70ap9LTrHsdbodlVJ/B0YAA4/HqJTqCrwBDAA8MP6/bq1lLLW53rRqn4vPFJ8QID0M0cQppbyAt4DZwAylVMBpmh4F2imlqv+bDwdSqi1X9iYqjhtQsV+9UkpdjHErbKLWOr/apg+BvRhvQvlg3N5StTxsba5XiHMiCUM0dW8DW7TWdwJLgZmnabcR47fofymlzBUPxv8GfFOtzRVKqWEVD5ZfADZorY/3LtKBjnUdfMUtr2+BW7TW8Sds9gbygUKlVCRwzwnbzxRTba5XiHMiCUM0WUqpicBYqn6QPgL0U0pNPbGt1roM4wfmOCAT+ADjh/Teas3mAc8B2UB/4KZq22YAcyveOLquDi9jNNAG+K7am1KxFdseBW7EeK7xMXDiw/bTxlTL6xXinMjAPSEwXjMFkrXWzzg6FiEaK+lhCCGEqBVJGEIIIWpFbkkJIYSoFelhCCGEqJVmNXAvKChIR0REODoMIYRoUrZu3ZqptW51tnbNKmFERESwZcsWR4chhBBNilLqcG3ayS0pIYQQtSIJQwghRK1IwhBCCFErzeoZxqmUl5eTnJyMxWJxdChNjpubG2FhYZjNZkeHIoRoBJp9wkhOTsbb25uIiAiUqm2hT6G1Jisri+TkZDp06ODocIQQjUCzvyVlsVgIDAyUZHGOlFIEBgZKz0wIUanZJwxAksV5ku+bEKK6FpEwhBCi2Sm3wLG9kH8UVv8/yDxQ76eUhNFIzJkzh/vvv/+c91uzZg3jx48/5bY777yTuLg4ABYsWED37t0ZOXIk27dv5+eff76geIUQDmQtgx/ugQ8GwhvdYe0rkLaj3k8rCaMZ++STT+jRowcAs2fP5uOPP2b16tWSMIRoymzl8MkoiF0ELl4w4Ha4aRH0vLreTy0Jo4FMmjSJ/v37ExUVxaxZswD47LPP6Nq1KzExMfzxxx+VbZcsWcLAgQOJjo5mzJgxpKenA7B27Vr69u1L3759iY6OpqCgAIDCwkKuueYaIiMjmTp1KscrEI8YMYItW7bw/PPPs27dOu644w7+8Y9/8OyzzzJ//nz69u3L/PknTuImhGi0Sgvh/4VA2i6Iugr+lQDj34TOoxvk9M3+tdrq/r0klrij+XV6zB4hPjz3t6iztvv0008JCAigpKSEiy66iCuvvJLnnnuOrVu34uvry8iRI4mOjgZg2LBhbNiwAaUUn3zyCa+++iqvv/46r732Gu+//z5Dhw6lsLAQNzc3ALZt20ZsbCwhISEMHTqUP/74g2HDhlWe+9lnn2XVqlW89tprDBgwgD59+rBlyxbee++9Ov1eCCHqSXoc7P8F/poLtjK46C648rUGD6NFJQxHeuedd/j+++8BSEpK4osvvmDEiBG0amUUiLz++uuJj48HjLEj119/PampqZSVlVWOgxg6dCiPPPIIU6dO5aqrriIsLAyAmJiYys99+/YlMTGxRsIQQjRhSx+FzR9XLYfFwGUvOiSUFpUwatMTqA9r1qxh5cqVrF+/Hg8PD0aMGEFkZGTlA+kTPfDAAzzyyCNMmDCBNWvWMGPGDACeeOIJrrzySn7++WeGDh3K8uXLAXB1da3c18nJCavVWu/XJISoRyW5YMmF3CNVyeLOVVCaDx0uAZNjnia0qIThKHl5efj7++Ph4cHevXvZsGEDJSUlrF27lqysLHx8fFiwYAF9+vSpbB8aGgrA3LlzK49z8OBBevXqRa9evdi8eTN79+7Fz8/vnOPx9vaufP4hhGgkSnJh6xxo2wu+uRGsFYNmTWZ4eCf4hDg0PJCH3g1i7NixWK1WunfvzhNPPMGgQYMIDg5mxowZDB48mKFDh9K9e/fK9jNmzODaa6+lf//+BAUFVa5/66236NmzJ71798ZsNjNu3LjzimfkyJHExcXJQ28hGoud38Ir7WHlc/DlVeDkAv4dICQabl3aKJIFNLM5vQcMGKBPnEBpz549NX4Yi3Mj3z8h6lG5Bb66BhJ/N5bbDQT3ABh8L3QY3mBhKKW2aq0HnK2d3JISQghHsJXDsn8ZycI9ACZ9CF0vh0ZckkcShhBCNKTcJFgwDVK2Vq17dD84Nf4fx40/QiGEaA5s5UaSmHcdlBaAmy+UFRnPKJpAsgBJGEIIUf/2LIH5N1Utj30FBt3tuHjOkyQMIYSoL7u+gx3fwIEVxnLXcTD8UQjt79i4zpMkDCGEqEs2K/z8TyNRWKtNQHbtHIia7LCw6oIkjEZizpw5Ut9JiKbMkgfrP4C1L1etc/WBK14zEoWzi+NiqyOSMIQQ4lwVZ8Pyp2Hkk+AXDtu+hB/vq9oedpFRctzVu1G/JnuuHJ4wlFJuwG+AK0Y832mtn1NKdQC+AQKBrcDNWusyx0V6YSZNmkRSUhIWi4WHHnqI6dOn89lnn/Gf//wHPz8/+vTpU1kTasmSJbz44ouUlZURGBjIV199RZs2bZgxYwYJCQkcOnSII0eO8Oabb7JhwwaWLVtGaGgoS5YswWw2O/hKhWjG7DZY8x/Y+zMci4Ud86DPjcZXMEqNB3SCjpc4Ns564vCEAZQCo7TWhUopM7BOKbUMeAR4U2v9jVJqJnAH8OEFnWnZE0Yd+brUtheMe/mszeqivDkY9aRWr15NXFwcgwcPZuHChbz66qtMnjyZpUuXMmnSpLq9PiGEIf8orH0Vtn5Wc/2OeaCc4P7NENjJMbE1EIcnDG3UJimsWDRX/NHAKODGivVzgRlcaMJwoLoobw4wbtw4zGYzvXr1wmazMXbsWAB69epFYmJiw16UEC2B1katp++nG8tRk2HCe+DiaVSU3fwJ9JkCvmGOjbMBODxhACilnDBuO3UG3gcOArla6+N1upOB0NPsOx2YDhAeHn7mE9WiJ1Af6qq8OVSVMjeZTJjNZlTF/VGTySRlzYWoD7sXViWLmL/DFa9WbXP3h+GPOSYuB2gU1Wq11jatdV8gDIgBIs9h31la6wFa6wHHf1tvbM5W3ry8vJwFCxbUaH+q8uZCiAZkyYMDv8JPjxjLty6tmSxaoEbRwzhOa52rlFoNDAb8lFLOFb2MMCDFsdGdv7FjxzJz5ky6d+9Ot27dTipv7ufnR9++fSvbHy9v7u/vz6hRo0hISHBg9EK0IOlxkH3QeAtqyYNV6+/bDK26Oi6uRsLh5c2VUq2A8opk4Q78ArwCTAMWVnvovVNr/cGZjiXlzeuefP9Ei1GQBm/2BHt5zfUT34fom069TzPRlMqbBwNzK55jmIBvtdY/KaXigG+UUi8C24DZjgxSCNHMpO0CbYfgPlCUBb88YySLDpdAq24w6v/AzcfRUTYqDk8YWuudQPQp1h/CeJ4hhBDnLj/VqAbr5Gw8hzj4q7G+17WQuR9St5+8z8B7HPZyTFPg8IQhhBB1bsnDJ4+XOG5XxQsmrj5Qml+1/tLnYfD99R9bEyYJQwjRfNjtsOAWo5w4GHNOWPLg0hdg6IPG9rwkKM6C0H5Gm4x48Ak2yniIM5KEIYRoPhLWViWLp9OMxHBoDfSdaqwzmcC/vfHnOHn7qdYaxTgMIYS4YGv/C19UlMa5cxWY3Y3R19E3NasCgI4kPQwBQEREBFu2bCEoKMjRoQhRe+UW2PYFpMdWPbPoOhbCmuYERY2dJIxmzGq14uwsf8Wimco/Ch+PhoKjVeuunQtRUoCzvsgtqQYyadIk+vfvT1RUFLNmzQLAy8uLxx57jKioKMaMGcOmTZsYMWIEHTt2ZPHixQDExsYSExND37596d27N/v37wfghRdeoFu3bgwbNowpU6bw2muvATBixAgefvhhBgwYwNtvv82SJUsYOHAg0dHRjBkzhvT0dACysrK47LLLiIqK4s4778TRAziFOCfH9hqVYwuOGrWcrvsC/v6bJIt61qJ+/Xxl0yvszd5bp8eMDIjk8ZjHz9ruxPLmV199NUVFRYwaNYr//ve/TJ48mWeeeYYVK1YQFxfHtGnTmDBhAjNnzuShhx5i6tSplJWVYbPZ2Lx5MwsXLmTHjh2Ul5fTr18/+vev6oKXlZVxfMR7Tk7OKUul//vf/2bYsGE8++yzLF26lNmzZVykaOSyDhrF/pY/BTu+NtZFXAyjnnFsXC1Ii0oYjnRiefP9+/fj4uJSozy5q6trZeny46XKBw8ezEsvvURycjJXXXUVXbp04Y8//mDixIm4ubnh5ubG3/72txrnuv766ys/n65U+m+//caiRYsAuPLKK/H396/vb4EQ589aBu/2q1r2agt9bpBxEw2sRSWM2vQE6sOpyptbLJaTypNXL11+vFT5jTfeyMCBA1m6dClXXHEFH3300VnP5+npWfn5TKXShWjUSguNQoBpu2BrRdVmN19jpPaIp8Az0LHxtUDyDKMBnKq8eW0dOnSIjh078uCDDzJx4kR27tzJ0KFDWbJkCRaLhcLCQn766acznvtUpdKHDx/OvHnGtJLLli0jJyfnPK9OiDqkNaTugK9vhP+EwkfDjbmykzcZU5/+KwGufF2ShYO0qB6Go5yqvHltffvtt3zxxReYzWbatm3LU089RUBAABMmTKB37960adOGXr164evre8r9T1cq/bnnnmPKlClERUUxZMiQs08+JUR9sdth9YuwebZRriPvSNW2i/8JJmfwbQedR4PJyXFxCseXN69LLam8eWFhIV5eXhQXFzN8+HBmzZpFv379zr7jOWqu3z/hIAVpRg+i86XGqOuUrbDgNsg9bGz3CQWv1tDzGhhwO7h4ODbeFqIplTcX52H69OnExcVhsViYNm1avSQLIWot5S/YOR+GPgSFx+DgKmMAXZseVW0Orq4aid22N3QbZ/QqTM4w9hWImW4kEdFoScJooo4/fxCiweQfhcx4CIsxfvNP2gxoCO4LX0+BwjTYOLOq/a//hkf2QvJmOBYHa/5jrHfzNR5kp+00lu9aBaEyMrspaBEJQ2td+TaSqL3mdLtSXIDibDi2B+ZcYSy7eEP/abD+vVO3D+wMbXtB3GJ4I7JqvUeQUTF26ENgt8GP9xtzVUiyaDKafcJwc3MjKyuLwMBASRrnQGtNVlYWbm5ujg5FOEp5CexbBt/dVrWu8xg4sLIqWTi7g7UE2g+DmxeBcjKSABi3qRbcajzI7jkZ+t8GHgHGNpMTTP6wQS9HXLhmnzDCwsJITk4mIyPD0aE0OW5uboSFhTk6DOEIB36FRXcZ5cEBOo0y5pRo2xMS14GtDDqONKrA2srByXzyMUL7wQNbjWcU8stas9DsE4bZbK4c3SyEqIVfX4DfXwPvYOP20dCHq3oGABHDarY/VbKozTbR5DT7hCGEOIuyIqPHYMmHb2+pmuv6tp8hoKNjYxONiiQMIVqCjH2g7aBMxliIQ6th25fGG07H9kB+stFOOUH0zTDyaWPaUiGqkYQhRHOXsQ8+HAJ268nbDqwAJxfwC4eQaOh9A0Re0fAxiiZBEoYQTVlpAZg9wJIHLp7gbBSwpNwCCb/Bxg+NQXRg9CbQEDkevNoYpTY8gsAsb8KJ2pGEIURTkX8Uvr8bBt9nDKArzoI/3jZuNQF4toK7/zASwMejIWt/1b7j34IBt536uELUkiQMIRqbtF3GjHJgPIAecLvRc/jyasjYCwlrT71fUQa82x/QUFZorLtvM7Tq2iBhi+ZPEoYQjUV5CSx5yKjJVF31EdWD7oMtnxoJpN/NMPAe8DXK17PiWaPH4dkaJn0IPSY0XOyiRWj21WqFaFSKs2HDB2AyG1VZLXnGoDZnd1j7ctVAuQF3gEegUaBv70+Q+AdETYZBd5/5+AXpRjJx96v/axHNhlSrFaKxsZXDO32NJHE6HkFw54qa4x9Cz6ESsXeb849PiLNweMJQSrUDPgfaABqYpbV+WykVAMwHIoBE4DqttUwLJ5qG0gJjpji/9tBvGuxaYPQgAMKHGK+u7v8Fyooh5i5jvU8otO4OnkGOi1uIM3D4LSmlVDAQrLX+SynlDWwFJgG3Atla65eVUk8A/lrrM07KLbekRKOgtTFies/ik7d1uQymLmj4mIQ4g9reknL4bCVa61St9V8VnwuAPUAoMBE4Pgn1XIwkIkTjl3XQSBYD7jAGw4Exxejl/4Fr5555XyEaMYffkqpOKRUBRAMbgTZa69SKTWkYt6yEaNy0hi+vMj4PuR/8IozPMpOcaAYaTcJQSnkBC4GHtdb51eeu0FprpdQp750ppaYD0wHCw8MbIlQhTma3Q8IaSI8z5qfufKkU7hPNTqNIGEopM0ay+EprvahidbpSKlhrnVrxnOPYqfbVWs8CZoHxDKNBAhYiOwE2fwLtBhpzROz4Bta+YmzrfCncOP/M+wvRBDk8YSijKzEb2KO1fqPapsXANODliq8/OiA8IWo6vB6SNsDKGcbyidOURo6HSR8YM8oJ0cw4PGEAQ4GbgV1KqYpC/DyFkSi+VUrdARwGrnNQfEIYs8z9cA/kHqlad8M8o1T40keM5YkfQPRUx8QnRANweMLQWq8DTjd/4+iGjEWIU4r9ARZMA99w6DsVWnWDsBhoP9jYftEdjo1PiAbi8IQhRKNQmAGuXmB2N5Yz9hm3nQ6uMmajA7jlBwjs5LAQhXA0SRiiZbPb4X+Pw6ZZRtG+S/5lPNDe8H5VGxdvuHmRJAvR4knCEC1PeQksuA3il0FQN8jcZ6wvzoSfH61qd/VsaNvbeIAtyUIISRiiBSnOhrWvGrPQHVd0DAbeDSOeMHoS2780pjLtOq6qbLgQApCEIZozu71qhHXiOpgzHqO+JdBjEkyeWfXM4rj+tzZkhEI0KZIwRPO0cwEsewzCLjLmmEjZaqwf91/oeAkEdAIn+ecvxLmQ/zGiaSvJgbwUaBNlTEQEkLQZFt1pfN7/i/HVqy2MfNIoNa5O9xa3EOJMJGGIpin7ECy8C1IqytkPuhcu/3/G58X3g9kD7vkDcg6DyRk6XOy4WIVoJiRhiKanJAfmToS8ilHXnq2MaU//+gKsJcZD64nvG8X/pACgEHVGEoZoWsqKYd4NUJAKE96DzmOMua+/ugYS1hptgrpC1FWOjVOIZkgShmgatDZGXf94PxQchUkfQt8bq7ZPW2y8CaVM0H6I4+IUohmThCGahl+eqaoMGzqgZrI4LmJYw8YkRAsjCUM0XulxELsIWkXC+vch+ibofYMx/4QQosFJwhCNT0kuHN1mVIi15FWsVDD2ZXD1dmhoQrRkMtGwaFyKsuCdvvDFJON12NHPGV/73yrJQggHkx6GaDzKLfDeAOO1Wb9wuH05+ITAwL8b4yqEEA4lCUM0HqtfgpJs6D4Brp1bVQfKxdOxcQkhAEkYwtGO7YG03cZkRfnJ0PcmmPT+WXcTQjQ8SRjCMXKTYOk/Yf/yqnWuPjDyKcfFJIQ4I0kYwjF+uAcSfzc+e7WBrpfD+LerbkMJIRodSRii/u38FnISofNoOLAKtnxqjNYGuOdPo9KsEKLRk4Qh6texPbDoLuPz6peq1nsHw02LoE0Px8QlhDhnkjBE3SrJhY0fgbsfZB2ETR8Z682e0GMCdBxp3H5y8QQns2NjFUKcE0kYom59PgFSd9RcN+RBuOwFx8QjhKgzkjBE3ck6aCSLy14ynkukbIX+t4FHgKMjE0LUAUkYom7YrDBrpPE58gpj4qJOIx0bkxCiTsk7jOL8aG2U8ABI3Qkzh0JpHnS+VGa5E6KZkh6GOD+7voNFdxqlxzP2GuuiJsO1cxwalhCi/kgPQ5yetQw2zITZlxsjs49b9aKRLMBIFpHj4b5NcM1njolTCNEgHN7DUEp9CowHjmmte1asCwDmAxFAInCd1jrHUTG2WCv+DzbOND6/dcKkRd4hxkC8XtdCx0saPjYhRINrDD2MOcDYE9Y9Afyqte4C/FqxLBpSQRps+wo6jTbedKqu13Xwj90w8T1JFkK0IA7vYWitf1NKRZyweiIwouLzXGAN8HiDBSXg1+ehrADGPAdtekH38RAcbbwiq5SjoxNCOIDDE8ZptNFap1Z8TgPanK6hUmo6MB0gPDy8AUJrAVK2wvavwCcU2vY2EkTnMY6OSgjhYOd0S0op9c9qn7vVfTgn01prQJ9h+yyt9QCt9YBWrVo1REjNm9aw5GFwcoGbv5fehBCiUq16GEopP+BNoJtSqgTYCdwB3HbGHc9fulIqWGudqpQKBo7V03lEddkJEL8c0nbC+DehVYP8TiCEaCJqlTC01rnAbUqpy4FMoDewqB7jWgxMA16u+PpjPZ6rZbLbobwYbGWwaDq4ekNsxV+pdwj0vMax8QkhGp2zJgyl1NvAw9pwfHq0rXUVgFLqa4wH3EFKqWTgOYxE8a1S6g7gMHBdXZ2vxbPbIG0X/PJM1QRG1UVdBVe+Dm4+DR+bEKJRq00PowBYrJS6QWtdVNHLeFZrPbQuAtBaTznNptF1cXxRzd6f4ZsTvt2uPtD/VrDkwrj/gtnNIaEJIRq/syYMrfUzSqkbgTVKqTKgEBkX0bQUZcE3N0LSBmN55DPg3QY6jTImMjI5OTY+IUSTUJtbUqOBu4AiIBi4XWu9r74DE3UkaRN8MxWKjkH7YTDxXSkOKIQ4L7W5JfU08H9a63VKqV7AfKXUI1rrVfUcm7hQ+1cYD7SdzHDHCmgX4+iIhBBNWG1uSY2q9nmXUmocsBAYUp+BifN0ZAPs/BZcveCPt41196yXubOFEBfsnEd6V4yNkAfSjU3cYvj25prrXLzg3vXgJyPghRAX7rxKg2itS+o6EHEByi1VycLND66aBaUF0H4I+IQ4NjYhRLPRWGtJiXNx8Ffja8+rYfD9ENrPsfEIIZolSRhNkbUUSnKNkdqFx2DLZ+DuD5M/Mh5wCyFEPZCE0dRoDd/eAvH/q7m+3y2SLIQQ9UoSRmNjLQVn11NvyzkMGz40koUygbaDqy90uRQue6lh4xRCtDiSMBqD/SugJAc2fwJJGyGwC2TtB6+2ENQFrBZo3QP2LDba9brWuP1kK5dSHkKIBiMJw1HiFhtFAEtyYPPHNbdl7Te+FqYZfwCSN0NIP5j6HYQNMNZJSQ8hRAOShFGfyophx9fQ+zqjfHjC7/DX5+DfHn77b822HkFw7RyIGGYsZ8Ybr8TGLzcShLs/uPk2+CUIIcRxkjDq0+qXYP178Oc7xuuuv71W1WPwCIRB90D/24zPJ85sd3zyol4yL4UQonGQhFFfMvfDhg+MzzmJ8POjxueL/wm+YcZzCFdvh4UnhBDnShJGXSsvMXoWG2aC2QMe3AYbZ0J6HLTqCsP/JQ+qhRBNkiSMuqQ1/Hgf7F4I/hFw6fPg1RpGP+voyIQQ4oJJwqhLv79uJItLnoCRTzo6GiGEqFOSMOrKwjth1wLocAlc8rijoxFCiDonCeNC2e2w9ycjWTi5wuSZYDI5OiohhKhzkjAuhLUU5oyH5E3G8j/3gkeAY2MSQoh6Ir8KX4i4H41k4R0M05ZIshBCNGvSwzgXWhtjK5I3Q3AfWDkDfMPhoR1yG0oI0exJwjgbux1spZAeCwvvMAbhAcR+b3ztMUGShRCiRZCEcSo2KxxcBYfWwNY5UF5Uta1tL4j5OxzdBnlJMPIpR0UphBANShLGiUpy4N3+UJxVtS58sDG73Zh/Q6eRxrp+NzsmPiGEcBBJGMdpbZT02PalkSzaDYKIodBuIHS93NHRCSGEw0nCOO7gKqPkeFiMUVl20L3ybEIIIapp1AlDKTUWeBtwAj7RWr9cLyey29HLn0R5B8OtP51+ilQhhGjBGu2v0EopJ+B9YBzQA5iilOpRH+f6YeldRHsW81mfcZSrRvstEUIIh2rMPx1jgANa60Na6zLgG2BifZzIGtQTm1K8kbKS+369n493fszyxOUUlxfzR8offL33a9KK0jicf5iUwpTaHdNuZV/2Pn5J/IUrF13JwviFpBWlseTgEmx2W31chhBC1CultXZ0DKeklLoGGKu1vrNi+WZgoNb6/hPaTQemA4SHh/c/fPjwOZ9r1m8HeXnlbzh57cOtzdIztvU0e3Jvn3vxd/OnvU97Ovt1xsPsgdaajWkb8XXxRSnFk78/yYHcA6c8hpfZi8iASAYGDyTMO4wrO1yJOnHGPSGEaCBKqa1a6wFtopzmAAAgAElEQVRnbdfUE0Z1AwYM0Fu2bDnnc63ee4wVe9LZcCiLhJxUtN0ZZ694nNyTsBWHY3LJxCPwL/qGdGBr+qmP7+vqS15pXuVyoFsgEztPxEk5ManzJPZm72V54nJWHVmFVVtr7Duw7UBuiLyB0eGjJXEIIRpcbRNGY37onQK0q7YcVrGuzo2MbM3IyNYAWMptWO2aBVuS6B3mR3Q7P5btTuO+eX/Ru10X/nO1N/ll+WSWZJJlyWJdyjrWpayje0B3BgYPxGwyc7TwKNOiphHiFVJ5jnCfcC6LuAwAu7aTX5qPyWTiy7gv+XDHh2xM28hl7S9jxpAZeLtUTd26M2MnX+/9moySDJxNzuRZ8nhl+Cu4OLmwPHE5YyPGYnYyE+AmdayEEPWrMfcwnIF4YDRGotgM3Ki1jj3dPufbw6iNR+ZvZ9G2FDxcnBjZrTXR4X7ceXHHOjl2UXkRc2LnMHPHTALcApgSOYUj+UdIKUxhe8Z27Np+xv29zF58M/4b2vu0r5N4hBAtS5O/JQWglLoCeAvjtdpPtdYvnal9fSaMvOJybp+7ma2HcyrX/frPS+gY5Flnt5FWH1nNw2serpEgBgcP5vrI60FDZGAkmSWZPPX7U5iUiR6BPQhwC2De3nnYtZ2but/EA9EP4GH2OOnYWmuWJixlV8YuJnWehNVupb1ve3xcfOokdiFE09UsEsa5qs+Ecdzn6xM5lFHEnD8TARjZrRUf3TwAF+e6eeFsX/Y+sixZ+Lj40NG34yl/+J/ofwn/Y0H8AjalGfNymHDBVN6ascF3o7QHO7M24uq3k/jcPTX283Hx4f3R79O3dd86iV0I0TRJwqhnTyzcyTebkwCYNrg9/57Ys0HOezolZTbu/ukV/ir4+pTb7eW+xIT05vbeU9ifu5+80jw+3f0pAFd3uZq7+9zNwv0L+eHAD6QVpdErqBeeZk/ySvPo16Yf245to8xWxg3dbmBi54m4ObudNhaL1cKyhGU4m5y5qO1FbEnfQlxWHGlFaVwcejGTOk+Sh/tCNCKSMOqZ3a7JKirjjRX7+HpTEmOj2vL8xCj8PV0wO1X1NlJyS7DbNe0Czt5TOJuDGYX8Fp/BFb2CCTjhPNM+3cTa+AycTIrf/jWS3WlJ7Mj+g81pG+jq24fvVren1ApzbruIjkFehPm7sz1jGy9vepk92XtOe05PsyeltlL8XP0Icg9ib/ZeWnu05uWLX6ajb0d8XH34JfEX0orSiPCJYE/2HpYlLONIwZEaxzGbzHi7eJNtySbUKxQXJxcyizN5atBTmDDh5eLFkJAhOJsa83sYQjRPkjAaiKXcxqVvriUpu6TG+suj2nAku4Q9qfkArHzEeN5hMlX9Zl1us/PaL/vYdjiXx8Z246II400nrTVvrojncHYxl/Zow+COgexIzuX2OVXX5mRS+HuYySwsw9vNmQKL8aruTw8Mo2eo70lxfrnhMM/8sLty+ZKurfj78I78EpfGskOryLXv5fXx19MrqB/pxal4upXT1rMtge6BaK3RaBSKX4/8yqNrH8WmTz34UKHo06oPfVv3xa7tbEzdyJ297qx8Q+zFDS+yIH7BKfcN9Qrlmq7XMCZ8DHtz9jK63WjMTuaz/h0IIS6MJIwGlJxTzNebjrB0ZyqJWcWnbefl6syIbq14fGwkX2w4zMaEbHYk5VZu7xHsw6COgWQVlfLj9qOnPMaz43vw3uoDZBeVMaZ7awosVv46kkN0uD8fTu1HoNfp62D9uiedO+bW7vvz2OXduKRrq1Mmn9isWO765S4KygoYEz6GEK8Qrut2HQdzDxLiFUJkQORpj6u1JqMkgyD3IJYnLkcphZfZi2UJy1h8cHGNtt0DuvNw/4fpHdQbLxevWsUthDh3kjAcpNRqY09qAX7uZtzMTrTxcWV5bDp3f7n1lO37tvPj/8Z359M/Elm6M7XGtpsGhWN2MrEjKRcnk+LJK7rTL9wfgAJLOd5u5/7bt6Xcxv70QmavO8SwLq1oH+hBSk4JxWU2folLY82+jBrtX76qFzfEhNdYtys5j3dX7cfPw8w1/dsR06FuxoDEZsby65FfKwdAfhv/beW2mLYxmJSJHEsOEb4ROJucSSlIwcPswe7M3YxoN4LBIYPJLM6kyFqEv6s/ge6B5JXmMbnzZLIt2WxM28j4juMxSb0wIWqQhNHIfPzbIdxcnHA3OzF/8xHG9w4hJbeE+0d1xqfiB//sdQmUWo0f6P8Y05XwwAt/7nEutNak5JbwS2w6325JYm9aAQCdW3sxdWA4Uwe258CxQq545/ca+z1zZXcm9Amhtc/pH4Sfj1VHVjE3di4+Lj5sTt+Mr4svR4uOolBojH+33mZv/N38SS5MPuN4FSflhE3buKvXXZhNZkqsJYwKH0XvVr0xKRNaa0qsJbV6K02I5kYShrhgSdnF3P/1tsrbZi7OJqw2O3YNU2La0TPUl6e/N56LBPu6sfaxkXX2evGJtNYopSi3lWN2MmOxWigsL8TXxRezk5l92fuIy4qjZ1BP3JzcOJB7gOTCZDJKMsgozuBQ3iHisuJOOm6PwB7cGnUrX+/9mm3HthHqFcqNkTdyS9QtxGbG0s6nHT4uPsTnxGO1W+kR2AO7tksvRTQrkjBEndBaU1JuY3lsGk8u2oWHizOvXt2bMT3aAMZzkQ/XHGRLxYDGuOcvx8Olcb7plJCXwLqUdXg4e9AzqCcrj6xk5o6ZALg5udHRr+NJScXT7Elbj7YczDtYuexicmFQyCAe6PsA7XzaYbVb+fPonxSVF9EjsAchniHn9LBea82G1A1E+EQQ7BWMXdvZnbkbD2cPOvt3BqCgrIDFBxcT6B7I6PDRmE2nPv7x/8+pRalYbBY6+tZNNQLRvEnCEHUut7gMdxcnXJ2daqzXWtNrxi8Ullr594Qoeob6kJhZzFX9QvklLh2AEF93eoUZD9D3pObTqZUXLs4mkrKLCfRycViS2Ze9j1VJq7iq81W08mhFYn4iKw+v5LPdn9HZrzM5pTm4OLkwoM0AdmbsJMuSRYhnCH8d+wuAPq36YLFa2Jezr/KY/q7++Lr6klmSyeCQwfy999/p4t+lRq8kqyQLu7bz/Ibn2Zmxk2xLNr6uvtzb514+j/uclMIUFIr7+t6HUop5e+aRZTHmmW/n3Y6nBz7NkJAhKKUoLi/Gpm0k5iXy/Ibn2Zu9t/I8A9oMIK8sj86+nZkWNY2ooKgG+s6KpkQShmhQpVYbUz/eWNnTAOgZ6sPulPzK5Y9u7o+72YlbPjVGpN87ohMfrDmIp4sTG54aTYHFSoCnC25mp5OO39hsTtvM3SvupsxeBsDFoRczKHgQB/MOsjV9K4fza5bZd1JOuDi54O7sTrh3ONsztlduC/YM5qK2F7E2eS15pXlE+ERwW8/beGvrW+SUGt/P/m36c3WXq/n1yK/E58STVJCE2WTm3r738vZfb9c4z+Quk8kuyabIWkSOJQcn5URKYQoFZQVc1eUqHuz3oBSrFDVIwhANLim7mFGvr6HcpvHzMJNbXA6As0lhtZ/531lbHzfS8i3ERATw7d2DGyLcC2bXdo7kH8Gu7XT0O/nWT35ZPv9L+B/5Zfkk5CVwIPcAaUVptHJvRVf/rvi4+jCq3ShigmMAOFZ8jL/S/2J42HA8zB4cLTzK+9vf59L2l3JJ2CWVo+MtVguvbXmNhfELK0vl9wzsSXvf9kyNnEqvVr1OiiW1MJU3/3qTZQnL8DJ7MfPSmfRp1acevzuiKZGEIRxif3oBbmYnfD3MzN+UxPg+wQR5ubIvrYDx764D4P0b+zEqsjUr9qTjpBTbk3L4+PeEymPcNjSCp67oXmMkuzhZUkESyxOXc23Xa/F1PXm8zKlsSdvCA6seoLC8kBFhI8guzabEWsJ1Xa/jhsgbanWMbEs229K34W52Z0jIkAu5BNFISMIQjU5hqRVXZ9MpE8GOpFz8PVyY82cin/6RgFIwbXAEkW29Gd29Da28XUnMNIo+PjS6C/6eLg64gubhaOFR3tn2DksP1Zxd0sfFh3J7Od5mb3q36s1FbS+iW0A3+rXuR05pDquPrGbxwcWVz2/AmChsztg5RPhGNPBViLokCUM0WT9sS+Hh+VX3+L1dnZl1ywCmfLwBADeziYX3DCEqpHa/VYuTFZcX88rmV7g84nL6tOrDPSvvYduxbSgUoV6hHC06WjmuZWDbgcRlx1FQZozLuan7TVwcejGvb32d+Jx4wChg2S2gG8PDhlNuK+dg3kGGhQ5DoXBxkuTe2EnCEE3aruQ8vtxwmM2HszmUUVS5PtTPnXxLOW5mJx4fG8nV/UKl8m0dOj7OpaCsgPiceN7Y+gY7M3YS6hXKP/r/g/Y+7StLv1isFl7e9DLrUtaRXpx+2mMODh5MuE84N3W/SXoijZQkDNFsfPZHAv/bncbV/cOY2DeET35P4L/LjddYbxwYzjX9wypLpoi6ZbVbSStKI9Tr9IlZa01yYTKphak8svYRvMxedA/ojq+rL/uy95FXlkdKYQp2bWfmmJkMDR3awFchzkYShmi2bHbNqr3HePr7XRwrKAXgs9suYlCHQNxdGv8ruc1Zflk+zsr5pBIr+7L3cc2Sa3A2OTOh0wSKy4vxdfXloX4P4WX2wmKz4O7s7qCohSQM0SLsTsmrfPsK4L6RnSgttzOmRxsGdQx0YGTiRLGZsdy+/HaKrcW4OrlitVsry+Q7K2fui76PUe1GUVheSO9WvR0cbcsiCUO0GCm5JTww7y/+OpJbY/2XdwxkaOfAk26laK3JLio7Yyl4UT8O5R0iNjOW8R3Hs/7oev48+ifHio8RnxNfWX4FjOrEsy6dhZNJeowNQRKGaHEOZhRyzYd/0jvMj7XxVWXavVyd+celXblpUDglZTae+n4Xy3an8fJVvbi2fzvKbHaSsosJ8HSRJOIgOZYcnlr3FN4u3hzJP0JsViwdfTtyRYcr6N+mP/ll+Xi7eNO/TX8p/FgPJGGIFslu15hMih+3p/DQN9trbDM7KcptZ/73fuewDjwzvgd2u2blnnR6hvoS4udOqdWG1WYUVfdybZzFFZsLrTWzd89m9q7ZFJYX1th2S49beOyixxwUWfMlCUO0eDa7ptRqY3NiDr/HZ/DTzlTySsq5Z0Qnbh7Unnu+2sqGQ9n0b+9Pj2Af5m06gs2u6dLai/3HCk973Bsuase0IRHsSc0nKbuEzq29GNezLSaTqizDLi6c1podGTtYcXgFMW1j+DzuczalbSLYMxg/Vz+mRU3jyo5XOjrMZkEShhBnobVmT2oBPUJ8AEjILGL8O79TVGajTzs//NzNZBaWklFQSlGpld5hfqw/lHXKYw3tHIil3I6Lk4nP74ipHM2enm8hIbOIgR0CGjSRrIhLJy2vhJsHR6C1pqjM1uR7Rja7jQ92fMDPh34muTAZMKbxfWfUO7T1bHvW/VMLU1mVtIpNqZs4kHuAlMIUPhzzIYNDatYu01qTX5Zf63IrzYEkDCHOg6XchtnJhJOp6od7qdWGs8lYtz+9gJzicmauPUi5zc7dl3Ri4V/JLPorpbL9hD4hvDMlmvj0Ai578zcALu4SxAOjutTZdLbVHSuwsHrvMbYn5REV4kOp1c4LPxnzenQI8iQh0xj46O3qzJDOgUwf3pH+7QPQWrMtKZcewT5sO5JL33Z+la8l2+0apWiUvSWb3UZifiLf7vuWb/Z9g9lkxsvsRWRAJO192jMlckrlAMFyWznF1mI2pG7g0bWPAhDkHoS3izcJeUb9sgmdJnBdt+voFdQLheLZP5/lhwM/0Mm3E6+PeJ31R9ezP3c/o8NHc3HoxeSW5uLv5s/Kwytp5dGqWRRxlIQhRANKyi4mr6Sct1bGs3LPMToGeXKo4gd1iK8bOcXleLo6s+7xkeddvv3AsUIufXMt7QM8eOSybjz7426KS22U2U4/Ne3pfH57DEk5xZUzJgIEerpwz4hOHM218OkfCQR5uXBjTDj3j+pSbzMpXqj9OftZtH8R+3P2syV9CzZtw9nkTKR/JLuzdp/U/q2RbzGy3UhMysTqI6t5cPWDNbY7m5yx2q3nFMP03tM5lHuIEK8QHoh+ADfnup2quCFIwhDCAcptdh6ev52lO1MBeHhMFx4e05X1B7OY8vEG3MwmxvUMpneYL85OJq4bEEahxYpNa1p7n/4HzY6kXO6Yu4XMwtJTbg/ycuGxy7uxIzmPDYeyKLRYWXTvEJJzSujf3p/iMhtFpVYOZxVX1uSqrRsHhvPSpJ6U2zTlNjuejfTWlsVqISEvgWn/m0aJtQSA6NbRbDu2DU+zJ99c+c1JpUnic+KxWC28ufVN2nq25VjxMfq16cfdve9mZ+ZOvov/jjHhY4huHc2nsZ+yM2MnWmtyS3OJ8IngSMERDuQeqDyeu7M7t/W8jas6X0UbzzYNefkXRBKGEA5UarVht1N5i0drzdUf/nnSWJHjPFyc+FvvEArLrLTz96Bzay/eXBHP/7uqF7tT8vjv8n208XFlzm0xzN+cxMaEbP49wZg9L8jLhY6tvCqPpbXGrqlxW626rzcd4clFuwC4fWgHYjr4c1mPtqzed4w1+zK4f1Rn3F2c8HJx5op3fmdvWgGhfu6k5Bo/hCPbevO3PiHcc0knTKc5hyNllmSSVJBEdOtowHh24WxyppVHqzo/1/Hk4ebsxvZj23n8t8fJKc1Boejbui+zL5+Ns3JmVdIqSq2lhHqHUm4rRynF6iOrSStOI8QzhDaebUjIS+Devvc6ZHIrSRhCNDIFlnIOZxXTPtCDrYdzWLozlQVbk2u1b3S4H69f26dGYrgQR7KK2ZWSx+jurc94i8xSbuPWzzax4VD2SdumDW6Pv6cL0eH+DO8S1CifdzS0o4VHmbdnHjsydlTOqhjoFlg5vW5tjAgbwZsj38TZ1HA9uSaRMJRS1wIzgO5AjNZ6S7VtTwJ3ADbgQa318rMdTxKGaGpsdo0C1sZn8N3WZF6+uhdWm2bOn4nsTskjt6Sc6cM7cnnU2d8Cqi+lVhvxaYV0au3JvI1H6NjKk4e+3k5BadW9/vG9g3l2fA9a+zS9+/f1QWvN+9vf56OdH+Ht4s0/+/8TLxcvEvMS8XfzJ78sn/Y+7RkaMpT1qevZk7WHyIBI1h9dz7fx39LZrzMP93uYS9pdctpzpBel8+WeL9mUtonMkkyeG/wcw8OGn1e8TSVhdAfswEfAo8cThlKqB/A1EAOEACuBrlpXFJ45DUkYQjSMfEs5mxOyySwsZcmOVNYdyCQ63I9F9wyRnkY1iXmJhHqFYnYy16q91poXN7zIov2LcDI5MWfsHNKL00nISyDMK6zyecyC+AXkllbd3hwXMY5bom6hZ1DP84qzSSSMyiCUWkPNhPEkgNb6PxXLy4EZWuv1ZzqOJAwhGp7WmleX7+PDNQd5aXJPRnRrTaifVJ69EFklWdy49EaOFh09bZsRYSO4redtRLeOvuAkXduE0Thfd4BQoPqrHMkV606ilJoOTAcIDw+v/8iEEDUopXh4TBfm/plY+ZrupqdHn/GtL3Fmge6BzL58Nm9sfYMufl24LOIyfk74mQC3AEa2G0lmSSZRgVENXpyx3nsYSqmVwKluwD6ttf6xos0aavYw3gM2aK2/rFieDSzTWn93pnNJD0MIx/l+WzL/mL8DgN5hvrxxXV9cnEy0CzB6G1rTKN+qEo2oh6G1HnMeu6UA7aoth1WsE0I0UpOjw5jUN5RZvx3i1eX7GPPG2pPa/PPSrjwwugv/253KjuQ8o7BjuY1+7f1l1sQmoLE+w4gC5lH10PtXoIs89BaiaTicVcSy3Wn8uP0oe1Lza7VP59Ze3DQwnFuHdqjn6MSJmsRDb6XUZOBdoBWQC2zXWl9ese1p4HbACjystV52tuNJwhCi8dl6OJvwAE9czSYue+M30vItOJsU706J5te9x+gR7ENqXglz1x+mzGrngVGduXdEZ5lutwE1iYRR1yRhCNG45ZWUA8abVX4eLjW2Hcoo5Ip3fsdSbueuizvw9JU9HBFii1TbhNE4K4oJIZolX3czvu7mk5IFQMdWXqx+dAQdgjz5+PcE5m08QnJO8VmPmZJbwiv/21tjlkVRPyRhCCEajWBfd96dYtSAeur7XQx7ZTW7U/JOapddVMaMxbH8cSCTWz/dxIdrDnL7nM0cyijEZm8+d00aG7klJYRodJJzirll9qbKEvGf3XoRIyNbU2q18eeBLG6bs7lG+1uHRDB3fSLHf5xFh/vx6tW96dLGG6iaulecmjzDEEI0ectj03jw622UWu0EebnWKO/eK9SX8EAPhnQKZOrA9nyxPpH/+zEWADezCW83M3cM68Cu5DyW7jLKzbubnXhnSjQD2vvj73nybbGWShKGEKJZ2J9ewM2zN5GWb6lct/TBYUSFnH4K1Q2Hsnjhpzhij57+ld7Rka3RQHiABzMqSsW3VJIwhBDNSkZBKRe9tJInx0Xy90s6nbW91povNxwmwNOV6HA/0vItRAR60u+FFSe1feqKSG4eFEFeSTl+HubznhWxqZKEIYRodorLrLg5O13w84gdSbks+isZu4bNidnsTSvA191c+drvxV2CePuGaAJayG0rSRhCCFELxWVWvtpwhI9/P8SxgqpnJN2DfVj6wLAW8bBcEoYQQpwjm12z/mAWe1LzeennPUSH+7Hg74NxdmreIxBk4J4QQpwjJ5NiWJcg7hjWgSkx4Ww7kkvnp5fx+fpESsrOWMquRZCEIYQQJzCZFM9PjKJ9oAcAz/4Yy8Pzt9Gc7sicj8Y6gZIQQjiU2cnE6n+OYP+xQn7aeZR3Vx1geWw6Y3s6bn51R5MehhBCnIbJpOjW1puHRnehtbcrd3+5lYgnlrI7JQ97CyxBIglDCCHOwtnJxJvX961cHv/uOoa9soqcojKOVRtQ2NzJW1JCCFFLZVY725Nyef6nWHanVI0if/P6PkyODnNgZBdG3pISQog65uJsIqZDAD89cDFPjIusXP+P+Tv4fH2iw+JqKNLDEEKI82Qpt1FuszPubWPip2+mD8TL1UxbXzdHh3ZOpIchhBD1zM3shLebmfdu7EdmYSlj3viNK975Ha01Vpudo7kllNvsrN53jNS8EsCoiXX8F/WjuSV8vj6R7KIyFu+o/fznjiI9DCGEqANvr9zPmyvjAegR7ENhqZUj2cUEerqQVVSGl6sz3m7OpOYZD8l7hvqQmFlMYam1xnGGdg7kyzsGolRVSRK7XfPnwSwGRPjXS2FEKQ0ihBANzG7XfL4+kRlL4irXdQ/2YUpMO56tmKujuo5Bnvi4m0nOKeGiCH+W7U4DwMfNmbsu7sj4PiE4KcWibcm8tXI/TibFuJ5tee5vUbg4m/Bxc66RWM6XJAwhhHCQQxmFZBSUMrBjYOW6GYtjmfNnIr//ayRh/u5Y7Rpnk6rxA99SbmPun4nM23SEw1lnn88cjJ7K13cNwtvNfN7xSsIQQohGxGqzk5RTQocgz7O21Vrz3dZk/rt8H8cKSukY5Mnc22Pw8zCzel8Gc/5IICLQk0XbUgB4fGwk94w4+xwhpyMJQwghmrkyq50J761jb1oB8+4cyJDOQed1HHlLSgghmjkXZxOzb72Ii7sE0dqn/l/lleKDQgjRhIX6ufPFHQMb5FzSwxBCCFErkjCEEELUiiQMIYQQteLQhKGU+q9Saq9SaqdS6nullF+1bU8qpQ4opfYppS53ZJxCCCEc38NYAfTUWvcG4oEnAZRSPYAbgChgLPCBUqrux8MLIYSoNYcmDK31L1rr44VUNgDHC8pPBL7RWpdqrROAA0CMI2IUQghhcHQPo7rbgWUVn0OBpGrbkivWCSGEcJB6H4ehlFoJnGrW9Ke11j9WtHkasAJfncfxpwPTAcLDwy8gUiGEEGdS7wlDaz3mTNuVUrcC44HRuqpOSQrQrlqzsIp1pzr+LGBWxbEylFKHzzPUICDzPPdtquSaWwa55pbhQq65fW0aObSWlFJqLPAGcInWOqPa+ihgHsZzixDgV6CL1tpWj7FsqU0tleZErrllkGtuGRrimh1dGuQ9wBVYUVHid4PW+m6tdaxS6lsgDuNW1X31mSyEEEKcnUMThta68xm2vQS81IDhCCGEOIPG9JaUo81ydAAOINfcMsg1twz1fs3Naj4MIYQQ9Ud6GEIIIWpFEoYQQohaaXEJQyk1tqKg4QGl1BOn2O6qlJpfsX2jUiqi4aOsW7W45keUUnEVRSB/VUrV6p3sxuxs11yt3dVKKa2UavKvYNbmmpVS11X8XccqpeY1dIx1rRb/tsOVUquVUtsq/n1f4Yg464pS6lOl1DGl1O7TbFdKqXcqvh87lVL96jQArXWL+QM4AQeBjoALsAPocUKbe4GZFZ9vAOY7Ou4GuOaRgEfF53tawjVXtPMGfsOoYzbA0XE3wN9zF2Ab4F+x3NrRcTfANc8C7qn43ANIdHTcF3jNw4F+wO7TbL8Co8SSAgYBG+vy/C2thxEDHNBaH9JalwHfYBQ6rG4iMLfi83fAaFUxSKSJOus1a61Xa62LKxarF4Fsqmrz9wzwAvAKYGnI4OpJba75LuB9rXUOgNb6WAPHWNf+f3v3G1p1Fcdx/P0xTSVLI3vQA2ESmZHlrAy1IiHxQYhlGGlKWYP+kAaWhdSDpEeCRUQQZRhSiFFWNky0B2KKKWpillYSFSL9QTJnJZStTw/O2byO6X7b7u71bt8XjP12f+ee3znbfvvunHPv9xTps4GL8vFQ4KcKtq/sbG8Bjp6lyB3AW052AMMkXVau6/e1gFEkqWFrGadMuk3AJRVpXc/obCLHBk4lgaxVHfY5D9VH2P64kg3rQUV+zqOAUZK2SdqRMy3UsiJ9XgLMlXQYWA8sqEzTqqZHE7dW+53e4RwiaS5wA3BrtdvSkyT1I6WkmVflplRaf9K01GTSKHKLpGtsH5uzjWcAAARaSURBVKtqq3rWbGCl7RclTQTeljTG9n/Vblgt6msjjCJJDVvLSOpPGsb+VpHW9YxCiRwlTQGeBabb/rtCbespHfX5QmAMsFnSj6S53sYaX/gu8nM+DDTaPum0z8xBUgCpVUX63AC8C2B7OzCIlKSvtyqcuLUr+lrA2AVcIWmkpPNJi9qNbco0Avfn45nAJufVpBrVYZ8ljQNeJwWLWp/Xhg76bLvJ9nDbdbbrSOs2023vrk5zy6LI7/Za0ugCScNJU1TfV7KRZVakz4eA2wAkXUUKGEfovRqB+/KrpSYATbZ/LlflfWpKyva/kuYDG0mvsHjTKdHh88Bu243ACtKw9TvS4tKs6rW4+wr2eRkwBHgvr+8fsj29ao3upoJ97lUK9nkjMFXSAaAZeMp2zY6eC/b5SeANSQtJC+DzavkfQEmrSUF/eF6XeQ4YAGD7NdI6ze2kXUpPAA+U9fo1/L0LIYRQQX1tSiqEEEIXRcAIIYRQSASMEEIIhUTACCGEUEgEjBBCqFEdJSNsU/YlSXvzx0FJnX7DZgSMEEpI+ix/rpN0b5nrfqa9a4XQDSuBQilebC+0XW+7HngF+KCzF4uAEUIJ25PyYR3QqYCRMwOczWkBo+RaIXRJe8kIJV0uaYOkzyVtlTS6nafOBlZ39noRMEIoIenPfLgUuCUP3xdKOk/SMkm78j4DD+fyk/NN2QgcyI+tzTfrfkkP5ceWAoNzfatKr5XflbtM0leSvpR0T0ndmyWtkfSNpFUtmZMlLdWpPUxeqOT3KJzzlgMLbF8PLAJeLT2ptN/NSGBTZyvuU+/0DqETFgOLbE8DyH/4m2yPlzQQ2Cbpk1z2OmBMzs8E8KDto5IGA7skvW97saT5eTqgrbuAemAsKc/RLklb8rlxwNWktNzbgJskfQ3MAEbbtqRhZe99qEmShgCTOJW1AWBgm2KzgDW2mztbfwSMEIqZClwraWb+eigpcd8/wM6SYAHwuKQZ+XhELne2FBw3A6vzDfyrpE+B8cDxXPdhAEl7SVNlO0h7eKyQtA5YV4b+hd6hH3DsDP+YtJgFPNbVykMIHRNpmF+fP0babhlh/NVaSJoMTAEm2h5L2uFuUDeuW5o5uBnon/dpuZG0wdc0YEM36g+9iO3jwA+S7obW6c6xLefzesbFwPau1B8BI4T2/UFKg95iI/CopAEAkkZJuqCd5w0Ffrd9It+cE0rOnWx5fhtbgXvyOsmlpG04d56pYXnaYajt9cBC0lRW6INyMsLtwJWSDktqAOYADZK+APZz+i6Es4B3upqAMaakQmjfPqA533QrgZdJ00F78sLzEeDOdp63AXgkrzN8S5o+arEc2Cdpj+05JY9/CEwk7Ult4Gnbv5zh1S2QAtlHkgaRRj5PdK2LodbZnn2GU+2+1Nb2ku5cL7LVhhBCKCSmpEIIIRQSASOEEEIhETBCCCEUEgEjhBBCIREwQgghFBIBI4QQQiERMEIIIRTyP4htgpjro5YRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"x optimization\")\n",
    "x = np.arange(len(adashift_x_history))\n",
    "plt.plot(xx, adashift_x_history, label='adashift')\n",
    "plt.plot(xx, x_history,label='adam')\n",
    "plt.plot(xx, ams_x_history, label='amsgrad')\n",
    "plt.legend()\n",
    "plt.xlabel(\"iterations\")\n",
    "plt.ylabel(r\"$x_t$\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}