latent_ode_tf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import numpy.random as npr\n",
    "import tensorflow as tf\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.contrib.eager as tfe\n",
    "\n",
    "keras = tf.keras\n",
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from neural_ode import NeuralODE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the pre-canned noisy data of the spiral.\n",
    "latent_dim = 4\n",
    "nhidden = 20\n",
    "rnn_nhidden = 25\n",
    "obs_dim = 2\n",
    "nspiral = 1000\n",
    "start = 0.\n",
    "stop = 6 * np.pi\n",
    "noise_std = .3\n",
    "a = 0.\n",
    "b = .3\n",
    "ntotal = 500\n",
    "nsample = 100\n",
    "orig_trajs = tf.cast(np.loadtxt(\"latent_ode_data/orig_trajs.csv\", delimiter=\",\").reshape((nspiral, ntotal, 2), order=\"F\"), dtype=tf.float32)\n",
    "samp_trajs = tf.cast(np.loadtxt(\"latent_ode_data/samp_trajs.csv\", delimiter=\",\").reshape((nspiral, nsample, 2), order=\"F\"), dtype=tf.float32)\n",
    "# orig_trajs = np.loadtxt(\"latent_ode_data/orig_trajs.csv\", delimiter=\",\").reshape((nspiral, ntotal, 2), order=\"F\")\n",
    "# samp_trajs = np.loadtxt(\"latent_ode_data/samp_trajs.csv\", delimiter=\",\").reshape((nspiral, nsample, 2), order=\"F\")\n",
    "orig_ts = np.loadtxt(\"latent_ode_data/orig_ts.csv\", delimiter=\",\")\n",
    "samp_ts = tf.cast(np.loadtxt(\"latent_ode_data/samp_ts.csv\", delimiter=\",\"), dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnXd4VGX6v+8zk0kmvfdCIJQAoQRCb4qFogJixYLu6mJfd131h7t+17a7uura0FWxsIhYVpCOoggqvQQIPZCQ3nufZMr5/ZGEDWQmdTL1va8rVyA5Z84zk3M+5znP+xRJlmUEAoFA4DgorG2AQCAQCMyLEHaBQCBwMISwCwQCgYMhhF0gEAgcDCHsAoFA4GAIYRcIBAIHQwi7QCAQOBhC2AUCgcDBEMIuEAgEDoaLNQ4aFBQkx8bGWuPQAoFAYLckJyeXyrIc3Nl2VhH22NhYDh8+bI1DCwQCgd0iSVJWV7YToRiBQCBwMISwCwQCgYMhhF0gEAgcDKvE2AUCgaAraLVacnNz0Wg01jbFoqjVaqKiolCpVD3aXwi7QCCwWXJzc/H29iY2NhZJkqxtjkWQZZmysjJyc3Pp379/j15DhGIEAoHNotFoCAwMdBpRB5AkicDAwF49pQhhFwgENo0ziXorvX3PQtgFAoHAwRAxdkGfsv5oHq9tSyW/soEIP3eemjWEBYmR1jbLLPT0vTnyZ+LMzJ07ly+++AI/Pz9rmyKEXdB3rD+axzPfnqBBqwcgr7KBZ749AWD3QtbT9+bIn4mzIssysiyzdetWa5tyERGKEfQZr21LvShgrTRo9by2LdVKFpmPnr43R/5MHJk33niDhIQEEhISeOutt8jMzGTIkCEsXryYhIQEcnJyiI2NpbS0FICXXnqJIUOGMHXqVBYtWsTrr79uUXuFxy7oM/IrG7r1c3uip+/NkT+TvuaFTac4nV9t1tccFuHDczcM73Cb5ORkVqxYwYEDB5BlmQkTJjBjxgzOnz/PypUrmThx4iXbHzp0iLVr15KSkoJWq2XMmDGMHTvWrHZ3hvDYBX1GhJ+70Z8rJIn+S7cw5ZUdrD+aZ2GrzIOp92bq573dT2A9du/ezY033oinpydeXl4sXLiQXbt20a9fv3aiDrBnzx7mz5+PWq3G29ubG264weI2C49d0Gc8NWvIJfHkVvSyDHQvvmxrC47G3pu7SslTs4b0yX4COvWsLY2np6e1TTCJ8NgFfcaCxEheXjiCSD93JEBpJDe3K/Hl1gXHvMoGZP53Q7Cmt3/5e4v0c+flhSM6vdn0dD+B9Zg2bRrr16+nvr6euro61q1bx7Rp00xuP2XKFDZt2oRGo6G2tpbNmzdb0NpmhMcu6FMWJEZeFK3+S7cY3aaz+HJHC459IYhdfTpo+966g6n9bO2pRNDMmDFjuPfeexk/fjwA999/P/7+/ia3HzduHPPmzWPkyJGEhoYyYsQIfH19LWUuIIRdYEEi/NzJMyLincWXLbngaK10RJEGads88cQTPPHEE5f87OTJk5f8PzMz8+K/n3zySZ5//nnq6+uZPn26WDwVOC5PzRqCu0p5yc+6El+25IJjX6Yjrj+ax5RXdhhdOBZpkI7FkiVLGD16NGPGjOGmm25izJgxFj2+8NgFFqPV8+xuuMGSC4599XTQmUcu0iAdiy+++MKqxxfCLugTTMWLexKX7ukNoTNbjNHTcFFndLZO0FfHFTgnQtgFZsdc8eK6Rh2ltY1U1GsJ8HRl6Zx4qjVaqht0XCit4+WtZ2jQ6tFo9TTpDOhlMMgyBoOMQZZRKiQKqjQcz6m6JMXyyW9S2H6miHGxAfi6q/D1UJGSU8mXB7IpqmlsZ4c5ng4688hFGqTAnAhhF5idrmSxaLR6MsvqyClvIL+y+SuvsoGCKg0lNY2U1jZS36Q39vIXUasUqFVK3FVKVEoFSoWEQmougFJIEgZZJqO07qKot6IzyGw+XsDm4wVdej8NWj1/+PoYL246zW+nxnLvlP54uXXv0unMI+/tU4lA0BazCbskSUrgMJAny/L15npdgeXpbdqdKe80r7KBOz7aT2ZpHflVlw4RcFUqiPBTE+7rTmKMH0Febi1frgR4ujZ71u4qfNxV+KhVqFWKLvWsNpViCXD42aupbtBy24f7Kalt76lfTnl9E6//cI7XfzhHiLcbA4I9GRDsRVywF0PDvIkP9yHA09Xovt3xyGWgsErDH74+xmvbUoXAC7qNOT32x4EzgI8ZX1NgYXoaRpFlmYIqDSfyqvByc6GmUdduG0mC+iY9EwYE0j/Ik9ggT2ICPIj0cyfQ0xWFwvwDFUx5ypF+7hdvHqVdEPW2+KhdmD44mAsltXx3ooCKeu3F34V4uxEf7sPISF9GRvkyOtqPEB91px755Z+7sercjvYX2A5//etfmT59OldffbXVbDCLsEuSFAVcB/wdeKKTzQU2TFeLgZp0Bk7mV5GcWcGmlHxO5ldhaIl4SC1fbQMgahcFLy8cwY1jojq1wZyFOl3xlE2JvylqNDpev2XUxf+X1DSSWljD2cJqzhTUcLqgmvd/SUff8oGE+agZ28+fcbH+fHj3WIaG+6BscxNbfzSPP/03pV3IqJUGrZ4XNp1CozWIPHc74MUXX7S2CWbz2N8Cnga8zfR6AithSuDyKhs4ml3B3vQy9qSVkpxVQaPOALQXcTcXBTcnRbHzbEmPhlCYs1CnK7FrUz1tTHF5pkqwtxvB3m5MHRR08WcNTXpOF1RxLKeKlJxKkrMq2HKiOabv7ebCuP4BTBkYRJNOz9vbz5sU9VbaPhVcPEYfVt/aK31RvZuZmcmcOXOYOnUqe/fuJTIykg0bNpCamsqDDz5IfX09cXFxfPrpp/j7+3Pvvfdy/fXXc/PNN7N06VI2btyIi4sL1157Lc899xwjR47k3LlzqFQqqqurGTVq1MX/m4teC7skSdcDxbIsJ0uSdEUH2y0BlgDExMT09rCCPkIpSSZF5sZ/7wUgPsybOybEMC42gBc3naaw+tJ4uUZn4MsDORhkudsXV1+0D+gsxfJy8fd1V1HXpEOrb/85dDVTxd1Vydh+AYztF3DxZ3mVDRzOLOdARjn70svYcba4B+/mUkSe+//oy+rd8+fP8+WXX/LRRx9x6623snbtWl599VWWLVvGjBkz+Otf/8oLL7zAW2+9dXGfsrIy1q1bx9mzZ5EkicrKSry9vbniiivYsmULCxYs4KuvvmLhwoVmFXUwj8c+BZgnSdJcQA34SJL0uSzLd7XdSJbl5cBygKSkpI7dE4FVkGW5Q89x2aJEJsUFEuTldvFnj6w+YnTbrnZwvNzDMvXE0NcCdrn4t9qVV9lw8WYX2QMP0JgH+Y8bRwCQW1HP1H/u7PQ13FVK3FwUVDa099pFnvv/6MueQv3792f06NEAjB07lvT0dCorK5kxYwYA99xzD7fccssl+/j6+qJWq7nvvvu4/vrruf765pyS+++/n1dffZUFCxawYsUKPvroo17ZZoxeC7ssy88AzwC0eOxPXi7qAtvFYJA5mlPBdycK+f5UocntlJKE3iBfIurQtfi0qYvLmId1eVin7XEsSU8bfLWlMw8yyr954bijz89VqWB2Qhijonz55/epIs+9A/qyetfN7X/nvVKppLKystN9XFxcOHjwID/99BNr1qzh3XffZceOHUyZMoXMzEx+/vln9Ho9CQkJvbbvckSvGCflXFENr35/lmmv7uSm9/excl8mg0K8uH1cNGqX9qeFXpaNtso11v/FGMYuLmMelkxzzL4t9ipgXen/Yqp/ztI58TxxzWAGBHuy7mgeL2w+TXSAO/4ezY/sot1veyzZU8jX1xd/f3927doFwKpVqy56763U1tZSVVXF3LlzefPNN0lJSbn4u8WLF3PHHXfwm9/8xuy2gZkLlGRZ/hn42ZyvKTAf5XVNrDuax5rkXM4UVKNUSEwdGMSTswZz1dBQfNTNojFxQKDRLA1jnvfl8WmFiRi9sYvLlCcl0yxc9p7W1xUPsrPF3d9fNYj0klo2HMtn3dFcKuq1uKuUTBgQQJS/O7IsX8znd/a2v5au3l25cuXFxdMBAwawYsWKS35fU1PD/Pnz0Wg0yLLMG2+8cfF3d955J88++yyLFi3qE9skuZPV+L4gKSlJPnz4sMWP6wh0N/ZrMMjsTS/jq0PZ/HCqiCa9gVFRvtyYGMl1IyMI9nYzcpTmwh5jZ4YEZLxyXYf2Gbu4jHmXU17ZYTLHfM/SmSaPYS+Y+/3JskxyVgVrj+SxKSWf2kYdg0K8WDQ+BrVKwUubz3Tpc7cnzpw5w9ChQ7u8vb3c3NasWcOGDRtYtWqVyW2MvXdJkpJlWU7q7PVFSwE7oitFLK0ncbVGy5rDuazan0VGaR1+HirunBjDbeOiiQ/rvIasp02pulMa7+j9Ucz9/iRJIik2gKTYAJ69bihbjhfwxcFsXtx82ujahDOmQ5pjbaSveeyxx/juu+/YunVrnx1DCLsV6a53YSxm20rrRTwyypdP92Tw7ZE86pv0jInx4w+3j2bW8DDURmLhpmzojSh19eJy9P4offn+PN1cuHVcNLeOi+ZkXhXXL9ttdDuRDml7LFu2rM+PIYTdSvQk57azizSvsoGr3vgFlVLBDSMjuHdyLCOiTI/k6ooNfS269uBh9QZLvL+ESF+T2TX+nq4YDPLFdg32EqpoS9t1BGehtyFyEWO3Eqbir37uKjzdXIxeeKb2aUUCHp05kMWTYk3Gzrtig6PEuJ0JY2sbreGZ+DBv/njNYOobdfx53Um7isNnZGTg7e1NYGCg04i7LMuUlZVRU1ND//79L/mdiLHbOKa878oG7cVClMs96M5K333dXfjTtV2P34qpPY6DsSesJ64ZjEIB7/yUxgOrklEppXbVtLYeh4+KiiI3N5eSkhJrm2JR1Go1UVGd91UyhRB2K9HVxlNtL7wFiZGkl9SybEea0W2rGtp3VOyJDaKa0T4xFfYxGOClzaeNVq6Cbd/IVSpVO69V0DmiQMlKdLWwB5ovvPNFNdy/8hDLdqRhqrttdwW5p8OlBfbDs+tP8OQ3KSZFHcSN3BERHruVMPboXN+kM9rFz91Vyay3fsXT1YWnZg0h2MuN5zae6nUanaNnpTg764/msXp/ttF6hLaMjvajUafHzaVrjobA9hGLpzbEs+tP8Pn+bKO/83RV8szcodw1sR9gn9kNAsvS2WJ7qI8boT5qjudWERvowQvzE5gxOPji78U5ZnuIxVM7ZOdZ0wtEdU16/r7lDF5uLhdjqeIiE3RER7HztplPv5wr4YWNp7jn04MsTIzk/64fxi/nSnrdAlfcGKyHiLHbEF3tkigQdAVTsXMJLgnbzRgczNbHp/HYzIFsTMnn6jd+4YVNpzptYNYRremXeZUNyPzvxnB5EzlB3yCE3QaQZZkNx/JMLoq2xZYzGAS2hbHFcQm4c2JMO89ZrVLyp2uHsOmxqUT5uxtd64Gun39d6Wwp6DtEKMbKlNU28ud1J9h2qoiYAA+KqjUXR84ZQ2QwCLpKdxfH2zaYM0VXzz9RI2FdhLBbkW2nCvnztyeo0eh4Zk48908bwKaUfJODjS9/hBYIOqOrazHGKlcvx0Xq+vknaiSsixD2PqCz1rr1TTpe2Hiarw/nMDzChy9+N5ohYc1zwBckRvLHr48ZfV0ZMZFe0D26uoDZUYO5i0gSk+ICu3RcR+/caesIYTcznbXWLahsYO3RPNJLann4ijj+cPVgXC+bWGTK24kU3o6gG3Sn0VxXQiQ6g8x17+zig7vGkhQb0On2apXi4rH93FU8P2+4cEwshBB2M9NZa91/bkslyMuNVb+dwNRBQUa3E96OwBx0Z7hzV1tcSEgs+mg/L85PYNH4GKD9U8GV8cGsTc675NgdrRsJzI8Q9jZ0N+/W2PZd8Xy+e3xah90XRUWowBx0ZwGzswZzrbgoJCbHBfHMtydILaxhZKQvf1l/8pKnAmPVrrbebMzREMLeQnf7o5va3tdd1XFfDl91l1rqigIkQW/pzgLm5c6EqXr0wmoNu5fO5JXvzvDRrgzULgo0l3njpvYVGTGWQ+Sxt9DdvFtT20sSJpt7uauUPD073jwGCwSd0N0mbwsSI9mzdCYZr1xncj3Hz0OFUiExPMIXX3dVO1HvCJERYzmEsLfQ3bxbUz+vqNeiVrX/WCP93G16oIHA8ViQGMnLC0cQ6eeORPfOwadmDUGlbF8xV6vR8ez6Ezzz7QmqOngyvXxPsUZkWUQopoXu5t2a2l6CS6r21C4KXrlppBB0gVXoaUhvQWIkz2881S6sqDXIfHkgx2idRSvuKiU3jY1k59kSsUZkJYSwt9DdTBRTi02Xn+4anYE/fH2M17alipNbYFeY8sg7EnWlJPHktYO5b9qAvjJL0AVEKKaF7j62LkiM5KaxkShb5jB29kGKJkgCe8PU06rSxOzRYC83vNQurNib2aXUSUHfIfqx95CulGAbQwyKFtgLxs7x1jDL5XnqrUOxBwR7cudHBwj0cuW/D04ixFttDdMdFtGPvY8xVYjUOhneFCLlq3vIskxRdSN5lfUUVTdSVK2hqLqR4moNFfVN1DXpqW/SUdeop65RR5PegARIktTSLVNCpZTwVrvgrVbhrXbBR63Cz0NFuK87EX7qi9/DfNS4KMVDbCsd1VMk9QswWWex8r7x3PnRAX77n0N8tWQSXm6mZUb0bO8bhMfeQ/ov3WJSwCM7qOITHrtpims0pORUca6ohvTiWtJKakkvrqWu6dIbqEopEeKtJsDTFU83JZ6uLni4ueDpqsTVRYEsg4yMLINBBq3eQK1GR7VGS41GR41GS3ldE9UaXbvXHRDkxaBQL4aEejMo1JvhET5E+bsjmQg/CIyzM7WY+1ceZnJcIJ/cM65d2www/UQgssdMIzz2PibcV01+labdz1uF29RJK1K+mtHpDRzPq+JIVgVHcyo5ll15yc0w3FdNXLAXtyRFExfsSXSAB6E+akJ91Ph7qMwitHWNOgqqGsir1FBQ2UBmWT3ni2pIya1k8/GCi9uF+rgxtp8/Y/sFMLafPwkRPsKz74Qrh4TwysIRPLXmOH/dcJKXF45o9zfrTssDQfcQwt4DZFlmcKh3O2FvK9yiLUB7csrr2XW+lF/PlbAnvZSaFo850s+d0TF+/GZKLKOj/YgP9+nw8d1ceLq5MDDEm4Eh3u1+V9eo43xxLcdzKzmcWUFyVgVbTxQC4OuuYvrgYGbGBzN9UDCBXp1XEjs6xkIqtyRFk1VWz7s704gP8+beKf0v2Uf0bO87en31SJIUDXwGhNIcXl4uy/LbvX1dW+bDXy/w87kSpg0KIr24loIqjVHhFm0B4HxRDZuOF7D1RAFpxbVAszc+NyGcaYODGN8/wCYX2DzdXBgd7cfoaD8WT4oFoLBKw8HMcn5JLeGXc8VsSslHkmBMjD/zRkVw/chwpxT5jtpxPHHNYFKLanhpyxkGhXozZeD/Gt+Jnu19R69j7JIkhQPhsiwfkSTJG0gGFsiyfNrUPvYcY19/NI8/fH2MeaMieOu20Si6Ms/Oycguq2fDsTw2Hy8gtagGSYIJ/QO4ZlgYMwYHERfsZfcxa4NB5mR+FTvOFvP9yULOFtagVEhMGxTEjYmRzBoehtpEawlHY8orO0y2md6zdCa1jTpufG8PFfVNbP39NEJ8mm/kIsbefSwWY5dluQAoaPl3jSRJZ4BIwKSw2yvJWRU8vfY4E/oH8Poto4Sot0GnN/DT2WI+35/FrvOlAIyL9eeFecOZkxB28WJ2FBQKiZFRfoyM8uMPVw/mbGE164/ms/FYHo9/dYxAT1cWjY/hzokxhPs6tgfaWUjFy82Ff985hnnv7uGxL4+y+v4JF9co3Fz+17Pd30PFczeInu3mwKyBTEmSYoFE4IA5X9cWyK2o54FVhwn3VfPBXWONrvI7IyU1jaw+kMVXB3MorNYQ5qPmj1cP5pakKKd6pI4P82HpHB+enjWEvell/GdvJu/9nMb7v6Qze3gYD18Zx/AIX2ub2Sd0JaQyKNSbvy1I4E/fpPDvn9OJCfBo561rtKJnu7kwW7qjJElewC/A32VZ/tbI75cASwBiYmLGZmVlmeW4lkCj1XPT+3vJLq9n3cNTGBjiZW2TrE5BVQMf/nKBLw9m06gzMH1wMHdNiGFmfIjIGGkhp7yez/dn8cXBbGo0OmYND+XxqwYzLMLH2qaZle6EVH7/5VG2nijA38OVktrGdq/VlXRgZ85972ooxizCLkmSCtgMbJNl+Y3Otre3GPvStcf56lAOn9yTxFVDQ61tjlXJLqvn/V/SWZOcgyzDjYmRPHhFHHHB4mZniqoGLSv2ZPDJ7gxqNDrmJITxzJyhxAR6WNs0s9FVsa2sb+LaN3+luKa9qENzgV/GK9d1eBxnjstbTNil5lWwlUC5LMt/6Mo+9iDsbQdSA1wzNJSP7un083RYquq1vLPjPJ/ty0SSJG5LiuaBGQOI8nccceprqhq0fLI7g493XUBnkHlg+gAevmIg7q6Ot8jakdDvTC3mNysOGd2vdei7qX07W6h1dCxZoDQFuBs4IUnSsZaf/VmW5a1meG2rYMwr2HW+hPVH85zCK2iLTm/gi4PZvPnjOSobtNyWFM0frxlMqIMthloCX3cVT1wzmDvGx/Dyd2dYtiONtcm5PD9vONcOD7O2eWajs2lkVw4JITHGj6PZlZfs565ScmV8cIf7itz3rtHrYKgsy7tlWZZkWR4py/Loli+bEfX1R/OY8soO+i/dwpRXdnSpu6KxijiNzmBympKjciiznDlv7+KvG04RH+bD5sem8spNI4Wo95IwXzVv357INw9OwsddxZJVyTz5TQrVGtODK+yJrkwj+/CusahVCtxakhBau6nuPFvS4b4dzUcQ/A+HXuVq9RzyWmY4drV1rrN7BfVNOl7YdIpbP9xHfZOe5XeP5YvfTXDYrA5rMS42gI2PTuX3Mwey7mgec97axb70Mmub1Wu6cv2E+Kh5elY8jToDK34zjj1LZ3bJI+/uuD9nxaGFvbtzTFsJMlE96Axewf4LZcx5excr9mRy98R+/PDH6Vw7PMzuC4psFVcXBU9cO4Q1D07C1UXBnR/vZ/mv6VijOZ+56KpXfdfEfsQGevCPLWfQ6Q1d2rc34/6cCYcW9p543lq9AZVLexFzdK9Ab5B5bdtZbl++H1mGL383kRfnJ+BpgZ4tAkiM8WfzY1OZkxDOP7ae5fGvjtHQ1L1e/7ZCV71qVxcFS+fEc764lm+P5HV537ZDt1s9fcGlOLSw9yQe9+nuDPIrNfx2SqzTeAWltY0s/vQA7+1M59akKL7/wzQmxQVa2yynw9PNhXfvSOTp2UPYdDyfWz7cS6mRXG9bpzte9azhYSRE+vDuzjR0eoPwyM2EQ/dj727Oa15lA1f962emDwpm+WLnSG1MzqrgkdVHKK9v4qX5w7ltXIy1TRIAO84W8fDqI0T4uvP5/RMcOgz4w6lClqxK5l+3jOKmsVHWNsem6Wq6o0N77N29+//zu7MAPDdveLeP1ZPsG2uzMSWf25fvw9VFwbcPTRaibkPMjA9l1X0TKKlp5JYP9pFRWmdtk3pFR9fHNcNCGRruwwe/2Pfagi3h0B57d0jOKuem9/fh7eZCbaOuW6XK9lgN9+nuDF7cfJrxsQF8tDgJXw+VtU0SGOFkXhWLPz2Iu0rJ2ocmE+Zrf6mmXbk+1iTn8uQ3Kay+f8IlrX0FlyI89m4gyzJPfJ0CQE2jrlupkdDz7BtrIMsy//z+LC9uPs2s4aF8dt94Ieo2TEKkL5/9djyV9U3cu+IgVQ32l+velevj+pHhBHq6smJPhqXNc0jsOuWhJ82A1h/N44VNp6iob75A/NxV3Dw2iqzy+nbbdnVMl73kvcuyzF83nGLV/izumBDDS/MTUIrWwzZPQqQvH9w9lt/+5xBLPjvM5/dPQGVHjda6cn2oVUpuHx/N+z+nU1ilscsnE1vCfs6Oy+hJ8dH6o3k8tSbloqgDVDZo+Xi3aS+hK+JsL9Vw//w+lVX7s1gyfQB/XyBE3Z6YNiiYV28eyYGMcpt8EuyIrl4ft4yNxiDDOjtYn7J17FbYexL+eG1bKlq98TUFUxLXFXG2h2q493am8cEv6dw5IYZn5sSLgiM75MbEKO6aGMPyXy+w/XSRtc3pMl29PmKDPBkX68/aI7liEbWX2K2w9yT80dHvZOixONt67u3qA1nNIaXREbw0P0GIuh3z7HXDGB7hw5++SaG4WtP5DjZAd66PeaMiSCuuvTgfV9Az7DbG3pNBuKb2gc7bhXaGrQ6uPphRznMbTnHlkGBeE+P87B61SsmyRYnMfmsXf9tyhncWJVrbJJP0ZA3smmFh/N+GU2w7VcigUG8LWep42K3H3pPwx1OzhqBSthc2F4mLJ50jlSoXVml4eHUyMQEevL0o0a4W3ASmGRDsxcNXxrExJZ9d50usbY5RetqAb/+FMlRKidd/OGc39SC2iN1e6T0JfyxIjOS1m0fh3ya9z81Fweu3jja6nz0WHbXSqNPz4OfJNDTp+fDusfioRUqjI/HgjDj6B3ny/MZT6A22F4/uyRpY682gdR2sOynHgkux21AM9Cz80brP29vP89ZP59j+xAyiA9pPAepsWICt8/q2VI7lVPL+nWPEI60Dom55On149RG2nijghlER1jbpEkytZ+VVNpgcWNPRzcAerjlbwm499p6y/mgek1/+iTe3n8NVqSA5q8LodvZUdHQ5R7Ir+GR3BndMiGHOiHBrmyPoI2YPD2NQiBfLdpzHYGNee0drXaa8cHupB7EHnErYW73w/KrmbIJGncHhTrJGnZ6n1xwnzEfNM3PirW2OoA9RKCQenTmQc0W1/GJjsXZja2CtmHKQOsp3t+ewqDVwKmHvihfeegKZ8n9srejocpb9lEZacS3/WDgCbxFXd3hmJ4Th56FibXKutU25hNY1MFMYc5CMJTe4KLg4B7W7C7HOjFMJe2deeNuVfGPYWtHR5eSU17P81wssTIzkiiEh1jZHYAHcXJTMGxXBD6eLbK6PzILESCK7W5V9uUclw5bjBXYbFrUWTiXsnZWqAcsgAAAgAElEQVQ2G/PoW7G1oiNjvLX9PEjw1GzbvfkIzM+NiZE06QzsPFtsbVPa0Z205Ne2paK9bK1AJ3NJC5C22HpY1Jo4lbA/NWtIu/4obU8yUyeKBDaf136+qIZ1R3O5Z1I/wn1tO1wkMC8jo/zwdVfZ5CDs7qQld1eobT0sak3sOt2xuyxIjOSV785SUd9Ek87QrhquJ9WstsK/fjiHh6sLD10x0NqmWBWd3kB5y9/XYABvtQs+7iqHbnimVEhM6B/A3gul1jbFKF1NSzZ1/fm5q2jUGdr1c7flsKi1cSphz69soLBaw1/mDuV30we0+/1Ts4YYHQhg6ydQdlk9204X8uiVAwnwdLW2ORZDqzdwMKOcgxnlHM2pJK2ohsJqDZdn/ikVEv0CPRga5sOkuECujA8xGfu1V8b3D+CH00WU1zXZ7Tlg7PpTSPB8y0Sznrb7cEacStj3tjyqTh1kfEJL64libyfQFwezUUgSd07oZ21TLMKxnEpW7cvix9OFVGt0KCQYHOrNxAGBRPq7E+LthpuLEkmCGo2O0tpG0ktqOZZTyZYTBUhScxvch2bEOczQ7gHBngBklNbZrbBffv25uiiI8HW/+HNbvw5tCacS9iPZFXi7uTCkg0pMW23mZYpGnZ5vDudwVXyIww8n2JtWyr9+PEdyVgVebi5cOyyU2QlhTB4YhJdb56eyLMukl9SxKSWfrw5ls+ij/cxJCOPlhSPw87BPMWwlNrBZ2DNL6xjbz9/K1nSfyxuGvXnbaLadKiS9RHR57AlOJexHsysZHePnUB0Ot50qoqyuibsmOq63XlDVwAsbT/P9qUIi/dz56/XDuHVcdJfEvC2SJDEwxIs/XjOYh66I45PdGby1/Rwn363iqyWT7Do807oOVFRjH61822KqfcfwCB/qm4xnqQk6xmmEva5RR2phNddc6ViLi1uPFxDuq2aqgw4A3plazBNfH0OjNfDktYO5f9oA1CYqGruDWqXkkSsHMikukHs+PchvVhxk46NTzfLa1sDNpTnBrVFrsLIl3cdU4eDpgmo8XI3/PXrSEtiZsEth784ftXXb1tX22kadJU3tU5p0BnanlTJvdIRDPYW08snuDF7afJqh4T68d0ciA4K9zH6MMTH+LFuUyL0rDvH5/izun9Z+Ud0ekCQJVxcFjTr7E3ZTaY71TXoURobC2HuDPktgd8Le2R+1rej7eaio1eguKXpYfSCbkVF+DnECHM4sp7ZRx5UOWGX675/TePX7VOYkhPHmbaP71JO+YkgIiTF+bErJt1thl2UZvUHGHu/vHQ3AqW3UtesGKbpAdo7dFSh19Ee9vLl/Rb22XSVbo87gMKXIO84W46pUMGWgY2R2tLIxJZ9Xv09l/ugIli1KtEh4ZGCwFwVV9hefbqVao0NvkO0yI6ajhmFAu+vVXhv0WRKzCLskSbMlSUqVJClNkqSl5nhNU3T0R+2oJUBXXsPe2J9Rxth+/ni42t2Dl0nOF9Xw/9YcJ6mfP6/dPAoXC0x90htkjuVU0i+wfV9+e6GirgkAfzvM7uluw7DOWoMIzCDskiQpgfeAOcAwYJEkScN6+7qm6OiP2lXBdoQTQKPVc7aghtExftY2xWzIsswz355ArVLw3p1jcHXpe1HXaPX8dcNJzhfXcvek2D4/Xl9xobQ5LdDY0Bh7oDsNw3oyFtPZMMeVMx5Ik2X5gizLTcBXwHwzvK5ROvqjdkWwHeUESCuuRWeQGR7hY21TzMYPp4s4nFXB/5sdT6hP3+XkZ5XV8d7ONJ759jjTX93J6gPZ/G5af24Yab9DSU7lVQMwNNx+p2UZu7YVLfOI29K2/wyAUpIuCcfaKiU1jby3M42ssro+P5Y5nuEjgZw2/88FJly+kSRJS4AlADExMT0+WGfVoZeXJKuUEh4qJVUaHd5qF16an+AQCywXSptPjrg+yBSxFst/vUBMgAc3j43q0+PklDeH7XzdVUwcEMDdE2NNViPbCyfyqugX6GHXPfjbXtt5lQ1INFcIG7tejV3vtp4dk15Sy2vbUhkd7Ue/loKyvsJiwVlZlpcDywGSkpJ6NcfLVHWoKdGfOiiIpL9td6hc1+yWu35sH58gliKrrI7krAqemRPf53H18f0DOPPibNxN5EjbG406PXvTy7hhlP0+cbTSem2X1zUx5qUfmTE42OS29pYdU1TdvDgf6uPW58cyh7DnAdFt/h/V8jOrYEz0M1u8W2+14ywyFtc04qN2cRhx+jm1ebTbnIS+FydLxO4tyYELzWmvVw8NtbYpZiOjZc2gowVtUymStpoc0SrsIX0YZmzFHGf4IWCQJEn9JUlyBW4HNprhdc1Gjaa5KMnLzX4fUy+nrLaJIK++v/NbimM5lYR4uxEdYP8L25Zmy/EC3FVKpjhQ9fGZghoA4sONryGtP5qHqZR9W02OKKpuxMNViXc3W2H0hF4fQZZlnSRJjwLbACXwqSzLp3ptmRmpb2oWdlPlyfZIXZMOTwucIJYip7yeAcGeSEYqDQWmqaxvYkNKHjcmRtptOwRjpBbW4K12IcJEY7vXtqUanUss0X6x1VYoqGogzEdtkXPcLMogy/JWYKs5XqsvcSTJ0Gj1HRZ12BuVDVoGhzrOQrCl+OZwLhqtgbsnxlrblB5jrEXI0ZwKEiJ8TYqgqXCLjG0unAKkF9fRP8gya2KOFWx0IiQkZKM+i32ikMBgf21OrEpdo44Pf73AxAEBDLPTtNfLq8XzKhtYuvY4p/KrGRdruv2wqXCLrXbo1BtkMkrrGBhiGefFqYTdcWQQXJQSustHBdkxfh6ulNU1WtsMu+KjXRcorW3k6dnx1jalxxjLbNHoDMgyjI0NMLmfvRUp5VbU06Q3WCw92SmE3a3lBGjUOU5vZy83F2o1jtOpMjbQg/SSOmTZcW5WfUluRT3Lf73A3BFhjImxv8EarXSUwVLUQe+e7gzJtgXOFTVn+cSFWCYU4zirbx3QOpChttFxhN3PQ0Vlg9baZpiNxBh//ns4l/SSWgaG2G/1pCUwGGSeXnMcCXhmzlBrm9MrOurs+PZP57l1XLTR34F9TTs7nluJUiEx1ESWj7lxCo/9orA7kIcb7K2mrLaRJjvsv22M1kEh204VWdkS22fV/iz2ppfx7PXD7LY3TCsdhU5sNR+9JxzLqWRwqLfFGvY5hbD7uDd/mFUO5OFG+bljkJtTqByB6AAPxscG8M3hHPQOtHZgbg5llvO3LaeZGR/C7R14s/bCgsRI/NyN15fYaj56d5FlmZScSkZH+1rsmE4h7B6uLnirXS5WfjkC/Vum0jvSsN/Fk/uRWVbP1hMF1jbFJsmrbODBVclE+Xvw5q2j7T7nf/3RPKa8soPKBm27VGRbXgjtLhdK66jW6BgVZblOrE4h7ABhPmoK7XiQwuUMDm2OQ58trLGyJeZjbkI4g0O9mjMlxBDjSyipaWTxJwdo0hn4aHESvh72XUXdNs0RLs1Yk4CbxvYuft560+i/dAtTXtlh1a6Pe9NKAZgwwHIDcZxi8RQgzFftMGELAF93FdEB7hzPqbK2KWZDoZB4YV4Ciz7az1vbz/HMXPteGDQX5XVN3PXxAfIrNaz87XiL5UL3JR0NxZGBtcl5JPULuLhtXmUDSklCL8tEdmHOsS11fdydVkqknzuxFhzk4jTCHhPgwebjBciybPePsK2M6xfAr+dLHeo9TYoLZNH4GD789QLj+wdwlQM1tuoJ+ZUN/GbFITLL6lhx7zjG9zed221PdLYw2qDV88KmU2i0hosCrW9JhW0V6sNZ5ew8W9KufbctdX3U6Q3sTS9jbkK4Ra9RpwnFDArxoqpBS0mt4xTBjO8fQGltI+eLHSfODvDcDcMYFu7DH74+xun8amubYzVO5lWx4L095Fc2sOI345jsQE2+urIwWlGvNenVN2j1rN6ffUnF6jPfnmD90TybmomakltFjUZn8X7/TiPsrbnRaUWOI4IzhjT3qt55ttjKlpgXtUrJR/ck4eXmwuJPD5DmYDeurrD+aB63frgPF4XEmocmMznOcUQdOh9g3RUuz51q9cptaSbqD6cKcVFITB9kuq98X+A0wt7aYOqMAy02hvu6MzTchx9OO17ud6SfO6vuax7EdcsHezmaXWFliyxDQ5OepWuP84evj5EQ4cu6R6YwJMzxCrYuH293Oe4qpck0yI7Ir2ywmXYDsizz3clCJg8Msvhit9MIe4iPmnBfNcdyKq1tilm5fmQ4yVkVJqv37JmBIV6sfWgy3moViz7az5rkXGub1KfsTStl9tu/8vXhHB69ciBf/G5Cn85+tSTGslQWJEay9ffT8Fa7MDzCp117gOfnDTfp1XfUi91W2g2cLqgmu7yeOQlhFj0uONHiKUBijJ/DeX43jIzgtW2prDuSy6MzB1nbHLPTL9CTbx+ezGNfHOXJb1I4cKGM/7thGD52PNvzckpqGnn1+7N8k5xLv0APVt8/waFCLx1lqZwtrKG2Ucfrt4wyWW5vLCvmyvhg1ibnXRKDb+uV20K7ge9OFKKQ4Nphlk8AcC5hj/Zn64lCims0hHg7hicUE+jB5LhAvjqUw0NXDESpcIzsmLYEebmx6r7xvLn9HO//nM6u86W8tCCBq4eG2HU2UF2jjo93ZbD813Q0OgMPzojjD1cPcqiBGWB6NunLW89QUa/lxsRIk6LekUAn9QswOdTe2hgMMuuO5jFlYBCBVph05lTC3poqti+9jPmjbeMEMAd3TIjh0S+OsvNsMVdbwTuwBC5KBU/NiufaYWE8tSaF3312mMlxgfx57lASIi1Xqm0Oqhq0fL4/ixV7MimtbWT28DCemj3EYi1dLY2pbJSimkY8XZUs7WHbYVvwyk2xN72subf8HOu0VHYqYU+I9MXPQ8Wu86UOJeyzhocR6efOh7+mO6ywtzIq2o8tv5/GFweyeWv7Oa5ftpur4kN46Io4kjro320LpBXX8OXBHL46mE1dk57pg4N5/KqBjO1n23b3lo46OP7+qkEWGe5saf57OAdfdxXXWOl6dCphVyokpgwMYtf5Eocq6lEpFdw/rT8vbDrNgQtlFi1dtgYqpYJ7JseyIDGSlXszWbEng5s/2MfwCB9uHx/D/NERNhODL69r4sfThfz3cC7JWRW4KCTmjgjngRkDGB5hX08aPeWpWUMuibG3ZeXeTEJ91DbrefeEyvomtp0q5LZx0VYLq0nWGGyQlJQkHz582OLHBfjvoRyeXnuczY9NtbtH+I7QaPXMeG0n0f4efPPgJIe5aXWF+iYda4/k8cWBbM4UVOOqVDBtUBCzE8K4YkgIwd6Wi3HKskx6SS27z5fyw+kiDmSUozfIDAj25PZx0dyYGGVRe2yF1rmmxjx3d5XSpodkdJd//5zGq9+n8t3j08zef12SpGRZlpM6286pPHaAq4eFolwn8d3JAocSdrVKyWMzB/Hs+pNsP1NstUdAa+Dh6sLdE/tx14QYjudWsSkln+9OFvJTS+HWkFBvJsUFMjraj4RIXwYEeaIw0yJzjUbL6fxqTuZXcyynkv0Xyiipaa5uHhjixUMz4pg1PIyESB+nutleTms8PPHFH6iov7R9trXK/fsCrd7AZ3uzmDIw0GJDNYzhdMIe4OnKxAEBbD1RyJPXDnGoi+22cdH8Z28mL20+zbRBQQ6XXdEZkiQxKtqPUdF+/OW6oZzMq2Z3Wil700v56lA2/9mbCTR7iP0CPYgN9CQm0IMgL1cCPN0I8FShdlHiplKgUirQGWSadAaadAaqGrRU1DdRVttEQVUDmWX1ZJfVU9imFXSojxuT4wKZNCCQSXGB9Au0zBi0ntDqQVsyoyS3or6dqLfSusBqDbvMydYTBRRWa/jHwgSr2uF0wg4wd0Q4f1l3ktMF1Q4V51QpFTx3wzDu/uQgy3+9wO+vcry89q4iSRIjonwZEeXLQ1fEodMbSCup5URuFWcKasgqq+N8cQ07Uou7PYUq2NuNfgEeTB4YyIAgT4ZH+jI8wsduUmit0f1Qo9Xz8OojSBgfKh/h525zXRm7iyzLfLwrgwFBnlwxOMSqtjilsM9JCOeFjadZk5zrUMIOMG1QMNePDOfdHWnMSQhjUKjjlaP3BBelgvgwH+LDLn08lmWZuiY9ZbWNlNc10agz0KgzoNUZUCol3JQKVC4KfNQqAjxd8fdQ4aK074Jta3Q/fG7DKY7nVnHf1P58cSDbaGGRLXVl7Ak/p5ZwIq+KlxeOMFuor6c4pbAHeLpyzbBQ1h3NY+mceNxcHCtk8cK84exJK+XJb1L45sHJuLrYtxD1JZIk4eXmgpebi02HTsyJpbsfrtybydeHc3hs5kD+dO0QRkT6Gg23/PHrYxa1y5zIssxb288R5e/OTWOirG2O8/SKuZzbxkVTWa/lRwdsoBXo5cbLC0eQklvFK9+dtbY5AhvDkt0Pd5wt4vlNp1C7KFi2I40pr+wAYM/SmWS8ch17ls686I3bUlfG7vJzagkpuVU8euVAm3CkrG+BlZg6MIgof3c+25dlbVP6hNkJ4dw7OZZP92Tw/clCa5sjsCEs1f3wRG4VD35+BGTQtKxjtO2bbi27zI3BIPPGj83e+kIb8NbBiYVdoZC4d3IsBzPKOZ7rWB0fW/nz3KGMivLlqTUpZJfVW9scgY1gie6H54tqWPzpAfR62WTfdGvY1RdsSMnjRF4VT1wz2Ca8dXDCAqW21Gi0THp5BzPjQ3hnUaK1zekTcsrrue6dXUT6e/DfBybibSMVmQLHJae8nls+2Ideli/m9F+OBGS8cp1lDesDGpr0zPzXzwR7u7H+4Sl9vmja1QIl27i9WAlvtYrbx0Wz5UQBOeWO6dFGB3iw7I4xnC+q4cHPk2nUGR81JhCYg6yyOm77cB8NWj2r7htvcpCGPcTNu8LHuy5QUKXh2euGWT0Tpi29EnZJkl6TJOmsJEnHJUlaJ0mSn7kMsxT3TxuAUiHx7o40a5vSZ8wYHMyrN49kT1oZT35zHIPB8k9pAscno7SO2z7cT4NWzxe/m0B8mI/dxs27Ql5lA+//ks7s4WE2N2S8tx77j0CCLMsjgXPAM703ybKE+aq5Y3wMa47kklVWZ21z+oyFY6JYOieeTSn5/G3LGawRghM4Lifzqrjlg7006Q188buJF+tD7DVu3hmyLPPX9SeRZXj2+qHWNqcdvRJ2WZZ/kGVZ1/Lf/YBtLAl3k4euiMNFIfH2T+etbUqf8sD0AfxmSnOmzCvfnRXiLjALe9NLuX35ftxclHzz4CSr9kixFN+39CJ64prBRPl7WNucdpizQOm3wNemfilJ0hJgCUBMTIwZD9t7Qn3ULJ7Uj493Z/DbKf0dqjlYWyRJ4v+uG4ZWb+DDXy9Q26jjpfkJNhUbFNg+bfu5+LqrqGnUERfsyWe/nUCYr7rdtvbcJsAYNRotz286xbBwH34zJdba5hilU49dkqTtkiSdNPI1v802fwF0wGpTryPL8nJZlpNkWU4KDg42j/Vm5NGZg/BzV/G3Lacd2pNVKCRemp/AgzPiWH0gmyf+ewytvnu9UgTOS6tQ51U2IAOVDVpkWebeybHtRB06bl9gr7y46TQlNY28vHCEzbaX6NQqWZavlmU5wcjXBgBJku4FrgfulO1YEX3dVTxxzWD2XyjnBwesRm2LJEksnRPPU7OGsP5YPg99foT6Jl3nOwqcHmNCbZDhvZ3pRre3dPuCvuaHU4V8k5zLQ1fEMSradnNFepsVMxt4Gpgny7Ld5wsuGh/D4FAvXtx02imE7pErB/Li/OH8dLaIm97f57ApnwLzYWrEnSmhtuc2AZdTWtvIM9+eYHiED49fNdja5nRIb58j3gW8gR8lSTomSdIHZrDJargoFfxtwQjyKht4a7tjL6S2snhSLCvuHUdeRT3z3t3N3rRSa5sksEFkWWbFngyTvzcl1I6S7ijLMkvXnqCmUcebt422mQpTU/Q2K2agLMvRsiyPbvl60FyGWYvx/QNYND6Gj3dd4GRelbXNsQhXDAlh46NTCfJy4+5PD/Lp7gyHXmcQdI/S2kbuW3mYFzadZniED+rLRK0joXaUdMdPdmew/UwRS2fHM9gOWmE7dUsBU1Q1aLn6jV8I9nJj/SNTbP7ubC5qG3U88fUxfjhdxKzhofzjxhEEejnffE7B/9hyvID/23CS2kYdf5k7lMWT+rHhWL5dTznqLslZ5dz24X6uGhrCB3eNterUta62FBDCboIfThWyZFUyD18Rx9Oz461tjsUwGGQ+3n2B17edw8ddxT9vGsFVQ51nfqqgmZKaRp7feIotJwoYGeXL67eMsgtP1dyU1zVx3Tu7UCkVbHpsKr7u1u21JHrF9JJrh4dxW1I0H/ySzqHMcmubYzEUCokl0+PY+NgUgr3duG/lYZauPU5to+MvJguab+yf789i5r9+5sfTRTw1awjfPjTZKUVdqzfw2JdHKKtr4t93jrG6qHcHIewd8H83DCPK34M/fn2MqgbjQ3gdlfgwH9Y/MpkHZ8Tx9eEcZr/1K9sdPA3U2UnOquCmD/by7PqTDI/wYevj03jkyoE2m6vd17y0+TR70sr4+4IEuytadM6/WBfxcnPhzdtGU1il4clvUpxuQdHNRcnSOfH894FJuKuU3P/ZYX77n0NkljpuTx1nJLO0jodXJ3PT+3vJKW/gX7eM4svfTWRgiJe1TbMaq/Zn8dm+LJZMH8AtSdHWNqfbiBh7F/hkdwYvbT7NM3PieWBGnLXNsQpavYGVezN5a/t5mnQGlkwfwMNXxuHh6pRjcx2Cirom3tlxns/3Z+GiULBk+gCWTB+Ap5tz/033pJWy+NODzBgczEeLk1DaUMsNsXhqRmRZ5uHVR/jhdBGr75/AxAGB1jbJahRXa3jlu7N8ezSPMB81j101kFvGRjtN5pAjUKPRsmp/Fu//nE5do47bxkXzx6sHE+LTviWAs3Eyr4rbl+8n0s+dNQ9NsrnBNELYzUyNRsv89/ZQUdfEhkemEhNoex3dLMmhzHJe3nqGI9mVRPq58/urBrJwTBQqJ43H2gOltY38Z08mn+3LpFqjY2Z8CEvn2EdetiXILqtn4ft7cVVKrH14MuG+tlcdK4S9D8gorWPBe3sI8XZj7cOT8bGxu7mlkWWZX86V8OaP50jJrSImwIPfXzWIBaMjnHbBzRbJKa/n410X+PpwDo06A7OHh/HgDNvudWJpSmoaufmDvVQ1aFnz4CQGhtjmzU4Iex+xN72UxZ8cZMrAID65J0kIGM0Cv+NsMW/8eI5T+dVE+Kq5a1I/bh8XQ4Cnq7XNc0pkWeZIdgWf789mY0o+CgluTIzkgRlxxAU776KoMSrrm1j00QEySmv54ncTGRPjb22TTCKEvQ/54kA2f153gluTovjnTSOtWolmS7QK/Io9mexOK8XNRcGC0ZHcMzmWYRGOP3zBFqisb+LbI3l8dSibc0W1eLoquX18DPdP62+ToQVrU1Wv5c5P9nOuqJaPFycxfbDttRRvS1eF3bmXv3vIHRNiKKzW8M5P5wn0cuP/OVFlakdIksRVQ0O5amgo54pqWLk3k2+P5PH14RzG9w/g5rFRzEkIs7kFKXtHlmUOZpTz5cFstp4spElnYFS0H/+8aQTXj4xw+iwXU1RrtCz+9ADnCmv58O6xNi/q3UF47D1ElmWeXX+S1Qeyefa6odw/bYC1TbJJquq1/PdwDqsPZJFZVo+bi4JrhoVyY2Ik0wcHi8XWHmIwyBzNqeS7EwV8d7KQvMoGvNUu3JgYye3jYsQTUidU1Wu59z8HOZlXxft3juXqYfbRNkOEYiyA3iDz2JdH2HqikBfnD2fxpFhrm2SzyHKzEK0/msemlHwq6rUEeLpy3YhwrhkWyoQBAbi5KDt/ISfGYJBJzq5gy/ECvj9ZSGG1BlelgmmDgpg7Ipy5I8JxdxWfYWeU1DSy+NODpBfXsuyORGYND7O2SV1GCLuFaNIZeHj1EbafKeJvCxK4a2I/a5tk82j1Bn49V8K3R/PYfrqIRp0BD1cl0wYFMTM+hCuHhIic6hYKqzTsSStlT1opv54vpbS2EVcXBTMGBzN3RBhXDQ11+uys7pBX2cDdHx+goErD8sVjmTbIvsIvQtgtSJPOwEOfJ/PT2WL+fmMCd04Q4t5VGpr07LtQyk9nitl5tpj8Kg0AIyJ9mRwXSFJsAGP7+TtNdk21RsuBC+XsSStld1opacW1AAR4ujI5LpBrhjWvYXiJuHm3SS+p5e6PD1DTqGPFveNIig2wtkndRgi7hWnU6Xno8yPsOFvMX+YO5XfTRcy9u8iyTGpRDT+dKebn1GJScqpoahm0HRfsybjYAJJiA0iM8SM20NOmSr17gkar51R+NSdyKzmeW0VKbiUXSuuQZVCrFIzvH8jUgYFMGRjE0DAfFHb+fq3JgQtlLFmVjItCYuVvx9tdU69WhLBbgSadgT/+9xhbjhfwyJVxPHntEJEK2Qs0Wj0n8qo4lFlOcmYFh7MqLnbZVKsUDArxZkiYN/Fhzd+HhHkT7OVmc595o05Pdlk96SV1ZJTWcaGkllP51ZwrqkFnaL7+gr3dGBXly8goP8bFBjCmn59YczATG47l8dQ3x4kKcOc/946366pxke5oBVxdFLxzeyI+ahfe25lOVYOWF+Yl2L1naS3UKiXjYgMY1/LIbDDIpJXUkpJTSWphDalFNfxyroQ1ybkX9/F0VRLp706kn3vLd4+W72oCPN3wdVfho3YxW2GZVm+grLaJkppGSmo1FFc3UlLTSHFNI9nl9WSU1pFbUY+hjf8U5OVGfJg3S6YPYGSUH6OifQnzUdvcDcnekWWZf/+czmvbUhnfP4Dld4/Fz8M5QnpC2M2MUiHxjxtH4Ovuyge/pFNYpeHt2xNFLrEZUCgkBod6t+ttUlbbSGpRDamFNWSV1ZNX2UB+ZQNHcyqprDfeR9/LzaVZ5FH1p5UAAA+ySURBVFuE3tVFgVIhoZQkFC3flYrmfzfp9NQ36dFom783aPU0NDX/u1qjxdhDr6+7iugAd0ZF+7EgMZIBQZ4MCPYkNshTLHZagIYmPU+vPc6mlHzmj47g1ZtHOtUTkAjF9CGr9mXy3MZTDIvw4ZN7xhEqMj0sTm2jjvzKBvIqG6isb6KqXktVg46qBu3Fr2qNFq3egMEgo5dl9AbQGwzoDTIGGVyVCtxdlXi4KnFXKXFv+e7hqsTf05VgbzeCvdwI8VET7O1GkJerU4mIrZFTXs8Dq5I5U1jNU7OG8NCMOId5GhIxdhthx9kiHv3iKH7uKj66J4nhEfa5aCMQ2AN700t5ZPURdAaZdxYlcuWQEGubZFbEzFMbYWZ8KP99YBIycNP7e9lwLM/aJgkEDofeIPP29vPc9fEBAr3c2PjoVIcT9e4ghN0CJET6svHRqYyM8uPxr47x0ubT6FrS+AQCQe8ortFw9ycHeHP7OeaPjmTDI1PoH+RpbbOsiljRsxDB3m6svn8Cf99yhk92Z3Ayr4q3b08kzFfE3QWCnrLrfAl//DqF2kYtr940kluSohwmnt4bhMduQVRKBc/PG86bt43iRF4Vc9/Zxc7UYmubJRDYHQ1Nep7feIq7PzmIn4eKjY9O5dZx0ULUWxDCbgVuTIxi46NTCfF24zcrDvGPrWdo0onQjEDQFY7nVnL9sl38Z28m906OZfNjU8V4v8sQwm4lBoZ4sf6RKdw1MYblv17gxn/vIbWwxtpmCQQ2S5POwFvbz7Hw33upb9Lz+X0TeH7ecNQqkVp6OULYrYhapeRvC0aw/O6xFFVruGHZbj78JR29wfIpqAKBLZOcVcH1y3bx1vbzXDcynO8fn87UQUHWNstmEYunNsC1w8MY28+fP687wcvfneXH00X88+aRYjalwOmpbdTx+rZUVu7LJMxHzSf3JHHVUPsYimFNzOKxS5L0J0mSZEmSxC20hwR6ufHBXWN549ZRnCuqYc5bu3jnp/Mi9i5wSmRZZuuJAq594xdW7stk8cR+/PjEDCHqXaTXHrskSdHAtUB2781xbiRJYuGYKKYOCuLFTad548dzbD6ez8sLRzK2n+1OThcIzElacQ3PbTzFnrQy4sO8WXZHImP72V/vdGvS65YCkiStAV4CNgBJsiyXdraPM7UU6A07zhbx7LqT5FdpuHlsFP9vdjzB3m7WNksg6BNqNFqW7Ujj090ZeLgq+dO1Q7hzQozZOnE6AhZp2ytJ0nwgT5blFJE/an5mxofy4xOBLNuRxie7L7DtZCGPXz2IeybHiiHQAodBqzfwxYFs3v7pPOV1TdyWFM1Ts4cQ5CWcmJ7SqccuSdJ2wNi0178AfwaulWW5SpKkTDrw2CVJWgIsAYiJiRmblZXVG7udjgsltby4+TQ/p5YQF+zJ0jlDuXpoiCjIENgtsizz/clCXt2WSkZpHRMHBPDnuUMZGeVnbdNslj7v7ihJ0gjgJ6C+5UdRQD4wXpblwo72FaGYniHLMj+dKeYfW89wobSO8bEBPDM3nsQYEX8X2A+yLLM3vYx//ZDKkexKBoV48czceK4cIhyVzrB4297OPPa2CGHvHVq9ga8P5fDW9vOU1jYyJyGMJ64ZzCBRfSewcfZfKOONH89xMKOccF81v79qELeMjRJx9C4iRuM5MCqlgrsm9uPGxEg+2nWBj369wPenCrlhZAS/v2ogA0OEwAtsB1mWOZBRzjs/nWdvehkh3m68MG84t42LFlWjfYQYtOEAlNc1sfzXC3y2L5MGrZ55oyJ4bKYQeIF1MRhktp8p4v1f0jmaXUmQlxsPXRHHnRNihKD3EDFByQkpq21k+a4LfLY3iwatnquHhvLQFQNEDrDAojTq9Gw8ls+Hv14grbiW6AB3lkyP45axUULQe4kQdiemvK6JlXszWbkvk8p6LUn9/HlgRhwz40NQKsTilKBvKK7RsHp/NqsPZFNa20h8mDcPXRHHdSPCRQzdTAhhF1DfpOPrQzl8vCuDvMoGYgI8WDypH7ckRePrrrK2eQIHQJZljudW8Z+9mWw+no9WL3PlkGDundKf6YOCRJaLmRHCLriIVm9g26lCVu7N5FBmBe4qJQvHRHLXxH4MDfextnkCO6Rao2XDsXy+PJDN6YJqvNxcuHlsFPdMjnX6sXR9iRB2gVFO5lXx2b5M1h/Lp0lnYFSUL7eNi+GGUeF4q4UXLzCNLMscya7gy4M5bDleQINWz7BwHxaNj2ZBYqQ4fyyAEHZBh1TUNbH+WB5fHcwhtagGd5WS60aGs3BMJBP7B6IQsXhBCxmldaw/msf6Y3lkldXj6apk3uhIFo2PZkSkrwi3WBAh7IIuIcsyKblVfH0om43H8qlr0hPmo2b+6Ajmj45kaLi3uHCdkKJqDd+fLGTd0TyO5VQiSTAlLoj5oyOYOyIcTzdRAmMNhLALuk1Dk57tZ4pYfzSPX86VoDPIDArxYs6IcOYkhBEfJkTekcmvbOD7k4V8d7KAw1kVyDLEh3mzcEwk80ZFEuartraJTo8QdkGvKK9rYsvxfDYdL+BQZjmyDLGBHsxOCGfW8FBGRfmJcI2dI8syZwpq2JlazI+niziWUwk0i/nclpu5aFNhWwhhF5iNkppGfjhdyPcnC9mbXvb/27u72LbuMo7j38d2HMeO47w4L21SmrTNEpV0ZXSt1pbCtHLRCkS5QiCBJsQlLwMhocENt1wgBBcICY3BJKYhVCYxIcSLVrRpg01t15G+JV2apq2Tpomb2EkcJ/HLw8Vxsq7SCKV2T3zyfKToHJ9azu8o7u/8/fc5NoWi0hIJ8qm+Vp7qb+NIb6udPlklMst53hq9w6mhKU4NTXErvQTAns4YxwY6OD7QwQ77SsYNy4rdVERqcYXXrkxzamiK165Mk1rM4fcJ+z7SxKFdLRzaGedj2xoJBuyClI0gVygymEjx5sgd3hhJcu7GLLmCEg76OdIb52h/O0/2tdLWYNMs1cCK3VRcvlDk3ZspTg1N8cZIkvPjaVQhHPSzv7uZgztb2N/dxEBnjNqAXUr+MCzlCgwm0pwem+HM2Aynx2ZZWM4jAh/d2sDhXXE+sSvOgZ5m+5tUISt289ClF3P8a/QO/7ya5M2RJFenMwAEAz72dsXYt72Zfdub2NsVsxFiGagqidksg4k0g4kUZ6/PMphIs1JwvgB9V1s9+7ubOdIb5+COFpoiQZcTmwdlxW5cl1xY5szYLGevOyPHixNpcgXn+dbeUMuezkYe7YqxpzNGX0eULbGQnXXzIQpF5fqdDMOT81y+Nce/E2nOj6eZyawAUOMXBjpjHOhu5vFu5wDabEXuOfZ57MZ18fpajg10cGzA+WbFpVyBC+NOIZ1PpBkcT/Pq0G1WxxYNoQB9HVEeaY/S3xFlR2s9PfEIHQ2hTXMGTq5Q5ObMIteSGUanM7w3Nc/Q5DxXbs+zlHNG4j6B3rYoR/vbeHRbI3u7nAOjTa2YVTZiN65aWM5zaWKO4dvzDE/OMTzpFNn8Un7tPqEaH90tEbpbImxvCbO1sY7Oxjpn2VRHQyhQNSN9VSW5sMJ4Ksv4bJbE7CLjqSyJ2SxjyQw3ZhbJF9//P9kSCdK/JUpfewP9HVH6t0TpbYtSF7QS34xsxG6qQn1tgAM9zRzoef8z41WV23PLjCYXuJbMMJbMcC2Z4crUPKeGp1jJFz/wGJGgn9ZoLfF652d1vSlSQ6yuhoZQDQ11q+sBQkE/oYCfGr/83wcEVWU5X2RxpUA2VyCznCe1mCOdzZFaXCGdddaTC8tMzy8zNe8skwvLa9NRq6KhAJ2NdfR1RDm+p4OeeD07WiPsiEdoDNt0irl/VuxmwxEROmIhOmIhDu2Mf+DfVke8E6ksE6ks46ksE6mltQK9Or3AW9fukFrMrft7/D6hrsZPqMZHwOfD7xN8PvCL4PMJgjO3nS8qxdIyX1SWck6Zr/diVwRaIs6BpjVayyPtUVqjtbRHa+lsCtPVtPqKw64BMOVlxW6qioisFeXebY0fer+VfJG5pdzayHmutJxfyrOUK5R+imRLJZ0vFCkUoahKoagUVEGd8g/4xFn6BZ84B4Nw0E8o6Cdc4yccDBCu9dNYFyRWemUQC9cQrQ1smvcGzMZixW48KRjwrU3NGLPZ2OWBxhjjMVbsxhjjMVbsxhjjMVbsxhjjMVbsxhjjMVbsxhjjMVbsxhjjMVbsxhjjMa58CJiITAPXK/DQcSBZgcfdKGz/qpeX9w1s/x6W7araut6dXCn2ShGRM//LJ59VK9u/6uXlfQPbv43GpmKMMcZjrNiNMcZjvFbsv3Q7QIXZ/lUvL+8b2P5tKJ6aYzfGGOO9Ebsxxmx6nih2ETkmIsMiMiIiz7qdp5xEZJuI/ENELonIRRF5xu1MlSAifhE5JyJ/cjtLuYlIo4icFJEhEbksIgfdzlROIvKd0nPzgoi8JCIhtzM9CBF5XkSmROTCXduaReTvIvJeadnkZsb1VH2xi4gf+DlwHNgNfElEdrubqqzywHdVdTfwBPB1j+3fqmeAy26HqJCfAX9R1X5gLx7aTxHpBL4FPK6qA4Af+KK7qR7Yb4Bj92x7FnhVVXuBV0u3N6yqL3bgADCiqqOqugL8DjjhcqayUdVbqvpOaX0epxQ63U1VXiLSBXwGeM7tLOUmIjHgk8CvAFR1RVVT7qYquwBQJyIBIAxMuJzngajq68DMPZtPAC+U1l8APv9QQ90nLxR7J3DzrtsJPFZ8q0SkG3gMeNvdJGX3U+B7QNHtIBXQA0wDvy5NNT0nIhG3Q5WLqo4DPwZuALeAtKr+zd1UFdGuqrdK65NAu5th1uOFYt8URKQe+APwbVWdcztPuYjIZ4EpVT3rdpYKCQAfB36hqo8BGTb4y/j7UZprPoFzANsKRETky+6mqix1TiXc0KcTeqHYx4Ftd93uKm3zDBGpwSn1F1X1ZbfzlNlh4HMiMoYzjfaUiPzW3UhllQASqrr6KuskTtF7xaeBa6o6rao54GXgkMuZKuG2iGwBKC2nXM7zX3mh2E8DvSLSIyJBnDduXnE5U9mIiODMz15W1Z+4nafcVPX7qtqlqt04f7tTquqZEZ+qTgI3RaSvtOkocMnFSOV2A3hCRMKl5+pRPPTm8F1eAZ4urT8N/NHFLOsKuB3gQalqXkS+AfwV5x3551X1osuxyukw8BXgvIi8W9r2A1X9s4uZzP35JvBiaeAxCnzV5Txlo6pvi8hJ4B2cM7jOUWVXad5LRF4CngTiIpIAfgj8CPi9iHwN55Npv+BewvXZlafGGOMxXpiKMcYYcxcrdmOM8RgrdmOM8RgrdmOM8RgrdmOM8RgrdmOM8RgrdmOM8RgrdmOM8Zj/AGPs92OoOIpBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.plot(orig_trajs[407, :, 0], orig_trajs[407, :, 1], label='orig')\n",
    "plt.scatter(samp_trajs[407, :, 0], samp_trajs[407, :, 1], label='noisy')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# simple network which is used to learn trajectory\n",
    "class ODEModel(tf.keras.Model):\n",
    "    def __init__(self, latent_dim=4, nhidden=20):\n",
    "        super(ODEModel, self).__init__()\n",
    "        self.elu = keras.layers.ELU()\n",
    "        self.linear1 = keras.layers.Dense(nhidden)\n",
    "        self.linear2 = keras.layers.Dense(nhidden)\n",
    "        self.linear3 = keras.layers.Dense(latent_dim)\n",
    "\n",
    "    def call(self, inputs, **kwargs):\n",
    "        t, y = inputs\n",
    "        out = self.linear1(y)\n",
    "        out = self.elu(out)\n",
    "        out = self.linear2(out)\n",
    "        out = self.elu(out)\n",
    "        out = self.linear3(out)\n",
    "        return out\n",
    "\n",
    "# Encoder / Recognition RNN\n",
    "class RecognitionRNN(tf.keras.Model):\n",
    "\n",
    "    def __init__(self, latent_dim=4, nhidden=25):\n",
    "        super(RecognitionRNN, self).__init__()\n",
    "        self.nhidden = nhidden\n",
    "        self.i2h = keras.layers.Dense(nhidden, activation=\"tanh\")\n",
    "        self.h2o = keras.layers.Dense(latent_dim * 2)\n",
    "\n",
    "    def call(self, inputs, **kwargs):\n",
    "        h = self.i2h(inputs)\n",
    "        out = self.h2o(h)\n",
    "        return out, h\n",
    "    \n",
    "    def initHidden(self):\n",
    "        return tf.zeros([1000, self.nhidden], dtype=tf.float32)\n",
    "\n",
    "# Decoder\n",
    "class Decoder(tf.keras.Model):\n",
    "    def __init__(self, obs_dim=2, nhidden=20):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.relu = keras.layers.ReLU()\n",
    "        self.fc1 = keras.layers.Dense(nhidden)\n",
    "        self.fc2 = keras.layers.Dense(obs_dim)\n",
    "\n",
    "    def call(self, inputs, **kwargs):\n",
    "        out = self.fc1(inputs)\n",
    "        out = self.relu(out)\n",
    "        out = self.fc2(out)\n",
    "        return out\n",
    "\n",
    "model = ODEModel()\n",
    "neural_ode = NeuralODE(model, t=samp_ts)\n",
    "rec = RecognitionRNN()\n",
    "dec = Decoder()\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_normal_pdf(x, mean, logvar):\n",
    "    const = tf.cast(np.array([2. * np.pi]), dtype=tf.float32)\n",
    "    const = tf.log(const)\n",
    "    return -.5 * (const + logvar + (x - mean) ** 2. / tf.exp(logvar))\n",
    "\n",
    "\n",
    "def normal_kl(mu1, lv1, mu2, lv2):\n",
    "    v1 = tf.exp(lv1)\n",
    "    v2 = tf.exp(lv2)\n",
    "    lstd1 = lv1 / 2.\n",
    "    lstd2 = lv2 / 2.\n",
    "\n",
    "    kl = lstd2 - lstd1 + ((v1 + (mu1 - mu2) ** 2.) / (2. * v2)) - .5\n",
    "    return kl\n",
    "\n",
    "def compute_gradients_and_update(samp_trajs, samp_ts):\n",
    "    \"\"\"Variation Autoencoder (VAE) which run backwards in time through the given sample trajectories\n",
    "       to make an intial predicition about z0.  NeuralODE then does a forward in time pass to get z1,..,zN.\n",
    "       The decoder then transforms the latent zi's to sample space xi's. Loss is calculated then backpropagated.\n",
    "    \"\"\"\n",
    "    \n",
    "    with tf.GradientTape(persistent=True) as tape:\n",
    "        \n",
    "        # Backward in time\n",
    "        h = rec.initHidden()\n",
    "        for t in reversed(range(samp_trajs.shape[1])):\n",
    "            obs = samp_trajs[:, t, :]\n",
    "            out, h = rec(tf.concat([obs, h], 1))\n",
    "        qz0_mean, qz0_logvar = out[:, :latent_dim], out[:, latent_dim:]\n",
    "        epsilon = tf.random.normal(qz0_mean.shape)\n",
    "        z0 = epsilon * tf.exp(.5 * qz0_logvar) + qz0_mean\n",
    "        \n",
    "        # Forward prediction of the latent state using the NeuralODE\n",
    "        outputs, states = neural_ode.forward(z0, return_states=\"tf\")\n",
    "#         print(outputs.shape)\n",
    "#         print(tf.concat(states, 1).shape)\n",
    "        pred_z = tf.reshape(tf.concat(states, 1), [states[0].shape[0], len(states), states[0].shape[1]])\n",
    "#         print(pred_z.shape)\n",
    "#         for i in range(100):\n",
    "#             if (not tf.reduce_all(tf.equal(pred_z[:, i, :], states[i])).numpy()):\n",
    "#                 print(\"not Equal ({})\".format(i))\n",
    "#         return\n",
    "        # Move from latent space to sample space\n",
    "        pred_x = dec(pred_z)\n",
    "        \n",
    "#         print(pred_x.shape)\n",
    "        \n",
    "        # Calculate KL divergence and loss\n",
    "        noise_std_ = tf.zeros(pred_x.shape) + noise_std\n",
    "        noise_logvar = 2. * tf.log(noise_std_)\n",
    "        logpx = tf.reduce_sum(tf.reduce_sum(log_normal_pdf(\n",
    "            samp_trajs, pred_x, noise_logvar), 2), 1)\n",
    "        pz0_mean = pz0_logvar = tf.zeros(z0.shape)\n",
    "        analytic_kl = tf.reduce_sum(normal_kl(qz0_mean, qz0_logvar,\n",
    "                                pz0_mean, pz0_logvar), -1)\n",
    "        loss = tf.reduce_mean(-logpx + analytic_kl)\n",
    "    \n",
    "    # Update the models\n",
    "    dLoss = tape.gradient(loss, outputs)\n",
    "    h_start, dfdh0, dWeights = neural_ode.backward(outputs, dLoss)\n",
    "    optimizer.apply_gradients(zip(dWeights, model.weights))\n",
    "    \n",
    "    dLoss_drec = tape.gradient(loss, rec.variables)\n",
    "    optimizer.apply_gradients(zip(dLoss_drec, rec.weights))\n",
    "    \n",
    "    dLoss_ddec = tape.gradient(loss, dec.variables)\n",
    "    optimizer.apply_gradients(zip(dLoss_ddec, dec.weights))\n",
    "    \n",
    "    del tape\n",
    "    return loss\n",
    "\n",
    "# Compile EAGER graph to static (this will be much faster)\n",
    "# compute_gradients_and_update = tfe.defun(compute_gradients_and_update)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/iancarter/Documents/JHU_EP/Independent_Study/kmkolasinski/kmkolasinskienv/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:642: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /Users/iancarter/Documents/JHU_EP/Independent_Study/github/test/neural_ode.py:103: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Iteration: 0, Loss: 20453.896484375, running average elbo: -20453.896484375\n",
      "Iteration: 1, Loss: 18172.603515625, running average elbo: -20431.083984375\n",
      "Iteration: 2, Loss: 16579.62890625, running average elbo: -20392.5703125\n",
      "Iteration: 3, Loss: 14713.0341796875, running average elbo: -20335.775390625\n",
      "Iteration: 4, Loss: 12453.4287109375, running average elbo: -20256.953125\n",
      "Iteration: 5, Loss: 10438.0615234375, running average elbo: -20158.763671875\n",
      "Iteration: 6, Loss: 9053.97265625, running average elbo: -20047.71484375\n",
      "Iteration: 7, Loss: 8078.265625, running average elbo: -19928.021484375\n",
      "Iteration: 8, Loss: 7481.15185546875, running average elbo: -19803.552734375\n",
      "Iteration: 9, Loss: 6816.35595703125, running average elbo: -19673.681640625\n",
      "Iteration: 10, Loss: 6378.7265625, running average elbo: -19540.732421875\n",
      "Iteration: 11, Loss: 6194.4482421875, running average elbo: -19407.271484375\n",
      "Iteration: 12, Loss: 6109.57421875, running average elbo: -19274.294921875\n",
      "Iteration: 13, Loss: 6102.46923828125, running average elbo: -19142.578125\n",
      "Iteration: 14, Loss: 6185.896484375, running average elbo: -19013.01171875\n",
      "Iteration: 15, Loss: 6209.296875, running average elbo: -18884.974609375\n",
      "Iteration: 16, Loss: 6250.3876953125, running average elbo: -18758.62890625\n",
      "Iteration: 17, Loss: 6045.00341796875, running average elbo: -18631.4921875\n",
      "Iteration: 18, Loss: 5851.3310546875, running average elbo: -18503.69140625\n",
      "Iteration: 19, Loss: 5715.26708984375, running average elbo: -18375.806640625\n",
      "Iteration: 20, Loss: 5726.373046875, running average elbo: -18249.3125\n",
      "Iteration: 21, Loss: 5794.28515625, running average elbo: -18124.763671875\n",
      "Iteration: 22, Loss: 5750.53466796875, running average elbo: -18001.021484375\n",
      "Iteration: 23, Loss: 5629.85888671875, running average elbo: -17877.310546875\n",
      "Iteration: 24, Loss: 5614.18310546875, running average elbo: -17754.6796875\n",
      "Iteration: 25, Loss: 5573.060546875, running average elbo: -17632.86328125\n",
      "Iteration: 26, Loss: 5571.72119140625, running average elbo: -17512.251953125\n",
      "Iteration: 27, Loss: 5470.13037109375, running average elbo: -17391.830078125\n",
      "Iteration: 28, Loss: 5448.80810546875, running average elbo: -17272.400390625\n",
      "Iteration: 29, Loss: 5427.58203125, running average elbo: -17153.951171875\n",
      "Iteration: 30, Loss: 5428.16650390625, running average elbo: -17036.693359375\n",
      "Iteration: 31, Loss: 5413.31982421875, running average elbo: -16920.458984375\n",
      "Iteration: 32, Loss: 5341.0634765625, running average elbo: -16804.6640625\n",
      "Iteration: 33, Loss: 5347.4599609375, running average elbo: -16690.091796875\n",
      "Iteration: 34, Loss: 5336.38330078125, running average elbo: -16576.5546875\n",
      "Iteration: 35, Loss: 5362.578125, running average elbo: -16464.4140625\n",
      "Iteration: 36, Loss: 5319.49755859375, running average elbo: -16352.9658203125\n",
      "Iteration: 37, Loss: 5310.26904296875, running average elbo: -16242.5390625\n",
      "Iteration: 38, Loss: 5254.1162109375, running average elbo: -16132.6552734375\n",
      "Iteration: 39, Loss: 5277.48388671875, running average elbo: -16024.103515625\n",
      "Iteration: 40, Loss: 5290.14697265625, running average elbo: -15916.763671875\n",
      "Iteration: 41, Loss: 5279.541015625, running average elbo: -15810.390625\n",
      "Iteration: 42, Loss: 5307.40380859375, running average elbo: -15705.361328125\n",
      "Iteration: 43, Loss: 5283.14892578125, running average elbo: -15601.138671875\n",
      "Iteration: 44, Loss: 5307.869140625, running average elbo: -15498.2060546875\n",
      "Iteration: 45, Loss: 5320.30908203125, running average elbo: -15396.427734375\n",
      "Iteration: 46, Loss: 5327.73486328125, running average elbo: -15295.7412109375\n",
      "Iteration: 47, Loss: 5358.51171875, running average elbo: -15196.369140625\n",
      "Iteration: 48, Loss: 5508.44384765625, running average elbo: -15099.4892578125\n",
      "Iteration: 49, Loss: 5415.337890625, running average elbo: -15002.6474609375\n",
      "Iteration: 50, Loss: 5419.12158203125, running average elbo: -14906.8125\n",
      "Iteration: 51, Loss: 5346.87255859375, running average elbo: -14811.212890625\n",
      "Iteration: 52, Loss: 5133.40576171875, running average elbo: -14714.4345703125\n",
      "Iteration: 53, Loss: 4865.92431640625, running average elbo: -14615.94921875\n",
      "Iteration: 54, Loss: 5137.61865234375, running average elbo: -14521.166015625\n",
      "Iteration: 55, Loss: 4681.40185546875, running average elbo: -14422.7685546875\n",
      "Iteration: 56, Loss: 4885.576171875, running average elbo: -14327.396484375\n",
      "Iteration: 57, Loss: 4754.18994140625, running average elbo: -14231.6650390625\n",
      "Iteration: 58, Loss: 4590.57373046875, running average elbo: -14135.25390625\n",
      "Iteration: 59, Loss: 4557.14306640625, running average elbo: -14039.47265625\n",
      "Iteration: 60, Loss: 4477.2041015625, running average elbo: -13943.8505859375\n",
      "Iteration: 61, Loss: 4516.55859375, running average elbo: -13849.578125\n",
      "Iteration: 62, Loss: 4307.18994140625, running average elbo: -13754.154296875\n",
      "Iteration: 63, Loss: 4050.589599609375, running average elbo: -13657.119140625\n",
      "Iteration: 64, Loss: 4057.89453125, running average elbo: -13561.126953125\n",
      "Iteration: 65, Loss: 3922.083251953125, running average elbo: -13464.736328125\n",
      "Iteration: 66, Loss: 3857.082275390625, running average elbo: -13368.66015625\n",
      "Iteration: 67, Loss: 3728.884033203125, running average elbo: -13272.2626953125\n",
      "Iteration: 68, Loss: 3600.98291015625, running average elbo: -13175.5498046875\n",
      "Iteration: 69, Loss: 3397.834716796875, running average elbo: -13077.7724609375\n",
      "Iteration: 70, Loss: 3153.741943359375, running average elbo: -12978.5322265625\n",
      "Iteration: 71, Loss: 3285.86376953125, running average elbo: -12881.60546875\n",
      "Iteration: 72, Loss: 3122.879150390625, running average elbo: -12784.017578125\n",
      "Iteration: 73, Loss: 2752.78271484375, running average elbo: -12683.705078125\n",
      "Iteration: 74, Loss: 2862.33447265625, running average elbo: -12585.4912109375\n",
      "Iteration: 75, Loss: 2709.285400390625, running average elbo: -12486.7294921875\n",
      "Iteration: 76, Loss: 2818.129150390625, running average elbo: -12390.0439453125\n",
      "Iteration: 77, Loss: 2824.801513671875, running average elbo: -12294.3916015625\n",
      "Iteration: 78, Loss: 2605.833984375, running average elbo: -12197.5068359375\n",
      "Iteration: 79, Loss: 2483.54296875, running average elbo: -12100.3671875\n",
      "Iteration: 80, Loss: 2453.560546875, running average elbo: -12003.8984375\n",
      "Iteration: 81, Loss: 2503.942138671875, running average elbo: -11908.8984375\n",
      "Iteration: 82, Loss: 2563.2509765625, running average elbo: -11815.4423828125\n",
      "Iteration: 83, Loss: 2360.257568359375, running average elbo: -11720.890625\n",
      "Iteration: 84, Loss: 2453.04638671875, running average elbo: -11628.2119140625\n",
      "Iteration: 85, Loss: 2388.933349609375, running average elbo: -11535.8193359375\n",
      "Iteration: 86, Loss: 2390.2333984375, running average elbo: -11444.36328125\n",
      "Iteration: 87, Loss: 2291.484130859375, running average elbo: -11352.8349609375\n",
      "Iteration: 88, Loss: 2212.403564453125, running average elbo: -11261.4306640625\n",
      "Iteration: 89, Loss: 2203.011962890625, running average elbo: -11170.8466796875\n",
      "Iteration: 90, Loss: 2208.677001953125, running average elbo: -11081.2255859375\n",
      "Iteration: 91, Loss: 2161.132080078125, running average elbo: -10992.0244140625\n",
      "Iteration: 92, Loss: 2214.973388671875, running average elbo: -10904.25390625\n",
      "Iteration: 93, Loss: 2096.376220703125, running average elbo: -10816.17578125\n",
      "Iteration: 94, Loss: 2043.175537109375, running average elbo: -10728.4453125\n",
      "Iteration: 95, Loss: 2010.2740478515625, running average elbo: -10641.263671875\n",
      "Iteration: 96, Loss: 1986.4840087890625, running average elbo: -10554.716796875\n",
      "Iteration: 97, Loss: 2017.627197265625, running average elbo: -10469.3466796875\n",
      "Iteration: 98, Loss: 2039.5753173828125, running average elbo: -10385.048828125\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 99, Loss: 2008.2352294921875, running average elbo: -10301.2802734375\n",
      "Iteration: 100, Loss: 2050.508056640625, running average elbo: -10218.7724609375\n",
      "Iteration: 101, Loss: 2135.15771484375, running average elbo: -10137.9365234375\n",
      "Iteration: 102, Loss: 2110.744384765625, running average elbo: -10057.6650390625\n",
      "Iteration: 103, Loss: 1948.7205810546875, running average elbo: -9976.576171875\n",
      "Iteration: 104, Loss: 2078.876708984375, running average elbo: -9897.599609375\n",
      "Iteration: 105, Loss: 1923.166748046875, running average elbo: -9817.85546875\n",
      "Iteration: 106, Loss: 2022.6719970703125, running average elbo: -9739.9033203125\n",
      "Iteration: 107, Loss: 1962.0235595703125, running average elbo: -9662.1240234375\n",
      "Iteration: 108, Loss: 1954.75048828125, running average elbo: -9585.05078125\n",
      "Iteration: 109, Loss: 1903.8294677734375, running average elbo: -9508.23828125\n",
      "Iteration: 110, Loss: 1905.6285400390625, running average elbo: -9432.212890625\n",
      "Iteration: 111, Loss: 1885.762451171875, running average elbo: -9356.748046875\n",
      "Iteration: 112, Loss: 1878.5679931640625, running average elbo: -9281.966796875\n",
      "Iteration: 113, Loss: 1898.367919921875, running average elbo: -9208.130859375\n",
      "Iteration: 114, Loss: 1873.375732421875, running average elbo: -9134.783203125\n",
      "Iteration: 115, Loss: 1876.4267578125, running average elbo: -9062.2001953125\n",
      "Iteration: 116, Loss: 1897.1707763671875, running average elbo: -8990.5498046875\n",
      "Iteration: 117, Loss: 1808.22802734375, running average elbo: -8918.7265625\n",
      "Iteration: 118, Loss: 1842.2777099609375, running average elbo: -8847.9619140625\n",
      "Iteration: 119, Loss: 1882.15771484375, running average elbo: -8778.3037109375\n",
      "Iteration: 120, Loss: 1832.9844970703125, running average elbo: -8708.8505859375\n",
      "Iteration: 121, Loss: 1820.845703125, running average elbo: -8639.9697265625\n",
      "Iteration: 122, Loss: 1823.023681640625, running average elbo: -8571.80078125\n",
      "Iteration: 123, Loss: 1855.8577880859375, running average elbo: -8504.6416015625\n",
      "Iteration: 124, Loss: 1878.957763671875, running average elbo: -8438.3857421875\n",
      "Iteration: 125, Loss: 1866.781005859375, running average elbo: -8372.669921875\n",
      "Iteration: 126, Loss: 1838.8082275390625, running average elbo: -8307.3310546875\n",
      "Iteration: 127, Loss: 1865.7862548828125, running average elbo: -8242.916015625\n",
      "Iteration: 128, Loss: 1921.1639404296875, running average elbo: -8179.6982421875\n",
      "Iteration: 129, Loss: 1839.8070068359375, running average elbo: -8116.29931640625\n",
      "Iteration: 130, Loss: 1929.256103515625, running average elbo: -8054.4287109375\n",
      "Iteration: 131, Loss: 2115.683349609375, running average elbo: -7995.041015625\n",
      "Iteration: 132, Loss: 2205.5625, running average elbo: -7937.146484375\n",
      "Iteration: 133, Loss: 1940.9954833984375, running average elbo: -7877.18505859375\n",
      "Iteration: 134, Loss: 1773.49169921875, running average elbo: -7816.14794921875\n",
      "Iteration: 135, Loss: 1822.2454833984375, running average elbo: -7756.208984375\n",
      "Iteration: 136, Loss: 2121.500732421875, running average elbo: -7699.86181640625\n",
      "Iteration: 137, Loss: 1696.718994140625, running average elbo: -7639.83056640625\n",
      "Iteration: 138, Loss: 1983.748291015625, running average elbo: -7583.26953125\n",
      "Iteration: 139, Loss: 1769.754638671875, running average elbo: -7525.134765625\n",
      "Iteration: 140, Loss: 1827.0296630859375, running average elbo: -7468.15380859375\n",
      "Iteration: 141, Loss: 1761.251220703125, running average elbo: -7411.08447265625\n",
      "Iteration: 142, Loss: 1707.731201171875, running average elbo: -7354.05078125\n",
      "Iteration: 143, Loss: 1856.061767578125, running average elbo: -7299.07080078125\n",
      "Iteration: 144, Loss: 2108.83154296875, running average elbo: -7247.16845703125\n",
      "Iteration: 145, Loss: 2243.284912109375, running average elbo: -7197.12939453125\n",
      "Iteration: 146, Loss: 1813.185546875, running average elbo: -7143.2900390625\n",
      "Iteration: 147, Loss: 1985.893310546875, running average elbo: -7091.71630859375\n",
      "Iteration: 148, Loss: 1719.8759765625, running average elbo: -7037.998046875\n",
      "Iteration: 149, Loss: 1758.4639892578125, running average elbo: -6985.20263671875\n",
      "Iteration: 150, Loss: 1654.70751953125, running average elbo: -6931.8974609375\n",
      "Iteration: 151, Loss: 1643.3394775390625, running average elbo: -6879.01220703125\n",
      "Iteration: 152, Loss: 1735.7288818359375, running average elbo: -6827.57958984375\n",
      "Iteration: 153, Loss: 1580.8175048828125, running average elbo: -6775.11181640625\n",
      "Iteration: 154, Loss: 1522.2215576171875, running average elbo: -6722.5830078125\n",
      "Iteration: 155, Loss: 1640.656005859375, running average elbo: -6671.76416015625\n",
      "Iteration: 156, Loss: 1731.56396484375, running average elbo: -6622.36181640625\n",
      "Iteration: 157, Loss: 1651.357666015625, running average elbo: -6572.65185546875\n",
      "Iteration: 158, Loss: 1522.916015625, running average elbo: -6522.154296875\n",
      "Iteration: 159, Loss: 1520.2587890625, running average elbo: -6472.13525390625\n",
      "Iteration: 160, Loss: 1541.33447265625, running average elbo: -6422.82763671875\n",
      "Iteration: 161, Loss: 1510.3477783203125, running average elbo: -6373.703125\n",
      "Iteration: 162, Loss: 1473.0301513671875, running average elbo: -6324.69677734375\n",
      "Iteration: 163, Loss: 1444.505126953125, running average elbo: -6275.89453125\n",
      "Iteration: 164, Loss: 1442.4232177734375, running average elbo: -6227.56005859375\n",
      "Iteration: 165, Loss: 1460.3582763671875, running average elbo: -6179.88818359375\n",
      "Iteration: 166, Loss: 1385.8487548828125, running average elbo: -6131.94775390625\n",
      "Iteration: 167, Loss: 1378.6910400390625, running average elbo: -6084.41552734375\n",
      "Iteration: 168, Loss: 1392.1085205078125, running average elbo: -6037.4921875\n",
      "Iteration: 169, Loss: 1371.997802734375, running average elbo: -5990.83740234375\n",
      "Iteration: 170, Loss: 1335.5960693359375, running average elbo: -5944.28515625\n",
      "Iteration: 171, Loss: 1405.8001708984375, running average elbo: -5898.900390625\n",
      "Iteration: 172, Loss: 1384.348876953125, running average elbo: -5853.7548828125\n",
      "Iteration: 173, Loss: 1363.7972412109375, running average elbo: -5808.85546875\n",
      "Iteration: 174, Loss: 1351.611083984375, running average elbo: -5764.283203125\n",
      "Iteration: 175, Loss: 1277.1519775390625, running average elbo: -5719.412109375\n",
      "Iteration: 176, Loss: 1373.302490234375, running average elbo: -5675.951171875\n",
      "Iteration: 177, Loss: 1302.4957275390625, running average elbo: -5632.216796875\n",
      "Iteration: 178, Loss: 1470.813232421875, running average elbo: -5590.6025390625\n",
      "Iteration: 179, Loss: 1874.5565185546875, running average elbo: -5553.4423828125\n",
      "Iteration: 180, Loss: 1994.9112548828125, running average elbo: -5517.857421875\n",
      "Iteration: 181, Loss: 1447.6212158203125, running average elbo: -5477.15478515625\n",
      "Iteration: 182, Loss: 1635.695556640625, running average elbo: -5438.740234375\n",
      "Iteration: 183, Loss: 1361.0081787109375, running average elbo: -5397.962890625\n",
      "Iteration: 184, Loss: 1548.43701171875, running average elbo: -5359.4677734375\n",
      "Iteration: 185, Loss: 1239.4217529296875, running average elbo: -5318.26708984375\n",
      "Iteration: 186, Loss: 1352.08251953125, running average elbo: -5278.60546875\n",
      "Iteration: 187, Loss: 1314.48291015625, running average elbo: -5238.96435546875\n",
      "Iteration: 188, Loss: 1271.5079345703125, running average elbo: -5199.28955078125\n",
      "Iteration: 189, Loss: 1282.4385986328125, running average elbo: -5160.12109375\n",
      "Iteration: 190, Loss: 1333.6875, running average elbo: -5121.85693359375\n",
      "Iteration: 191, Loss: 1203.12255859375, running average elbo: -5082.66943359375\n",
      "Iteration: 192, Loss: 1200.084228515625, running average elbo: -5043.84375\n",
      "Iteration: 193, Loss: 1095.3809814453125, running average elbo: -5004.35888671875\n",
      "Iteration: 194, Loss: 1131.9718017578125, running average elbo: -4965.63525390625\n",
      "Iteration: 195, Loss: 1140.5150146484375, running average elbo: -4927.38427734375\n",
      "Iteration: 196, Loss: 1195.6112060546875, running average elbo: -4890.06640625\n",
      "Iteration: 197, Loss: 1169.165771484375, running average elbo: -4852.85791015625\n",
      "Iteration: 198, Loss: 1091.268310546875, running average elbo: -4815.2421875\n",
      "Iteration: 199, Loss: 1057.9454345703125, running average elbo: -4777.66943359375\n",
      "Iteration: 200, Loss: 1045.3546142578125, running average elbo: -4740.34619140625\n",
      "Iteration: 201, Loss: 1054.3587646484375, running average elbo: -4703.486328125\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 202, Loss: 1092.6875, running average elbo: -4667.37841796875\n",
      "Iteration: 203, Loss: 1087.137451171875, running average elbo: -4631.576171875\n",
      "Iteration: 204, Loss: 975.5923461914062, running average elbo: -4595.01611328125\n",
      "Iteration: 205, Loss: 1009.1722412109375, running average elbo: -4559.15771484375\n",
      "Iteration: 206, Loss: 1025.1419677734375, running average elbo: -4523.81787109375\n",
      "Iteration: 207, Loss: 1055.8082275390625, running average elbo: -4489.1376953125\n",
      "Iteration: 208, Loss: 1025.574951171875, running average elbo: -4454.50244140625\n",
      "Iteration: 209, Loss: 1042.5826416015625, running average elbo: -4420.38330078125\n",
      "Iteration: 210, Loss: 978.067138671875, running average elbo: -4385.96044921875\n",
      "Iteration: 211, Loss: 965.08056640625, running average elbo: -4351.751953125\n",
      "Iteration: 212, Loss: 1034.767822265625, running average elbo: -4318.58203125\n",
      "Iteration: 213, Loss: 1018.3462524414062, running average elbo: -4285.580078125\n",
      "Iteration: 214, Loss: 987.1912841796875, running average elbo: -4252.59619140625\n",
      "Iteration: 215, Loss: 1026.6929931640625, running average elbo: -4220.33740234375\n",
      "Iteration: 216, Loss: 1097.4625244140625, running average elbo: -4189.10888671875\n",
      "Iteration: 217, Loss: 1058.05615234375, running average elbo: -4157.79833984375\n",
      "Iteration: 218, Loss: 1072.904296875, running average elbo: -4126.94921875\n",
      "Iteration: 219, Loss: 1119.16064453125, running average elbo: -4096.87109375\n",
      "Iteration: 220, Loss: 1066.7493896484375, running average elbo: -4066.56982421875\n",
      "Iteration: 221, Loss: 943.9866943359375, running average elbo: -4035.343994140625\n",
      "Iteration: 222, Loss: 1265.4722900390625, running average elbo: -4007.645263671875\n",
      "Iteration: 223, Loss: 1914.1705322265625, running average elbo: -3986.71044921875\n",
      "Iteration: 224, Loss: 1619.885498046875, running average elbo: -3963.042236328125\n",
      "Iteration: 225, Loss: 1588.8087158203125, running average elbo: -3939.300048828125\n",
      "Iteration: 226, Loss: 1550.5819091796875, running average elbo: -3915.412841796875\n",
      "Iteration: 227, Loss: 1477.499267578125, running average elbo: -3891.03369140625\n",
      "Iteration: 228, Loss: 1088.450927734375, running average elbo: -3863.0078125\n",
      "Iteration: 229, Loss: 1186.6719970703125, running average elbo: -3836.244384765625\n",
      "Iteration: 230, Loss: 1338.064208984375, running average elbo: -3811.2626953125\n",
      "Iteration: 231, Loss: 1413.812744140625, running average elbo: -3787.288330078125\n",
      "Iteration: 232, Loss: 1243.36669921875, running average elbo: -3761.84912109375\n",
      "Iteration: 233, Loss: 1204.0841064453125, running average elbo: -3736.271484375\n",
      "Iteration: 234, Loss: 1210.1429443359375, running average elbo: -3711.010009765625\n",
      "Iteration: 235, Loss: 1275.63720703125, running average elbo: -3686.65625\n",
      "Iteration: 236, Loss: 1239.614990234375, running average elbo: -3662.18603515625\n",
      "Iteration: 237, Loss: 1332.03271484375, running average elbo: -3638.884521484375\n",
      "Iteration: 238, Loss: 1598.4908447265625, running average elbo: -3618.48046875\n",
      "Iteration: 239, Loss: 1640.3887939453125, running average elbo: -3598.699462890625\n",
      "Iteration: 240, Loss: 1466.3995361328125, running average elbo: -3577.37646484375\n",
      "Iteration: 241, Loss: 1310.8992919921875, running average elbo: -3554.711669921875\n",
      "Iteration: 242, Loss: 1184.8740234375, running average elbo: -3531.01318359375\n",
      "Iteration: 243, Loss: 1242.466796875, running average elbo: -3508.127685546875\n",
      "Iteration: 244, Loss: 1398.55126953125, running average elbo: -3487.031982421875\n",
      "Iteration: 245, Loss: 1476.195556640625, running average elbo: -3466.923583984375\n",
      "Iteration: 246, Loss: 1401.20458984375, running average elbo: -3446.266357421875\n",
      "Iteration: 247, Loss: 1291.8140869140625, running average elbo: -3424.721923828125\n",
      "Iteration: 248, Loss: 1123.6551513671875, running average elbo: -3401.71142578125\n",
      "Iteration: 249, Loss: 1017.7670288085938, running average elbo: -3377.8720703125\n",
      "Iteration: 250, Loss: 1012.079345703125, running average elbo: -3354.214111328125\n",
      "Iteration: 251, Loss: 1057.0555419921875, running average elbo: -3331.24267578125\n",
      "Iteration: 252, Loss: 1149.332275390625, running average elbo: -3309.423583984375\n",
      "Iteration: 253, Loss: 1222.498291015625, running average elbo: -3288.554443359375\n",
      "Iteration: 254, Loss: 1238.6195068359375, running average elbo: -3268.05517578125\n",
      "Iteration: 255, Loss: 1086.878173828125, running average elbo: -3246.24365234375\n",
      "Iteration: 256, Loss: 911.1279296875, running average elbo: -3222.892578125\n",
      "Iteration: 257, Loss: 924.0126342773438, running average elbo: -3199.90380859375\n",
      "Iteration: 258, Loss: 1108.5059814453125, running average elbo: -3178.98974609375\n",
      "Iteration: 259, Loss: 1209.864990234375, running average elbo: -3159.298583984375\n",
      "Iteration: 260, Loss: 1184.135498046875, running average elbo: -3139.546875\n",
      "Iteration: 261, Loss: 1117.9835205078125, running average elbo: -3119.331298828125\n",
      "Iteration: 262, Loss: 988.2973022460938, running average elbo: -3098.02099609375\n",
      "Iteration: 263, Loss: 844.0061645507812, running average elbo: -3075.480712890625\n",
      "Iteration: 264, Loss: 808.3280639648438, running average elbo: -3052.80908203125\n",
      "Iteration: 265, Loss: 817.5185546875, running average elbo: -3030.456298828125\n",
      "Iteration: 266, Loss: 807.379638671875, running average elbo: -3008.2255859375\n",
      "Iteration: 267, Loss: 831.806396484375, running average elbo: -2986.46142578125\n",
      "Iteration: 268, Loss: 878.0610961914062, running average elbo: -2965.37744140625\n",
      "Iteration: 269, Loss: 770.906005859375, running average elbo: -2943.4326171875\n",
      "Iteration: 270, Loss: 668.1473388671875, running average elbo: -2920.6796875\n",
      "Iteration: 271, Loss: 639.404541015625, running average elbo: -2897.866943359375\n",
      "Iteration: 272, Loss: 643.2805786132812, running average elbo: -2875.321044921875\n",
      "Iteration: 273, Loss: 935.3630981445312, running average elbo: -2855.92138671875\n",
      "Iteration: 274, Loss: 1928.2044677734375, running average elbo: -2846.644287109375\n",
      "Iteration: 275, Loss: 2778.720947265625, running average elbo: -2845.965087890625\n",
      "Iteration: 276, Loss: 2171.370361328125, running average elbo: -2839.218994140625\n",
      "Iteration: 277, Loss: 1272.7635498046875, running average elbo: -2823.554443359375\n",
      "Iteration: 278, Loss: 1163.9927978515625, running average elbo: -2806.958740234375\n",
      "Iteration: 279, Loss: 1607.02197265625, running average elbo: -2794.95947265625\n",
      "Iteration: 280, Loss: 1626.137451171875, running average elbo: -2783.271484375\n",
      "Iteration: 281, Loss: 1549.6734619140625, running average elbo: -2770.935546875\n",
      "Iteration: 282, Loss: 1291.74072265625, running average elbo: -2756.143798828125\n",
      "Iteration: 283, Loss: 1115.3897705078125, running average elbo: -2739.736083984375\n",
      "Iteration: 284, Loss: 1266.1202392578125, running average elbo: -2725.0\n",
      "Iteration: 285, Loss: 1188.10498046875, running average elbo: -2709.631103515625\n",
      "Iteration: 286, Loss: 1054.835693359375, running average elbo: -2693.083251953125\n",
      "Iteration: 287, Loss: 1027.351806640625, running average elbo: -2676.42578125\n",
      "Iteration: 288, Loss: 944.233642578125, running average elbo: -2659.10400390625\n",
      "Iteration: 289, Loss: 985.141845703125, running average elbo: -2642.3642578125\n",
      "Iteration: 290, Loss: 1032.767822265625, running average elbo: -2626.268310546875\n",
      "Iteration: 291, Loss: 970.80224609375, running average elbo: -2609.713623046875\n",
      "Iteration: 292, Loss: 1000.5772705078125, running average elbo: -2593.622314453125\n",
      "Iteration: 293, Loss: 944.1951293945312, running average elbo: -2577.1279296875\n",
      "Iteration: 294, Loss: 886.7463989257812, running average elbo: -2560.22412109375\n",
      "Iteration: 295, Loss: 889.872314453125, running average elbo: -2543.5205078125\n",
      "Iteration: 296, Loss: 808.1810913085938, running average elbo: -2526.1669921875\n",
      "Iteration: 297, Loss: 782.237060546875, running average elbo: -2508.7275390625\n",
      "Iteration: 298, Loss: 788.4953002929688, running average elbo: -2491.525390625\n",
      "Iteration: 299, Loss: 778.3400268554688, running average elbo: -2474.3935546875\n",
      "Iteration: 300, Loss: 787.2006225585938, running average elbo: -2457.521728515625\n",
      "Iteration: 301, Loss: 737.4031372070312, running average elbo: -2440.320556640625\n",
      "Iteration: 302, Loss: 706.3004150390625, running average elbo: -2422.98046875\n",
      "Iteration: 303, Loss: 730.4358520507812, running average elbo: -2406.05517578125\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 304, Loss: 730.2133178710938, running average elbo: -2389.296875\n",
      "Iteration: 305, Loss: 759.1660766601562, running average elbo: -2372.995361328125\n",
      "Iteration: 306, Loss: 744.329833984375, running average elbo: -2356.708740234375\n",
      "Iteration: 307, Loss: 715.8958740234375, running average elbo: -2340.300537109375\n",
      "Iteration: 308, Loss: 736.7698974609375, running average elbo: -2324.26513671875\n",
      "Iteration: 309, Loss: 755.1797485351562, running average elbo: -2308.57421875\n",
      "Iteration: 310, Loss: 760.4645385742188, running average elbo: -2293.09326171875\n",
      "Iteration: 311, Loss: 765.8658447265625, running average elbo: -2277.821044921875\n",
      "Iteration: 312, Loss: 735.4044799804688, running average elbo: -2262.39697265625\n",
      "Iteration: 313, Loss: 724.3623657226562, running average elbo: -2247.0166015625\n",
      "Iteration: 314, Loss: 721.9452514648438, running average elbo: -2231.765869140625\n",
      "Iteration: 315, Loss: 729.5244750976562, running average elbo: -2216.743408203125\n",
      "Iteration: 316, Loss: 718.9035034179688, running average elbo: -2201.764892578125\n",
      "Iteration: 317, Loss: 713.4133911132812, running average elbo: -2186.88134765625\n",
      "Iteration: 318, Loss: 693.025146484375, running average elbo: -2171.942626953125\n",
      "Iteration: 319, Loss: 669.6220703125, running average elbo: -2156.91943359375\n",
      "Iteration: 320, Loss: 648.6586303710938, running average elbo: -2141.8369140625\n",
      "Iteration: 321, Loss: 646.1414794921875, running average elbo: -2126.8798828125\n",
      "Iteration: 322, Loss: 647.9827270507812, running average elbo: -2112.0908203125\n",
      "Iteration: 323, Loss: 660.8054809570312, running average elbo: -2097.578125\n",
      "Iteration: 324, Loss: 649.0165405273438, running average elbo: -2083.092529296875\n",
      "Iteration: 325, Loss: 630.5519409179688, running average elbo: -2068.567138671875\n",
      "Iteration: 326, Loss: 603.7013549804688, running average elbo: -2053.91845703125\n",
      "Iteration: 327, Loss: 571.6607666015625, running average elbo: -2039.0958251953125\n",
      "Iteration: 328, Loss: 580.82177734375, running average elbo: -2024.5130615234375\n",
      "Iteration: 329, Loss: 572.6146240234375, running average elbo: -2009.994140625\n",
      "Iteration: 330, Loss: 533.2324829101562, running average elbo: -1995.2264404296875\n",
      "Iteration: 331, Loss: 526.6686401367188, running average elbo: -1980.5408935546875\n",
      "Iteration: 332, Loss: 530.9727172851562, running average elbo: -1966.045166015625\n",
      "Iteration: 333, Loss: 502.6997375488281, running average elbo: -1951.4117431640625\n",
      "Iteration: 334, Loss: 505.9718017578125, running average elbo: -1936.9573974609375\n",
      "Iteration: 335, Loss: 504.6936950683594, running average elbo: -1922.6348876953125\n",
      "Iteration: 336, Loss: 499.6474914550781, running average elbo: -1908.405029296875\n",
      "Iteration: 337, Loss: 498.2715148925781, running average elbo: -1894.3037109375\n",
      "Iteration: 338, Loss: 502.13525390625, running average elbo: -1880.382080078125\n",
      "Iteration: 339, Loss: 492.1436462402344, running average elbo: -1866.4996337890625\n",
      "Iteration: 340, Loss: 482.4248962402344, running average elbo: -1852.6588134765625\n",
      "Iteration: 341, Loss: 509.2872009277344, running average elbo: -1839.22509765625\n",
      "Iteration: 342, Loss: 519.4048461914062, running average elbo: -1826.0269775390625\n",
      "Iteration: 343, Loss: 494.614990234375, running average elbo: -1812.712890625\n",
      "Iteration: 344, Loss: 468.4052429199219, running average elbo: -1799.2698974609375\n",
      "Iteration: 345, Loss: 455.65673828125, running average elbo: -1785.833740234375\n",
      "Iteration: 346, Loss: 458.0684509277344, running average elbo: -1772.55615234375\n",
      "Iteration: 347, Loss: 483.890869140625, running average elbo: -1759.66943359375\n",
      "Iteration: 348, Loss: 523.5319213867188, running average elbo: -1747.30810546875\n",
      "Iteration: 349, Loss: 507.8268737792969, running average elbo: -1734.913330078125\n",
      "Iteration: 350, Loss: 471.6387634277344, running average elbo: -1722.2806396484375\n",
      "Iteration: 351, Loss: 461.53155517578125, running average elbo: -1709.6732177734375\n",
      "Iteration: 352, Loss: 471.69512939453125, running average elbo: -1697.29345703125\n",
      "Iteration: 353, Loss: 515.7992553710938, running average elbo: -1685.478515625\n",
      "Iteration: 354, Loss: 541.999755859375, running average elbo: -1674.0438232421875\n",
      "Iteration: 355, Loss: 545.65087890625, running average elbo: -1662.7598876953125\n",
      "Iteration: 356, Loss: 511.6833190917969, running average elbo: -1651.2491455078125\n",
      "Iteration: 357, Loss: 443.3610534667969, running average elbo: -1639.1702880859375\n",
      "Iteration: 358, Loss: 427.8341064453125, running average elbo: -1627.056884765625\n",
      "Iteration: 359, Loss: 479.9569396972656, running average elbo: -1615.5859375\n",
      "Iteration: 360, Loss: 533.4298706054688, running average elbo: -1604.764404296875\n",
      "Iteration: 361, Loss: 581.9887084960938, running average elbo: -1594.5367431640625\n",
      "Iteration: 362, Loss: 531.5521240234375, running average elbo: -1583.906982421875\n",
      "Iteration: 363, Loss: 434.6087646484375, running average elbo: -1572.4139404296875\n",
      "Iteration: 364, Loss: 417.22161865234375, running average elbo: -1560.862060546875\n",
      "Iteration: 365, Loss: 477.63226318359375, running average elbo: -1550.02978515625\n",
      "Iteration: 366, Loss: 568.0537719726562, running average elbo: -1540.2100830078125\n",
      "Iteration: 367, Loss: 611.6511840820312, running average elbo: -1530.9244384765625\n",
      "Iteration: 368, Loss: 570.770263671875, running average elbo: -1521.322998046875\n",
      "Iteration: 369, Loss: 485.97314453125, running average elbo: -1510.969482421875\n",
      "Iteration: 370, Loss: 413.7716064453125, running average elbo: -1499.9974365234375\n",
      "Iteration: 371, Loss: 403.7019348144531, running average elbo: -1489.034423828125\n",
      "Iteration: 372, Loss: 482.7976379394531, running average elbo: -1478.9720458984375\n",
      "Iteration: 373, Loss: 563.4573974609375, running average elbo: -1469.81689453125\n",
      "Iteration: 374, Loss: 583.725341796875, running average elbo: -1460.9560546875\n",
      "Iteration: 375, Loss: 491.3063049316406, running average elbo: -1451.2596435546875\n",
      "Iteration: 376, Loss: 397.93304443359375, running average elbo: -1440.7264404296875\n",
      "Iteration: 377, Loss: 387.3447265625, running average elbo: -1430.192626953125\n",
      "Iteration: 378, Loss: 458.59881591796875, running average elbo: -1420.4766845703125\n",
      "Iteration: 379, Loss: 559.6223754882812, running average elbo: -1411.8681640625\n",
      "Iteration: 380, Loss: 582.126708984375, running average elbo: -1403.57080078125\n",
      "Iteration: 381, Loss: 524.61181640625, running average elbo: -1394.78125\n",
      "Iteration: 382, Loss: 446.38189697265625, running average elbo: -1385.29736328125\n",
      "Iteration: 383, Loss: 368.5335693359375, running average elbo: -1375.1297607421875\n",
      "Iteration: 384, Loss: 381.0505065917969, running average elbo: -1365.18896484375\n",
      "Iteration: 385, Loss: 441.30224609375, running average elbo: -1355.9500732421875\n",
      "Iteration: 386, Loss: 523.8465576171875, running average elbo: -1347.629150390625\n",
      "Iteration: 387, Loss: 503.8676452636719, running average elbo: -1339.1915283203125\n",
      "Iteration: 388, Loss: 416.5062561035156, running average elbo: -1329.9647216796875\n",
      "Iteration: 389, Loss: 356.10662841796875, running average elbo: -1320.22607421875\n",
      "Iteration: 390, Loss: 351.086181640625, running average elbo: -1310.53466796875\n",
      "Iteration: 391, Loss: 414.6728820800781, running average elbo: -1301.5760498046875\n",
      "Iteration: 392, Loss: 459.288330078125, running average elbo: -1293.1531982421875\n",
      "Iteration: 393, Loss: 467.52923583984375, running average elbo: -1284.89697265625\n",
      "Iteration: 394, Loss: 424.0965270996094, running average elbo: -1276.2889404296875\n",
      "Iteration: 395, Loss: 366.997802734375, running average elbo: -1267.196044921875\n",
      "Iteration: 396, Loss: 340.4149475097656, running average elbo: -1257.92822265625\n",
      "Iteration: 397, Loss: 366.7061767578125, running average elbo: -1249.01611328125\n",
      "Iteration: 398, Loss: 428.1883850097656, running average elbo: -1240.807861328125\n",
      "Iteration: 399, Loss: 423.8590087890625, running average elbo: -1232.638427734375\n",
      "Iteration: 400, Loss: 397.3523864746094, running average elbo: -1224.2855224609375\n",
      "Iteration: 401, Loss: 336.95263671875, running average elbo: -1215.4122314453125\n",
      "Iteration: 402, Loss: 325.6737365722656, running average elbo: -1206.514892578125\n",
      "Iteration: 403, Loss: 351.63507080078125, running average elbo: -1197.966064453125\n",
      "Iteration: 404, Loss: 382.7060546875, running average elbo: -1189.8134765625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 405, Loss: 394.57275390625, running average elbo: -1181.861083984375\n",
      "Iteration: 406, Loss: 360.75048828125, running average elbo: -1173.6500244140625\n",
      "Iteration: 407, Loss: 324.71728515625, running average elbo: -1165.1607666015625\n",
      "Iteration: 408, Loss: 326.05792236328125, running average elbo: -1156.769775390625\n",
      "Iteration: 409, Loss: 338.1510009765625, running average elbo: -1148.5836181640625\n",
      "Iteration: 410, Loss: 359.1460876464844, running average elbo: -1140.689208984375\n",
      "Iteration: 411, Loss: 373.7081298828125, running average elbo: -1133.0194091796875\n",
      "Iteration: 412, Loss: 332.486572265625, running average elbo: -1125.0140380859375\n",
      "Iteration: 413, Loss: 374.5949401855469, running average elbo: -1117.5098876953125\n",
      "Iteration: 414, Loss: 475.7188720703125, running average elbo: -1111.092041015625\n",
      "Iteration: 415, Loss: 400.4853820800781, running average elbo: -1103.9859619140625\n",
      "Iteration: 416, Loss: 544.2857666015625, running average elbo: -1098.3890380859375\n",
      "Iteration: 417, Loss: 319.4696350097656, running average elbo: -1090.599853515625\n",
      "Iteration: 418, Loss: 367.3561706542969, running average elbo: -1083.367431640625\n",
      "Iteration: 419, Loss: 573.1923828125, running average elbo: -1078.2657470703125\n",
      "Iteration: 420, Loss: 376.0404968261719, running average elbo: -1071.2435302734375\n",
      "Iteration: 421, Loss: 517.9673461914062, running average elbo: -1065.7108154296875\n",
      "Iteration: 422, Loss: 820.5795288085938, running average elbo: -1063.259521484375\n",
      "Iteration: 423, Loss: 606.0451049804688, running average elbo: -1058.6873779296875\n",
      "Iteration: 424, Loss: 1006.572509765625, running average elbo: -1058.1661376953125\n",
      "Iteration: 425, Loss: 775.4323120117188, running average elbo: -1055.3387451171875\n",
      "Iteration: 426, Loss: 991.817626953125, running average elbo: -1054.70361328125\n",
      "Iteration: 427, Loss: 475.1836853027344, running average elbo: -1048.908447265625\n",
      "Iteration: 428, Loss: 929.7422485351562, running average elbo: -1047.716796875\n",
      "Iteration: 429, Loss: 562.0864868164062, running average elbo: -1042.8604736328125\n",
      "Iteration: 430, Loss: 809.992431640625, running average elbo: -1040.5318603515625\n",
      "Iteration: 431, Loss: 598.9150390625, running average elbo: -1036.11572265625\n",
      "Iteration: 432, Loss: 675.9567260742188, running average elbo: -1032.5140380859375\n",
      "Iteration: 433, Loss: 424.1340026855469, running average elbo: -1026.4302978515625\n",
      "Iteration: 434, Loss: 574.2943725585938, running average elbo: -1021.908935546875\n",
      "Iteration: 435, Loss: 369.2921142578125, running average elbo: -1015.3828125\n",
      "Iteration: 436, Loss: 528.00634765625, running average elbo: -1010.5090942382812\n",
      "Iteration: 437, Loss: 425.9316101074219, running average elbo: -1004.663330078125\n",
      "Iteration: 438, Loss: 365.14447021484375, running average elbo: -998.2681274414062\n",
      "Iteration: 439, Loss: 439.9786376953125, running average elbo: -992.6852416992188\n",
      "Iteration: 440, Loss: 432.417724609375, running average elbo: -987.0825805664062\n",
      "Iteration: 441, Loss: 374.94146728515625, running average elbo: -980.961181640625\n",
      "Iteration: 442, Loss: 444.7146301269531, running average elbo: -975.5986938476562\n",
      "Iteration: 443, Loss: 409.6229248046875, running average elbo: -969.93896484375\n",
      "Iteration: 444, Loss: 351.9331359863281, running average elbo: -963.7589111328125\n",
      "Iteration: 445, Loss: 379.5912780761719, running average elbo: -957.917236328125\n",
      "Iteration: 446, Loss: 380.7286682128906, running average elbo: -952.1453857421875\n",
      "Iteration: 447, Loss: 321.8321228027344, running average elbo: -945.84228515625\n",
      "Iteration: 448, Loss: 355.69354248046875, running average elbo: -939.9407958984375\n",
      "Iteration: 449, Loss: 323.5325927734375, running average elbo: -933.7767333984375\n",
      "Iteration: 450, Loss: 300.55682373046875, running average elbo: -927.4445190429688\n",
      "Iteration: 451, Loss: 336.7716979980469, running average elbo: -921.537841796875\n",
      "Iteration: 452, Loss: 284.8325500488281, running average elbo: -915.1707763671875\n",
      "Iteration: 453, Loss: 279.934326171875, running average elbo: -908.8184204101562\n",
      "Iteration: 454, Loss: 290.61175537109375, running average elbo: -902.6363525390625\n",
      "Iteration: 455, Loss: 268.2040100097656, running average elbo: -896.2920532226562\n",
      "Iteration: 456, Loss: 280.402587890625, running average elbo: -890.1331787109375\n",
      "Iteration: 457, Loss: 270.7502746582031, running average elbo: -883.9393920898438\n",
      "Iteration: 458, Loss: 256.3852844238281, running average elbo: -877.6638793945312\n",
      "Iteration: 459, Loss: 272.762939453125, running average elbo: -871.6148681640625\n",
      "Iteration: 460, Loss: 249.83316040039062, running average elbo: -865.3970947265625\n",
      "Iteration: 461, Loss: 260.8605651855469, running average elbo: -859.3516845703125\n",
      "Iteration: 462, Loss: 250.57809448242188, running average elbo: -853.2639770507812\n",
      "Iteration: 463, Loss: 242.396240234375, running average elbo: -847.1552734375\n",
      "Iteration: 464, Loss: 248.56121826171875, running average elbo: -841.1693115234375\n",
      "Iteration: 465, Loss: 238.81631469726562, running average elbo: -835.1458129882812\n",
      "Iteration: 466, Loss: 239.00497436523438, running average elbo: -829.1844482421875\n",
      "Iteration: 467, Loss: 240.9256591796875, running average elbo: -823.3018798828125\n",
      "Iteration: 468, Loss: 235.85418701171875, running average elbo: -817.4273681640625\n",
      "Iteration: 469, Loss: 236.2830047607422, running average elbo: -811.615966796875\n",
      "Iteration: 470, Loss: 235.58216857910156, running average elbo: -805.8556518554688\n",
      "Iteration: 471, Loss: 229.83078002929688, running average elbo: -800.0953979492188\n",
      "Iteration: 472, Loss: 232.98541259765625, running average elbo: -794.4242553710938\n",
      "Iteration: 473, Loss: 228.92149353027344, running average elbo: -788.7692260742188\n",
      "Iteration: 474, Loss: 225.84402465820312, running average elbo: -783.1399536132812\n",
      "Iteration: 475, Loss: 229.96112060546875, running average elbo: -777.608154296875\n",
      "Iteration: 476, Loss: 221.56028747558594, running average elbo: -772.0476684570312\n",
      "Iteration: 477, Loss: 224.9547119140625, running average elbo: -766.5767822265625\n",
      "Iteration: 478, Loss: 222.85791015625, running average elbo: -761.1395874023438\n",
      "Iteration: 479, Loss: 217.9171905517578, running average elbo: -755.7073974609375\n",
      "Iteration: 480, Loss: 218.54708862304688, running average elbo: -750.3358154296875\n",
      "Iteration: 481, Loss: 218.24533081054688, running average elbo: -745.014892578125\n",
      "Iteration: 482, Loss: 215.02931213378906, running average elbo: -739.7150268554688\n",
      "Iteration: 483, Loss: 216.1285400390625, running average elbo: -734.4791259765625\n",
      "Iteration: 484, Loss: 212.33694458007812, running average elbo: -729.2576904296875\n",
      "Iteration: 485, Loss: 213.30735778808594, running average elbo: -724.0982055664062\n",
      "Iteration: 486, Loss: 210.06381225585938, running average elbo: -718.9578857421875\n",
      "Iteration: 487, Loss: 210.5668182373047, running average elbo: -713.8739624023438\n",
      "Iteration: 488, Loss: 209.66250610351562, running average elbo: -708.8318481445312\n",
      "Iteration: 489, Loss: 210.005126953125, running average elbo: -703.8435668945312\n",
      "Iteration: 490, Loss: 208.74893188476562, running average elbo: -698.892578125\n",
      "Iteration: 491, Loss: 206.95339965820312, running average elbo: -693.9732055664062\n",
      "Iteration: 492, Loss: 207.3282470703125, running average elbo: -689.1068115234375\n",
      "Iteration: 493, Loss: 205.90341186523438, running average elbo: -684.2747802734375\n",
      "Iteration: 494, Loss: 206.59854125976562, running average elbo: -679.498046875\n",
      "Iteration: 495, Loss: 205.1042022705078, running average elbo: -674.7540893554688\n",
      "Iteration: 496, Loss: 204.1421661376953, running average elbo: -670.0479736328125\n",
      "Iteration: 497, Loss: 203.57443237304688, running average elbo: -665.3832397460938\n",
      "Iteration: 498, Loss: 203.1425323486328, running average elbo: -660.7608642578125\n",
      "Iteration: 499, Loss: 203.87777709960938, running average elbo: -656.1920166015625\n",
      "Iteration: 500, Loss: 201.0819091796875, running average elbo: -651.6409301757812\n",
      "Iteration: 501, Loss: 200.5320587158203, running average elbo: -647.1298217773438\n",
      "Iteration: 502, Loss: 200.8743438720703, running average elbo: -642.667236328125\n",
      "Iteration: 503, Loss: 200.32057189941406, running average elbo: -638.2438354492188\n",
      "Iteration: 504, Loss: 199.6401824951172, running average elbo: -633.8577880859375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 505, Loss: 197.30313110351562, running average elbo: -629.4922485351562\n",
      "Iteration: 506, Loss: 197.96975708007812, running average elbo: -625.177001953125\n",
      "Iteration: 507, Loss: 196.44552612304688, running average elbo: -620.8897094726562\n",
      "Iteration: 508, Loss: 195.20462036132812, running average elbo: -616.6328735351562\n",
      "Iteration: 509, Loss: 196.4641876220703, running average elbo: -612.4312133789062\n",
      "Iteration: 510, Loss: 196.7870635986328, running average elbo: -608.2747802734375\n",
      "Iteration: 511, Loss: 194.866943359375, running average elbo: -604.1406860351562\n",
      "Iteration: 512, Loss: 194.47637939453125, running average elbo: -600.0440673828125\n",
      "Iteration: 513, Loss: 194.7376251220703, running average elbo: -595.9910278320312\n",
      "Iteration: 514, Loss: 193.61849975585938, running average elbo: -591.96728515625\n",
      "Iteration: 515, Loss: 192.76516723632812, running average elbo: -587.9752807617188\n",
      "Iteration: 516, Loss: 191.94540405273438, running average elbo: -584.0149536132812\n",
      "Iteration: 517, Loss: 191.21661376953125, running average elbo: -580.0869750976562\n",
      "Iteration: 518, Loss: 190.33644104003906, running average elbo: -576.1895141601562\n",
      "Iteration: 519, Loss: 190.9069061279297, running average elbo: -572.336669921875\n",
      "Iteration: 520, Loss: 190.68878173828125, running average elbo: -568.5201416015625\n",
      "Iteration: 521, Loss: 191.28091430664062, running average elbo: -564.7477416992188\n",
      "Iteration: 522, Loss: 189.9990997314453, running average elbo: -561.000244140625\n",
      "Iteration: 523, Loss: 190.11534118652344, running average elbo: -557.2913818359375\n",
      "Iteration: 524, Loss: 189.10186767578125, running average elbo: -553.6094360351562\n",
      "Iteration: 525, Loss: 188.3850555419922, running average elbo: -549.9572143554688\n",
      "Iteration: 526, Loss: 188.31756591796875, running average elbo: -546.3408203125\n",
      "Iteration: 527, Loss: 189.12025451660156, running average elbo: -542.7686157226562\n",
      "Iteration: 528, Loss: 186.81309509277344, running average elbo: -539.2090454101562\n",
      "Iteration: 529, Loss: 186.2562255859375, running average elbo: -535.6795043945312\n",
      "Iteration: 530, Loss: 186.66419982910156, running average elbo: -532.1893310546875\n",
      "Iteration: 531, Loss: 186.58265686035156, running average elbo: -528.7332763671875\n",
      "Iteration: 532, Loss: 186.00665283203125, running average elbo: -525.3059692382812\n",
      "Iteration: 533, Loss: 185.2964324951172, running average elbo: -521.9058837890625\n",
      "Iteration: 534, Loss: 185.72763061523438, running average elbo: -518.5441284179688\n",
      "Iteration: 535, Loss: 186.67337036132812, running average elbo: -515.2254638671875\n",
      "Iteration: 536, Loss: 184.90660095214844, running average elbo: -511.9222717285156\n",
      "Iteration: 537, Loss: 183.19296264648438, running average elbo: -508.6349792480469\n",
      "Iteration: 538, Loss: 182.04067993164062, running average elbo: -505.3690490722656\n",
      "Iteration: 539, Loss: 183.52493286132812, running average elbo: -502.1506042480469\n",
      "Iteration: 540, Loss: 183.98931884765625, running average elbo: -498.968994140625\n",
      "Iteration: 541, Loss: 182.6548614501953, running average elbo: -495.80584716796875\n",
      "Iteration: 542, Loss: 181.1178436279297, running average elbo: -492.6589660644531\n",
      "Iteration: 543, Loss: 181.07427978515625, running average elbo: -489.5431213378906\n",
      "Iteration: 544, Loss: 180.5448760986328, running average elbo: -486.4531555175781\n",
      "Iteration: 545, Loss: 180.2965850830078, running average elbo: -483.3916015625\n",
      "Iteration: 546, Loss: 179.413818359375, running average elbo: -480.351806640625\n",
      "Iteration: 547, Loss: 180.59532165527344, running average elbo: -477.3542175292969\n",
      "Iteration: 548, Loss: 179.40530395507812, running average elbo: -474.374755859375\n",
      "Iteration: 549, Loss: 180.6746063232422, running average elbo: -471.437744140625\n",
      "Iteration: 550, Loss: 178.5572509765625, running average elbo: -468.5089416503906\n",
      "Iteration: 551, Loss: 177.90823364257812, running average elbo: -465.6029357910156\n",
      "Iteration: 552, Loss: 178.23507690429688, running average elbo: -462.729248046875\n",
      "Iteration: 553, Loss: 178.22576904296875, running average elbo: -459.88421630859375\n",
      "Iteration: 554, Loss: 179.4524688720703, running average elbo: -457.07989501953125\n",
      "Iteration: 555, Loss: 178.2625274658203, running average elbo: -454.2917175292969\n",
      "Iteration: 556, Loss: 177.28305053710938, running average elbo: -451.5216369628906\n",
      "Iteration: 557, Loss: 176.57528686523438, running average elbo: -448.7721862792969\n",
      "Iteration: 558, Loss: 176.45065307617188, running average elbo: -446.0489501953125\n",
      "Iteration: 559, Loss: 176.88552856445312, running average elbo: -443.3573303222656\n",
      "Iteration: 560, Loss: 174.169189453125, running average elbo: -440.66546630859375\n",
      "Iteration: 561, Loss: 177.01443481445312, running average elbo: -438.0289611816406\n",
      "Iteration: 562, Loss: 177.11001586914062, running average elbo: -435.4197692871094\n",
      "Iteration: 563, Loss: 177.3546600341797, running average elbo: -432.8391418457031\n",
      "Iteration: 564, Loss: 176.32652282714844, running average elbo: -430.2740173339844\n",
      "Iteration: 565, Loss: 174.77438354492188, running average elbo: -427.7190246582031\n",
      "Iteration: 566, Loss: 174.91775512695312, running average elbo: -425.1910095214844\n",
      "Iteration: 567, Loss: 175.50624084472656, running average elbo: -422.6941833496094\n",
      "Iteration: 568, Loss: 174.74049377441406, running average elbo: -420.21466064453125\n",
      "Iteration: 569, Loss: 173.65066528320312, running average elbo: -417.7490234375\n",
      "Iteration: 570, Loss: 172.92449951171875, running average elbo: -415.30078125\n",
      "Iteration: 571, Loss: 172.70040893554688, running average elbo: -412.874755859375\n",
      "Iteration: 572, Loss: 173.07803344726562, running average elbo: -410.4767761230469\n",
      "Iteration: 573, Loss: 174.4553680419922, running average elbo: -408.1165771484375\n",
      "Iteration: 574, Loss: 173.33929443359375, running average elbo: -405.768798828125\n",
      "Iteration: 575, Loss: 175.32362365722656, running average elbo: -403.46435546875\n",
      "Iteration: 576, Loss: 174.2055206298828, running average elbo: -401.1717834472656\n",
      "Iteration: 577, Loss: 172.7727508544922, running average elbo: -398.8877868652344\n",
      "Iteration: 578, Loss: 173.22422790527344, running average elbo: -396.63116455078125\n",
      "Iteration: 579, Loss: 171.72610473632812, running average elbo: -394.3821105957031\n",
      "Iteration: 580, Loss: 169.9439697265625, running average elbo: -392.1377258300781\n",
      "Iteration: 581, Loss: 169.62156677246094, running average elbo: -389.9125671386719\n",
      "Iteration: 582, Loss: 170.7315673828125, running average elbo: -387.72076416015625\n",
      "Iteration: 583, Loss: 171.3246307373047, running average elbo: -385.55682373046875\n",
      "Iteration: 584, Loss: 170.6915283203125, running average elbo: -383.4081726074219\n",
      "Iteration: 585, Loss: 171.68832397460938, running average elbo: -381.2909851074219\n",
      "Iteration: 586, Loss: 170.4529266357422, running average elbo: -379.1826171875\n",
      "Iteration: 587, Loss: 168.3192596435547, running average elbo: -377.0740051269531\n",
      "Iteration: 588, Loss: 168.1404266357422, running average elbo: -374.98468017578125\n",
      "Iteration: 589, Loss: 169.2571563720703, running average elbo: -372.9273986816406\n",
      "Iteration: 590, Loss: 169.23104858398438, running average elbo: -370.89044189453125\n",
      "Iteration: 591, Loss: 169.67965698242188, running average elbo: -368.87835693359375\n",
      "Iteration: 592, Loss: 169.1745147705078, running average elbo: -366.8813171386719\n",
      "Iteration: 593, Loss: 168.74615478515625, running average elbo: -364.89996337890625\n",
      "Iteration: 594, Loss: 169.08987426757812, running average elbo: -362.9418640136719\n",
      "Iteration: 595, Loss: 167.08303833007812, running average elbo: -360.9832763671875\n",
      "Iteration: 596, Loss: 166.6236114501953, running average elbo: -359.0396728515625\n",
      "Iteration: 597, Loss: 165.7742919921875, running average elbo: -357.1070251464844\n",
      "Iteration: 598, Loss: 166.02781677246094, running average elbo: -355.19622802734375\n",
      "Iteration: 599, Loss: 166.90330505371094, running average elbo: -353.31329345703125\n",
      "Iteration: 600, Loss: 166.9832000732422, running average elbo: -351.4499816894531\n",
      "Iteration: 601, Loss: 164.1784210205078, running average elbo: -349.5772705078125\n",
      "Iteration: 602, Loss: 164.70614624023438, running average elbo: -347.72857666015625\n",
      "Iteration: 603, Loss: 165.39468383789062, running average elbo: -345.9052429199219\n",
      "Iteration: 604, Loss: 163.75146484375, running average elbo: -344.0837097167969\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 605, Loss: 165.0226287841797, running average elbo: -342.2931213378906\n",
      "Iteration: 606, Loss: 164.36546325683594, running average elbo: -340.5138244628906\n",
      "Iteration: 607, Loss: 164.97149658203125, running average elbo: -338.7584228515625\n",
      "Iteration: 608, Loss: 163.2282257080078, running average elbo: -337.0031433105469\n",
      "Iteration: 609, Loss: 163.49276733398438, running average elbo: -335.2680358886719\n",
      "Iteration: 610, Loss: 162.42269897460938, running average elbo: -333.5395812988281\n",
      "Iteration: 611, Loss: 163.2440643310547, running average elbo: -331.8366394042969\n",
      "Iteration: 612, Loss: 164.39540100097656, running average elbo: -330.1622314453125\n",
      "Iteration: 613, Loss: 162.06365966796875, running average elbo: -328.48126220703125\n",
      "Iteration: 614, Loss: 162.40213012695312, running average elbo: -326.8204650878906\n",
      "Iteration: 615, Loss: 162.5950164794922, running average elbo: -325.17822265625\n",
      "Iteration: 616, Loss: 161.8248748779297, running average elbo: -323.5447082519531\n",
      "Iteration: 617, Loss: 162.00057983398438, running average elbo: -321.92926025390625\n",
      "Iteration: 618, Loss: 163.93751525878906, running average elbo: -320.3493347167969\n",
      "Iteration: 619, Loss: 165.4160919189453, running average elbo: -318.8000183105469\n",
      "Iteration: 620, Loss: 168.84873962402344, running average elbo: -317.3005065917969\n",
      "Iteration: 621, Loss: 172.08924865722656, running average elbo: -315.848388671875\n",
      "Iteration: 622, Loss: 179.93902587890625, running average elbo: -314.4892883300781\n",
      "Iteration: 623, Loss: 181.2107696533203, running average elbo: -313.156494140625\n",
      "Iteration: 624, Loss: 176.78546142578125, running average elbo: -311.79278564453125\n",
      "Iteration: 625, Loss: 167.2279052734375, running average elbo: -310.3471374511719\n",
      "Iteration: 626, Loss: 160.15509033203125, running average elbo: -308.8452453613281\n",
      "Iteration: 627, Loss: 164.73631286621094, running average elbo: -307.4041748046875\n",
      "Iteration: 628, Loss: 170.91358947753906, running average elbo: -306.0392761230469\n",
      "Iteration: 629, Loss: 177.3740692138672, running average elbo: -304.75262451171875\n",
      "Iteration: 630, Loss: 173.89964294433594, running average elbo: -303.444091796875\n",
      "Iteration: 631, Loss: 163.00665283203125, running average elbo: -302.03973388671875\n",
      "Iteration: 632, Loss: 161.28887939453125, running average elbo: -300.6322326660156\n",
      "Iteration: 633, Loss: 170.05836486816406, running average elbo: -299.3265075683594\n",
      "Iteration: 634, Loss: 176.9511260986328, running average elbo: -298.1027526855469\n",
      "Iteration: 635, Loss: 166.018310546875, running average elbo: -296.78192138671875\n",
      "Iteration: 636, Loss: 161.38677978515625, running average elbo: -295.427978515625\n",
      "Iteration: 637, Loss: 161.8889617919922, running average elbo: -294.09259033203125\n",
      "Iteration: 638, Loss: 167.7338104248047, running average elbo: -292.8290100097656\n",
      "Iteration: 639, Loss: 166.4982452392578, running average elbo: -291.5657043457031\n",
      "Iteration: 640, Loss: 159.80296325683594, running average elbo: -290.2480773925781\n",
      "Iteration: 641, Loss: 158.1085662841797, running average elbo: -288.92669677734375\n",
      "Iteration: 642, Loss: 160.48892211914062, running average elbo: -287.642333984375\n",
      "Iteration: 643, Loss: 160.7120361328125, running average elbo: -286.3730163574219\n",
      "Iteration: 644, Loss: 158.61643981933594, running average elbo: -285.0954284667969\n",
      "Iteration: 645, Loss: 156.86221313476562, running average elbo: -283.8131103515625\n",
      "Iteration: 646, Loss: 158.3368377685547, running average elbo: -282.558349609375\n",
      "Iteration: 647, Loss: 158.1353302001953, running average elbo: -281.3141174316406\n",
      "Iteration: 648, Loss: 158.14125061035156, running average elbo: -280.0823974609375\n",
      "Iteration: 649, Loss: 155.68475341796875, running average elbo: -278.83843994140625\n",
      "Iteration: 650, Loss: 156.20761108398438, running average elbo: -277.61212158203125\n",
      "Iteration: 651, Loss: 157.0603790283203, running average elbo: -276.4066162109375\n",
      "Iteration: 652, Loss: 157.6791534423828, running average elbo: -275.2193298339844\n",
      "Iteration: 653, Loss: 156.16844177246094, running average elbo: -274.02880859375\n",
      "Iteration: 654, Loss: 155.88502502441406, running average elbo: -272.84735107421875\n",
      "Iteration: 655, Loss: 154.58363342285156, running average elbo: -271.6647033691406\n",
      "Iteration: 656, Loss: 156.50387573242188, running average elbo: -270.5130920410156\n",
      "Iteration: 657, Loss: 156.05760192871094, running average elbo: -269.3685302734375\n",
      "Iteration: 658, Loss: 155.03836059570312, running average elbo: -268.2252197265625\n",
      "Iteration: 659, Loss: 153.3829803466797, running average elbo: -267.0768127441406\n",
      "Iteration: 660, Loss: 154.22801208496094, running average elbo: -265.94830322265625\n",
      "Iteration: 661, Loss: 153.53207397460938, running average elbo: -264.8241271972656\n",
      "Iteration: 662, Loss: 153.46377563476562, running average elbo: -263.7105407714844\n",
      "Iteration: 663, Loss: 154.38978576660156, running average elbo: -262.6173095703125\n",
      "Iteration: 664, Loss: 152.8290252685547, running average elbo: -261.5194396972656\n",
      "Iteration: 665, Loss: 153.52984619140625, running average elbo: -260.4395446777344\n",
      "Iteration: 666, Loss: 152.6849365234375, running average elbo: -259.36199951171875\n",
      "Iteration: 667, Loss: 153.06100463867188, running average elbo: -258.2989807128906\n",
      "Iteration: 668, Loss: 154.38919067382812, running average elbo: -257.2598876953125\n",
      "Iteration: 669, Loss: 154.22811889648438, running average elbo: -256.22955322265625\n",
      "Iteration: 670, Loss: 153.04840087890625, running average elbo: -255.19775390625\n",
      "Iteration: 671, Loss: 151.25698852539062, running average elbo: -254.15835571289062\n",
      "Iteration: 672, Loss: 151.6632537841797, running average elbo: -253.13340759277344\n",
      "Iteration: 673, Loss: 151.71646118164062, running average elbo: -252.11924743652344\n",
      "Iteration: 674, Loss: 150.7819366455078, running average elbo: -251.10586547851562\n",
      "Iteration: 675, Loss: 153.05384826660156, running average elbo: -250.12533569335938\n",
      "Iteration: 676, Loss: 152.61927795410156, running average elbo: -249.15028381347656\n",
      "Iteration: 677, Loss: 150.34979248046875, running average elbo: -248.1622772216797\n",
      "Iteration: 678, Loss: 149.75750732421875, running average elbo: -247.17823791503906\n",
      "Iteration: 679, Loss: 148.9779510498047, running average elbo: -246.19622802734375\n",
      "Iteration: 680, Loss: 151.2245330810547, running average elbo: -245.24652099609375\n",
      "Iteration: 681, Loss: 152.49380493164062, running average elbo: -244.31898498535156\n",
      "Iteration: 682, Loss: 152.19151306152344, running average elbo: -243.397705078125\n",
      "Iteration: 683, Loss: 152.037353515625, running average elbo: -242.48410034179688\n",
      "Iteration: 684, Loss: 150.6411895751953, running average elbo: -241.565673828125\n",
      "Iteration: 685, Loss: 149.8755645751953, running average elbo: -240.64877319335938\n",
      "Iteration: 686, Loss: 148.62144470214844, running average elbo: -239.728515625\n",
      "Iteration: 687, Loss: 147.91978454589844, running average elbo: -238.81044006347656\n",
      "Iteration: 688, Loss: 148.9317169189453, running average elbo: -237.91165161132812\n",
      "Iteration: 689, Loss: 149.95484924316406, running average elbo: -237.03207397460938\n",
      "Iteration: 690, Loss: 152.4976806640625, running average elbo: -236.18673706054688\n",
      "Iteration: 691, Loss: 151.83871459960938, running average elbo: -235.34326171875\n",
      "Iteration: 692, Loss: 151.47520446777344, running average elbo: -234.5045928955078\n",
      "Iteration: 693, Loss: 149.87449645996094, running average elbo: -233.65829467773438\n",
      "Iteration: 694, Loss: 148.3347625732422, running average elbo: -232.80506896972656\n",
      "Iteration: 695, Loss: 147.4693145751953, running average elbo: -231.9517059326172\n",
      "Iteration: 696, Loss: 147.14540100097656, running average elbo: -231.1036376953125\n",
      "Iteration: 697, Loss: 147.34219360351562, running average elbo: -230.26602172851562\n",
      "Iteration: 698, Loss: 146.75953674316406, running average elbo: -229.4309539794922\n",
      "Iteration: 699, Loss: 147.2923126220703, running average elbo: -228.60955810546875\n",
      "Iteration: 700, Loss: 150.2587432861328, running average elbo: -227.82606506347656\n",
      "Iteration: 701, Loss: 155.10491943359375, running average elbo: -227.09886169433594\n",
      "Iteration: 702, Loss: 156.68577575683594, running average elbo: -226.39474487304688\n",
      "Iteration: 703, Loss: 154.87814331054688, running average elbo: -225.67958068847656\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 704, Loss: 152.0966339111328, running average elbo: -224.94375610351562\n",
      "Iteration: 705, Loss: 149.99961853027344, running average elbo: -224.19432067871094\n",
      "Iteration: 706, Loss: 148.1426544189453, running average elbo: -223.43380737304688\n",
      "Iteration: 707, Loss: 145.88967895507812, running average elbo: -222.6583709716797\n",
      "Iteration: 708, Loss: 146.30699157714844, running average elbo: -221.89486694335938\n",
      "Iteration: 709, Loss: 145.74891662597656, running average elbo: -221.13340759277344\n",
      "Iteration: 710, Loss: 146.2139434814453, running average elbo: -220.38421630859375\n",
      "Iteration: 711, Loss: 146.11611938476562, running average elbo: -219.64154052734375\n",
      "Iteration: 712, Loss: 147.28858947753906, running average elbo: -218.9180145263672\n",
      "Iteration: 713, Loss: 146.80950927734375, running average elbo: -218.19692993164062\n",
      "Iteration: 714, Loss: 146.0430450439453, running average elbo: -217.47540283203125\n",
      "Iteration: 715, Loss: 144.697021484375, running average elbo: -216.74761962890625\n",
      "Iteration: 716, Loss: 144.85116577148438, running average elbo: -216.02865600585938\n",
      "Iteration: 717, Loss: 145.5294952392578, running average elbo: -215.32366943359375\n",
      "Iteration: 718, Loss: 144.1387481689453, running average elbo: -214.61183166503906\n",
      "Iteration: 719, Loss: 143.8774871826172, running average elbo: -213.9044952392578\n",
      "Iteration: 720, Loss: 144.202392578125, running average elbo: -213.2074737548828\n",
      "Iteration: 721, Loss: 143.81228637695312, running average elbo: -212.51351928710938\n",
      "Iteration: 722, Loss: 143.8536834716797, running average elbo: -211.82691955566406\n",
      "Iteration: 723, Loss: 143.81846618652344, running average elbo: -211.14683532714844\n",
      "Iteration: 724, Loss: 143.95947265625, running average elbo: -210.47496032714844\n",
      "Iteration: 725, Loss: 142.39736938476562, running average elbo: -209.794189453125\n",
      "Iteration: 726, Loss: 141.78318786621094, running average elbo: -209.11407470703125\n",
      "Iteration: 727, Loss: 143.92718505859375, running average elbo: -208.4622039794922\n",
      "Iteration: 728, Loss: 145.81515502929688, running average elbo: -207.83572387695312\n",
      "Iteration: 729, Loss: 150.33509826660156, running average elbo: -207.26072692871094\n",
      "Iteration: 730, Loss: 163.1131591796875, running average elbo: -206.8192596435547\n",
      "Iteration: 731, Loss: 176.5039825439453, running average elbo: -206.51611328125\n",
      "Iteration: 732, Loss: 195.55966186523438, running average elbo: -206.40655517578125\n",
      "Iteration: 733, Loss: 162.49449157714844, running average elbo: -205.96743774414062\n",
      "Iteration: 734, Loss: 148.35406494140625, running average elbo: -205.39129638671875\n",
      "Iteration: 735, Loss: 186.92166137695312, running average elbo: -205.20660400390625\n",
      "Iteration: 736, Loss: 198.4123992919922, running average elbo: -205.138671875\n",
      "Iteration: 737, Loss: 184.31982421875, running average elbo: -204.93048095703125\n",
      "Iteration: 738, Loss: 147.7543487548828, running average elbo: -204.3587188720703\n",
      "Iteration: 739, Loss: 224.7684783935547, running average elbo: -204.5628204345703\n",
      "Iteration: 740, Loss: 267.4441833496094, running average elbo: -205.19163513183594\n",
      "Iteration: 741, Loss: 196.92596435546875, running average elbo: -205.10897827148438\n",
      "Iteration: 742, Loss: 238.19244384765625, running average elbo: -205.4398193359375\n",
      "Iteration: 743, Loss: 191.167236328125, running average elbo: -205.29708862304688\n",
      "Iteration: 744, Loss: 184.3810577392578, running average elbo: -205.0879364013672\n",
      "Iteration: 745, Loss: 200.6399688720703, running average elbo: -205.04345703125\n",
      "Iteration: 746, Loss: 201.25424194335938, running average elbo: -205.0055694580078\n",
      "Iteration: 747, Loss: 161.07720947265625, running average elbo: -204.56629943847656\n",
      "Iteration: 748, Loss: 210.04843139648438, running average elbo: -204.62112426757812\n",
      "Iteration: 749, Loss: 178.50823974609375, running average elbo: -204.36000061035156\n",
      "Iteration: 750, Loss: 161.95712280273438, running average elbo: -203.93597412109375\n",
      "Iteration: 751, Loss: 176.3202362060547, running average elbo: -203.65982055664062\n",
      "Iteration: 752, Loss: 170.41944885253906, running average elbo: -203.32742309570312\n",
      "Iteration: 753, Loss: 172.68585205078125, running average elbo: -203.02099609375\n",
      "Iteration: 754, Loss: 187.0966339111328, running average elbo: -202.86175537109375\n",
      "Iteration: 755, Loss: 170.1259307861328, running average elbo: -202.53440856933594\n",
      "Iteration: 756, Loss: 161.9764862060547, running average elbo: -202.1288299560547\n",
      "Iteration: 757, Loss: 167.5508270263672, running average elbo: -201.78305053710938\n",
      "Iteration: 758, Loss: 166.1310272216797, running average elbo: -201.42654418945312\n",
      "Iteration: 759, Loss: 165.3134002685547, running average elbo: -201.06541442871094\n",
      "Iteration: 760, Loss: 160.03147888183594, running average elbo: -200.6550750732422\n",
      "Iteration: 761, Loss: 151.64422607421875, running average elbo: -200.16497802734375\n",
      "Iteration: 762, Loss: 165.23040771484375, running average elbo: -199.8156280517578\n",
      "Iteration: 763, Loss: 161.378662109375, running average elbo: -199.43125915527344\n",
      "Iteration: 764, Loss: 165.56906127929688, running average elbo: -199.09263610839844\n",
      "Iteration: 765, Loss: 158.17564392089844, running average elbo: -198.6834716796875\n",
      "Iteration: 766, Loss: 154.98306274414062, running average elbo: -198.24647521972656\n",
      "Iteration: 767, Loss: 163.66297912597656, running average elbo: -197.900634765625\n",
      "Iteration: 768, Loss: 165.06166076660156, running average elbo: -197.57225036621094\n",
      "Iteration: 769, Loss: 148.51202392578125, running average elbo: -197.08164978027344\n",
      "Iteration: 770, Loss: 157.46046447753906, running average elbo: -196.68544006347656\n",
      "Iteration: 771, Loss: 153.7030792236328, running average elbo: -196.255615234375\n",
      "Iteration: 772, Loss: 161.38385009765625, running average elbo: -195.9069061279297\n",
      "Iteration: 773, Loss: 157.32492065429688, running average elbo: -195.52108764648438\n",
      "Iteration: 774, Loss: 151.1402130126953, running average elbo: -195.0772705078125\n",
      "Iteration: 775, Loss: 148.82252502441406, running average elbo: -194.61471557617188\n",
      "Iteration: 776, Loss: 162.89218139648438, running average elbo: -194.2974853515625\n",
      "Iteration: 777, Loss: 151.12496948242188, running average elbo: -193.86575317382812\n",
      "Iteration: 778, Loss: 163.04852294921875, running average elbo: -193.5575714111328\n",
      "Iteration: 779, Loss: 155.65806579589844, running average elbo: -193.17857360839844\n",
      "Iteration: 780, Loss: 156.9674835205078, running average elbo: -192.81646728515625\n",
      "Iteration: 781, Loss: 161.16847229003906, running average elbo: -192.49998474121094\n",
      "Iteration: 782, Loss: 160.80918884277344, running average elbo: -192.18307495117188\n",
      "Iteration: 783, Loss: 148.80892944335938, running average elbo: -191.74932861328125\n",
      "Iteration: 784, Loss: 165.40194702148438, running average elbo: -191.48585510253906\n",
      "Iteration: 785, Loss: 155.4303436279297, running average elbo: -191.12530517578125\n",
      "Iteration: 786, Loss: 157.4027099609375, running average elbo: -190.78807067871094\n",
      "Iteration: 787, Loss: 163.18434143066406, running average elbo: -190.51202392578125\n",
      "Iteration: 788, Loss: 155.8603057861328, running average elbo: -190.16551208496094\n",
      "Iteration: 789, Loss: 162.82388305664062, running average elbo: -189.89208984375\n",
      "Iteration: 790, Loss: 174.45571899414062, running average elbo: -189.7377166748047\n",
      "Iteration: 791, Loss: 161.3421630859375, running average elbo: -189.45376586914062\n",
      "Iteration: 792, Loss: 173.18006896972656, running average elbo: -189.29103088378906\n",
      "Iteration: 793, Loss: 174.17349243164062, running average elbo: -189.1398468017578\n",
      "Iteration: 794, Loss: 151.97537231445312, running average elbo: -188.76820373535156\n",
      "Iteration: 795, Loss: 180.24110412597656, running average elbo: -188.6829376220703\n",
      "Iteration: 796, Loss: 158.81175231933594, running average elbo: -188.3842315673828\n",
      "Iteration: 797, Loss: 171.8433074951172, running average elbo: -188.2188262939453\n",
      "Iteration: 798, Loss: 185.4665069580078, running average elbo: -188.19129943847656\n",
      "Iteration: 799, Loss: 175.75149536132812, running average elbo: -188.06689453125\n",
      "Iteration: 800, Loss: 169.54302978515625, running average elbo: -187.8816680908203\n",
      "Iteration: 801, Loss: 199.61618041992188, running average elbo: -187.99900817871094\n",
      "Iteration: 802, Loss: 156.45703125, running average elbo: -187.68359375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 803, Loss: 197.1900634765625, running average elbo: -187.77865600585938\n",
      "Iteration: 804, Loss: 174.2152862548828, running average elbo: -187.6430206298828\n",
      "Iteration: 805, Loss: 177.83290100097656, running average elbo: -187.544921875\n",
      "Iteration: 806, Loss: 177.11920166015625, running average elbo: -187.440673828125\n",
      "Iteration: 807, Loss: 187.898193359375, running average elbo: -187.44525146484375\n",
      "Iteration: 808, Loss: 159.74917602539062, running average elbo: -187.1682891845703\n",
      "Iteration: 809, Loss: 213.12049865722656, running average elbo: -187.42782592773438\n",
      "Iteration: 810, Loss: 175.46493530273438, running average elbo: -187.30819702148438\n",
      "Iteration: 811, Loss: 197.10275268554688, running average elbo: -187.40614318847656\n",
      "Iteration: 812, Loss: 195.64793395996094, running average elbo: -187.4885711669922\n",
      "Iteration: 813, Loss: 198.32205200195312, running average elbo: -187.59690856933594\n",
      "Iteration: 814, Loss: 171.62939453125, running average elbo: -187.43724060058594\n",
      "Iteration: 815, Loss: 221.12741088867188, running average elbo: -187.77413940429688\n",
      "Iteration: 816, Loss: 166.2891845703125, running average elbo: -187.5592803955078\n",
      "Iteration: 817, Loss: 208.02098083496094, running average elbo: -187.76390075683594\n",
      "Iteration: 818, Loss: 211.5538330078125, running average elbo: -188.00180053710938\n",
      "Iteration: 819, Loss: 200.71331787109375, running average elbo: -188.12890625\n",
      "Iteration: 820, Loss: 185.4493408203125, running average elbo: -188.10211181640625\n",
      "Iteration: 821, Loss: 227.476318359375, running average elbo: -188.49586486816406\n",
      "Iteration: 822, Loss: 169.9784393310547, running average elbo: -188.31068420410156\n",
      "Iteration: 823, Loss: 235.8831024169922, running average elbo: -188.78640747070312\n",
      "Iteration: 824, Loss: 199.38429260253906, running average elbo: -188.8923797607422\n",
      "Iteration: 825, Loss: 190.33482360839844, running average elbo: -188.9068145751953\n",
      "Iteration: 826, Loss: 179.59298706054688, running average elbo: -188.8136749267578\n",
      "Iteration: 827, Loss: 205.22772216796875, running average elbo: -188.97781372070312\n",
      "Iteration: 828, Loss: 184.9067840576172, running average elbo: -188.93711853027344\n",
      "Iteration: 829, Loss: 212.84341430664062, running average elbo: -189.17617797851562\n",
      "Iteration: 830, Loss: 228.77684020996094, running average elbo: -189.5721893310547\n",
      "Iteration: 831, Loss: 167.95591735839844, running average elbo: -189.3560333251953\n",
      "Iteration: 832, Loss: 216.434326171875, running average elbo: -189.62681579589844\n",
      "Iteration: 833, Loss: 232.15817260742188, running average elbo: -190.0521240234375\n",
      "Iteration: 834, Loss: 202.63955688476562, running average elbo: -190.17800903320312\n",
      "Iteration: 835, Loss: 202.97427368164062, running average elbo: -190.30596923828125\n",
      "Iteration: 836, Loss: 222.4947967529297, running average elbo: -190.6278533935547\n",
      "Iteration: 837, Loss: 173.02528381347656, running average elbo: -190.4518280029297\n",
      "Iteration: 838, Loss: 234.0865478515625, running average elbo: -190.88818359375\n",
      "Iteration: 839, Loss: 257.3146667480469, running average elbo: -191.55245971679688\n",
      "Iteration: 840, Loss: 224.7844696044922, running average elbo: -191.88478088378906\n",
      "Iteration: 841, Loss: 194.0518341064453, running average elbo: -191.90646362304688\n",
      "Iteration: 842, Loss: 236.81228637695312, running average elbo: -192.3555145263672\n",
      "Iteration: 843, Loss: 209.29478454589844, running average elbo: -192.52490234375\n",
      "Iteration: 844, Loss: 237.77166748046875, running average elbo: -192.9773712158203\n",
      "Iteration: 845, Loss: 281.164794921875, running average elbo: -193.85923767089844\n",
      "Iteration: 846, Loss: 216.1596221923828, running average elbo: -194.08224487304688\n",
      "Iteration: 847, Loss: 195.17074584960938, running average elbo: -194.09312438964844\n",
      "Iteration: 848, Loss: 217.43075561523438, running average elbo: -194.3264923095703\n",
      "Iteration: 849, Loss: 241.41233825683594, running average elbo: -194.79734802246094\n",
      "Iteration: 850, Loss: 231.58880615234375, running average elbo: -195.16526794433594\n",
      "Iteration: 851, Loss: 284.8441162109375, running average elbo: -196.0620574951172\n",
      "Iteration: 852, Loss: 265.3485107421875, running average elbo: -196.7549285888672\n",
      "Iteration: 853, Loss: 193.0331268310547, running average elbo: -196.71771240234375\n",
      "Iteration: 854, Loss: 255.93768310546875, running average elbo: -197.30990600585938\n",
      "Iteration: 855, Loss: 322.0023498535156, running average elbo: -198.55682373046875\n",
      "Iteration: 856, Loss: 275.62994384765625, running average elbo: -199.3275604248047\n",
      "Iteration: 857, Loss: 275.284912109375, running average elbo: -200.08714294433594\n",
      "Iteration: 858, Loss: 292.6570129394531, running average elbo: -201.01284790039062\n",
      "Iteration: 859, Loss: 212.02540588378906, running average elbo: -201.1229705810547\n",
      "Iteration: 860, Loss: 253.0926513671875, running average elbo: -201.64266967773438\n",
      "Iteration: 861, Loss: 312.357666015625, running average elbo: -202.7498321533203\n",
      "Iteration: 862, Loss: 313.3867492675781, running average elbo: -203.856201171875\n",
      "Iteration: 863, Loss: 272.7778625488281, running average elbo: -204.54542541503906\n",
      "Iteration: 864, Loss: 258.62957763671875, running average elbo: -205.0862579345703\n",
      "Iteration: 865, Loss: 310.21844482421875, running average elbo: -206.13758850097656\n",
      "Iteration: 866, Loss: 262.96051025390625, running average elbo: -206.70582580566406\n",
      "Iteration: 867, Loss: 315.44305419921875, running average elbo: -207.79319763183594\n",
      "Iteration: 868, Loss: 350.70526123046875, running average elbo: -209.22232055664062\n",
      "Iteration: 869, Loss: 294.7645568847656, running average elbo: -210.07774353027344\n",
      "Iteration: 870, Loss: 231.87400817871094, running average elbo: -210.29571533203125\n",
      "Iteration: 871, Loss: 261.0803527832031, running average elbo: -210.80357360839844\n",
      "Iteration: 872, Loss: 287.8319091796875, running average elbo: -211.57386779785156\n",
      "Iteration: 873, Loss: 290.0788879394531, running average elbo: -212.35891723632812\n",
      "Iteration: 874, Loss: 371.593994140625, running average elbo: -213.95127868652344\n",
      "Iteration: 875, Loss: 401.25537109375, running average elbo: -215.82432556152344\n",
      "Iteration: 876, Loss: 722.4616088867188, running average elbo: -220.8907012939453\n",
      "Iteration: 877, Loss: 1843.654052734375, running average elbo: -237.1183319091797\n",
      "Iteration: 878, Loss: 1132.94775390625, running average elbo: -246.07662963867188\n",
      "Iteration: 879, Loss: 1483.8955078125, running average elbo: -258.4548034667969\n",
      "Iteration: 880, Loss: 1342.12890625, running average elbo: -269.2915344238281\n",
      "Iteration: 881, Loss: 1099.614013671875, running average elbo: -277.59478759765625\n",
      "Iteration: 882, Loss: 1106.6246337890625, running average elbo: -285.8851013183594\n",
      "Iteration: 883, Loss: 1082.7718505859375, running average elbo: -293.8539733886719\n",
      "Iteration: 884, Loss: 906.6276245117188, running average elbo: -299.9817199707031\n",
      "Iteration: 885, Loss: 839.3892211914062, running average elbo: -305.37579345703125\n",
      "Iteration: 886, Loss: 1018.9600219726562, running average elbo: -312.51165771484375\n",
      "Iteration: 887, Loss: 915.9783935546875, running average elbo: -318.54632568359375\n",
      "Iteration: 888, Loss: 1029.73876953125, running average elbo: -325.65826416015625\n",
      "Iteration: 889, Loss: 868.5071411132812, running average elbo: -331.08673095703125\n",
      "Iteration: 890, Loss: 762.0843505859375, running average elbo: -335.396728515625\n",
      "Iteration: 891, Loss: 770.8993530273438, running average elbo: -339.7517395019531\n",
      "Iteration: 892, Loss: 656.082275390625, running average elbo: -342.9150390625\n",
      "Iteration: 893, Loss: 713.8934326171875, running average elbo: -346.6248474121094\n",
      "Iteration: 894, Loss: 727.2767333984375, running average elbo: -350.4313659667969\n",
      "Iteration: 895, Loss: 723.464599609375, running average elbo: -354.1617126464844\n",
      "Iteration: 896, Loss: 732.9385375976562, running average elbo: -357.949462890625\n",
      "Iteration: 897, Loss: 607.8045043945312, running average elbo: -360.447998046875\n",
      "Iteration: 898, Loss: 549.858642578125, running average elbo: -362.3421325683594\n",
      "Iteration: 899, Loss: 584.2920532226562, running average elbo: -364.5616455078125\n",
      "Iteration: 900, Loss: 606.704833984375, running average elbo: -366.98309326171875\n",
      "Iteration: 901, Loss: 595.6738891601562, running average elbo: -369.2699890136719\n",
      "Iteration: 902, Loss: 605.9217529296875, running average elbo: -371.6365051269531\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 903, Loss: 548.92919921875, running average elbo: -373.409423828125\n",
      "Iteration: 904, Loss: 556.7247314453125, running average elbo: -375.2425842285156\n",
      "Iteration: 905, Loss: 578.744873046875, running average elbo: -377.2776184082031\n",
      "Iteration: 906, Loss: 625.1893920898438, running average elbo: -379.7567443847656\n",
      "Iteration: 907, Loss: 568.0974731445312, running average elbo: -381.64013671875\n",
      "Iteration: 908, Loss: 538.6737060546875, running average elbo: -383.2104797363281\n",
      "Iteration: 909, Loss: 517.1455688476562, running average elbo: -384.5498352050781\n",
      "Iteration: 910, Loss: 518.0814208984375, running average elbo: -385.8851623535156\n",
      "Iteration: 911, Loss: 526.4854736328125, running average elbo: -387.2911682128906\n",
      "Iteration: 912, Loss: 535.755126953125, running average elbo: -388.77581787109375\n",
      "Iteration: 913, Loss: 524.2623291015625, running average elbo: -390.13067626953125\n",
      "Iteration: 914, Loss: 512.0689086914062, running average elbo: -391.3500671386719\n",
      "Iteration: 915, Loss: 520.6000366210938, running average elbo: -392.642578125\n",
      "Iteration: 916, Loss: 545.5728149414062, running average elbo: -394.171875\n",
      "Iteration: 917, Loss: 584.733154296875, running average elbo: -396.0774841308594\n",
      "Iteration: 918, Loss: 630.1614990234375, running average elbo: -398.4183044433594\n",
      "Iteration: 919, Loss: 592.1239013671875, running average elbo: -400.3553466796875\n",
      "Iteration: 920, Loss: 539.9024047851562, running average elbo: -401.7508239746094\n",
      "Iteration: 921, Loss: 512.403076171875, running average elbo: -402.8573303222656\n",
      "Iteration: 922, Loss: 504.9938659667969, running average elbo: -403.8786926269531\n",
      "Iteration: 923, Loss: 513.8961791992188, running average elbo: -404.9788818359375\n",
      "Iteration: 924, Loss: 529.7792358398438, running average elbo: -406.2268981933594\n",
      "Iteration: 925, Loss: 552.9607543945312, running average elbo: -407.6942443847656\n",
      "Iteration: 926, Loss: 578.0126342773438, running average elbo: -409.3974304199219\n",
      "Iteration: 927, Loss: 571.394287109375, running average elbo: -411.01739501953125\n",
      "Iteration: 928, Loss: 549.8797607421875, running average elbo: -412.4060363769531\n",
      "Iteration: 929, Loss: 560.32373046875, running average elbo: -413.8852233886719\n",
      "Iteration: 930, Loss: 590.7322387695312, running average elbo: -415.6536865234375\n",
      "Iteration: 931, Loss: 656.0996704101562, running average elbo: -418.05816650390625\n",
      "Iteration: 932, Loss: 660.2058715820312, running average elbo: -420.4796447753906\n",
      "Iteration: 933, Loss: 688.6406860351562, running average elbo: -423.1612548828125\n",
      "Iteration: 934, Loss: 662.390869140625, running average elbo: -425.5535583496094\n",
      "Iteration: 935, Loss: 672.34814453125, running average elbo: -428.0215148925781\n",
      "Iteration: 936, Loss: 657.8882446289062, running average elbo: -430.3201904296875\n",
      "Iteration: 937, Loss: 742.515380859375, running average elbo: -433.442138671875\n",
      "Iteration: 938, Loss: 742.0255126953125, running average elbo: -436.5279846191406\n",
      "Iteration: 939, Loss: 744.8834228515625, running average elbo: -439.6115417480469\n",
      "Iteration: 940, Loss: 782.5263061523438, running average elbo: -443.0406799316406\n",
      "Iteration: 941, Loss: 679.4772338867188, running average elbo: -445.4050598144531\n",
      "Iteration: 942, Loss: 733.6389770507812, running average elbo: -448.28741455078125\n",
      "Iteration: 943, Loss: 810.843994140625, running average elbo: -451.9129638671875\n",
      "Iteration: 944, Loss: 809.2431030273438, running average elbo: -455.48626708984375\n",
      "Iteration: 945, Loss: 774.7572021484375, running average elbo: -458.678955078125\n",
      "Iteration: 946, Loss: 728.0673828125, running average elbo: -461.3728332519531\n",
      "Iteration: 947, Loss: 784.1763305664062, running average elbo: -464.6008605957031\n",
      "Iteration: 948, Loss: 737.1273803710938, running average elbo: -467.3261413574219\n",
      "Iteration: 949, Loss: 726.30126953125, running average elbo: -469.9158935546875\n",
      "Iteration: 950, Loss: 751.7693481445312, running average elbo: -472.73443603515625\n",
      "Iteration: 951, Loss: 679.235595703125, running average elbo: -474.7994689941406\n",
      "Iteration: 952, Loss: 646.1067504882812, running average elbo: -476.5125427246094\n",
      "Iteration: 953, Loss: 605.506103515625, running average elbo: -477.802490234375\n",
      "Iteration: 954, Loss: 571.6920776367188, running average elbo: -478.74139404296875\n",
      "Iteration: 955, Loss: 582.0272216796875, running average elbo: -479.7742614746094\n",
      "Iteration: 956, Loss: 555.9389038085938, running average elbo: -480.5359191894531\n",
      "Iteration: 957, Loss: 520.385498046875, running average elbo: -480.9344177246094\n",
      "Iteration: 958, Loss: 481.00872802734375, running average elbo: -480.9351806640625\n",
      "Iteration: 959, Loss: 462.3942565917969, running average elbo: -480.749755859375\n",
      "Iteration: 960, Loss: 470.831787109375, running average elbo: -480.65057373046875\n",
      "Iteration: 961, Loss: 466.7568054199219, running average elbo: -480.51165771484375\n",
      "Iteration: 962, Loss: 474.3661193847656, running average elbo: -480.4501953125\n",
      "Iteration: 963, Loss: 450.5333251953125, running average elbo: -480.1510314941406\n",
      "Iteration: 964, Loss: 411.4834899902344, running average elbo: -479.46435546875\n",
      "Iteration: 965, Loss: 418.3631286621094, running average elbo: -478.85333251953125\n",
      "Iteration: 966, Loss: 439.8971862792969, running average elbo: -478.4637451171875\n",
      "Iteration: 967, Loss: 441.94525146484375, running average elbo: -478.09857177734375\n",
      "Iteration: 968, Loss: 431.4717712402344, running average elbo: -477.63232421875\n",
      "Iteration: 969, Loss: 410.2484436035156, running average elbo: -476.95849609375\n",
      "Iteration: 970, Loss: 385.23748779296875, running average elbo: -476.0412902832031\n",
      "Iteration: 971, Loss: 369.65740966796875, running average elbo: -474.9774475097656\n",
      "Iteration: 972, Loss: 391.8353271484375, running average elbo: -474.14605712890625\n",
      "Iteration: 973, Loss: 392.29461669921875, running average elbo: -473.3275451660156\n",
      "Iteration: 974, Loss: 387.9354248046875, running average elbo: -472.4736328125\n",
      "Iteration: 975, Loss: 390.17742919921875, running average elbo: -471.6506652832031\n",
      "Iteration: 976, Loss: 371.2999572753906, running average elbo: -470.6471862792969\n",
      "Iteration: 977, Loss: 351.6147766113281, running average elbo: -469.45684814453125\n",
      "Iteration: 978, Loss: 336.556396484375, running average elbo: -468.12786865234375\n",
      "Iteration: 979, Loss: 340.20062255859375, running average elbo: -466.8486022949219\n",
      "Iteration: 980, Loss: 344.9793701171875, running average elbo: -465.6299133300781\n",
      "Iteration: 981, Loss: 335.3569641113281, running average elbo: -464.32720947265625\n",
      "Iteration: 982, Loss: 312.0889892578125, running average elbo: -462.8048095703125\n",
      "Iteration: 983, Loss: 299.7022399902344, running average elbo: -461.17376708984375\n",
      "Iteration: 984, Loss: 299.08050537109375, running average elbo: -459.5528564453125\n",
      "Iteration: 985, Loss: 301.6747741699219, running average elbo: -457.9740905761719\n",
      "Iteration: 986, Loss: 312.0694274902344, running average elbo: -456.5150451660156\n",
      "Iteration: 987, Loss: 304.6573181152344, running average elbo: -454.9964599609375\n",
      "Iteration: 988, Loss: 294.94921875, running average elbo: -453.39599609375\n",
      "Iteration: 989, Loss: 290.830810546875, running average elbo: -451.77032470703125\n",
      "Iteration: 990, Loss: 286.37091064453125, running average elbo: -450.1163330078125\n",
      "Iteration: 991, Loss: 290.8160705566406, running average elbo: -448.5233459472656\n",
      "Iteration: 992, Loss: 285.0445251464844, running average elbo: -446.8885498046875\n",
      "Iteration: 993, Loss: 273.0375061035156, running average elbo: -445.1500549316406\n",
      "Iteration: 994, Loss: 263.0411071777344, running average elbo: -443.3289489746094\n",
      "Iteration: 995, Loss: 262.69219970703125, running average elbo: -441.5225830078125\n",
      "Iteration: 996, Loss: 270.24493408203125, running average elbo: -439.809814453125\n",
      "Iteration: 997, Loss: 270.72149658203125, running average elbo: -438.1189270019531\n",
      "Iteration: 998, Loss: 262.32318115234375, running average elbo: -436.3609619140625\n",
      "Iteration: 999, Loss: 257.1523742675781, running average elbo: -434.5688781738281\n",
      "Iteration: 1000, Loss: 247.33985900878906, running average elbo: -432.69659423828125\n",
      "Iteration: 1001, Loss: 260.1949462890625, running average elbo: -430.9715881347656\n",
      "Iteration: 1002, Loss: 254.56178283691406, running average elbo: -429.2074890136719\n",
      "Iteration: 1003, Loss: 259.55523681640625, running average elbo: -427.5109558105469\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 1004, Loss: 262.2915954589844, running average elbo: -425.8587646484375\n",
      "Iteration: 1005, Loss: 245.43768310546875, running average elbo: -424.0545654296875\n",
      "Iteration: 1006, Loss: 264.8268127441406, running average elbo: -422.4622802734375\n",
      "Iteration: 1007, Loss: 248.5911407470703, running average elbo: -420.72357177734375\n",
      "Iteration: 1008, Loss: 251.5757293701172, running average elbo: -419.0320739746094\n",
      "Iteration: 1009, Loss: 245.93045043945312, running average elbo: -417.3010559082031\n",
      "Iteration: 1010, Loss: 232.31642150878906, running average elbo: -415.4512023925781\n",
      "Iteration: 1011, Loss: 232.93890380859375, running average elbo: -413.6260681152344\n",
      "Iteration: 1012, Loss: 235.05709838867188, running average elbo: -411.84039306640625\n",
      "Iteration: 1013, Loss: 234.72337341308594, running average elbo: -410.0692138671875\n",
      "Iteration: 1014, Loss: 237.44393920898438, running average elbo: -408.3429870605469\n",
      "Iteration: 1015, Loss: 231.6840057373047, running average elbo: -406.5763854980469\n",
      "Iteration: 1016, Loss: 227.38050842285156, running average elbo: -404.784423828125\n",
      "Iteration: 1017, Loss: 231.22897338867188, running average elbo: -403.0488586425781\n",
      "Iteration: 1018, Loss: 230.00628662109375, running average elbo: -401.31842041015625\n",
      "Iteration: 1019, Loss: 233.54022216796875, running average elbo: -399.640625\n",
      "Iteration: 1020, Loss: 231.85496520996094, running average elbo: -397.9627685546875\n",
      "Iteration: 1021, Loss: 224.63259887695312, running average elbo: -396.2294921875\n",
      "Iteration: 1022, Loss: 228.46299743652344, running average elbo: -394.5518493652344\n",
      "Iteration: 1023, Loss: 222.17770385742188, running average elbo: -392.8280944824219\n",
      "Iteration: 1024, Loss: 236.109619140625, running average elbo: -391.2608947753906\n",
      "Iteration: 1025, Loss: 231.74234008789062, running average elbo: -389.66571044921875\n",
      "Iteration: 1026, Loss: 242.86012268066406, running average elbo: -388.1976318359375\n",
      "Iteration: 1027, Loss: 249.90615844726562, running average elbo: -386.8147277832031\n",
      "Iteration: 1028, Loss: 264.698974609375, running average elbo: -385.5935974121094\n",
      "Iteration: 1029, Loss: 305.4351806640625, running average elbo: -384.7920227050781\n",
      "Iteration: 1030, Loss: 307.9658203125, running average elbo: -384.02374267578125\n",
      "Iteration: 1031, Loss: 312.1405334472656, running average elbo: -383.3049011230469\n",
      "Iteration: 1032, Loss: 256.7077331542969, running average elbo: -382.0389404296875\n",
      "Iteration: 1033, Loss: 220.09580993652344, running average elbo: -380.4195251464844\n",
      "Iteration: 1034, Loss: 220.31918334960938, running average elbo: -378.8185119628906\n",
      "Iteration: 1035, Loss: 249.00640869140625, running average elbo: -377.5203857421875\n",
      "Iteration: 1036, Loss: 259.9966735839844, running average elbo: -376.34515380859375\n",
      "Iteration: 1037, Loss: 234.22366333007812, running average elbo: -374.9239196777344\n",
      "Iteration: 1038, Loss: 215.39981079101562, running average elbo: -373.32867431640625\n",
      "Iteration: 1039, Loss: 228.10125732421875, running average elbo: -371.87640380859375\n",
      "Iteration: 1040, Loss: 237.8234405517578, running average elbo: -370.535888671875\n",
      "Iteration: 1041, Loss: 240.79833984375, running average elbo: -369.238525390625\n",
      "Iteration: 1042, Loss: 220.51443481445312, running average elbo: -367.75128173828125\n",
      "Iteration: 1043, Loss: 218.86666870117188, running average elbo: -366.2624206542969\n",
      "Iteration: 1044, Loss: 233.25930786132812, running average elbo: -364.932373046875\n",
      "Iteration: 1045, Loss: 233.12918090820312, running average elbo: -363.6143493652344\n",
      "Iteration: 1046, Loss: 219.08738708496094, running average elbo: -362.1690979003906\n",
      "Iteration: 1047, Loss: 212.55003356933594, running average elbo: -360.67291259765625\n",
      "Iteration: 1048, Loss: 219.37127685546875, running average elbo: -359.2599182128906\n",
      "Iteration: 1049, Loss: 228.8528594970703, running average elbo: -357.95587158203125\n",
      "Iteration: 1050, Loss: 230.1162567138672, running average elbo: -356.6774597167969\n",
      "Iteration: 1051, Loss: 226.21359252929688, running average elbo: -355.3728332519531\n",
      "Iteration: 1052, Loss: 218.90208435058594, running average elbo: -354.0081481933594\n",
      "Iteration: 1053, Loss: 221.06602478027344, running average elbo: -352.6787414550781\n",
      "Iteration: 1054, Loss: 229.29571533203125, running average elbo: -351.4449157714844\n",
      "Iteration: 1055, Loss: 232.45506286621094, running average elbo: -350.2550354003906\n",
      "Iteration: 1056, Loss: 227.01341247558594, running average elbo: -349.02264404296875\n",
      "Iteration: 1057, Loss: 221.18869018554688, running average elbo: -347.7442932128906\n",
      "Iteration: 1058, Loss: 214.7764434814453, running average elbo: -346.41461181640625\n",
      "Iteration: 1059, Loss: 218.2909698486328, running average elbo: -345.1333923339844\n",
      "Iteration: 1060, Loss: 226.59384155273438, running average elbo: -343.947998046875\n",
      "Iteration: 1061, Loss: 230.96009826660156, running average elbo: -342.818115234375\n",
      "Iteration: 1062, Loss: 232.30438232421875, running average elbo: -341.71295166015625\n",
      "Iteration: 1063, Loss: 226.9478759765625, running average elbo: -340.5653076171875\n",
      "Iteration: 1064, Loss: 225.9513702392578, running average elbo: -339.419189453125\n",
      "Iteration: 1065, Loss: 225.6697235107422, running average elbo: -338.28167724609375\n",
      "Iteration: 1066, Loss: 224.55409240722656, running average elbo: -337.1444091796875\n",
      "Iteration: 1067, Loss: 217.54522705078125, running average elbo: -335.94842529296875\n",
      "Iteration: 1068, Loss: 216.72044372558594, running average elbo: -334.75616455078125\n",
      "Iteration: 1069, Loss: 222.08880615234375, running average elbo: -333.6294860839844\n",
      "Iteration: 1070, Loss: 229.8961944580078, running average elbo: -332.5921325683594\n",
      "Iteration: 1071, Loss: 235.74046325683594, running average elbo: -331.62359619140625\n",
      "Iteration: 1072, Loss: 227.59481811523438, running average elbo: -330.58331298828125\n",
      "Iteration: 1073, Loss: 226.08287048339844, running average elbo: -329.538330078125\n",
      "Iteration: 1074, Loss: 226.67678833007812, running average elbo: -328.50970458984375\n",
      "Iteration: 1075, Loss: 233.031005859375, running average elbo: -327.554931640625\n",
      "Iteration: 1076, Loss: 230.20770263671875, running average elbo: -326.5814514160156\n",
      "Iteration: 1077, Loss: 225.1436004638672, running average elbo: -325.56707763671875\n",
      "Iteration: 1078, Loss: 228.37403869628906, running average elbo: -324.59515380859375\n",
      "Iteration: 1079, Loss: 235.0493621826172, running average elbo: -323.69970703125\n",
      "Iteration: 1080, Loss: 244.91152954101562, running average elbo: -322.9118347167969\n",
      "Iteration: 1081, Loss: 257.6951904296875, running average elbo: -322.2596740722656\n",
      "Iteration: 1082, Loss: 244.72628784179688, running average elbo: -321.4843444824219\n",
      "Iteration: 1083, Loss: 245.741455078125, running average elbo: -320.7269287109375\n",
      "Iteration: 1084, Loss: 251.57325744628906, running average elbo: -320.035400390625\n",
      "Iteration: 1085, Loss: 272.1650390625, running average elbo: -319.55670166015625\n",
      "Iteration: 1086, Loss: 304.7049560546875, running average elbo: -319.408203125\n",
      "Iteration: 1087, Loss: 319.1109924316406, running average elbo: -319.40521240234375\n",
      "Iteration: 1088, Loss: 381.05267333984375, running average elbo: -320.02166748046875\n",
      "Iteration: 1089, Loss: 350.6039733886719, running average elbo: -320.3274841308594\n",
      "Iteration: 1090, Loss: 347.1136779785156, running average elbo: -320.5953369140625\n",
      "Iteration: 1091, Loss: 321.6471252441406, running average elbo: -320.6058349609375\n",
      "Iteration: 1092, Loss: 352.5038757324219, running average elbo: -320.9248046875\n",
      "Iteration: 1093, Loss: 351.3583068847656, running average elbo: -321.2291259765625\n",
      "Iteration: 1094, Loss: 280.3885498046875, running average elbo: -320.82073974609375\n",
      "Iteration: 1095, Loss: 294.00506591796875, running average elbo: -320.5526123046875\n",
      "Iteration: 1096, Loss: 305.4047546386719, running average elbo: -320.401123046875\n",
      "Iteration: 1097, Loss: 292.7152404785156, running average elbo: -320.124267578125\n",
      "Iteration: 1098, Loss: 289.6822509765625, running average elbo: -319.8198547363281\n",
      "Iteration: 1099, Loss: 286.97198486328125, running average elbo: -319.49139404296875\n",
      "Iteration: 1100, Loss: 351.0996398925781, running average elbo: -319.8074645996094\n",
      "Iteration: 1101, Loss: 362.9021911621094, running average elbo: -320.2384338378906\n",
      "Iteration: 1102, Loss: 376.09033203125, running average elbo: -320.79693603515625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 1103, Loss: 369.5279235839844, running average elbo: -321.28424072265625\n",
      "Iteration: 1104, Loss: 316.16778564453125, running average elbo: -321.23309326171875\n",
      "Iteration: 1105, Loss: 345.1992492675781, running average elbo: -321.4727478027344\n",
      "Iteration: 1106, Loss: 373.1341247558594, running average elbo: -321.9893798828125\n",
      "Iteration: 1107, Loss: 370.34423828125, running average elbo: -322.4729309082031\n",
      "Iteration: 1108, Loss: 307.54833984375, running average elbo: -322.32366943359375\n",
      "Iteration: 1109, Loss: 302.5639343261719, running average elbo: -322.1260681152344\n",
      "Iteration: 1110, Loss: 399.70782470703125, running average elbo: -322.9018859863281\n",
      "Iteration: 1111, Loss: 498.08111572265625, running average elbo: -324.6536865234375\n",
      "Iteration: 1112, Loss: 549.5157470703125, running average elbo: -326.9023132324219\n",
      "Iteration: 1113, Loss: 428.38482666015625, running average elbo: -327.9171447753906\n",
      "Iteration: 1114, Loss: 396.21905517578125, running average elbo: -328.60015869140625\n",
      "Iteration: 1115, Loss: 468.8910827636719, running average elbo: -330.0030517578125\n",
      "Iteration: 1116, Loss: 424.5954284667969, running average elbo: -330.948974609375\n",
      "Iteration: 1117, Loss: 477.82562255859375, running average elbo: -332.4177551269531\n",
      "Iteration: 1118, Loss: 446.86444091796875, running average elbo: -333.5622253417969\n",
      "Iteration: 1119, Loss: 368.4043884277344, running average elbo: -333.91064453125\n",
      "Iteration: 1120, Loss: 369.84234619140625, running average elbo: -334.26995849609375\n",
      "Iteration: 1121, Loss: 426.4042663574219, running average elbo: -335.1913146972656\n",
      "Iteration: 1122, Loss: 466.03350830078125, running average elbo: -336.499755859375\n",
      "Iteration: 1123, Loss: 652.60498046875, running average elbo: -339.66082763671875\n",
      "Iteration: 1124, Loss: 445.3865661621094, running average elbo: -340.71807861328125\n",
      "Iteration: 1125, Loss: 397.4023132324219, running average elbo: -341.2849426269531\n",
      "Iteration: 1126, Loss: 482.541259765625, running average elbo: -342.697509765625\n",
      "Iteration: 1127, Loss: 475.18536376953125, running average elbo: -344.02239990234375\n",
      "Iteration: 1128, Loss: 429.9224853515625, running average elbo: -344.88140869140625\n",
      "Iteration: 1129, Loss: 603.460693359375, running average elbo: -347.4671936035156\n",
      "Iteration: 1130, Loss: 488.67401123046875, running average elbo: -348.8792724609375\n",
      "Iteration: 1131, Loss: 417.9891052246094, running average elbo: -349.57037353515625\n",
      "Iteration: 1132, Loss: 421.9252624511719, running average elbo: -350.2939147949219\n",
      "Iteration: 1133, Loss: 500.3150329589844, running average elbo: -351.79412841796875\n",
      "Iteration: 1134, Loss: 418.26336669921875, running average elbo: -352.4588317871094\n",
      "Iteration: 1135, Loss: 538.8432006835938, running average elbo: -354.3226623535156\n",
      "Iteration: 1136, Loss: 470.91558837890625, running average elbo: -355.4886169433594\n",
      "Iteration: 1137, Loss: 362.9234924316406, running average elbo: -355.56298828125\n",
      "Iteration: 1138, Loss: 349.68756103515625, running average elbo: -355.5042419433594\n",
      "Iteration: 1139, Loss: 340.8575744628906, running average elbo: -355.3577575683594\n",
      "Iteration: 1140, Loss: 371.6715393066406, running average elbo: -355.5208740234375\n",
      "Iteration: 1141, Loss: 360.4184875488281, running average elbo: -355.5698547363281\n",
      "Iteration: 1142, Loss: 443.97833251953125, running average elbo: -356.4539489746094\n",
      "Iteration: 1143, Loss: 470.6264953613281, running average elbo: -357.5956726074219\n",
      "Iteration: 1144, Loss: 396.56036376953125, running average elbo: -357.9853210449219\n",
      "Iteration: 1145, Loss: 350.2119140625, running average elbo: -357.9075622558594\n",
      "Iteration: 1146, Loss: 369.82659912109375, running average elbo: -358.0267639160156\n",
      "Iteration: 1147, Loss: 415.7565612792969, running average elbo: -358.60406494140625\n",
      "Iteration: 1148, Loss: 449.20343017578125, running average elbo: -359.51007080078125\n",
      "Iteration: 1149, Loss: 450.0511779785156, running average elbo: -360.4154968261719\n",
      "Iteration: 1150, Loss: 344.4969787597656, running average elbo: -360.2563171386719\n",
      "Iteration: 1151, Loss: 314.2895202636719, running average elbo: -359.796630859375\n",
      "Iteration: 1152, Loss: 292.3191833496094, running average elbo: -359.1218566894531\n",
      "Iteration: 1153, Loss: 301.7496643066406, running average elbo: -358.5481262207031\n",
      "Iteration: 1154, Loss: 330.4460754394531, running average elbo: -358.2671203613281\n",
      "Iteration: 1155, Loss: 356.68719482421875, running average elbo: -358.2513122558594\n",
      "Iteration: 1156, Loss: 365.5423889160156, running average elbo: -358.32421875\n",
      "Iteration: 1157, Loss: 326.82196044921875, running average elbo: -358.0091857910156\n",
      "Iteration: 1158, Loss: 269.16644287109375, running average elbo: -357.1207580566406\n",
      "Iteration: 1159, Loss: 256.3293151855469, running average elbo: -356.11285400390625\n",
      "Iteration: 1160, Loss: 281.2750549316406, running average elbo: -355.3644714355469\n",
      "Iteration: 1161, Loss: 297.4637145996094, running average elbo: -354.78546142578125\n",
      "Iteration: 1162, Loss: 327.2230529785156, running average elbo: -354.50982666015625\n",
      "Iteration: 1163, Loss: 334.93988037109375, running average elbo: -354.3141174316406\n",
      "Iteration: 1164, Loss: 305.8233947753906, running average elbo: -353.8291931152344\n",
      "Iteration: 1165, Loss: 295.6560974121094, running average elbo: -353.2474670410156\n",
      "Iteration: 1166, Loss: 272.55743408203125, running average elbo: -352.4405822753906\n",
      "Iteration: 1167, Loss: 256.777587890625, running average elbo: -351.48394775390625\n",
      "Iteration: 1168, Loss: 260.4788513183594, running average elbo: -350.57391357421875\n",
      "Iteration: 1169, Loss: 282.1656494140625, running average elbo: -349.88983154296875\n",
      "Iteration: 1170, Loss: 291.1734619140625, running average elbo: -349.30267333984375\n",
      "Iteration: 1171, Loss: 287.0456848144531, running average elbo: -348.68011474609375\n",
      "Iteration: 1172, Loss: 287.0261535644531, running average elbo: -348.0635986328125\n",
      "Iteration: 1173, Loss: 281.47967529296875, running average elbo: -347.39776611328125\n",
      "Iteration: 1174, Loss: 267.9678649902344, running average elbo: -346.6034851074219\n",
      "Iteration: 1175, Loss: 255.17027282714844, running average elbo: -345.68914794921875\n",
      "Iteration: 1176, Loss: 257.0356140136719, running average elbo: -344.8026123046875\n",
      "Iteration: 1177, Loss: 275.7287902832031, running average elbo: -344.11187744140625\n",
      "Iteration: 1178, Loss: 285.1867980957031, running average elbo: -343.52264404296875\n",
      "Iteration: 1179, Loss: 286.9635009765625, running average elbo: -342.9570617675781\n",
      "Iteration: 1180, Loss: 271.209228515625, running average elbo: -342.2395935058594\n",
      "Iteration: 1181, Loss: 251.87535095214844, running average elbo: -341.3359680175781\n",
      "Iteration: 1182, Loss: 238.29246520996094, running average elbo: -340.3055419921875\n",
      "Iteration: 1183, Loss: 242.5672149658203, running average elbo: -339.3281555175781\n",
      "Iteration: 1184, Loss: 259.6658020019531, running average elbo: -338.5315246582031\n",
      "Iteration: 1185, Loss: 268.42742919921875, running average elbo: -337.8304748535156\n",
      "Iteration: 1186, Loss: 270.2487487792969, running average elbo: -337.1546630859375\n",
      "Iteration: 1187, Loss: 267.28045654296875, running average elbo: -336.4559020996094\n",
      "Iteration: 1188, Loss: 256.17645263671875, running average elbo: -335.6531066894531\n",
      "Iteration: 1189, Loss: 246.16824340820312, running average elbo: -334.75823974609375\n",
      "Iteration: 1190, Loss: 243.22486877441406, running average elbo: -333.8429260253906\n",
      "Iteration: 1191, Loss: 249.81243896484375, running average elbo: -333.00262451171875\n",
      "Iteration: 1192, Loss: 259.85552978515625, running average elbo: -332.2711486816406\n",
      "Iteration: 1193, Loss: 262.3350524902344, running average elbo: -331.57177734375\n",
      "Iteration: 1194, Loss: 256.3550720214844, running average elbo: -330.8196105957031\n",
      "Iteration: 1195, Loss: 245.54803466796875, running average elbo: -329.9668884277344\n",
      "Iteration: 1196, Loss: 235.88941955566406, running average elbo: -329.026123046875\n",
      "Iteration: 1197, Loss: 239.55697631835938, running average elbo: -328.1314392089844\n",
      "Iteration: 1198, Loss: 249.91952514648438, running average elbo: -327.3493347167969\n",
      "Iteration: 1199, Loss: 258.4270324707031, running average elbo: -326.66009521484375\n",
      "Iteration: 1200, Loss: 262.4143371582031, running average elbo: -326.01763916015625\n",
      "Iteration: 1201, Loss: 256.34918212890625, running average elbo: -325.32098388671875\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 1202, Loss: 249.89378356933594, running average elbo: -324.56671142578125\n",
      "Iteration: 1203, Loss: 246.98455810546875, running average elbo: -323.7908935546875\n",
      "Iteration: 1204, Loss: 246.47616577148438, running average elbo: -323.0177307128906\n",
      "Iteration: 1205, Loss: 252.8504180908203, running average elbo: -322.3160705566406\n",
      "Iteration: 1206, Loss: 263.1680603027344, running average elbo: -321.724609375\n",
      "Iteration: 1207, Loss: 264.92919921875, running average elbo: -321.1566467285156\n",
      "Iteration: 1208, Loss: 259.7781677246094, running average elbo: -320.5428466796875\n",
      "Iteration: 1209, Loss: 253.4205780029297, running average elbo: -319.87164306640625\n",
      "Iteration: 1210, Loss: 249.7723388671875, running average elbo: -319.170654296875\n",
      "Iteration: 1211, Loss: 248.8461151123047, running average elbo: -318.4674072265625\n",
      "Iteration: 1212, Loss: 253.81246948242188, running average elbo: -317.82086181640625\n",
      "Iteration: 1213, Loss: 263.35418701171875, running average elbo: -317.2762145996094\n",
      "Iteration: 1214, Loss: 273.4949951171875, running average elbo: -316.8384094238281\n",
      "Iteration: 1215, Loss: 275.5977478027344, running average elbo: -316.4259948730469\n",
      "Iteration: 1216, Loss: 267.0664367675781, running average elbo: -315.9324035644531\n",
      "Iteration: 1217, Loss: 260.9859924316406, running average elbo: -315.3829345703125\n",
      "Iteration: 1218, Loss: 257.5360412597656, running average elbo: -314.804443359375\n",
      "Iteration: 1219, Loss: 256.3528747558594, running average elbo: -314.2199401855469\n",
      "Iteration: 1220, Loss: 267.7409973144531, running average elbo: -313.7551574707031\n",
      "Iteration: 1221, Loss: 280.6844787597656, running average elbo: -313.4244689941406\n",
      "Iteration: 1222, Loss: 285.484130859375, running average elbo: -313.1450500488281\n",
      "Iteration: 1223, Loss: 281.0688781738281, running average elbo: -312.8243103027344\n",
      "Iteration: 1224, Loss: 270.8529052734375, running average elbo: -312.40460205078125\n",
      "Iteration: 1225, Loss: 264.0918273925781, running average elbo: -311.9214782714844\n",
      "Iteration: 1226, Loss: 262.9259338378906, running average elbo: -311.4315490722656\n",
      "Iteration: 1227, Loss: 270.58819580078125, running average elbo: -311.0231018066406\n",
      "Iteration: 1228, Loss: 291.895263671875, running average elbo: -310.8318176269531\n",
      "Iteration: 1229, Loss: 312.5434265136719, running average elbo: -310.84893798828125\n",
      "Iteration: 1230, Loss: 319.515869140625, running average elbo: -310.93560791015625\n",
      "Iteration: 1231, Loss: 307.7436218261719, running average elbo: -310.9036865234375\n",
      "Iteration: 1232, Loss: 293.64825439453125, running average elbo: -310.73114013671875\n",
      "Iteration: 1233, Loss: 283.0467529296875, running average elbo: -310.4543151855469\n",
      "Iteration: 1234, Loss: 277.1671447753906, running average elbo: -310.1214294433594\n",
      "Iteration: 1235, Loss: 284.8478698730469, running average elbo: -309.86871337890625\n",
      "Iteration: 1236, Loss: 312.4617919921875, running average elbo: -309.8946228027344\n",
      "Iteration: 1237, Loss: 338.2784729003906, running average elbo: -310.178466796875\n",
      "Iteration: 1238, Loss: 337.83917236328125, running average elbo: -310.455078125\n",
      "Iteration: 1239, Loss: 316.8599853515625, running average elbo: -310.5191345214844\n",
      "Iteration: 1240, Loss: 298.6478576660156, running average elbo: -310.4004211425781\n",
      "Iteration: 1241, Loss: 291.4640808105469, running average elbo: -310.2110595703125\n",
      "Iteration: 1242, Loss: 299.31121826171875, running average elbo: -310.10205078125\n",
      "Iteration: 1243, Loss: 328.90130615234375, running average elbo: -310.2900390625\n",
      "Iteration: 1244, Loss: 371.1595764160156, running average elbo: -310.89874267578125\n",
      "Iteration: 1245, Loss: 403.1972961425781, running average elbo: -311.8217468261719\n",
      "Iteration: 1246, Loss: 398.409423828125, running average elbo: -312.6876220703125\n",
      "Iteration: 1247, Loss: 374.62890625, running average elbo: -313.3070373535156\n",
      "Iteration: 1248, Loss: 352.8481140136719, running average elbo: -313.70245361328125\n",
      "Iteration: 1249, Loss: 332.2118835449219, running average elbo: -313.8875427246094\n",
      "Iteration: 1250, Loss: 323.4093322753906, running average elbo: -313.9827575683594\n",
      "Iteration: 1251, Loss: 345.239990234375, running average elbo: -314.2953186035156\n",
      "Iteration: 1252, Loss: 395.8158264160156, running average elbo: -315.11053466796875\n",
      "Iteration: 1253, Loss: 437.72662353515625, running average elbo: -316.3367004394531\n",
      "Iteration: 1254, Loss: 438.3218688964844, running average elbo: -317.5565490722656\n",
      "Iteration: 1255, Loss: 402.44049072265625, running average elbo: -318.4053955078125\n",
      "Iteration: 1256, Loss: 356.0379638671875, running average elbo: -318.78173828125\n",
      "Iteration: 1257, Loss: 336.73944091796875, running average elbo: -318.9613342285156\n",
      "Iteration: 1258, Loss: 350.97454833984375, running average elbo: -319.2814636230469\n",
      "Iteration: 1259, Loss: 400.93487548828125, running average elbo: -320.0979919433594\n",
      "Iteration: 1260, Loss: 484.1780700683594, running average elbo: -321.73876953125\n",
      "Iteration: 1261, Loss: 549.8829956054688, running average elbo: -324.0202331542969\n",
      "Iteration: 1262, Loss: 551.2881469726562, running average elbo: -326.29290771484375\n",
      "Iteration: 1263, Loss: 533.5621337890625, running average elbo: -328.3656005859375\n",
      "Iteration: 1264, Loss: 496.28546142578125, running average elbo: -330.0447998046875\n",
      "Iteration: 1265, Loss: 457.93511962890625, running average elbo: -331.3236999511719\n",
      "Iteration: 1266, Loss: 406.8645935058594, running average elbo: -332.0791015625\n",
      "Iteration: 1267, Loss: 410.0836181640625, running average elbo: -332.859130859375\n",
      "Iteration: 1268, Loss: 467.52496337890625, running average elbo: -334.205810546875\n",
      "Iteration: 1269, Loss: 552.4854125976562, running average elbo: -336.38861083984375\n",
      "Iteration: 1270, Loss: 657.7442626953125, running average elbo: -339.6021728515625\n",
      "Iteration: 1271, Loss: 689.3782348632812, running average elbo: -343.09991455078125\n",
      "Iteration: 1272, Loss: 580.4398803710938, running average elbo: -345.47332763671875\n",
      "Iteration: 1273, Loss: 463.1359558105469, running average elbo: -346.6499328613281\n",
      "Iteration: 1274, Loss: 413.291748046875, running average elbo: -347.31634521484375\n",
      "Iteration: 1275, Loss: 447.4162292480469, running average elbo: -348.31732177734375\n",
      "Iteration: 1276, Loss: 569.1541137695312, running average elbo: -350.5256652832031\n",
      "Iteration: 1277, Loss: 756.0096435546875, running average elbo: -354.58050537109375\n",
      "Iteration: 1278, Loss: 857.10986328125, running average elbo: -359.6058044433594\n",
      "Iteration: 1279, Loss: 898.7055053710938, running average elbo: -364.9967956542969\n",
      "Iteration: 1280, Loss: 890.9757080078125, running average elbo: -370.256591796875\n",
      "Iteration: 1281, Loss: 812.880859375, running average elbo: -374.6828308105469\n",
      "Iteration: 1282, Loss: 729.8785400390625, running average elbo: -378.23480224609375\n",
      "Iteration: 1283, Loss: 665.7890625, running average elbo: -381.1103515625\n",
      "Iteration: 1284, Loss: 584.1180419921875, running average elbo: -383.14044189453125\n",
      "Iteration: 1285, Loss: 542.598876953125, running average elbo: -384.73504638671875\n",
      "Iteration: 1286, Loss: 649.3786010742188, running average elbo: -387.3814697265625\n",
      "Iteration: 1287, Loss: 909.3836059570312, running average elbo: -392.60150146484375\n",
      "Iteration: 1288, Loss: 1126.38330078125, running average elbo: -399.9393005371094\n",
      "Iteration: 1289, Loss: 1120.2779541015625, running average elbo: -407.1427001953125\n",
      "Iteration: 1290, Loss: 911.2242431640625, running average elbo: -412.18353271484375\n",
      "Iteration: 1291, Loss: 649.0659790039062, running average elbo: -414.5523681640625\n",
      "Iteration: 1292, Loss: 527.7181396484375, running average elbo: -415.6840515136719\n",
      "Iteration: 1293, Loss: 619.7570190429688, running average elbo: -417.72479248046875\n",
      "Iteration: 1294, Loss: 753.9827270507812, running average elbo: -421.0873718261719\n",
      "Iteration: 1295, Loss: 905.882568359375, running average elbo: -425.9353332519531\n",
      "Iteration: 1296, Loss: 1114.4169921875, running average elbo: -432.8201599121094\n",
      "Iteration: 1297, Loss: 1224.4991455078125, running average elbo: -440.7369689941406\n",
      "Iteration: 1298, Loss: 1261.4490966796875, running average elbo: -448.944091796875\n",
      "Iteration: 1299, Loss: 1194.9735107421875, running average elbo: -456.4043884277344\n",
      "Iteration: 1300, Loss: 1119.5611572265625, running average elbo: -463.0359802246094\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 1301, Loss: 1074.4403076171875, running average elbo: -469.1500549316406\n",
      "Iteration: 1302, Loss: 1019.39306640625, running average elbo: -474.6524963378906\n",
      "Iteration: 1303, Loss: 869.8328857421875, running average elbo: -478.60430908203125\n",
      "Iteration: 1304, Loss: 679.1024780273438, running average elbo: -480.6092834472656\n",
      "Iteration: 1305, Loss: 578.2855224609375, running average elbo: -481.5860595703125\n",
      "Iteration: 1306, Loss: 610.2195434570312, running average elbo: -482.8724060058594\n",
      "Iteration: 1307, Loss: 729.5202026367188, running average elbo: -485.3388977050781\n",
      "Iteration: 1308, Loss: 866.8258666992188, running average elbo: -489.15374755859375\n",
      "Iteration: 1309, Loss: 969.218994140625, running average elbo: -493.95440673828125\n",
      "Iteration: 1310, Loss: 1000.8727416992188, running average elbo: -499.0235900878906\n",
      "Iteration: 1311, Loss: 954.8018798828125, running average elbo: -503.58135986328125\n",
      "Iteration: 1312, Loss: 808.0004272460938, running average elbo: -506.6255798339844\n",
      "Iteration: 1313, Loss: 652.7709350585938, running average elbo: -508.0870361328125\n",
      "Iteration: 1314, Loss: 611.7640991210938, running average elbo: -509.1238098144531\n",
      "Iteration: 1315, Loss: 686.5228881835938, running average elbo: -510.8977966308594\n",
      "Iteration: 1316, Loss: 776.5735473632812, running average elbo: -513.5545654296875\n",
      "Iteration: 1317, Loss: 870.251953125, running average elbo: -517.12158203125\n",
      "Iteration: 1318, Loss: 998.86376953125, running average elbo: -521.9390258789062\n",
      "Iteration: 1319, Loss: 1112.4200439453125, running average elbo: -527.8438720703125\n",
      "Iteration: 1320, Loss: 1152.2266845703125, running average elbo: -534.0877075195312\n",
      "Iteration: 1321, Loss: 1116.687744140625, running average elbo: -539.9136962890625\n",
      "Iteration: 1322, Loss: 1052.9512939453125, running average elbo: -545.0441284179688\n",
      "Iteration: 1323, Loss: 1011.9800415039062, running average elbo: -549.7135009765625\n",
      "Iteration: 1324, Loss: 962.4545288085938, running average elbo: -553.8409423828125\n",
      "Iteration: 1325, Loss: 903.5018920898438, running average elbo: -557.3375854492188\n",
      "Iteration: 1326, Loss: 779.3433837890625, running average elbo: -559.5576782226562\n",
      "Iteration: 1327, Loss: 713.826904296875, running average elbo: -561.100341796875\n",
      "Iteration: 1328, Loss: 689.9694213867188, running average elbo: -562.3890380859375\n",
      "Iteration: 1329, Loss: 669.6915283203125, running average elbo: -563.4620361328125\n",
      "Iteration: 1330, Loss: 626.2798461914062, running average elbo: -564.0902099609375\n",
      "Iteration: 1331, Loss: 583.7012329101562, running average elbo: -564.286376953125\n",
      "Iteration: 1332, Loss: 554.2404174804688, running average elbo: -564.1859130859375\n",
      "Iteration: 1333, Loss: 556.0796508789062, running average elbo: -564.1048583984375\n",
      "Iteration: 1334, Loss: 575.469482421875, running average elbo: -564.218505859375\n",
      "Iteration: 1335, Loss: 619.9754028320312, running average elbo: -564.776123046875\n",
      "Iteration: 1336, Loss: 702.0348510742188, running average elbo: -566.148681640625\n",
      "Iteration: 1337, Loss: 829.7186279296875, running average elbo: -568.7843627929688\n",
      "Iteration: 1338, Loss: 756.653076171875, running average elbo: -570.6630249023438\n",
      "Iteration: 1339, Loss: 676.6505737304688, running average elbo: -571.722900390625\n",
      "Iteration: 1340, Loss: 595.5053100585938, running average elbo: -571.9607543945312\n",
      "Iteration: 1341, Loss: 685.73681640625, running average elbo: -573.0985107421875\n",
      "Iteration: 1342, Loss: 907.5635986328125, running average elbo: -576.4431762695312\n",
      "Iteration: 1343, Loss: 447.2621154785156, running average elbo: -575.1513671875\n",
      "Iteration: 1344, Loss: 941.0191040039062, running average elbo: -578.81005859375\n",
      "Iteration: 1345, Loss: 1764.3175048828125, running average elbo: -590.6651611328125\n",
      "Iteration: 1346, Loss: 1215.3997802734375, running average elbo: -596.9125366210938\n",
      "Iteration: 1347, Loss: 1234.76220703125, running average elbo: -603.291015625\n",
      "Iteration: 1348, Loss: 401.0038146972656, running average elbo: -601.2681274414062\n",
      "Iteration: 1349, Loss: 630.7942504882812, running average elbo: -601.5633544921875\n",
      "Iteration: 1350, Loss: 415.5791320800781, running average elbo: -599.7034912109375\n",
      "Iteration: 1351, Loss: 530.010009765625, running average elbo: -599.006591796875\n",
      "Iteration: 1352, Loss: 382.78082275390625, running average elbo: -596.8443603515625\n",
      "Iteration: 1353, Loss: 459.9935607910156, running average elbo: -595.475830078125\n",
      "Iteration: 1354, Loss: 425.7823181152344, running average elbo: -593.7788696289062\n",
      "Iteration: 1355, Loss: 336.2171325683594, running average elbo: -591.2032470703125\n",
      "Iteration: 1356, Loss: 445.99603271484375, running average elbo: -589.7511596679688\n",
      "Iteration: 1357, Loss: 348.5350646972656, running average elbo: -587.3389892578125\n",
      "Iteration: 1358, Loss: 331.5054931640625, running average elbo: -584.7806396484375\n",
      "Iteration: 1359, Loss: 363.9389953613281, running average elbo: -582.572265625\n",
      "Iteration: 1360, Loss: 285.63037109375, running average elbo: -579.6028442382812\n",
      "Iteration: 1361, Loss: 336.2078552246094, running average elbo: -577.1688842773438\n",
      "Iteration: 1362, Loss: 275.3684997558594, running average elbo: -574.15087890625\n",
      "Iteration: 1363, Loss: 286.5349426269531, running average elbo: -571.2747192382812\n",
      "Iteration: 1364, Loss: 276.9737548828125, running average elbo: -568.3316650390625\n",
      "Iteration: 1365, Loss: 267.53851318359375, running average elbo: -565.3237915039062\n",
      "Iteration: 1366, Loss: 288.0430603027344, running average elbo: -562.5509643554688\n",
      "Iteration: 1367, Loss: 251.78408813476562, running average elbo: -559.4432983398438\n",
      "Iteration: 1368, Loss: 266.4020080566406, running average elbo: -556.5128784179688\n",
      "Iteration: 1369, Loss: 256.5342712402344, running average elbo: -553.5131225585938\n",
      "Iteration: 1370, Loss: 246.5963134765625, running average elbo: -550.4439086914062\n",
      "Iteration: 1371, Loss: 272.9359436035156, running average elbo: -547.6688232421875\n",
      "Iteration: 1372, Loss: 245.6232452392578, running average elbo: -544.6483764648438\n",
      "Iteration: 1373, Loss: 246.38497924804688, running average elbo: -541.665771484375\n",
      "Iteration: 1374, Loss: 242.21275329589844, running average elbo: -538.6712646484375\n",
      "Iteration: 1375, Loss: 245.94737243652344, running average elbo: -535.7440185546875\n",
      "Iteration: 1376, Loss: 249.88746643066406, running average elbo: -532.885498046875\n",
      "Iteration: 1377, Loss: 232.86000061035156, running average elbo: -529.88525390625\n",
      "Iteration: 1378, Loss: 244.81349182128906, running average elbo: -527.0345458984375\n",
      "Iteration: 1379, Loss: 230.06947326660156, running average elbo: -524.06494140625\n",
      "Iteration: 1380, Loss: 242.7740020751953, running average elbo: -521.2520141601562\n",
      "Iteration: 1381, Loss: 230.68353271484375, running average elbo: -518.3463134765625\n",
      "Iteration: 1382, Loss: 227.81643676757812, running average elbo: -515.4409790039062\n",
      "Iteration: 1383, Loss: 226.01475524902344, running average elbo: -512.5466918945312\n",
      "Iteration: 1384, Loss: 221.25099182128906, running average elbo: -509.6337585449219\n",
      "Iteration: 1385, Loss: 228.10986328125, running average elbo: -506.8185119628906\n",
      "Iteration: 1386, Loss: 217.6966094970703, running average elbo: -503.92730712890625\n",
      "Iteration: 1387, Loss: 223.59718322753906, running average elbo: -501.1239929199219\n",
      "Iteration: 1388, Loss: 217.47897338867188, running average elbo: -498.2875671386719\n",
      "Iteration: 1389, Loss: 217.8064727783203, running average elbo: -495.4827575683594\n",
      "Iteration: 1390, Loss: 212.89849853515625, running average elbo: -492.65692138671875\n",
      "Iteration: 1391, Loss: 210.537109375, running average elbo: -489.8357238769531\n",
      "Iteration: 1392, Loss: 209.14962768554688, running average elbo: -487.02886962890625\n",
      "Iteration: 1393, Loss: 206.3961181640625, running average elbo: -484.2225646972656\n",
      "Iteration: 1394, Loss: 208.82125854492188, running average elbo: -481.46856689453125\n",
      "Iteration: 1395, Loss: 202.44358825683594, running average elbo: -478.6783447265625\n"
     ]
    }
   ],
   "source": [
    "runningAvg = None\n",
    "momentum = 0.99\n",
    "loss_history = []\n",
    "niters = 1395\n",
    "for step in range(niters+1):\n",
    "    \n",
    "    loss = compute_gradients_and_update(samp_trajs, samp_ts)    \n",
    "    loss_history.append(loss.numpy())\n",
    "    \n",
    "    runningAvg = loss if runningAvg is None else (runningAvg * momentum + loss * (1 - momentum))\n",
    "    print(\"Iteration: {}, Loss: {}, running average elbo: {}\".format(step, loss, -runningAvg))\n",
    "#     if step % 500 == 0:        \n",
    "#         yN, states_history_model = neural_ode_test.forward(true_y0, return_states=\"numpy\")        \n",
    "#         plot_spiral([true_y, np.concatenate(states_history_model)])        \n",
    "#         plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 4)\n",
      "2000\n",
      "(4,)\n",
      "(2000, 2)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x129efdf28>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xd4VEXbwOHfSTbJpodUSEIaCYEk1NARkKYIgoog2F7EglhRQREr+ooFURHLJyj6qggIIkhVadJraAJJSO+9182W+f4IRJAAoWx2k8x9XedK2d0zTzbJc+bMmfOMIoRAkiRJav4sTB2AJEmS1DhkwpckSWohZMKXJElqIWTClyRJaiFkwpckSWohZMKXJElqIWTClyRJaiFkwpckSWohZMKXJElqIVTGbkBRlGSgDNADOiFEj0s9193dXQQEBBg7JEmSpGYlKioqXwjhcaXnGT3hnzVYCJF/pScFBARw+PDhxohHkiSp2VAUJaUhz5NDOpIkSS1EYyR8AfypKEqUoihTGqE9SZIkqR6NMaRzkxAiQ1EUT2CzoigxQoid5x48exCYAuDn59cI4UiSJLVMRu/hCyEyzn7MBVYDvf71+CIhRA8hRA8Pjytec5AkSZKukVETvqIo9oqiOJ77HLgFOGnMNiVJkqT6GXtIxwtYrSjKubaWCiF+N3KbkiRJUj2MmvCFEIlAF2O2IUmSJDVMY83DlyTpcoSA6mooKal/q6oCvR50ugs3IUClAiurCz+e+9zeHpydwckJPDygbVuwsTH1TyuZiEz4kmQEBmEgryKPrPIsCqsKKctNxxB3BsuERNTJ6dhl5GJfUIZzYQWtCqtwqNBipW+E9aUVBdq0AX//2i04GMLCarfQUFCrjR+DZDIy4UvSNaioqSChKIH4wnjiC+NJK0kjtyAV67hEnJMycc8oJqjAQHAhdCoEj8oLX5/rYEGuixVpLtYcb+tMhaMajYOaGgc1NQ52aOxtqLBVUWlnRaWdFRprC/QqC7SKoMJQTbm+mgpDNRW6SrSaKmo0lVRVllJVXY6VAVQGsNKDvRacNOBcDSE6R8Iq7QkptcS3KBWvnTE4r1yBhd5QG5SFBbRrV5v8u3aFnj2hRw/w8mr8N1gyCpnwJekycityOZFzghM5JziZe5L4wnhScuNwSskmPBfC8yA8F+7KtyCowIDleZ30Mk8XqgJ80d7kT1ZwCFahYdiFdca2fTieDg54GiHeGn0NRVVFFFYVUlBVQGFVIXkVeWSUZZBRmsH6snQySjNIL02noKoIax20L4CwPOhZrKZ7USEdDv9Fm7VrUcTZH8bXtzb59+wJgwbVHgSsrY0QvWRsihCNcBrZQD169BCylo5kCkIIkoqT2J++n6jMKE7knuDvnL+pKMyhazZ0z4L+eWp6ZFvgn1WFpaH2/0ZYWGBoF4Rlp84QHl67hYXVDpXY2pr4p7q8Km0ViUWJxBXGcabgDHEFccQV1m4l+Zl0y4aeGXBTrg29sizxzak9TTHYqlH69Ue5+ebaA0Dv3vIAYGKKokRdrjBl3fNkwpdaooqaCg5lHmJf2j72Z+xnf/p+Kgpz6ZkJfXJU3FzgSJcMPV5ZZf/0dNu0ge7doUuXf5J7Mx33Lqwq5ETOCY5nH+d4Tu2Wk3SSXok13JwMw9JUhGXpANA72KHccisWo26H226rfZ+kRiUTviSdR2fQcSjjEFsSt7AlaQv7UvfiU6ijXxqMzHOhf4YFfslFWJztuePnV5vcz99aeCLT6rWcyDnB3rS97Evfx6mYXQT9nc5tcTAqXsGntPa9q+4cjs34iSj33APt25s46pZBJnypxcsozWBt7Fp+T/idHYnbaZtextAkGJ3jQs8ULU6FFbVPdHCoHZbo1w/69q0dq3Z3N23wTURmWSZ7UvewJWEzaXs20jkqg9FnoH9a7eOlHYKwu/8hVBPvrR3mkoxCJnypxRFCEJMfw5qYNayJWU3h34cYkgRjMuwZkKjHqaS69olBQbXJ/dwWEQGWlqYNvplIKExgc+Jmog7+huvGbdxxvIZ+6bWP5XfvgOPUadjc+0DtQVa6YWTCl1qM+MJ4fjz+I+uilhJwKJ5RcXB7sjVeRTUACB8flKFDYcgQGDy4drhGMrpqXTVbErewY+ePOPy6nokHKwktgCq1iqIxw2k9/U0sevU2dZjNgkz4UrNWVFXEilMr2LZ5IW12HGVUHNycomClFxicnbAYfgsMG1ab5IODa284kkxGZ9CxK3knh1bOp83K3xl7Qou9FpIj2qJ+cRat758iz7Kug0z4UrMjhGBv2l5Wr3oHl7V/MvakgbCzC2dqQ0OwGn0H3H577TCNlZVpg5UuqUpbxYaoZeR98SEj1scQWAyZXnaUTp1M6PT3UBwdTR1ikyMTvtRsVOuq2fTbR+R+9zkDDmQTlg8GBSr6ROIw4UGU0aNrx+WlJiejMIU9n71I4Her6Zmio8TWgvgHRhL+7teo3VubOrwmQyZ8qckrKM/jz4+eJOD7NfRN0mFQILtbe9z+8zg2E+6D1jIhNBcanYatP72D+pPPGHK8hGJbhdgHRtL1vW+xcTPGPcnNi0z4UpOVX5zJjnceo9P3m2ifL8j2tKPskQcJfvoNFG9vU4cnGZEQgkMbv6b69VkMPFpIob0FJ5+ZSJ/ZX2NtY2fq8MxWQxN+YyxiLkkNUpCXyoYnhqHx9+XujzZi5ehM2qJ5tM4sJeTdr2SybwEURaHXqCkMiMrnwG9fkuzjwMD3l5IY1IqDP801dXhNnkz4kslpy0rY/exdGAIDGPXVVorbepCyfCGBCYW0fWy6nL3RAimKQu8xT9AtuoioBbOwrzbQ64GZ/DE0gKSU46YOr8mSCV8yqS2JWxjzUSR9Pl9DSmArEtb9QPjJHPwnTJFTKSUUCwsin3kXz+Q8Dtw7kGHbU7Dq3JWfP34EvUFv6vCaHDmGL5lEdnk2T218il+jfyWoVRBfhc5g2C1TUVpYktcb9BRVF1FQWUBRdREanQatQUuNvoYafQ1avRaBQK1So1apsbG0qfvc1dYVD3sP1KrmV7ztUnK3rUPzn/tpm1HGj6PaMuB/2whwlyUbGjqGL+vhS41uxakVPLHhCSpqKpgzZA4v9H2hWSYtIQSZZZkkFiWSVJxU9zGpKImcihzyK/MpqipCcH2dLgdrBzztPfGw88Db0Ztg12CCXYNp16odwa7B+Dr5YmnRPIbFPIeMRpzJJu6BkTy4ege7e3bg8OL5jBvytKlDaxJkD19qNIVVhTy18SmWn1xOT++efH/n93T06GjqsG4IrV5LdH40R7OOciz7GEezaz+WaErqnqOg4OvkS2CrQNo4tMHN1g03Ozfc7dxxs3WjlW0r1Co11pbWWFtaY2VhhbVlbZ15jV5Dta6aal01Gp2GSm1l7eImlXnkVeSRW5lLXkUeaaVpJBYlUqOvqWvXVmVL19Zd6d6mO5FtIon0jqSje0esLJv2zWn5X32Cw7QZZNsaWDF3EtMfXdxsDmxXS07LlMzKoYxD3PXzXeRU5PDGwDeYNWAWKoumdYKZV1PD3tJSjpeXE11RzrGSPLJqNFTrNNToaxBCD7oKLPSVOKss8bC2w9/OgWB7Z8KdvOjayocgOwc8ra2xNOLQld6gJ6Msg4TC2iUYT+ed5kj2EY5kHaG8phyoPSsY5D+IYUHDGBo4lAjPiCY5nKY7uJ/KEUMxVFby7vSevPr6ZpzVzqYOq9HJhC+ZjR+O/8CUdVNo7dCaVfesItI70tQhNYheCPaUlPBrXh4bCwuJq6qqe0zR5CIq06CmAFe1C63tvXCz98Ta2gWdhZoSvZ4irZasmhpq/vU/plIUvK2tCVCr6evkxEAXF/o7O+OsMu4B0CAMxBXEEZUVxe7U3WxJ3EJcYRwAXvZe3BF6BxMiJjDIf1DT6imnpFA4uC+2aVk8+1Qg7717EHe7llXeWiZ8yeR0Bh0v/vki8w/M5+aAm1kxbgUe9h6mDuuy9Dode5KTWWEw8EteHjlaLSoMOFbGUZT9F5ScxM+yhtHBw7kt+DZu8rvpsj1KIQT5Wi3pGs1F25mqKqLKytAKgQJ0cXDgPk9Pnvf1RWXROBPoUktS2Zq4lT8S/mD9mfVUaCvwsvdiXNg4Hur6ED28r5hDzENeHmUDeqNKSOLZJwN5772WlfRlwpdMqkZfw32r7mNV9Cqe7fUs826ZZ95jxhkZnHj7Y55v78u2yG7YKAK3qjiyklYgCvbSyS2YCeETGNtxLB3cO9yw4Y9KvZ4DpaXsLClha1ERu0pK6OPkxA8dOhBi17h3llZqK9kYt5HlJ5ezIW4D1bpqenr35KmeT3Fvp3vrrieYrbw8ym/qhUhOZvKMYBa9dgBXW1dTR9UoZMKXTEaj03DPL/ewNnYtn9z6Cc/1ec7UIV3awYMwdy6sWQN6Pe/cNZpNHfTstd2Gp60Tk7tOZlKXSY12cXl5Tg5PxsWhMRiY264dT3h7Y2GCsfWS6hJ+PPEj/3f4/zidd5q2Tm15qf9LPNb9MWxUNo0eT4NlZFDdoyt5lfm8+N8BLHl6W5O7VnQtGprwEUKYzRYZGSmkpq1KWyVuW3KbYDbiy4Nfmjqci/y0P1n0nrNZbP30ByEGDxYChMHFWeya2E90fM5BMBvh/UE3sfLUSqHRaUwSY3p1tbj12DHB9u1i2LFjIrWqyiRxCCGEwWAQG89sFP0W9xPMRgTODxQrT60UBoPBZDFd0dGjosbWRuxui3hp4wumjqZRAIdFA3KsvNNWumEMwsB/Vv+H3+N/55vR3/BEzycaPYalB1Lo895Wlh5IufhBITg1/xu+WTCVIdP+Q+Xfp9j9zJ0EvmDJgA57yW7VmdaaD/HTf8i4sHEmG8LwsbFhU+fOLGzfnn0lJXQ+fJi9JSVXfqERKIrCbSG3sXvybv544A8crB0Yv3I8ty65ldSSVJPEdEVdu2L1zbf0TwPbDz5mdfRqU0dkNmTCl26Y/+74LytPr2Tu8Lk80v0Rk8SwYFs82SXVfLYt/sIHsrNh5EjmLH8HW62G50eOw3OKFQPc1hDk14VDjx3i81uXEuDYjWeGmP7OTUVRmOLtzfGePfGwsmL48eP8VVRk0nhuaXcLRx8/yue3fc7etL1EfBnB8pPLTRbTZd13H4YH7uf1nfDV11Mo05SZOiLz0JDTgMba5JBO07Xi5ArBbMSk1ZNMerr/0/5k0efdLeKn/cn/fHPzZiHc3YVQq8XeF14Tbm+MEsxGWL7hKULfece8hyeEENkajQg7cEDY7dgh/ioqMnU4QgghEgsTRf/F/QWzEdP/mC70Br2pQ7pYcbGocW8l9vgintvwrKmjMSrkkI7UWOIL45n822T6+vZl4e0LTXoDz329/dk3ayj39fav/caiRTBiBHh5Eb9lBUOdl1CgbMRd3EV3m8W8MeyBungvOxxkQl7W1mzr2hUD8E6KecQW2CqQbZO28VTPp/ho30c8tOYhdAadqcO6kLMzVvM+oV865H37GcnFyaaOyORkwpeui96gZ9KaSVhZWrFi/ArzmsExbx48/jjccgsbv3+NLjsmgkUJrbVvc3+H1zn4yu3/HBi4zHCQGfCytqaLvT0VevOpEGltac1nt33GO4Pf4ccTP/LkhicRZjTrD4AHH6QmrAOzdgo+37fA1NGYnEz40nVZfHQxe9P2smDEAnydfE0dzj+++AJefBEmTGDx7DsYveF+wjzC6GDxf9jou3EgqfCilzw7JJg2zmqzGMOvz3BXVw6UlpJbU3PlJzcSRVF4deCrzLppFl8f+ZovDn1h6pAuZGGB9auvE54H6cu+avFj+XIevnTNSjWlBH0aRIRnBNsnbTefWizbtmEYPpzdob1Z+sbtLIx9lc7ug7AumUH/IB8OJBXyzJDgC3r3TUF0RQVhhw4xNyiIF/38TB3OBQzCwOhlo9metJ0TT5wg2NWMDpo6HRpvLza4FcKvqxjbcaypI7rh5BKHktHN3z+fgqoCPrrlI/NJ9nl5MHEiye5tmTRyAAtjX2dE8AhsSl4mr1RhW0zudRYjNp2O9vb0d3Lim6wssxs6sVAsWHT7IiwtLHnzrzdNHc6FVCpU9z3I7XGw+/g6U0djUjLhS9ekSlvFggMLGBM6xqyKoSU+9AS6gkK+mTadVMfPaOfciV/G/8JzQzvSxrm25r65jtOfozPoyCjN4EjWEXan7mZ36m72pu0lvjCe/3i6c6aqil0mmpd/OT5OPjzR4wmWn1xOemm6qcO5gOW4cVjroWrzJlOHYlLN/55jyShWnFpBQVUBz/d5vtHbXnoghQXb4nn238Myx48TtHEVX/S5m890X+KktuHZrl8y9KP9PDskmH2zhrL0QAqfbYs3q3H6rLIsNiduZmvSVo5lHyM6LxqtQVv/ky1ssOj7C/ce+JX3Wqu4q8NdONo4Nm7AlzG562Q+3Psh62LXmeTGu0vq1QutjRUdTuVQqa3Ezqpx6xSZC5nwpWvy098/EewazCD/QY3e9vmzaS5I+HPnorVzYNEQT8rFKpaNWsb8dVUXPPfcZmp6g55fo39l0ZFFbEncAoCHnQc9fXoyot0IAlsF4mXvhYO1Q+3zhZ7cilzSStJYVh7DKbseTPrzIab9Po3Zg2bzbO9nzWJYrYN7BzztPYnKijJ1KBeytqY41I+I3ASyyrJo59rO1BGZhNETvqIoI4BPAUvgGyHE+8ZuUzKu4upitiVtY2b/mY2eZJYeSKFSo8NJrbqwl15eDr/+ivLwQ+S7/UZf575MCJ+AoSLV7Hr0mxM2M/3P6fyd+zd+zn7MHjSbOzrcQWevzlgoVx5lnarV4r9vHz1v/h71mfd57o/nyK3IZc7QOY0Q/eUpikJbp7ZklmWaOpSLVPn7ErIjgcKaljtTx6hj+IqiWAJfALcBYcC9iqKEGbNNyfh2pexCL/TcGnxro7e9YFs8pdU67G1UF/TUd37+E1RX83kbKzLLMnl78NsoilJ3I9bBpEKCZm3gueVHGz3mc/QGPdP/mM4tS26hQlvBz+N+JvHZRN68+U26tu7aoGQP4GZlxUt+fvxVIZg5ciljQscw/8B8I0ffcBq9xixLKVe5OuJWBZZKE1rc5QYz9kXbXkC8ECJRCFEDLAfuMHKbkpEdyDiApWJJL59ejd72pebKJ2/aTo2Findr9tHWqS1DAodc8Pja45kYRO1HUxBC8Pj6x/l4/8c83fNpTj95mnvC77mmlaWEENxpr8FLBROO7+OvlL10dDePtYFr9DXEFcQR4hpi6lAuUmahQ60Dd1s3U4dykdN5p8mvzDd6O8Ye0vEB0s77Oh3obeQ2JSM7lXeKELcQ1Cp1o7d9/hj8uYu3fQJdGZOTSoqbD6XWp5gQ+shFveUxXbxZezyTMV28Gz1mgKV/L2Xx0cW8ctMr1z30oigKfb/uTqVDOHSei2/HGawYMP4GRXp9NidsRqPXMMB/gKlDuUhRaQ5aS/C09zR1KBe5e8XddHTvyK8TfjVqOya/aKsoyhRgCoCfmd1MItUvvTSdAJcAU4dRd/F27fFMJlaWUWJvg0ZfVe800fkTuzF/YjcTRFnbI39n1zt0b9Odtwe/fUP2+eNdP+Jg7cDP1c58S2+itA4E3ZA9XzshBB/v/xhPe09uC77NxNFcrDIzhVInGzwtTZ72LqDVa4kvjGdsB+PfEGbsIZ0MoO15X/ue/V4dIcQiIUQPIUQPDw/zXu9UqlVQWYCbGZwWnxveGdPFG7UFuDrWDo8EtTJ16rtQZlkmMfkxTOoy6YYtDj6241huaXcLX3ToTH8nJx6Mjmafiefm/3L6F7YlbeO1Aa+Z3XKWRVVFuGUUUuVjfr372IJYdAZdo6yqZuyEfwgIURQlUFEUa2AisNbIbUpGZhAGs1g27twF2fkTu9G1SxDuNVUAZnfBsLi6GMAoi2qrLS1ZExFBW7Wa206cYL+Jkn5sfiyPrXuMXj69mNpjqkligEtXPF1y9Hu6ZoFdr/4miuzSdqfuBqCvb1+jt2XUhC+E0AFPA38A0cAKIcQpY7YpGZ+1pTXVumpTh3Gh4GDskzOx1HNBgaxzCeC55UdNVvo4sFUgloolhzONUyfK3dqarV264G5lxfATJ9hceHFhOGNKLEpk+I/DsVHZsPzu5Sbt3ddX8VQIwYFVn+KgBY/hd5ostkvZmrQVb0fvRjkzNXppBSHERiFEeyFEOyGE6ScKS9fN3c6dvMo8U4dxochILDU1dM6BuMK4um+fP86fXVLN+5tiGj3x21nZcXfY3Xx79FsySjOu/IJr4KdWs7NbNwLUakacOMGHqamNUm9nf/p++i7uS3lNOX888AeBrQKN3ubl1DeLa23sWrruS0avsoTbzOvaQqW2kk1xmxjTfkyj3NMia+lIVy2wVSDxheZVi2ZVq1D0igUT4tR1p8hw4Ti/KWvp/Hfwf9EatEz4ZQLlNeVGacPbxoZ93box1sODlxITGXTsGEfLjHOTkc6g4+0db3PTtzdhZ2XH3kf20rV1V6O0dTXODfMB9HlvK9/vO8PLG57j4b8tUUaOAieni15jyoVv1sWuo0JbwbiwcY3Snkz40lWL8IggtSSVoirTrbH6bx8eK+ZA2wgmHlX4PWY9NframvHnj/PvmzWUl2/rUNcDbMx/9PZu7fl2zLfsS9/HrUtuNdqdqA4qFSvCwljYvj2nKyqIjIpiSmwseTewhv7u1N30W9yPN/96k4kREzn6+FE6uHe4Yfv/t2v5PZ07s3tt63vctCMZ13I9FtOmXfa5piio93+H/49Al0AGBw5ulPZkwpeuWn+/2gtfu1J3mTSO8xPBs0OCWTtgLP7FVQw7VsZPJ36q9zXnL4HY2P/oEyIm8PO4nzmWfYwuX3Vh+cnlRhl2ObcAelzv3kzz9eW77GxCDhzgk7Q0agyGa97v8ezj3Ln8TgZ8N4CMsgyW372cJWOX4KJ2uYHRX+xafk/PDgnG3vEMxdU/8MEeW+jVCwbXn1RNtfDNkawj7EjZwdQeUxt8l/X1kglfumq9fXpjb2XPhjMbTBrHvxPBzo59KfELYu4uG+Zufwetvrbi5KV6iKb4Rx8XNo7Djx3Gz9mPe1fdS79v+7EpbhMGce2J+FJaWVnxSXAwJ3r0oI+TEy8kJND50CE2FRQ0eB9avZZfo3/l5v/dTNeFXdmWtI05Q+YQ90wcEyIm3PCY63Mtv6eh4WqK1B8y95AzroVV8PHHcN4Y+fl/Exetg9xI3trxFi5qFx6PfLzR2pQJX7pqNiobRoeO5teYX+uGTkzh/ESwYFs8mWVa3r75YQJyNIz6PZG5e+YCl+4h/nu8t7HGcDt6dOTgowf5dsy3pJWkMXLpSCK+jOCTfZ8YZaino709mzp3Zl1EBHpg5N9/M+rECc5UVtb7fJ1Bx7akbTy+7nHafNSGu1fcTXJxMh8O/5CU51J4ZcArjVpe+GoS8tIDKfR49zf6fD2MDrEFTN1SApMmQf8Lp2Oaev3iXSm7WBu7lhf6vICz2rnR2pVLHErXZGPcRkYtHcXyu5c3Wk/vcs6vc3/fe9Oo2biOvo9Z8N0bUZxMdv7nsXqSRp/3tpJdUk0bZ3XdAaCx1OhrWHFqBZ8e+JTDmYdRUOjv159b293K8KDhRHpH3tB7HmoMBhakp/N2SgrVBgPP+/oyy8+X9KI4/kr+i61JW9metJ0STQn2VvaMDh3NvRH3MjJkpFnce3ElEbN/I97wCr7FsZz6yQEbFzc4evSii7UX/L00cs9eb9ATuSiSwqpCYp6OuSEHz4YucSgTvnRNDMJA+8/a42rryoFHD5hFLfY6eXnoO0WQIAq4+wUftjx1EC8Hr0sunGLKf/7zxebHsuzkMtbGruVodm1VT1uVLV1bdyWyTSThnuG0a9WOYNdgfJ18r3q+e5W2itSSVBKKEjhakMKScltiVAEomjxE3CdQsI9Al0CGBg7l1uBbGRky0mwXCqnvd1lUVUTbDwbhWHaSPf9rRZCiwJ49EBpq4mgv9MHuD3h568usGLeC8eE3pgaSTPiS0X0d9TVT1k9h3b3ruL397aYO50LbtiFuuYXNgYJ77gvh41t+ZdHOLJP15K9WXkUe25K2sT99P1FZURzNPnrRdM5W6lZ42HvgauuKWqXGxtIGG5UNeoOeGn0NGr2GSm0l+ZX55FXkUaGtuOD1jtaOBPqPIbX1RLSWtpzoHGh2ZSku5d9nZYlFiYz8aSTaxES2fu+AT5UGqy2boV8/U4d6gb9z/qbH1z0Y3X40K8evvGEdJZnwJaPT6rV0/KIjVpZWHJ963OxKGrB4MTz6KP/rAtPu7MC8YT/z7a48k/fkr4VBGMgsyyS+MJ74wngyyzLJrcglrzKPoqoiqnXVaPQaNDoNlhaWWFtaY2Npg62VLe527njYeeBu505bp7a0c21Hu1bt8LT3RFEUHoqOZltxMal9jX9r/41y/lmZj1cy41eOJzxdw6Zf1KgrNLBpE5jZz1OmKaPn1z0pqi7i5BMn8bC/cbXDGprwEUKYzRYZGSmkpmV97HrBbMS7O981dSj1Ov7Y80KA+LGzIrp/3lmkFqde8PhP+5NF73e3iJ/2J1/weUvS6/BhMfDIEVOH0SDn/460eq14betrQnkT8cYET6G3thLC11cIM/xZ9Aa9uGflPcLiLQuxPWn7Dd8/cFg0IMfKWTrSdRnVfhR3d7yb2TtmcyLnhKnDuUjnRR/DO+/wwAnBq1+cpOf8rmxP2l73+PmzNd7fFFNXfqEhTHmH5o1SoddzrLyc3vXcgWpKl3pvz/2O3t60jYHfDeSTLe/wxyY/3vo5l+zu/Wov0HYzTRnsy3lj+xusOLWC94e+z80BN5ssDpnwpev21e1f0Urdiom/TDRa2YDr8uqrfDLqCe6MFmxYWMYDnw9l/E/T0eq19c7xbuioqqmn9t0I6/LzqRGCUW6mL3d9vku9twZqKFYtJZbH0R4/TsZyb4YcSmfuwP8w7vZXwf3GVyS9XgsPL2TOrjk81v0xZvSbYdJYZMKXrpu7nTtL715KbEEsD//2sFFuIrpeXq/P5KX//JcOBZYcXmRF/uaP6f1NbyICSurmeJ8ruzDztoaVCTDVHZo30tdZWfja2DDAufHmgl/OuZ6g9RC5AAAgAElEQVR9n0DXC95bIQSb4jaRb/8sZZZLeWW3L3u+rMG5ysD2z39i9Yj/0Kudu9mdcf1w/Aee2PAEo0JG8cXIL0w/m60h4z6Ntckx/KZt7u65gtmIWVtmmTqUS1q/5HeR7N5W6BVFfDrYTqjftBTPbXpOLNp1XPR+d4uYtuzIZcfxm9M4/97iYsH27WJeauqVn9xIer+7RfjPXC/6vLul7nu7U3aLAd8OEMxGDHndTySFtRcCROqgW4XIybnsa03pu6PfCYu3LMSwH4aJKm2VUdtCjuFLjW1GvxlM6T6F93a/x/z9800dTr1G3X8r/snRWDz2GM9ur+TM984c+GU+T269iTPlK/nteMplh2nqG2poimP5BiF4MSEBdysrHm/TxtTh1Dn/rGl/+n5GLR3FTd/dRGp2LAfTbmPL3BwCMnLgxx9pu30TeP6zglWfQFcsFOgd6GrCn6DWggMLmPzbZIYGDmXNhDUmWf+5PjLhSzeMoih8MeoLxnYcy/N/PM8XB78wepvXlGzt7WHhQvjtN1wrbdi7GL5aa4WFdiH5Do+DwyamDPKpt61KjQ4nteqCYZymOJb/bVYWe0pLmRsUhIPKeHfQXs3vZ+mBFD7dGkuf8Hi+PHU/fRf3ZW/qHn6xfpDEhbb0XLwJZexYOHUKHnjggto4APuTCjEIOJDUuAvAnE9v0DPjzxlM+30ad3W4i3X3rsPe2t5k8fybTPjSDaWyULHs7mWMCR3D05ueZvZfs426EMd1JdsxY7j9iUV83fNOJh0tJuVTaz47aEtRxRe8vHcQb/31FlllWRe0VVqtw95GdcE8/qY2lh9XWckLCQkMdHbmodatjdpWQ38/uRW5zNr8Lkc0DzH/6FQyyjJY2fpZ8teHcfcrP1JqacOTj85j6bT3wOfigzGY/vdQpilj3MpxfLTvI57u+TQrxq/ARmVjklguRd54JRmFVq/l8fWP892x73i468N8dftXRln67nrLIiw9kML7m2IIyktlQcwa/LZtROdgx7qbPHkhNJl0NxVjO45lauRUMnIC+GJ7YpO8ceucKr2efkePklpdzdEePfBTG3eo4XK/H51Bxysbl/DNkcWUsh+90OGohPOG/She2Hwciz/+gDZt4PXX6V8QREa5zmzvko7Jj+Gun+/iTMEZPr7lY6b1qb/2vrHIO20lkxNCMPuv2by9821GBI/g53E/42RjXvO9L3LkCHz0EaxYgTAYONknkLdDs1ntW4GPqx/3RdzH/Z3vJ8Izoq6eS59AV/YnFV5Uo8fcGITg3tOnWZGXx/pOnUwyFVNn0LE9aTsrT69kdcxq8ivzsRAueFkMY3/7Yfgt+hm2bwdXV5g1C558EuzszKbeUX1+OvETUzdMxVZly8/jfm60xUzOJxO+ZDa+jvqaJzY8QYBLACvGr6B7m+4mi+VSBdQukpEBX35ZO9ZfUEClhwsberfibf8UTnoYCPcIp6igC4bKSNSEIISF2fY+ofbg+3x8PJ9mZPBhUBAz/Pwu+dwGv0cNVFRVxOw/l7HsxDqqLA9Qri3CwdqB0e1H46frg+PyRJ46sQmXxDO1wzXPPQdTptRb4fJGxnW9SqpLeHrT0yw5sYSb/G5i2d3L8HXyNUksMuFLZmV36m7uXXUvuRW5zBs+j6d7PW2SOclXXQq5pgY2bID//a/2o15PYUBrNkZY86lPOodbG7BTuWEvejA+4jZeHTYeb0fvBsfTGElMCMHs5GTeTknhOV9fPm7X7rLv/fWWi9boNBzOPMy2pG38nvA7+9P3YxAGLIQ9rSx78824pxlR7I76ux9h2TIoL4dOnWDGDJg4Eazrr8lkyjLW/7YmZg1Pb3ya7PJs3hz0JrMGzDJp+WiZ8CWzU1BZwEO/PcT6M+u5q8NdLLx94Q0tINUQ1zI0cC4pv9TFmbGJ+2H1atixA/R6Ktq4sS/cmSWe2azzraTQDkLdQhkcMJi+bfvS26c3IW4h9S5ht/RACq+tOYlBcFESa+iBoL5hJaDutff28uOlxETmpaXxcOvWfB0aisUVDrRX+x7llOdwOPMwe9L2sCt1FwfSD6I11KCg0MO7ByOCRxCbGID+rzweL4tlePzB2pk2trZwzz3w2GO1VS1vcFzGkFmWyTObnuHX6F/p7NWZr0d/TS+fXiaJ5Xwy4UtmSQjB/P3zmbllJk42Tsy7ZR6Tukwy/R2Il1Fvz7KgANatgzVrYNs2KCtDKAo5oT7sDrZmlUsm21pXk+sAzjbO9PTpSffW3enk1YkIzwg6uHfg5g/3kF1SjYUC79wZUZfELncguFRsFgp1zxdAdkk1Xi5qut7ly6KsLJ7y9mZBSMhFyf5qzjBq9DUkFCYQnR/N0ayjrI/Zy6m842ipXTJRZaEisk0kKVl+6CtD6Fvjw2/hati2jeLV63CpLEVnYYFq4EAYPx7uuw9cjLse7o1iEAYWRS1i5paZ1OhreHPQm0zvO90oExGuhUz4klk7nXeaKeumsCdtD4MDBvPV7V/R3q29qcOq1xV7ljodHDwIW7bUbvv3g7Z2Pd0yb3digl3Y6VnFFrtsjnjqyXUAC8UCJ1VbNNWehLoF81Dv3gS1CiKwVSAPfRNHXqkVlopywYHgcrH1DnTlQFIhzwwJ5mBSIatPZSJucibNRsfLfn68GxhY70H1/IPZ3peHUFhVSGpJKqklqaSVppFcnExsQSyx+bEkFiWiF3qgNn4b/LDQBuJuE8oP4++iV6Et6mMnSV7zO/b79+BRWnsgKLR3IaF7f37z6UaXR8YzfljnG/BbaTxbE7fy0paXOJJ1hCGBQ1h4+0KCXc1rCq5M+JLZMwgDX0d9zcwtM6nWVTOz/0xm9JuBo42jqUO7PtXVtVUb9+2rTf779kF6+j8PuzqR6e/KZusaYlyqSXCtIM5VQ7ILVJ/tMCpY4ar2JNjNFy8HL4rKrIjLFvQN9GVAuwCc1c6oVeq6uvfWltZ1c74fWXWClNBg9HZqnnPVMsCmgoqaCiq0/3wsqS4hrzKPE5lpxBdkYqeupFxbQJWu6oIfxcbShhC3EELdQtFp2hAVb0NAqSOvBrSl/FgsIuoI/UpT8clIrD3wQe1UykGDONg2nNeK3Djj2pY2LrYmH3e/WsezjzNzy0z+SPgDP2c/5gyZw/2d7jfLs1GZ8KUmI6ssixf+fIHlJ5fjae/JGwPf4LHIx8xvQZXrkZ8Pf/8NJ07Ufvz7b7QnT2FVeeEqVJWerpR4OFLoZEWOo0KGnZ5kdTUnRDl5ag1FthqK1VCkhnJrEP++NOA5FEJeAKGFU29CyfF6w7GzssPDzgMPe4+6BVI87Dzwt/cmWO9MQJkK7xIDTvllxByKJuNELL55afgXpGOn1dTtp8DWiTjf9vQZNxwiI2s3f39QlAuGm650pmJOUopTeH376yw5sQQXtQuvDXyNJ3s+aTblEeojE77U5BxIP8DMLTPZkbKDdq3aMWfIHMaHj6/3gqc5uuoZN0JAXh4PvLIUt5x02pfn8pS/BWRmQnZ27ZaXd+mXW1igd3JA7+RIqZsLMybezw+9+tIrM42vNq/Hu7oKVCosrK2xUtlgpQcrnQGV1oCFVgsaDZSUQFHRP1v5xeWtNSorshzcSXf3IdnNl2Q3X/qO7EepXxBz/y7jmaEh9f685nCR9WpE50Uzd+9clpxYgspCxbTe03j5ppdxUZv/dQaZ8KUmSQjB7/G/8/LWlzmRc4JOnp14qf9LTAifYDYXyC7lWqcNXjYxarWQmws5OVBcXLsVFf3zeXExMUJwz6BB/O3hwaw//uDtFStQabW1QyznNr0ebGxqpzxaW//zubMztGp1wTY3qoBYK2d0bbz5/o27WZpQwWfbE5pM4r5a+9L28f6e91kbuxZblS2Pdn+UF/u9SFvntqYOrcHkEodSk6Y36MWS40tE+BfhgtkIv0/8xPx980WZpszUoV3ST/uTRZ8Glk7+d5nlaym7bDAYxMKMDGG/Y4dw371bbMrPv+bY/x1bQ3+Opkqn14m1MWvFwO8GCmYjXD9wFW9uf1PkVeSZOrRrQgPLI8sevmTWDMLAprhNzN07l50pO2mlbsWTPZ9kSuQU/Jwvfbeoufv32cDVnh1kaTQ8GhvLxsJChrq48H3HjvjYmFehLnOUXprO4iOL+eboN6SXptPWqS3T+07n0e6PmlVVy6vV0B5+0xgclVosC8WCUe1HseOhHex7ZB83B9zMu7veJfDTQEYtHcVvMb+hM+hMHeZV+3dlx6up9LgyN5eIQ4fYVlzMguBg/uzSpUHJvinW7b8R9AY968+sZ8yyMfjP9+etHW8R7hHOqntWkfBsAtP6TGvSyf5qyB6+1OQkFyez+Mhivj32LZllmXg7ejO562Qe6fYIga0CTR2e0eTW1PBcfDzLcnPp6ejIDx060MG+4YnKnEoTGJsQgiNZR1h+cjnLTy0nvTQdL3svHu72MI91f6zZ/Z3Ii7ZSs6cz6NgYt5FFUYvYFL8JgzDQ26c3E8InMD58vMkKWd1oQgi+z85mekICZXo9r/n7M8vPDyuLqztBb2qzZq7FydyTtUn+5HISihJQWai4pd0tPNz1YcaEjjH7C//XSiZ8qUVJK0lj6d9L+fnUzxzNPgrATX43MSF8AuPCxtHawbgLfRhLfGUlj585w7biYvo7ObEoNJSwq+jVN3cGYeBI1hE2nNnAytMrOZV3CgvFgiGBQ5gQPoGxHcfiamv6JQ+NTSZ8qcU6U3CGFadW8POpnzmZexIFhV4+vRgZMpKRISPp3qa72c/t1xgMfJyWxtspKVgrCh8EBTHF2/uKhc9agpLqEjYnbmZD3AY2xW0ipyIHBYX+fv2ZGD6RcWHj8HLwMnWYjUomfEkCTuWeYlX0KjbFb+JA+gEEAk97T24Lvo2RISMZFjTM7HqAGwoKeC4+nviqKsa6u/NZSAjeLXgGjlavJSorih3JO/g94Xd2p+5GZ9DhonZhRPAIRgaPZETwiEavvGpOZMKXpH/Jr8znj/g/2Bi/kd/jf6ewqhAFhU5enRjoN5CB/gMZ4D/AZMM/sZWVPB8fz6bCQkJtbZkfHMwIE6xKZWoanYZDmYfYkbyDHSk72Ju2lwptbQmKCM8IRoWMYlTIKPq27WvSGvTmRCZ8SboMvUHPgYwDbE/azs7UnexJ3VOXVNq7tWeg30D6tu1LD+8ehHmEGTWxlOp0/DclhU/T07G1sODNgACe9vHB+iovyjZFQgiSipOIyozicOZhDmYeZH/6fqp11UBtgh/kP4hB/oMY6D+wxQ3VNJTJE76iKLOBx4BzxUBeEUJsvNxrZMKXTEWr13I0+yg7U3ayM2Unu1J3UVxdDIBapaaLVxd6ePcgsk0kkd6RN+wgsK+khHtPnyZFo+Hh1q15NygIr0us+NTU6Q16kouTOZ5znMOZh+u2ouoiAKwsrOjs1Zmb/G5ikP8gBvgPwN3O3cRRNw3mkvDLhRDzGvoamfAlc2EQBuIK4ojKiiIqM6r2Y1YU5TW1xcWsLa0JcQ2ho0dHOrqf3Tw6EuoWiq2V7RX3rxeCD1JTeSMpibZqNUs7dqSvs7Oxf6xGodVriS+MJzo/mtN5pzmdd5ro/Ghi8mPqeu4qCxWdPDvVHUR7ePcgwjOirsSzdHUamvDlAJgk1cNCsSDUPZRQ91Du63QfcOFB4Hj2caLzozmWfYxfo3/FIAwAKCgEuAQQ1CoIf2d//F38CXAJqPvc18mXPK2eB6Oj2VpczAQPDxaGhuKsajr/inqDnqzyLJKLk+vdUkpSLrj72d/ZnzCPMIYEDCHMI4xOXp3o7NXZrMsNN1fG7uE/BJQCh4HpQoiiep43BZgC4OfnF5mS0rJu+5aavmpdNXEFcUTnRxOdF01MQQxJRUmklKSQXZ59wXMtXPtAh1kISzXhJVvpruTQ2t4LT3vPus3Nzg1Ha0ccbRxxtHbE3treaNNIDcJAeU05pZpSyjRllGpKKdGUkFOeQ25FLjkV//p49vtag/aC/bR2aE2ASwABLgEEugTS0b0jYR5hhLqH4mDtYJTYpX80ypCOoihbgPqmNLwK7AfyAQH8F2gjhHj4cvuTQzpSc1OtqyatJI24omTm55azuaYVrfSFhOevpqokui6Z1uhrLrkPBQV7a/u6g4C9lT0qC9VFm5WlFSoLFQoKWoMWrV6LzqCr+1xrqP1ao9PUJviasrohqkuxtrTGy94LL4fag5KXvRde9l51yT3AJQA/Z78GDWNJxtMoQzpCiGENDOZrYP31tCVJTZFapUax9eHN5FIO11jxpLc389oNwNZybN1zhBCU1ZSRW5FLbkUuBZUFlNWU1fW4z31eVlO7VWor0Rv06Ay6uq1KV1X3uUEYsLKwwsrSqvajhRV2VnZYWdQeEGxUNjhZO+Fo44iTjROO1mc/nv3a2ca57mzDycbJLJf0k66N0QYOFUVpI4TIOvvlXcBJY7UlSeZqaU4Oj585g0pR+DU8nLs8Lr45SFEUnGyccLJxMrvFsaXmxZhXiuYqitKV2iGdZOBxI7YlSWalXKfjmfh4/pedTX8nJ5aGheGnlhcpJdMyWsIXQjxorH1Lkjk7VlbGxNOnOVNVxev+/rzh74+qBdxEJZm/pjMXTJLMnBCCzzMymJGQgLuVFVu7dGFwq1amDkuS6siEL0k3QLZGwyNnlxy83c2N70JDcW+md8xKTZdM+JJ0ndbk5fHYmTOU6/UsCA7maR8fObNFMksy4UvSNSrT6ZgWH8932dlEOjiwpGPHq1pyUJIam0z4knQNdhcX85+YGFKqq3nVz483AgJaRHVLqWmTCV+SrkKNwcCbycl8kJpKoFrNrm7d6NdMip5JzZ9M+JLUQKcrKnggOpqj5eU82qYNH7drh2MTKnomSfKvVZKuQGcw8FF6Om8kJeGsUrEmIoI73GWddqnpkQlfki7jdEUFk2NiOFhWxt3u7nzZvj2ecrql1ETJhC9J9dAZDMxLS+PN5GScVCp+DgvjHk9PU4clSddFJnxJ+pfTFRU8FBPDobIyxnl48EVIiOzVS82CTPiSdNa/e/UrwsIYL3v1UjMiE74kAafOjtUfKitjvIcHn8tevdQMyYQvtWgag4F3U1J4LzUVZ9mrl5o5mfClFmtXcTFTzpwhprKSB7y8+LhdOzxkr15qxmTCl1qcYq2WmYmJLMrKIkCt5vfOnbnV1dXUYUmS0cmEL7UYQgh+zc/nmbg4cmpqmO7ry1uBgdhbWpo6NElqFDLhSy1ChkbDU2fO8FtBAd0cHFjXqRORjo6mDkuSGpVM+FKzpheChZmZvJyYiE4I5gYF8byvr1xyUGqRZMKXmq3DpaU8ERfH4bIyhrVqxVft29PO1tbUYUmSyciELzU7RVotryYl8VVmJl7W1izt2JGJnp5yFSqpxZMJX2o2hBD8kJPDiwkJFGi1POvjw1uBgTjLEsaSBMiELzUTJ8vLeTIujl0lJfRxcuLPzp3pKi/KStIFZMKXmrQynY63kpOZn56Os0rFN6GhTG7dGgs5fCNJF5EJX2qShBCszMvj+fh4MmtqeLRNG94LDMRd3ikrSZckE77U5BwvL2daXBw7SkroYm/PL+Hh9JXrykrSFcmELzUZ+TU1vJ6czKLMTFxUKr4MCeGxNm3knHpJaiCZ8CWzpzMY+L/MTN5ITqZMp+MpHx9mBwTgamVl6tAkqUmRCV8ya1uLipgWF8epykqGurgwPziYCAcHU4clSU2STPiSWUqqqmJ6QgKr8/MJVKtZHR7OHe7u8uYpSboOMuFLZqVUp+P91FQ+TkvDUlGYExjIC76+qGVFS0m6bjLhS2ZBZzDwdVYWbyYnk6fVcr+nJx+0a4ePjY2pQ5OkZkMmfMmkhBBsKCjgxcREYiorGejszMZ27ejh5GTq0CSp2ZEJXzKZI2VlzEhIYHtxMe1tbVkTEcEYNzc5Ti9JRiITvtTo0qqreTUpiR9zcnC3suLzkBCmtGmDlZxPL0lGJRO+1GjKzl2QTU9HCMHMtm2Z5e8vq1lKUiOR/2lSo1idl8cTZ86Qc/aC7JygIPzValOHJUktynWdQyuKMl5RlFOKohgURenxr8dmKYoSryhKrKIot15fmFJTVaTV8mB0NGNPncLbxoaD3buzJCxMJntJMoHr7eGfBMYCC8//pqIoYcBEIBzwBrYoitJeCKG/zvakJuT3ggIeiY0lp6aGN/39edXfX47TS5IJXVfCF0JEA/XNqrgDWC6E0ABJiqLEA72AfdfTntQ0lOl0TE9I4OusLMLs7FjbqRORcjESSTI5Y43h+wD7z/s6/ez3pGbur6IiJsfGklJdzUtt2/JWQIC8S1aSzMQVE76iKFuA1vU89KoQ4rfrDUBRlCnAFAA/P7/r3Z1kIpV6PbMSE1mQkUGwrS27unWjv6xRL0lm5YoJXwgx7Br2mwG0Pe9r37Pfq2//i4BFAD169BDX0JZkYvtKSpgUE0NcVRXP+PjwXlAQ9rJXL0lmx1hX0NYCExVFsVEUJRAIAQ4aqS3JRDQGA7MSE7np6FE0BgNbu3RhQUiITPaSZKauawxfUZS7gM8AD2CDoijHhBC3CiFOKYqyAjgN6ICn5Ayd5uVoWRn/iYnhZEUFj7RuzcfBwTjJG6gkyaxd7yyd1cDqSzw2B5hzPfuXzI/WYODd1FTeSUnBw8qK9Z06McrNzdRhSZLUALJLJjXY3+XlTIqJ4Wh5Ofd7erIgJEQuMyhJTYhM+NIV6QwGPkxL483kZFxUKn4ND+cuDw9Th2VWtFot6enpVFdXmzoUqRlTq9X4+vpidY0dLZnwpcuKrqjgoZgYDpaVMd7Dgy9CQvCwtjZ1WGYnPT0dR0dHAgICZHlnySiEEBQUFJCenk5gYOA17UPe5y7VSy8E81JT6Xb4MAlVVfwcFsaK8HCZ7C+huroaN1nLXzIiRVFwc3O7rrNI2cOXLhJXWcnkmBj2lJZyh5sbC0ND8ZKJ/opkspeM7Xr/xmQPX6pjEIIF6el0OXyYU5WV/NihA6sjImSybwKKi4v58ssvjbb/v/76i71791716w4fPsyzzz57TW3Onz+fysrKa3qtVD+Z8CUAEquqGHLsGNPi4xns4sKpnj15oHVr2WttIi6X8HU63XXv/3IJ/3L779GjBwsWLLimNq8l4ev18nafy5EJv4XTC8H8tDQ6HTrEkfJyFoeGsr5TJ7xtbEwdmnQVXn75ZRISEujatSsvvvgif/31FwMGDGDMmDGEhYWRnJxMRERE3fPnzZvH7NmzAUhISGDEiBFERkYyYMAAYmJiLth3cnIyX331FZ988gldu3Zl165dPPTQQ0ydOpXevXvz0ksvcfDgQfr27Uu3bt3o168fsbGxQO2B4vbbbwegoqKChx9+mF69etGtWzd++622FJder2fGjBlERETQuXNnPvvsMxYsWEBmZiaDBw9m8ODBACxbtoxOnToRERHBzJkz6+JzcHBg+vTpdOnShTlz5nDnnXfWPbZ582buuuuuG/+GN1FyDL8Fi66o4JHYWPaVljLS1ZWv2renrVyY5Lo99/tzHMs+dkP32bV1V+aPmH/Jx99//31OnjzJsWO17f71118cOXKEkydPEhgYSHJy8iVfO2XKFL766itCQkI4cOAATz75JNu2bat7PCAggKlTp+Lg4MCMGTMAWLx4Menp6ezduxdLS0tKS0vZtWsXKpWKLVu28Morr7Bq1aoL2pkzZw5Dhgzh22+/pbi4mF69ejFs2DB++OEHkpOTOXbsGCqVisLCQlxdXfn444/Zvn077u7uZGZmMnPmTKKiomjVqhW33HILa9as4c4776SiooLevXvz0UcfIYSgY8eO5OXl4eHhwXfffcfDDz98He988yITfgukNRiYm5bG28nJOFpasqRjR+7z9JTDN81Mr169rjh9r7y8nL179zJ+/Pi672k0mgbtf/z48VierZtUUlLCpEmTiIuLQ1EUtFrtRc//888/Wbt2LfPmzQNqZzalpqayZcsWpk6diupsaQ5XV9eLXnvo0CFuvvlmPM7e/3H//fezc+dO7rzzTiwtLbn77ruB2ouaDz74IEuWLGHy5Mns27ePH374oUE/T0sgE34Lc6SsjIdjYjheUcE9Hh58FhKCp7woe0NdrifemOzt7es+V6lUGAyGuq/PTe0zGAy4uLjUnRlc6/5ff/11Bg8ezOrVq0lOTubmm2++6PlCCFatWkVoaOhVt3U5arW67sADMHnyZEaPHo1arWb8+PF1BxJJjuG3GNVn69X3iooiV6tldXg4P4eHy2TfTDg6OlJWVnbJx728vMjNzaWgoACNRsP69esBcHJyIjAwkJUrVwK1Sfn48eNXvf+SkhJ8fGrXOPrf//5X73NuvfVWPvvsM4SorYJ+9OhRAIYPH87ChQvrLv4WFhZe1GavXr3YsWMH+fn56PV6li1bxqBBg+ptx9vbG29vb9555x0mT558yZhbIpnwW4DdxcV0OXyY91NTmdS6Nad69uROWRqhWXFzc6N///5ERETw4osvXvS4lZUVb7zxBr169WL48OF06NCh7rGffvqJxYsX06VLF8LDw+supp5v9OjRrF69uu6i7b+99NJLzJo1i27dul00a+fcUOHrr7+OVqulc+fOhIeH8/rrrwPw6KOP4ufnR+fOnenSpQtLly4Faq8tjBgxgsGDB9OmTRvef/99Bg8eTJcuXYiMjOSOO+645Ptx//3307ZtWzp27NiAd6/lUM4dbc1Bjx49xOHDh00dRrNRrtMxKymJLzIy8FerWdS+PcPrGR+Vrl90dLRMLvVYtWoVa9eu5fvvv2/Udp9++mm6devGI4880qjtNob6/tYURYkSQvS40mvl4FYz9WdhIVNiY0nVaHjGx4c5gYE4yLFMqRGtXbuWV199lW+//bZR242MjMTe3p6PPvqoUdttCmQGaGaKtFpeSEjgf9nZhMq1ZSUTGjNmDGPGjNwdzxQAABebSURBVGn0dqOiohq9zaZCJvxmZHVeHk/GxZFXU8Mrfn687u+PWi43KEnSWTLhNwM5NTU8ExfHyrw8ujo4sLFTJ7o5Opo6LEmSzIxM+E2YEIIlOTk8Fx9PuV7PnMBAXmzbFisLOflKkqSLyYTfRCVVVTH1zBn+LCqir5MTi0ND+f/27j2sqjrf4/j7p2MqdAM1x8QBTZQtdxTFHMVRCc/B0exyusiUZNbRRhtPT6ZZmNac4yRqOeM001Oip5s2auXx6UKMoDJ5AVEMQUHRVMTcgqLcROR7/gBXXDZKsHFt4Pd6nv3IXntdvsD2uxe/tdZnWWpcCKNpmlaX3hVsZSoqK1l+8iQ+ycl8d/Eif+7fnx2BgbrZa9x6661ml2AYPXo0tk6xbmrkcXR0NPHx8T97uePHjxvn9beEvLw8Ixxu//79fPnllz9r+bVr1+Lp6Ymnp2etU1fHjRvH+fPn7Vor6IbfqqQVFTF83z5eOHqUMS4uZAQH83s3NzrqDBzNjloyYvh6Df962128eDHjxo372dtrSsP/OXHSy5cvZ/r06UD9hl9YWFgrzqKugoICFi1axO7du9mzZw+LFi0ymvzvfve7Frm/gW74rUDp1au8nJPD4JQUTpSVsW7QIDb7+OhkS61BS5cuJTg4GD8/PxYuXGhMv//++xk8eDDe3t68++67xvSaEcM7d+7Ew8ODhQsXEhQUhK+vrxGZ3FDEcWlpKY8++igWi4XJkydTWlparyZbkcd1t7t48WKCg4Px8fHhmWeeMWIYpk6dyoYNG4Cq0y5DQ0MZPHgw4eHh5OXlAXDkyBHGjRuHv78/QUFBHD16lHnz5rFjxw4CAgJYsWIFZWVlREVF4evrS2BgIAkJCUBVHMTEiRMZM2YMY8eO5YknnuDzzz83ap8yZYrNK5A3btzI+PHjKS8vJzo6mvXr1xMQEMD69etJSkpi4MCBvPbaa5w4caLest988w1hYWG4urri4uJCWFgYX3/9NVB1Susnn3zS2F9344mIwzwGDx4sWm1bCwqk/65dQkKCRGVmSn55udklaTZkZGT89OT550VCQ+37eP75G9bg7OwsIiLffPONTJ8+XSorK+Xq1asSEREh27ZtExGR/Px8EREpKSkRb29vOXfunIiIALJ+/XpjXe7u7rJy5UoREVm1apVMmzZNRETmz58vH3zwgYiInD9/Xjw9PaWoqEiWLVsmUVFRIiKSlpYmHTt2lOTk5Ho1uru7i9VqNZ7X3e61+kREIiMjZfPmzSIi8uSTT8o//vEPKS8vl+HDh8vZs2dFRGTdunXGdocOHSqbNm0SEZHS0lIpLi6WhIQEiYiIMNYZExNjzJ+ZmSl9+vSR0tJSiY2Nld69exvbT0xMlEmTJomIyIULF8TDw0OuXLlS63vJycmRoKAg43lsbKw899xzteaxWq2yfPly8ff3l/DwcPn000/l8uXLIiKydOlSef311415Fy9eLEuXLjWe9+/f3/j91FTrvfbTzzFFGtFj9R6+gzp/5QrTDx9mTFoalSLE+/uz2ssL106dzC5Nc3BxcXHExcURGBhIUFAQhw4dIjs7G6jay/b39yckJISTJ08a02tGDF/zwAMPAFVXrl7L04+Li2PJkiUEBAQwevRoI+J4+/btREZGAuDn54efn1+jaq273YSEBIYNG4avry9bt27l4MGDteY/fPgw6enphIWFERAQwBtvvMGpU6e4dOkSubm5xs1OunTpgpOTU73tJSUlGXV6eXnh7u5OVlYWgLG3DRAaGkp2djZWq5VPPvmEBx98sF7qZl5enhHX3JDu3bszZ84c9u/fz8KFC4mOjmbIkBsmIABw1113cfr06UbN21j6LB0HIyJstFr5fXY2565cYW6fPiz08MBJX0DVerxlbjyyiDB//nyeffbZWtMTExOJj49n586dODk5GQ0b6kcMA3SuvutZx44djXFtsXPEcc3tlpWVMXPmTFJSUujTpw+vvfaaUV/N783b25udO3fWmn69JM/Gcq5z4sMTTzzBhx9+yLp164iNja03f9euXevVZ0tGRgaxsbF8/vnnhIaGGmP+vXv3JjEx0Zjv1KlTtWKly8rK6Nq1a9O+mQboPXwHknv5MpPT03k4I4PenTuTPHgwf7rnHt3stZ8lPDyc1atXU1RUBEBubi5nz56lsLAQFxcXnJycOHToELt27WrSum1FHI8aNco4OJqens6BAwdsLn+9mOVrzbN79+4UFRUZY/Y1DRw4EKvVajT8K1eucPDgQW677Tbc3NyMcffLly9TUlJSb3sjR47ko48+AiArK4sTJ040+OE1depU3qr+8B40aFC91wcMGFDrTmJ1t5WamkpISAhPP/00Xl5e7Nu3j/fee49hw4YBVT/LuLg4zp8/z/nz54mLiyM8PByo+mA7c+YMHh4eNmtrKt3wHUClCO/k5mLZs4e48+dZ2q8fu4OC9NWyWpPcd999PP744wwfPhxfX18eeughLl26xPjx46moqMBisTBv3jxCQkJ+9robijieMWMGRUVFWCwWoqOjGTx4sM3la0Ye13XnnXcyffp0fHx8CA8PJzg4uNbrSiluueUWNmzYwEsvvYS/vz8BAQHGzdU/+OADVq5ciZ+fH/feey9nzpzBz8+Pjh074u/vz4oVK5g5cyaVlZX4+vryyCOPsGbNGuMvmbp69uyJxWJpMFPf2dmZe+65hyNHjgDwm9/8hoyMDOOgbdeuXYmNjeW7775j2rRp9U6bdXV15dVXXyU4OJjg4GCio6ONIaW9e/cSEhJi/5u3NGag/2Y92uNB24yiIhmxd6+QkCDj9u+XoyUlZpekNYGtA2ma/UyYMEG2bt16U7dZXFws/fr1kwsXLjQ4z6ZNm2TBggV23/bs2bMlPj7e5mv6oG0rVF5ZyeLjxwlISSGjpIQ1Xl7E+fnRz85jdprW2j311FOUlJTw61//+qZtMz4+HovFwqxZs7jjOmmzkydPtvuwC4CPjw9jx461+3r1DVBMsLOwkOmHD3OwpITH7rqLt/r317cabOX0DVC0m0XfAKWVuFRRwcvVd6By69yZLb6+RHTrZnZZmqa1E7rh3yRbzp1jRnY2udV3oHqjb19u03eg0jTtJtIdp4X9WF7O89nZrLda8XZy4h+BgYToO1BpmmYC3fBbiIiw5swZXjh6lOKrV3ndw4O5v/oVt+isek3TTKK7Tws4WlpKWFoaTx0+jI+zM2lDhvCKh4du9lqL0vHI9el45Np0B7KjispKlp44gW9yMsmXLvG3AQNIDAjAS2fVa62Ijke2TzxyQwoKCox/dTxyK5V66RJDU1OZm5NDuKsrGUOH8uzdd9NBZ9VrJtDxyObHI9dUUVHB5s2bmThxohHwZkY8crPG8JVSDwOvARZgqIikVE/3ADKBw9Wz7hKR/2zOthxVydWrvHb8OMtPnqTHLbew0dubB26QoKe1bX/IzmZ/dY6NvQTceitveXo2at64uDiys7PZs2cPIsLEiRPZvn07o0aNYvXq1bi6ulJaWkpwcDAPPvgg3bp1o7i4mGHDhrFs2TJjPd27dyc1NZW//vWvxMTE8N577/HHP/6RMWPGsHr1ai5cuMDQoUMZN24cf//733FyciIzM5MDBw4QFBRUr67Zs2ezfPlyEhIS6N69O0C97Q4aNIjo6Gigai93y5Yt/Pa3vzXWceXKFWbNmsUXX3xBjx49WL9+PQsWLGD16tVMmTKFefPmMXnyZMrKyqisrGTJkiXExMSwZcsWAJYtW4ZSiu+//55Dhw5x3333GWmZqampHDhwAFdXV7Zt28aKFSu4//77KSws5Lvvvqs15AJw7NgxXFxcjGiGxYsXk5KSwl/+8hdjniNHjvD++++zYcMG7r33Xl544QVCQ0OBqoyjPn36GPO6ubmRm5sLgIuLC5cvXyY/P59udjx1u7kHbdOBB4C/23jtqIgENHP9Di2+oIBns7LIKStjeq9evNmvH3fq+GLNZDXjkQGKiorIzs5m1KhRrFy5ks8++wzAiEfu1q3bDeORN23aZKx78+bNxMTEANSKR549ezbQ/HjkN998k5KSEgoKCvD29q7V8GvGI0PVMFCvXr1sxiPbkpSUxKxZs4AbxyPPnDkTq9XKxo0bmxSPvHHjRh555BEWLFhAamoqt/3MbKxr8cgO0/BFJBOqQo3aE2t5OS8cPcoHP/6IZ9euJAYEEHrnnWaXpTmIxu6JtxTR8chNYu945LCwMN5++21iY2PZuXMnUVFRTJ482fgwamvxyH2VUvuUUtuUUiNbcDs3jYjwv2fOYNmzh3Vnz/KquzsHhgzRzV5zKDoe2THikW+//Xaee+45UlJS+NOf/kRSUhIWi4W5c+cCDhqPrJSKV0ql23hMus5iecCvRCQQ+C/gY6XU7Q2s/xmlVIpSKsVqtTbtu7gJjpSUEJaWxpOHDjHQyYl9Q4awuG9fuuises3B6Hhkx4hHrikwMJBVq1aRmZlp7MW32nhkIBEY0tTXrz0cMR65/OpV+e/jx6XLtm1y+/bt8rfcXLlaWWl2WZqD0fHILUvHI/+kOfHILXKlrVKqB1AgIleVUv0ATyCnJbbVknYVFjI9K4v04mIe6tGDt/v35+4G9gY0TWsZZsUjT5s2jTlz5twwHjk/P9/u23fIeGSl1GTgz0AP4AKwX0TClVIPAouBK0AlsFBE/u9G63OUeOSLFRW8nJPDX0+fxq1zZ1Z5evLb6tPINM0WHY+s3SymxSOLyGfAZzambwQ2NmfdZvms+gbieeXlzO7dm9d1qqWmaW2E7mTVTpWVMevIET4/dw5/Z2c+9/Eh+Habx5k1TdNapXbf8K9W30D85WPHqBDhzX79+IObG5100JmmaW1Mu274B4qKeObwYXZfukS4iwvvDBhAX31PWU3T2qh2uRtbevUq83NyGLx3LzllZXxksfCVn59u9prWSA3FHzckMTHRiBFu7nqbGrGstcOGH19QgG9yMktOnOCJnj3JHDqUx3v2bHfxEJrWWumG33TtpuFby8t5IjOTsAMH6KAUW/39ed/Li2467ExrA4qLi4mIiMDf3x8fHx/jSs+G4oZHjx7NnDlzGDJkCBaLheTkZB544AE8PT155ZVXgKoseS8vL6ZMmYLFYuGhhx6y2Wjj4uIYPnw4QUFBPPzww0akw9dff42XlxdBQUFG+Fpd14tVnjFjBkOGDMHb29uIeLYVsWxrPq0Bjbk662Y9WuJK28rKSlmblyfdduyQTomJ8mpOjpRWVNh9O1r7ZvaVths2bJCnn37aeH7t6tD8/HxjWmRkpGzevFlEREJDQ2Xu3LkiIvLWW29Jr1695PTp01JWVia9e/eWc+fOybFjxwSQpKQkERGJioqSpUuXGssnJyeL1WqVkSNHSlFRkYiILFmyRBYtWiSlpaXi5uYmWVlZUllZKQ8//LBERETUq3vZsmUSFRUlIiJpaWnSsWNHSU5OrlV7RUWFhIaGSlpamoiIuLu7i9VqNdbR0HxtVXOutG3Te/g6/0ZzZB/v/oGQ//knH+/+odnr8vX15dtvv+Wll15ix44dxtWhCQkJDBs2DF9fX7Zu3crBgweNZSZOnGgs6+3tTa9evejcuTP9+vXj5MmTAPTp04cRI0YAEBkZSVJSUq3t7tq1i4yMDEaMGEFAQABr167lhx9+4NChQ/Tt2xdPT0+UUkRGRtqse/v27cZrdWOVP/30U4KCgggMDOTgwYNkZGTYXEdj59Pa6Fk6VyoriTl5ksU//MAtSvG3AQOY3quXvvuU5lBWbj3CmcIy/rz1CI8Pc2/WugYMGEBqaipffvklr7zyCmPHjmXu3LnXjRu+FhrWoUOHWgFiHTp0MOKQ6x7bqvtcRAgLC6t3d6b9+/c36/s5duwYMTExJCcn4+LiwtSpU21GETd2Pq1Km9vD31VYSNDevbx87BgTunUjU99qUHNQs8f0p9cdXZg1pn+z13X69GmcnJyIjIzkxRdfJDU1tVFxwzdy4sQJI4r4448/rpdnExISwr/+9S8jMbK4uJisrCy8vLw4fvw4R48eBWjwdn0NxSpfvHgRZ2dn7rjjDn788Ue++uorY5maMcTXm0+rr83s4dfMv+nduTNf+PgwUeffaA7s8WHuzd6zv+b777/nxRdfpEOHDnTq1Il33nmnVtzwL3/5y3pxw40xcOBAVq1axVNPPcWgQYOYMWNGrdd79OjBmjVreOyxx7h8+TIAb7zxBgMGDODdd98lIiICJycnRo4caTMHf8aMGURFRWGxWLBYLEassr+/P4GBgXh5edUaVoKfIpbvvvtuEhISGpxPq69Z4Wn21tTwtJSLF7k/PZ3TOv9GM0lbDE87fvw4EyZMID093exStBpMC09zFP26dsXb2ZnPdP6Npmlag9pEw3ft1Ilv/P3NLkPT2hQPDw+9d9/GtLmDtpqmaZptuuFrmp040vEwrW1q7ntMN3xNs4MuXbqQn5+vm77WYkSE/Px8unTp0uR1tIkxfE0zm5ubG6dOncJqtZpditaGdenSBTc3tyYvrxu+ptlBp06d6Nu3r9llaNp16SEdTdO0dkI3fE3TtHZCN3xN07R2wqGiFZRSVqD5WbFVugPn7LSulqJrtA9do33oGu3DjBrdRaTHjWZyqIZvT0qplMZkS5hJ12gfukb70DXahyPXqId0NE3T2gnd8DVN09qJttzw3zW7gEbQNdqHrtE+dI324bA1ttkxfE3TNK22tryHr2maptXQLhq+UuoFpZQopRzunodKqaVKqUNKqQNKqc+UUneaXdM1SqnxSqnDSqkjSql5ZtdTl1Kqj1IqQSmVoZQ6qJR63uyaGqKU6qiU2qeU2mJ2LbYope5USm2ofi9mKqWGm11TXUqpOdW/53Sl1CdKqaaniNmvptVKqbNKqfQa01yVUt8qpbKr/3Uxs8aa2nzDV0r1Ae4DTphdSwO+BXxExA/IAuabXA9Q1aCAVcC/AYOAx5RSg8ytqp4K4AURGQSEAM85YI3XPA9kml3EdbwNfC0iXoA/DlarUqo3MBsYIiI+QEfgUXOrAmANML7OtHnAP0XEE/hn9XOH0OYbPrACmAs45MEKEYkTkYrqp7uApkfh2ddQ4IiI5IhIObAOmGRyTbWISJ6IpFZ/fYmqJtXb3KrqU0q5ARHAe2bXYotS6g5gFPA+gIiUi8gFc6uy6RdAV6XULwAn4LTJ9SAi24GCOpMnAWurv14L3H9Ti7qONt3wlVKTgFwRSTO7lkZ6CvjK7CKq9QZO1nh+CgdsptcopTyAQGC3uZXY9BZVOx2VZhfSgL6AFYitHnZ6TynlbHZRNYlILhBD1V/qeUChiMSZW1WDeopIXvXXZ4CeZhZTU6tv+Eqp+OoxvbqPScDLQLSD13htngVUDVF8ZF6lrZNS6lZgI/AHEblodj01KaUmAGdFZK/ZtVzHL4Ag4B0RCQSKcaBhCIDqcfBJVH043Q04K6Uiza3qxqTqNEiHGV1o9Xn4IjLO1nSllC9Vb440pRRUDZWkKqWGisiZm1higzVeo5SaCkwAxorjnCebC/Sp8dyteppDUUp1oqrZfyQim8yux4YRwESl1L8DXYDblVIfiogjNatTwCkRufbX0QYcrOED44BjImIFUEptAu4FPjS1Ktt+VEr1EpE8pVQv4KzZBV3T6vfwGyIi34vIXSLiISIeVL2pg252s78RpdR4qv7cnygiJWbXU0My4KmU6quUuoWqA2SbTa6pFlX1Sf4+kCkiy82uxxYRmS8ibtXvwUeBrQ7W7Kn+P3FSKTWwetJYIMPEkmw5AYQopZyqf+9jcbADyzVsBp6s/vpJ4AsTa6ml1e/htwF/AToD31b/JbJLRP7T3JJARCqUUr8HvqHqjIjVInLQ5LLqGgH8DvheKbW/etrLIvKliTW1VrOAj6o/3HOAKJPrqUVEdiulNgCpVA197sMBrmhVSn0CjAa6K6VOAQuBJcCnSqlpVKX//od5Fdamr7TVNE1rJ9rskI6maZpWm274mqZp7YRu+Jqmae2EbviapmnthG74mqZp7YRu+Jqmae2EbviapmnthG74mqZp7cT/A+EZ+2h8x04OAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Test:\n",
    "# Sample from trajectorys' approx. posterior\n",
    "h = rec.initHidden()\n",
    "for t in reversed(range(samp_trajs.shape[1])):\n",
    "    obs = samp_trajs[:, t, :]\n",
    "    out, h = rec(tf.concat([obs, h], 1))\n",
    "qz0_mean, qz0_logvar = out[:, :latent_dim], out[:, latent_dim:]\n",
    "epsilon = tf.random.normal(qz0_mean.shape)\n",
    "z0 = epsilon * tf.exp(.5 * qz0_logvar) + qz0_mean\n",
    "\n",
    "# take first trajectory for visualization\n",
    "print(tf.reshape(z0[0], [1, 4]).shape)\n",
    "z0 = tf.reshape(z0[0], [1, 4])\n",
    "\n",
    "ts_pos = np.linspace(0., 2. * np.pi, num=2000)\n",
    "ts_neg = np.linspace(-np.pi, 0., num=2000)[::-1].copy()\n",
    "ts_pos = tf.cast(ts_pos, dtype=tf.float32)\n",
    "ts_neg = tf.cast(ts_neg, dtype=tf.float32)\n",
    "\n",
    "neural_ode_pos = NeuralODE(model, t=ts_pos)\n",
    "outputs, zs_pos = neural_ode_pos.forward(z0, return_states=\"tf\")\n",
    "neural_ode_neg = NeuralODE(model, t=ts_neg)\n",
    "outputs, zs_neg = neural_ode_neg.forward(z0, return_states=\"tf\")\n",
    "\n",
    "zs_pos = tf.reshape(tf.concat(zs_pos, 1), [len(zs_pos), zs_pos[0].shape[1]])\n",
    "zs_neg = tf.reshape(tf.concat(zs_neg, 1), [len(zs_neg), zs_neg[0].shape[1]])\n",
    "\n",
    "print(len(zs_pos))\n",
    "print(zs_pos[0].shape)\n",
    "\n",
    "\n",
    "xs_pos = dec(zs_pos)\n",
    "xs_neg = tf.reverse(dec(zs_neg), [0])\n",
    "print(xs_pos.shape)\n",
    "\n",
    "xs_pos = xs_pos.numpy()\n",
    "xs_neg = xs_neg.numpy()\n",
    "orig_traj = orig_trajs[0].numpy()\n",
    "samp_traj = samp_trajs[0].numpy()\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(orig_traj[:, 0], orig_traj[:, 1], 'g', label='true trajectory')\n",
    "plt.plot(xs_pos[:, 0], xs_pos[:, 1], 'r', label='learned trajectory (t>0)')\n",
    "plt.plot(xs_neg[:, 0], xs_neg[:, 1], 'c', label='learned trajectory (t<0)')\n",
    "plt.scatter(samp_traj[:, 0], samp_traj[:, 1], label='sampled data', s=3)\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}