Lecture8_Variational_Autoencoder.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "[FULL CODE] Lecture 8_Variational Autoencoder",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMp8dy6tmtegtHEWVR6HtXq",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "7a128abdefa3476dbed0ff1430e2970f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3b3ed20fac054cf8b8606094815b36c6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a05441d5fa5748719a6da5bfda08077b",
              "IPY_MODEL_ba4c9fc4f6d6416682c4a6ed7de601aa"
            ]
          }
        },
        "3b3ed20fac054cf8b8606094815b36c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a05441d5fa5748719a6da5bfda08077b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cad9fa4c88e341d5894e98518bd72405",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_979853b9a2b5427496bcde99846ab9e5"
          }
        },
        "ba4c9fc4f6d6416682c4a6ed7de601aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_98a70b8830194852a7c19bb0f15c3269",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 20000/? [00:04&lt;00:00, 4447.25it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f4b186dfef134bc6a8612944edf572b3"
          }
        },
        "cad9fa4c88e341d5894e98518bd72405": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "979853b9a2b5427496bcde99846ab9e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "98a70b8830194852a7c19bb0f15c3269": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f4b186dfef134bc6a8612944edf572b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4115a30faee349a38f166d52f3aa81b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9af51c6ec46f4dafae417402e7db0fca",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e3f114a544254e2fb275dacf0ce758a3",
              "IPY_MODEL_789fa16922474c9c917f2c60106f6868"
            ]
          }
        },
        "9af51c6ec46f4dafae417402e7db0fca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e3f114a544254e2fb275dacf0ce758a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3e8037a2b9b9438182e33875eb4190c1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2e69e24610874d53a21d9e7cb4c36896"
          }
        },
        "789fa16922474c9c917f2c60106f6868": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4393e1f81fae4cbebb77fb1e06b9742d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10000/? [00:01&lt;00:00, 6843.08it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_83a60818e4bf49a882351efbaa6c9085"
          }
        },
        "3e8037a2b9b9438182e33875eb4190c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2e69e24610874d53a21d9e7cb4c36896": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4393e1f81fae4cbebb77fb1e06b9742d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "83a60818e4bf49a882351efbaa6c9085": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "22b56bebbb4742088f953644f03056da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4630f78ca78f47e7ba5cc021a50bab26",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b67645d91dc1419d80da3064c7bb9f79",
              "IPY_MODEL_d96cdb4ab6a64f5285108b93f58548e7"
            ]
          }
        },
        "4630f78ca78f47e7ba5cc021a50bab26": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b67645d91dc1419d80da3064c7bb9f79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5cb6a1b936c44818b2d70378f4734570",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 100,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1dd5a953f3114f46af6631ccf552132e"
          }
        },
        "d96cdb4ab6a64f5285108b93f58548e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_21c72341add84a2886a31a243bd672e9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100/100 [02:39&lt;00:00,  1.60s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6b46387327344c76b2c0c3cc210bacdb"
          }
        },
        "5cb6a1b936c44818b2d70378f4734570": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1dd5a953f3114f46af6631ccf552132e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "21c72341add84a2886a31a243bd672e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6b46387327344c76b2c0c3cc210bacdb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cotton-ahn/TUM_20WS_Intro2Deep/blob/main/Lecture8_Variational_Autoencoder.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kmIOkV65f5S0"
      },
      "source": [
        "### Import Packages\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X1f7bWu9lJtX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4aa4128d-ea05-4665-883d-f7a6a66bfdf3"
      },
      "source": [
        "import os\n",
        "import csv\n",
        "import numpy as np\n",
        "from tqdm.notebook import tqdm\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import torch\n",
        "import torchvision\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "print(torch.__version__)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.7.0+cu101\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TTj1RldNHgSQ"
      },
      "source": [
        "### MNIST dataset is provided in default"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pcMuF_m9Ev8W"
      },
      "source": [
        "train_path = './sample_data/mnist_train_small.csv'\n",
        "test_path = './sample_data/mnist_test.csv'\n",
        "img_size = 28"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PYiku-SxHnA6"
      },
      "source": [
        "### Choose your device - use GPU or not?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k44xmCOHHmF4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d58031ba-eb0b-40c9-a012-5d5abbf7ab87"
      },
      "source": [
        "# device = 'cpu'\n",
        "device = 'cuda'\n",
        "print('Current Device : {}'.format(device))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Current Device : cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1lukzx9CHuNv"
      },
      "source": [
        "### Read the MNIST dataset\n",
        "- total 1+ 28x28 dimension. 1 For Label, 28**2 for image\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KACiPSPiFOyF"
      },
      "source": [
        "def mnist_reader(file_path, img_size=img_size):\n",
        "    total_img = list()\n",
        "    total_label = list()\n",
        "    with open(file_path) as csvfile:\n",
        "        mnist_reader = csv.reader(csvfile)\n",
        "        for row in tqdm(mnist_reader):\n",
        "            img = np.array(row[1:], dtype=np.uint16).reshape(img_size, img_size)\n",
        "            img = img / 255\n",
        "            label = int(row[0])\n",
        "\n",
        "            total_img.append(img)\n",
        "            total_label.append(label)\n",
        "\n",
        "    total_img = np.asarray(total_img)\n",
        "    total_label = np.asarray(total_label)\n",
        "    \n",
        "    return total_img, total_label"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sghlSXb_Gd9-",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 116,
          "referenced_widgets": [
            "7a128abdefa3476dbed0ff1430e2970f",
            "3b3ed20fac054cf8b8606094815b36c6",
            "a05441d5fa5748719a6da5bfda08077b",
            "ba4c9fc4f6d6416682c4a6ed7de601aa",
            "cad9fa4c88e341d5894e98518bd72405",
            "979853b9a2b5427496bcde99846ab9e5",
            "98a70b8830194852a7c19bb0f15c3269",
            "f4b186dfef134bc6a8612944edf572b3",
            "4115a30faee349a38f166d52f3aa81b7",
            "9af51c6ec46f4dafae417402e7db0fca",
            "e3f114a544254e2fb275dacf0ce758a3",
            "789fa16922474c9c917f2c60106f6868",
            "3e8037a2b9b9438182e33875eb4190c1",
            "2e69e24610874d53a21d9e7cb4c36896",
            "4393e1f81fae4cbebb77fb1e06b9742d",
            "83a60818e4bf49a882351efbaa6c9085"
          ]
        },
        "outputId": "e794b8b9-0efd-43ea-9232-b68b44b9aa7e"
      },
      "source": [
        "train_img, train_label = mnist_reader(train_path)\n",
        "test_img, test_label = mnist_reader(test_path)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7a128abdefa3476dbed0ff1430e2970f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4115a30faee349a38f166d52f3aa81b7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XwsvttO0IJRL"
      },
      "source": [
        "### Define the Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "naa0-u1dGc6H"
      },
      "source": [
        "class MNIST_Dataset(Dataset):\n",
        "    def __init__(self, target_img, target_label):\n",
        "        self.target_img = target_img\n",
        "        self.target_label = target_label\n",
        "        self.num_data = target_img.shape[0]\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.num_data\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        sample = dict()\n",
        "        sample['img'] = self.target_img[idx]\n",
        "        sample['label'] = self.target_label[idx]\n",
        "\n",
        "        return sample"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j3S8BdL5IMcb"
      },
      "source": [
        "### Define the DataLoader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qu2Vn-qbIeTa"
      },
      "source": [
        "batch_size = 64\n",
        "\n",
        "train_dataset = MNIST_Dataset(train_img, train_label)\n",
        "train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sT_N_IqnIQgQ"
      },
      "source": [
        "### Try to sample out one data point from dataloader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RAiU23NUKFl7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "416ba254-fd05-4c9a-cba1-ae1991851edf"
      },
      "source": [
        "sample = next(iter(train_dataloader))\n",
        "print(sample['img'].shape)\n",
        "print(sample['label'].shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([64, 28, 28])\n",
            "torch.Size([64])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XnDU90xVTpqC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "outputId": "1b71b03b-990f-495d-cc85-42347e8790f4"
      },
      "source": [
        "plot_idx = 0\n",
        "plt.imshow(sample['img'][plot_idx])\n",
        "plt.title('LABEL : {}'.format(sample['label'][plot_idx]))\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEICAYAAACZA4KlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAARh0lEQVR4nO3de7BddXnG8e9DbphAICEQYogQNFFBJZRjBKGK0lKMWrDTUlKgoeP0MKOI1wqDzhg7Y2VQQCoWGyQSlKpUxGBFLk2hlJHbgUYIoAkyARJCYhogCUI4J3n7x15xNuGs3z5n33N+z2dmz9lnvWvt9WYnT9bae11+igjMbOTbo9MNmFl7OOxmmXDYzTLhsJtlwmE3y4TDbpYJh90sEw77bkbSakl/kqhL0hOSHh2kdoeklyVtlfSCpDslvb2qvlBSf1Hf+Xi+qh6S3tSEP8M4Sd+WtF7SJkk/kzS90de1NId95HkPcABwqKR3DlI/JyL2AiYDdwDf26X+o4jYq+qxbwt6/CRwDPAO4PXAc8A3W7Aeq+KwjzwLgKXATcXzQUXEduCHwGFt6qvaTOCWiFgfES8DPwIO70AfWXHYRxBJ44G/BK4tHqdJGlsy71jgdOCeFvRxXPXu/yCuAo6V9Pqi59OBXzS7D3s1h31k+QtgG3Ar8HNgDPDBXeb55yKIW4BzgC/vUj9V0vNVj9uH20RE3FVj938V8DSwFtgMvBX4x+Gux4bHYR9ZFgDXRcRAsXt8Pa/dlT+3COLrgA8BP5b0jqr6dRGxb9XjfS3o81vAOGA/YALwE7xlbzmHfYSQdBDwfuAMSc9KepbKLv08SVN2nT8idkTE/wCPAye2t1vmAFdHxKaI2Ebly7m5g/VpzeOw757GSNqz6jEaOBNYCbyZSpjmALOBNcD8wV5E0jFUvqB7ZBjrHrvLukfV0f/9wN9K2kfSGOBjwDMRsbGO17Ihcth3TzcBL1U9FlLZXf+XiHi2+gF8m1fvyl++8xg6lcNuX4yI6l3ov97lOPtWSQdU1R/ZZd1/t2tzkv64eP0ynwNepvLZ/XfAPOAjw3kDbPjkm1eY5cFbdrNMOOxmmXDYzTLhsJtlYnQ7VzZW42JPJrRzlWZZeZkXeSW2abBaQ2GXdBJwGTAK+E5EXJiaf08m8C6d0MgqzSzh3lhWWqt7N744meJbwAeonJgxX1InrqAysyFo5DP7XODxiHgiIl6hcrnkyc1py8yarZGwT6dy5dJOa4ppryKpV1KfpL5+tjWwOjNrRMu/jY+IRRHRExE9YxjX6tWZWYlGwr4WmFH1+0HFNDPrQo2E/X5glqSZxV1PTgNubE5bZtZsdR96i4gBSecAt1A59LY4IoZzqaSZtVFDx9kj4iYql1uaWZfz6bJmmXDYzTLhsJtlwmE3y4TDbpYJh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TDbpYJh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TDbpYJh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TDbpaJhkZxNWspKVkefeDUul96+6bnkvXYtq3u1+5WDYVd0mpgC7AdGIiInmY0ZWbN14wt+/siYmMTXsfMWsif2c0y0WjYA7hV0gOSegebQVKvpD5Jff2MvM9BZruLRnfjj4uItZIOAG6T9OuIuLN6hohYBCwCmKjJ0eD6zKxODW3ZI2Jt8XMDcAMwtxlNmVnz1R12SRMk7b3zOXAisKJZjZlZczWyGz8VuEGVY6GjgX+LiJub0pUNT+J49B5HvLWNjbzWxiP3Ka29dED6OHot/3vuN+te9uSVH07WX7z4oGR9/JqtyXqsWJmuDwwk661Qd9gj4gngiCb2YmYt5ENvZplw2M0y4bCbZcJhN8uEw26WCV/i2gVGT399sv7S4en6lnM3l9Z+eeT36uqpG/TH9mT97m1jk/VjxpUvv3T2z9Ir/9d0uZbDvn9Osn7oeXc3toI6eMtulgmH3SwTDrtZJhx2s0w47GaZcNjNMuGwm2XCx9nboP/E9E1391/4eLL+04MbPOjbgMf6+xta/pmB8ktcL1/z/uSyL1z2hmR9wlPpy0xfPHivZD3ludmjkvWbP3ZRsv7oGZcn6x8676hh99Qob9nNMuGwm2XCYTfLhMNulgmH3SwTDrtZJhx2s0z4OHsTbDz7mGT9xxd8LVk/aPTrGlr/3L7TS2tTvt7Ya4/ekh6y6/m3TkzWNx9Svj056Ku/TC47nnXJeq3hhcY/UGOG5LrT/uusQ5L1+Xuvr3/lLeItu1kmHHazTDjsZplw2M0y4bCbZcJhN8uEw26WCR9nH6JRh7+5tLb0C+nj6FNHpY91f3fzjGT98u+ckqxPu/Te8uKO9L3Xa9lRoz5xeY16Q2vvnI296XMnZo9NX6/ejWpu2SUtlrRB0oqqaZMl3SZpVfFzUmvbNLNGDWU3/mrgpF2mnQ8si4hZwLLidzPrYjXDHhF3Apt2mXwysKR4vgRI72eaWcfV+5l9akTsPHH5WWBq2YySeoFegD1rnnFsZq3S8LfxEREkrkmIiEUR0RMRPWMY1+jqzKxO9YZ9vaRpAMXPDc1rycxaod6w3wgsKJ4vAJY2px0za5Wan9kl/QA4HpgiaQ3wJeBC4DpJHwWeBE5tZZPdIMaU30e81nH0Wr71m/cm69MuTl/3bSX2KP870xFvSS76lc8vTtaPqvGJtNbY8p1QM+wRMb+kdEKTezGzFvLpsmaZcNjNMuGwm2XCYTfLhMNulglf4jpUA+UXez634+XkopP22DNZ7519V7L+89lzk/XtK3+brI9Uo6bsl6yv+vzs0tqjpzd2ieqnn3l3sv7wl49I1vfkvobWXw9v2c0y4bCbZcJhN8uEw26WCYfdLBMOu1kmHHazTPg4+xDtWPHr0tqfffUfkssuPf+iZL13n9Xplf+0Rrm3/ALEPe6qca/nFhs19YDS2vaZByaXffIz6UGZP/G2O5L13n1uTdZTrtta3jfAA5cemaxP/I976l53q3jLbpYJh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlQpUBXdpjoibHu5TfTWlXfveoZP2/T7gsWZ9W41bVOxIDK8+58pPJZQ/oG0jWGzX1/PJr7a+dWf9x8KG46oU3lNauWPme5LLTz/6/ZH37+u4cF+XeWMbm2KTBat6ym2XCYTfLhMNulgmH3SwTDrtZJhx2s0w47GaZ8HH2LrD6K8ck6yvOauwe57ureb8+JVlfe/uMZH3mkqdKawNPr6mrp27X0HF2SYslbZC0omraQklrJS0vHvOa2bCZNd9QduOvBk4aZPqlETGneNzU3LbMrNlqhj0i7gQ2taEXM2uhRr6gO0fSQ8Vu/qSymST1SuqT1NfPtgZWZ2aNqDfsVwBvBOYA64CLy2aMiEUR0RMRPWMYV+fqzKxRdYU9ItZHxPaI2AFcCaSHGTWzjqsr7JKmVf36EWBF2bxm1h1q3jde0g+A44EpktYAXwKOlzQHCGA1cHYLe+wKqbHAH/9c+TjgAJ/4cPpgxaxxi+vqaXd3xN0LkvUZf/VIuh5PJ+utvVJ/91Mz7BExf5DJV7WgFzNrIZ8ua5YJh90sEw67WSYcdrNMOOxmmfCQzUO07tQ3l9YeOfObLV33P218e7J+9X3HltYm96X/is889xfJ+sf3Lb8VdKNunvvtZL13bPpy6Njm06+Hw1t2s0w47GaZcNjNMuGwm2XCYTfLhMNulgmH3SwTPs5eWHX5u5L1Fad8o7R260sT06+97cBk/crvp2/OO+Nr9yXrswfuT9ZTbr5qSrJ+y1uOTtafXJj+J9R3dPnlu7WGoh59y+Rkvf/4dcm6vZq37GaZcNjNMuGwm2XCYTfLhMNulgmH3SwTDrtZJjxkc+HoX/Un65/Zr6+01nPtZ5LLHnre3XX1NBI89e/l1+I/9O6r08sOvJSsf+zg4+ppaURraMhmMxsZHHazTDjsZplw2M0y4bCbZcJhN8uEw26WiaEM2TwDuAaYSmWI5kURcZmkycCPgEOoDNt8akQ817pWW+vL+6eHB/7pi/uX1mZdkr63+va6OuoOoyZNStYf++qsZP2md5bfBwDG1dGR1WsoW/YB4LMRcRhwNPBxSYcB5wPLImIWsKz43cy6VM2wR8S6iHiweL4FeAyYDpwMLClmWwKc0qomzaxxw/rMLukQ4EjgXmBqROy8L9CzVHbzzaxLDTnskvYCrgc+FRGbq2tROcF+0JPsJfVK6pPU14/H5jLrlCGFXdIYKkG/NiJ+UkxeL2laUZ8GbBhs2YhYFBE9EdEzxl/ImHVMzbBLEnAV8FhEXFJVuhFYUDxfACxtfntm1ixDuZX0scCZwMOSlhfTLgAuBK6T9FHgSeDU1rTYHttjR7L+wfEvlNZ+deua5LLXPJy+TfW0G8Ym66300n7p/++PP/veZH3pgVfUWEP9e3Nnnve5ZH1v7qn7tXNUM+wRcRcw6PWxQHdenG5mr+Ez6Mwy4bCbZcJhN8uEw26WCYfdLBMOu1kmfCvpwprrD0/Wlx99TZs6GVlueLF82OUvXv83yWVnXlDjOHob/+3uLnwraTNz2M1y4bCbZcJhN8uEw26WCYfdLBMOu1kmhnI9exYOOTd9F+w/n3JG3a+96ox90zNMTw9NfMKbfpOsXz79rtLap595d3LZ+za8IVk/cK8tyfqjaw9M1t/4jfIbac+8L9+hrDvBW3azTDjsZplw2M0y4bCbZcJhN8uEw26WCYfdLBO+nn03sMf48en6xL1Lazu2bE0uG9tqDMk1alRjy1tb+Xp2M3PYzXLhsJtlwmE3y4TDbpYJh90sEw67WSZqXs8uaQZwDTAVCGBRRFwmaSHw98DvilkviIibWtVoznb8/vcN1RsyMNC617a2GsrNKwaAz0bEg5L2Bh6QdFtRuzQivt669sysWWqGPSLWAeuK51skPQZMb3VjZtZcw/rMLukQ4Ejg3mLSOZIekrRY0qSSZXol9Unq68enVpp1ypDDLmkv4HrgUxGxGbgCeCMwh8qW/+LBlouIRRHRExE9YxjXhJbNrB5DCrukMVSCfm1E/AQgItZHxPaI2AFcCcxtXZtm1qiaYZck4CrgsYi4pGr6tKrZPgKsaH57ZtYsQ/k2/ljgTOBhScuLaRcA8yXNoXI4bjVwdks6NLOmGMq38XcBg10f62PqZrsRn0FnlgmH3SwTDrtZJhx2s0w47GaZcNjNMuGwm2XCYTfLhMNulgmH3SwTDrtZJhx2s0w47GaZcNjNMtHWIZsl/Q54smrSFGBj2xoYnm7trVv7AvdWr2b2dnBE7D9Yoa1hf83Kpb6I6OlYAwnd2lu39gXurV7t6s278WaZcNjNMtHpsC/q8PpTurW3bu0L3Fu92tJbRz+zm1n7dHrLbmZt4rCbZaIjYZd0kqTfSHpc0vmd6KGMpNWSHpa0XFJfh3tZLGmDpBVV0yZLuk3SquLnoGPsdai3hZLWFu/dcknzOtTbDEm3S3pU0iOSPllM7+h7l+irLe9b2z+zSxoFrAT+FFgD3A/Mj4hH29pICUmrgZ6I6PgJGJLeA2wFromItxXTLgI2RcSFxX+UkyLivC7pbSGwtdPDeBejFU2rHmYcOAU4iw6+d4m+TqUN71sntuxzgccj4omIeAX4IXByB/roehFxJ7Bpl8knA0uK50uo/GNpu5LeukJErIuIB4vnW4Cdw4x39L1L9NUWnQj7dODpqt/X0F3jvQdwq6QHJPV2uplBTI2IdcXzZ4GpnWxmEDWH8W6nXYYZ75r3rp7hzxvlL+he67iI+CPgA8DHi93VrhSVz2DddOx0SMN4t8sgw4z/QSffu3qHP29UJ8K+FphR9ftBxbSuEBFri58bgBvovqGo1+8cQbf4uaHD/fxBNw3jPdgw43TBe9fJ4c87Efb7gVmSZkoaC5wG3NiBPl5D0oTiixMkTQBOpPuGor4RWFA8XwAs7WAvr9Itw3iXDTNOh9+7jg9/HhFtfwDzqHwj/1vgC53ooaSvQ4FfFY9HOt0b8AMqu3X9VL7b+CiwH7AMWAX8JzC5i3r7HvAw8BCVYE3rUG/HUdlFfwhYXjzmdfq9S/TVlvfNp8uaZcJf0JllwmE3y4TDbpYJh90sEw67WSYcdrNMOOxmmfh/vHlgBFrPMmkAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LEjuX7lWIYlb"
      },
      "source": [
        "### Define the VAE (encoder + decoder)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cx-fxCW5JD_w"
      },
      "source": [
        "class VAE(nn.Module):\n",
        "    def __init__(self, dim_hidden_1, dim_hidden_2, dim_z, dim_input=img_size**2):\n",
        "        super(VAE, self).__init__()\n",
        "        \n",
        "        # define hidden dimension\n",
        "        self.dim_hidden_1 = dim_hidden_1\n",
        "        self.dim_hidden_2 = dim_hidden_2\n",
        "        self.dim_z = dim_z\n",
        "        self.dim_input = dim_input\n",
        "\n",
        "        # define encoder\n",
        "        self.enc_1 = nn.Linear(dim_input, dim_hidden_1)\n",
        "        self.enc_2 = nn.Linear(dim_hidden_1, dim_hidden_2)\n",
        "        self.enc_z_mean = nn.Linear(dim_hidden_2, dim_z)\n",
        "        self.enc_z_logvar = nn.Linear(dim_hidden_2, dim_z)\n",
        "\n",
        "        # define decoder\n",
        "        self.dec_1 = nn.Linear(dim_z, dim_hidden_2)\n",
        "        self.dec_2 = nn.Linear(dim_hidden_2, dim_hidden_1)\n",
        "        self.dec_3 = nn.Linear(dim_hidden_1, dim_input)\n",
        "    \n",
        "        self.activation = nn.ReLU()\n",
        "        # self.dropout = nn.Dropout(0.5)\n",
        "\n",
        "    def encoder(self, img):\n",
        "        batch_size = img.shape[0]\n",
        "        input = img.view(batch_size, -1)\n",
        "\n",
        "        tmp_z = self.activation(self.enc_1(input))\n",
        "        tmp_z = self.activation(self.enc_2(tmp_z))\n",
        "        z_mean = self.enc_z_mean(tmp_z)\n",
        "        z_logvar = self.enc_z_logvar(tmp_z)\n",
        "\n",
        "        return z_mean, z_logvar\n",
        "\n",
        "    def sampling(self, mean, logvar):\n",
        "        std = torch.exp(0.5*logvar)\n",
        "        eps = torch.randn_like(std)\n",
        "        z_sample = torch.mul(eps, std) + mean\n",
        "        return z_sample \n",
        "        \n",
        "    def decoder(self, z_sample):\n",
        "        out = self.activation(self.dec_1(z_sample))\n",
        "        out = self.activation(self.dec_2(out))\n",
        "        out = torch.sigmoid(self.dec_3(out))\n",
        "        return out\n",
        "    \n",
        "    def forward(self, img):\n",
        "        z_mean, z_logvar = self.encoder(img)\n",
        "        z_sample = self.sampling(z_mean, z_logvar)\n",
        "        return self.decoder(z_sample), z_mean, z_logvar"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r4hjFl4-IsqO"
      },
      "source": [
        "### Define the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZWtw-y-MKuv1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "75db62d1-cc28-40dd-b0f6-363a705e7b37"
      },
      "source": [
        "dim_hidden_1 = 512\n",
        "dim_hidden_2 = 256\n",
        "dim_z = 128\n",
        "model = VAE(dim_hidden_1, dim_hidden_2, dim_z)\n",
        "model = model.to(device)\n",
        "\n",
        "optimizer = optim.Adam(model.parameters(), lr=1e-4)\n",
        "print(model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "VAE(\n",
            "  (enc_1): Linear(in_features=784, out_features=512, bias=True)\n",
            "  (enc_2): Linear(in_features=512, out_features=256, bias=True)\n",
            "  (enc_z_mean): Linear(in_features=256, out_features=128, bias=True)\n",
            "  (enc_z_logvar): Linear(in_features=256, out_features=128, bias=True)\n",
            "  (dec_1): Linear(in_features=128, out_features=256, bias=True)\n",
            "  (dec_2): Linear(in_features=256, out_features=512, bias=True)\n",
            "  (dec_3): Linear(in_features=512, out_features=784, bias=True)\n",
            "  (activation): ReLU()\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-KjpE_mygY6U"
      },
      "source": [
        "#### Note that the KL dirvergence $D_{KL}(q_\\phi(z|x^{(i)}\\|p_\\theta(z))$ is calculated as\n",
        "#### -$\\frac{1}{2}\\sum_{j=1}^{J}\\Big( 1 + \\log((\\sigma_j^{(i)})^2) - (\\mu_j^{(i)})^2 - (\\sigma_j^{(i)})^2 \\Big)$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BHTp8_5BiBsH"
      },
      "source": [
        "def criterion(recon_img, img, z_mean, z_logvar):\r\n",
        "    BCE_loss = F.binary_cross_entropy(recon_img, img.view(-1, img_size**2), reduction='sum')\r\n",
        "    KL_loss = -0.5 * torch.sum(1 + z_logvar - z_mean**2 - torch.exp(z_logvar))\r\n",
        "    return BCE_loss + KL_loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DUSpLgVXIwqG"
      },
      "source": [
        "### Define functions for train/test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q9pXkpAMK42y"
      },
      "source": [
        "def train(model, optimizer, sample):\n",
        "    model.train()\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    img = sample['img'].float().to(device)\n",
        "    \n",
        "    recon_img, z_mean, z_logvar = model(img)\n",
        "\n",
        "    curr_loss = criterion(recon_img, img, z_mean, z_logvar)\n",
        "\n",
        "    curr_loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "    return curr_loss.item()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MhcKwq8WOUGu"
      },
      "source": [
        "def test(model):\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        z_sample = torch.randn(1, dim_z).to(device)\n",
        "        gen_out = model.decoder(z_sample)\n",
        "\n",
        "    return gen_out.view(img_size, img_size).detach().cpu().numpy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T23Toz8vI7Gu"
      },
      "source": [
        "### Run Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-kTrpt4YMRS6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "22b56bebbb4742088f953644f03056da",
            "4630f78ca78f47e7ba5cc021a50bab26",
            "b67645d91dc1419d80da3064c7bb9f79",
            "d96cdb4ab6a64f5285108b93f58548e7",
            "5cb6a1b936c44818b2d70378f4734570",
            "1dd5a953f3114f46af6631ccf552132e",
            "21c72341add84a2886a31a243bd672e9",
            "6b46387327344c76b2c0c3cc210bacdb"
          ]
        },
        "outputId": "8c53bb92-a8ed-4d1c-a586-36d652cbf4ac"
      },
      "source": [
        "max_epoch = 100\n",
        "\n",
        "for epoch in tqdm(range(max_epoch)):        \n",
        "    ###Train Phase\n",
        "    \n",
        "    # Initialize Loss\n",
        "    train_loss = 0.0\n",
        "\n",
        "    # Iterate over the train_dataloader\n",
        "    for idx, sample in enumerate(train_dataloader):\n",
        "        curr_loss = train(model, optimizer, sample)\n",
        "        train_loss += curr_loss / len(train_dataloader)\n",
        "        \n",
        "    print('[EPOCH {}] SAMPLED TRAINING LOSS : {}'.format(epoch, train_loss))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "22b56bebbb4742088f953644f03056da",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "[EPOCH 0] SAMPLED TRAINING LOSS : 16234.163747379196\n",
            "[EPOCH 1] SAMPLED TRAINING LOSS : 12703.19549190046\n",
            "[EPOCH 2] SAMPLED TRAINING LOSS : 12377.06413488418\n",
            "[EPOCH 3] SAMPLED TRAINING LOSS : 12000.800069888175\n",
            "[EPOCH 4] SAMPLED TRAINING LOSS : 11441.448739828775\n",
            "[EPOCH 5] SAMPLED TRAINING LOSS : 10847.804069738419\n",
            "[EPOCH 6] SAMPLED TRAINING LOSS : 10479.248221595446\n",
            "[EPOCH 7] SAMPLED TRAINING LOSS : 10260.231279952075\n",
            "[EPOCH 8] SAMPLED TRAINING LOSS : 10048.832385370899\n",
            "[EPOCH 9] SAMPLED TRAINING LOSS : 9883.209528816391\n",
            "[EPOCH 10] SAMPLED TRAINING LOSS : 9729.910159370002\n",
            "[EPOCH 11] SAMPLED TRAINING LOSS : 9601.566747890882\n",
            "[EPOCH 12] SAMPLED TRAINING LOSS : 9472.845394244212\n",
            "[EPOCH 13] SAMPLED TRAINING LOSS : 9349.035657011282\n",
            "[EPOCH 14] SAMPLED TRAINING LOSS : 9225.047999762879\n",
            "[EPOCH 15] SAMPLED TRAINING LOSS : 9111.24748527356\n",
            "[EPOCH 16] SAMPLED TRAINING LOSS : 8999.984852361225\n",
            "[EPOCH 17] SAMPLED TRAINING LOSS : 8887.106701153152\n",
            "[EPOCH 18] SAMPLED TRAINING LOSS : 8783.954524323586\n",
            "[EPOCH 19] SAMPLED TRAINING LOSS : 8683.451530675924\n",
            "[EPOCH 20] SAMPLED TRAINING LOSS : 8580.334806497101\n",
            "[EPOCH 21] SAMPLED TRAINING LOSS : 8476.719037040744\n",
            "[EPOCH 22] SAMPLED TRAINING LOSS : 8388.852905663442\n",
            "[EPOCH 23] SAMPLED TRAINING LOSS : 8298.92572976987\n",
            "[EPOCH 24] SAMPLED TRAINING LOSS : 8211.325463633184\n",
            "[EPOCH 25] SAMPLED TRAINING LOSS : 8123.304523699578\n",
            "[EPOCH 26] SAMPLED TRAINING LOSS : 8049.260882587859\n",
            "[EPOCH 27] SAMPLED TRAINING LOSS : 7968.76856248752\n",
            "[EPOCH 28] SAMPLED TRAINING LOSS : 7894.798921725238\n",
            "[EPOCH 29] SAMPLED TRAINING LOSS : 7830.713168305711\n",
            "[EPOCH 30] SAMPLED TRAINING LOSS : 7761.962832904847\n",
            "[EPOCH 31] SAMPLED TRAINING LOSS : 7709.333765974441\n",
            "[EPOCH 32] SAMPLED TRAINING LOSS : 7656.5393386207\n",
            "[EPOCH 33] SAMPLED TRAINING LOSS : 7601.038436938396\n",
            "[EPOCH 34] SAMPLED TRAINING LOSS : 7559.308522769819\n",
            "[EPOCH 35] SAMPLED TRAINING LOSS : 7513.238295290032\n",
            "[EPOCH 36] SAMPLED TRAINING LOSS : 7468.401266567244\n",
            "[EPOCH 37] SAMPLED TRAINING LOSS : 7428.142996206069\n",
            "[EPOCH 38] SAMPLED TRAINING LOSS : 7390.787780176715\n",
            "[EPOCH 39] SAMPLED TRAINING LOSS : 7354.785855911792\n",
            "[EPOCH 40] SAMPLED TRAINING LOSS : 7320.4031892721605\n",
            "[EPOCH 41] SAMPLED TRAINING LOSS : 7288.115721096246\n",
            "[EPOCH 42] SAMPLED TRAINING LOSS : 7257.691219049524\n",
            "[EPOCH 43] SAMPLED TRAINING LOSS : 7229.614701633637\n",
            "[EPOCH 44] SAMPLED TRAINING LOSS : 7201.2521372054725\n",
            "[EPOCH 45] SAMPLED TRAINING LOSS : 7171.238338190144\n",
            "[EPOCH 46] SAMPLED TRAINING LOSS : 7148.034231947633\n",
            "[EPOCH 47] SAMPLED TRAINING LOSS : 7125.908733526359\n",
            "[EPOCH 48] SAMPLED TRAINING LOSS : 7099.376432083665\n",
            "[EPOCH 49] SAMPLED TRAINING LOSS : 7075.009809305115\n",
            "[EPOCH 50] SAMPLED TRAINING LOSS : 7052.434295751798\n",
            "[EPOCH 51] SAMPLED TRAINING LOSS : 7028.073663388581\n",
            "[EPOCH 52] SAMPLED TRAINING LOSS : 7005.663310578068\n",
            "[EPOCH 53] SAMPLED TRAINING LOSS : 6988.433542269871\n",
            "[EPOCH 54] SAMPLED TRAINING LOSS : 6966.480571710264\n",
            "[EPOCH 55] SAMPLED TRAINING LOSS : 6950.7317807133595\n",
            "[EPOCH 56] SAMPLED TRAINING LOSS : 6935.805507282097\n",
            "[EPOCH 57] SAMPLED TRAINING LOSS : 6916.298154203277\n",
            "[EPOCH 58] SAMPLED TRAINING LOSS : 6897.533756926417\n",
            "[EPOCH 59] SAMPLED TRAINING LOSS : 6882.978604545224\n",
            "[EPOCH 60] SAMPLED TRAINING LOSS : 6866.580378425765\n",
            "[EPOCH 61] SAMPLED TRAINING LOSS : 6855.408520585811\n",
            "[EPOCH 62] SAMPLED TRAINING LOSS : 6841.089186988319\n",
            "[EPOCH 63] SAMPLED TRAINING LOSS : 6827.662892497007\n",
            "[EPOCH 64] SAMPLED TRAINING LOSS : 6814.701380915533\n",
            "[EPOCH 65] SAMPLED TRAINING LOSS : 6802.6193200254575\n",
            "[EPOCH 66] SAMPLED TRAINING LOSS : 6788.7095405164255\n",
            "[EPOCH 67] SAMPLED TRAINING LOSS : 6772.154328855085\n",
            "[EPOCH 68] SAMPLED TRAINING LOSS : 6767.661221732732\n",
            "[EPOCH 69] SAMPLED TRAINING LOSS : 6750.284886369313\n",
            "[EPOCH 70] SAMPLED TRAINING LOSS : 6742.502132525458\n",
            "[EPOCH 71] SAMPLED TRAINING LOSS : 6728.235608183156\n",
            "[EPOCH 72] SAMPLED TRAINING LOSS : 6721.38516576603\n",
            "[EPOCH 73] SAMPLED TRAINING LOSS : 6709.684517272366\n",
            "[EPOCH 74] SAMPLED TRAINING LOSS : 6699.434296531795\n",
            "[EPOCH 75] SAMPLED TRAINING LOSS : 6691.01083032772\n",
            "[EPOCH 76] SAMPLED TRAINING LOSS : 6682.022669198033\n",
            "[EPOCH 77] SAMPLED TRAINING LOSS : 6675.15526017747\n",
            "[EPOCH 78] SAMPLED TRAINING LOSS : 6666.330216965359\n",
            "[EPOCH 79] SAMPLED TRAINING LOSS : 6655.683416689541\n",
            "[EPOCH 80] SAMPLED TRAINING LOSS : 6652.192806353584\n",
            "[EPOCH 81] SAMPLED TRAINING LOSS : 6636.331562468802\n",
            "[EPOCH 82] SAMPLED TRAINING LOSS : 6632.041528866314\n",
            "[EPOCH 83] SAMPLED TRAINING LOSS : 6627.299544166834\n",
            "[EPOCH 84] SAMPLED TRAINING LOSS : 6615.412910437052\n",
            "[EPOCH 85] SAMPLED TRAINING LOSS : 6604.985091821835\n",
            "[EPOCH 86] SAMPLED TRAINING LOSS : 6604.284965929508\n",
            "[EPOCH 87] SAMPLED TRAINING LOSS : 6598.809231791628\n",
            "[EPOCH 88] SAMPLED TRAINING LOSS : 6590.919354033542\n",
            "[EPOCH 89] SAMPLED TRAINING LOSS : 6583.414428320934\n",
            "[EPOCH 90] SAMPLED TRAINING LOSS : 6578.422355481229\n",
            "[EPOCH 91] SAMPLED TRAINING LOSS : 6567.340735292281\n",
            "[EPOCH 92] SAMPLED TRAINING LOSS : 6562.999501578723\n",
            "[EPOCH 93] SAMPLED TRAINING LOSS : 6560.632977860425\n",
            "[EPOCH 94] SAMPLED TRAINING LOSS : 6549.001747204478\n",
            "[EPOCH 95] SAMPLED TRAINING LOSS : 6544.000318240817\n",
            "[EPOCH 96] SAMPLED TRAINING LOSS : 6541.175060528151\n",
            "[EPOCH 97] SAMPLED TRAINING LOSS : 6535.33423397564\n",
            "[EPOCH 98] SAMPLED TRAINING LOSS : 6528.680315401611\n",
            "[EPOCH 99] SAMPLED TRAINING LOSS : 6522.159877789287\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 98
        },
        "id": "Zsve2TFhkE1I",
        "outputId": "63af7b80-2e11-4258-9042-ee393ef160c9"
      },
      "source": [
        "fig, ax = plt.subplots(1, 10, figsize=(10, 3))\r\n",
        "for i in range(10):\r\n",
        "    ax[i].imshow(test(model))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlIAAABRCAYAAAAZ1Ej0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9WYyk2XXn97v3flt8sWbkvlRmVdZevXdzF1uiSFsUqWUkjT0Wh9YMhgYED7y8GTBswPaDxoYfbGBgwTDmwTOeMSxrLM3Y2ihR3MRmU2wuTXZ39Vp7ZeUeGfvyrff64YvIyt63qsysYv6BRHcFIjO+E3f733P+5xxhjOEIRzjCEY5whCMc4QjvH/KgH+AIRzjCEY5whCMc4V7FEZE6whGOcIQjHOEIR/iAOCJSRzjCEY5whCMc4QgfEEdE6ghHOMIRjnCEIxzhA+KISB3hCEc4whGOcIQjfEAcEakjHOEIRzjCEY5whA+ID0WkhBC/LIR4VQhxWQjxX96phzpMOLLx3sf9bh8c2Xi/4H638X63D45s/JmEMeYD/QAKuAIsAw7wHHDhg/69w/hzZOO9/3O/23dk48E/25GNR/Yd2Xh/2fh+fz6MR+pjwGVjzFVjTAT838Df+RB/7zDiyMZ7H/e7fXBk4/2C+93G+90+OLLxZxLWh/jdeWBlz79vAR9/45uEEL8L/C6AQj3hU/oQH7m/8MiTElMSVRPQA/gH/AzaeL/YN3ypA/zLN77vfrHxZ3mewv1v4/1i3/Clo7XIkY2HHQE9IhOKd3rPhyFS7wnGmH8G/DOAkqiaj4vP3e2PvGPYNLfYYYML4iM8Y75BTPSW77vfbbxf7AP4uvmj2lu9736x8Wd5nsL9b+P9Yh8crUWObLwn8Iz5xru+58MQqVXg2J5/Lwxfu2/gkiNgsPelIxvvMbyFfQ73kX1wj46h2HPBM+/e7/PQ2ygVQgpQCiGy/5KmmCTBpOn9YeOHxNFavD/ws2Dj+8WH0Uj9EDgthDghhHCA3wb+5M481uFAiTEGdBmYHgYDRzbec9hrnzYaoMp9ZB/cg2MoBEIphGWDkK8nVW+DQ2ujEAjbQeY8ZCGPLBYQ5RKyUkYU8gjHyWx8Dzi0Nt4hHK3F+wM/Cza+X3xgj5QxJhFC/KfAX5Gp+P93Y8yLd+zJDgGkkJw1j/ITnmJAH+Bf31c2CoHk/rZx7xgOF339frIPDvE8FQKEzDw1gHBdhOciyiWMa2NcG9XoYPoBhCEmSdBBCEa/yYNz6GwUAlUswswknQcnaC4rgimDDEHGAhVC6YamfLGJ2thGd7qYaBgCeRvv1KGz8Q7jaC3eH/hZsPH94kNppIwxfwH8xR16lg+O4Y1WKPW6l402CCkw2rzl5vxeMCFmmWCWZ8w3aJv6P7kjz/tBMLq1D2+3Qikw+rZt8P7sGx5yoJngAGwceiVQCuE4CNdBWBa4Dsa2MJ6LMCazKU4QcYKpN9FhiAnD9/VRozEE+Lr5o427YQ7wOuIgLAvhuTBRzcI81nBuGgNRjAgidL2BiSJMknzojz408/QNEEohlATbRpZLGN8jnCuTuhLtSJyCg+rFqHob+gFSa0yqMWm6S8BGc/xA5umbDBIIy0Z4LmZxjsFikfo5RX8ppjDVo9vMIfoW7pYidUTm8xcSIQRGyNtr9W1wWMfxTmHf1uIB4n4fQ/jZsPH94K6Lze8qRgeXbWUHVyGf6RMAk+rbm1aq0f1+pll4j3qFQ4E95EkoBaMD2rYQnpcdwKnGDAbZ4ZPE7822Pd+biRMw6d21442QCmFbyGIBqhWSySK9eY+wJBhMCcKqxl7qkSaSNJGYuotXk8x9dwJntYW+euNwjaNUGWHw3IwMOjaMVwgWStz4ZQtdSLEKMUIYtJaw4ZJfkcx9s4jcrJNu74De5zG42xiF72wLWSpiinl6Z8YZjFu0TkHqgnYNqp/D6vlMPevj7gRYqzYmCCBOwM62JxPFGeF8nwT6jmM4b9XUJOlUmZtfLDNYDvnyY08BEBvFV2+cpzMoMn4xJb86QKzvYDoddBTfu2O8J/T6xssq8PrL3Jt+V+594+FZs0c4wh3EvUek9hxaspDHFPOk1Txx3qa95JC6kHoCGYOMDf62xu6muBtdZGeAabYz4vE+RKD7jhHRGR5EmafGRTg2plIkzTuEkx4yNshI49xqILt90kbzvZHFUbhF76PtQmSeJ8dBHJslHs+zfcEjGBcEkxpdibFzMePlHuO5Pg+XVwm1RahtrveqrLVLrLhVKq9NMW4pWNskbXcP9nAaCoxlIY8oFEhnxugu5ulPSvqzgrCacuHhG0x6XSacLlIYEi25dmyca6fGeW2xQuFGleKtE5Sfvk6608DEb50Bc+gxWpdKgpRD7VA2b03Bx+QczF4tlABjGbQLqYH+lAXCw6o5iDTNDufRAa31/s7Vt4IQqHIJxis0H5uiO6/QD3U4O1Fn2m5zJZjkZm+M3vUyhVWJv9FH7XQxQfD6S909AGFZoBSyUoZKiWCpQuJLEleSOgKjwEiwBganqxGJQWiDFaSIWCPDBNkJEIMQlARjML0+ZhBkF9rDuOe+Hd4YnraGR6aUmDh575fXI3x47HUsyGFEZbiuPkzU6U7g3iJSw81ZOA6yWiGdLDOY9WkvWoRVsJ5oMF3scrq0TS3MsxPkuX5xDrfmULlcwd/0cVYkoilhMNRlaHOwh/FbZS+NSJTnZiEvz8Pkc+icQ2+pwKCq6C4JZAjWACZkFafmIoMAE2S39rckU6MQ6HBT2LfNfbgZyZyHKBRoPlilfVwy8/kVPj5+nS+UnqMqA/JSU5YKhcAWinT4/DUdsZL4/OHcx/nTqUex+2OUByFiEGDCAxq7kcfFsRGVMslMhcbZPLWPapbOrvGV+Wc57WzwuNvEE4qccFDD23lqNKFJqD8W8d+u/TLfvnyawo1JRK9/bxKpkYfRdTMvkrwdzjM5F+O7aM9CGINMDDKWpK7BSIN2wAjoz0jAwr+eESmSBKI4q6ScpgdORIRSiLEyg+VxNj4lqJyq8bsn/5aiHBAbi7VBmUs7k4y9KCjejLBv1jJPVK9/b3liRheenIdemKR7osDmxyTJeEyu3KfkB+SshIITcr1epbeRR0YKkYDTclABOC1DfjOPuxOhXYVMNPZWF9loY8Lw8F5g92LvZVbJLDwvJcLzYLR/9gcQcESm9gN7pSCj/1rWruNApGlWZTxODmS93TtEarRZLy2QTBbZvpAjHBMEVUNaTDG5lOVCj7IzQGIIUpt+bGOqEQPHQqQWnUUP8fgsxZVpvJ0Y9+IKpj8YhsYOYHG/Sfe059Yz0pTkc6RFj3AiR1yQdBYU4RiEiyGEChFKwMXbcahKiegOEK0upt/PQn+jm7wUWdhTyuzvS5HdlKPo7pKRkSh3fpqtT0/QPGdYeGiDJ8c2+I2xHzNjdZhRKa6QKCwkEjUUwSOyw7MoJMesPr9a+SnXz47zUuc4ue0qdrtDGkX7P257Qjx6vMTaz1XoLhmWn1jhCxPXeMK/xmm7RlFqbKGQSDQGbVIk2RjbQlGVDl+Z+g6PFlf4/d/8ImMvP0DlD5+9d8iUVEjHRk6MY8oF4qoPxiAMyH6MSFNEfHtuJTlJ4g09GpYBRyO6FioQ5NcM/laM6A12vRcYA1rf9h4fEITtoMbHaD0+zdqTggcfvc7Hxq7TSnye6Z3gb68tw80c/rqgerGHvdlCN1uZuPweIVHCspDFImKszNZnZukuCpIzfSbGavz61DVKVoCvQjyR6flSBI2xPBsLJXqJSz+xWWmPEcQWncCmPrAhdFB9idOQzH1X4CQp1HaGHyjgMH0te7RvyeOn6E+71M9JwnGNmgxIejZiIMmvKJy2wd/SFK51kKvb2Vgfkak7hyGZl66LGCuTTpSIKx71cy5hBQYLCcJP8PIRlpWiRPa99/ou6UaOuacMpb+9QbpduyP60/eCe4NICZF5M3yfcKFCf9qmvQxJIYViAsIglaETusSpohu7bPfy9AYuaAHKEJcNWoF2NUYoYt/BvVVCCoGOY+BtvDh33bYhidpbg8a2szCe76ELLknRISwroqIgKkNc0niFiNSTpKmkH3qkniBXy+M0bSylkJbKNvLRASTk7k1K2Hb2WpJkB9Vd1J4Ix0GMlemdqNA8Zzj56C3+8eK3OW1vs2CBQgCKFENsND2TTXwFKCGwEaQYPCE4aTd4sLTGK3PTREUXxx6mz++zxkvI7HZkSnmi8RzdRYNzss3vHvsOx+0a0yrCEQIQ1NMUTUqKwMZgC6hKJ7NNKC7YATPqRf63c5+mFZcZc+xsHt4DehphW4hcLiNREz6DCQeZmsw7IUCFEpFojBIYKUg8QZITaMtgJKAFMhRYfYHXSHBaUeZRjeLbG6A2B+vBkJlnWE9X6cwrxk/WeKJykxPuNk+3T/NKfRp5LUfhBhRXY+z1BqbVwQT3iOcFMkLs+zAxRjhfoXHBkD/V5HeWn2XJrXHOWScwFoGxiY1FZBTN1MeXERNWl07q0dcOBTsk0ZmGKkgt+onDWrtEZ6NIXLCw3WEpCPEOmqr9xjAkLXMeolxClwvsnPfoz0H+sRqfmFzn82MXuThY4OZgjO+WTzGoOWhb4bRyeDUHoSQm3f996F0xigQ49m5SD0pmYzDKkh0MDtccHRJaWSwgSkXi2Qq9OZfBhKT5UII/2ePXll7juLfDKXeDcdXFEwm20LwSTfMntcd4tnaB3Po0qtvDdDr78tiHn0gNb73po6dpncyx9WRCYaLNqUqT9XaJZq1A/pKLVzNYXRcCTdJKKHqKvCNIHUFUkLTODElUMaF7RtNblMAk+fUK/vMC3e1hen3gA2TAfVAYA+I2iRqFRkQul2U3LVSIi4qoIAkrksSHuKjRniaOFdPVNvOFFv0Fh52Bz825CZxGHm8nT2FtHLuTIpPbdqgg8xAYKzvgVKOHEMMw550+uEcLYmmB2sensL68ye/MvsIvFl9iRvXwhKGloaMVTe2ymoyxGVf4k42HSbQkZ8WcKW1x0tvmIW+FogxQGHwVUS4OiPM5yHnDrMw7++jvZhdKIVyXYL5IZ8EmGY+Zyg9IEawmFbbTlFfCWdajCs815rPDpJ7HLw8Yyw/4z5e/waJVZ8Ea4AlBWQr+h4f/Lf9L5XPIf16FWh3d6+2jUe8Tw8NHzU6TjhVpnylmJMkHGYOKwO5KTKIxliSueIRjFq1lSVw0pPkU1VPkbtqUrmlytRj/Ug36A3SrvZu1t4uD2uiHuii9PMelLxcpnazzdxafpxYX+NPaI/zgp6fxVxTHngmw6wNku387s3QUjhTicB1Ub4CwLOR4ld5Hj7PzoEX8WJevnH+aXyi8zFl7gAZ2UsGlaJpXg1nWwjKNyOdKY5xB6BAObHSsQBimp1sslhp8ceIF5HAf/X7xFD+0FunNjOM0PdRhIVDDUJEcG8PMjtN4sMLGpzUPXFjhP5n+NsedbZatFp4AT0g+nVuhoyXfqyzzVzsP8GNxmty2jXvTycJ+xIdnrEcEKu8j8z7BAwv0p21aJ7L1pz1D+VVBflNTfPoapts7NPuNLBSQpSKtTx6jvajoPhIwN73NY5UtHi/dYNJqM281sEWKg6aqYjwhcIXkmFrnEwur/LN/f4e/+uR5xv+7JcQLl/YlSeVwEykhUIU8zE/TOJujeRZmFupM+j0skdIbODjrNoUVjb+dYLcj5CBGtno4ORfj2qQ5G6fsEIxn3ou4CG4lQGtBd76A0Da5fA4Rx4ggzC4VB7DYxW6YT+z+CJ1pSoQGoQ0YgbYBR+P7IbP5NmcKW4TaYtMtUp/OE6kcMlaEnczTJeOMrGklsAKJjC3ivELGhlysUcHdCSMJx0EWC7QfnqB1Gn5l8jpnvXVKImQjzVNPCzzdOc213jjXmlU6PY8ksLHWHTBgLHh5Yo58ZcBnjl1mKVfjrLtOP3VwrQT97jUc7y6MRg1S7L6FCBSNrs93WucYpDa91OHi5iz9jovadLE7glIbkrzDVrHE74vP8nB1jf9w/HtMqwEVKVm0Gsz6LbbLc4h+AIdkY3srjLRhJjfSPoGKDDLNBMgqNFidEBGmiDgBk80/7RiMZZChxGkL/E2Dvxnj1rJwHmF4m0QZ/Tox6b5jGL7Vy3O0zhSZPFfjofF1TrjbvNSZ5VJ9gtwtRX7V4Gx2b4ckR17gD1KSZD8xJBJqYY54vsrWExbhyYBfXLrKI/4NZlSfvoGVxOe7vbM80zjOjdYYrY5POrCwdmxkCN5AIDRoBZtphTiVNMd8KqqPJ2JyMsJSKfHe9XqQZGp0CViYJa0WqD1UYjAh6C2lnD23yq9PP8enclcZV4aydHd1mikxsdD0tctWv4i/LvEaESKIbpP+gxjr4bkhhzXamJogreYJJjzCiiTOC7qLEJc1udkWE35AwYm4XJqms2Vj5AkK17qIi5eHoeiDna9CCLAUiSeICzA73eSB6jrn8+sU5YBOmuPrg3m6qUsvdXkgv8oxe4ePult4QlKWio/lr9CddXm++iheqUS6vX3Xn/tQEymhFMxMsv3JCRqfC/h7F37MlNOmlfh8t3YSvZJn4emE/Mub6O2dLE3aaJI0zX5XSJTnYpdLTJhZ6ucc4gnB8uQOs7k2T7FMM5en8koBK0kRYQSDQUam3mcLiw8Mo8kCWXsgBWiN1Yt3X9K2lWlLHI1TiLgwucnPVa7wkLdCU/ts5UpoI/mJNU/UKxH1BEZJ7L5BW4LYJyNSiaE3I5EJiNTHD+O7srHJYgEzP8XaryZcOL7GFyvP4YmYwFh8p3uOHzSO89q3lylfNkz9pMHMztZuGYdMqOygF6YIZny++kuP4Sz0+M1Tz7EdFcnZcdYqMxlmd+0nhuJnE8XY600KSQnvVJ4oKPDnnQdROzZOUzL1bIy3NUDdWsXEcZbOrxQi77P9uUX+8sEZlr5Q4xfyrzCpUqoqZtZrc3P+LH4Qwz4s/g8EIbIkCD+HLnhoV2H3UtRAY3UjVH1IKqJo6JW0kJMFwMZYgAG3LineNIy91EFtNDDdLnoQZN/r67JvDk5fJB0bUSyy9nNlWg/E/H8X/k9sNH1j8U+bn6VxtcriczG5tR5mZR0Tx68ngXDgh9I7QSiFyOVoPzZL/Zzit3/r2zzmX+dRdwuA2MCz4Qxfaz7IV597EG/FIbdpmKlprL7GrXWRUYqIEhACnbPZ+kiR9vI4Pxo/zun8FgtOnVDbpFoiYxDpnu/jgEiHdGxEIc/Op2ZpnhX8x3/3qzyeu86y1SUvJLaQyOGxmBqDRpNiCIyhrh2+tXOGm69Nc+7Pa7DTJN3Vwh3MWI+E13Ksgp4aY/NTFZrnNQ8/cp0vTr3AOXeNZauLJwTeUK+phGD7dMjFaJzfe/RXuPWtGRZvljKt1yHRZxopSH3DRydvci63zpzd4OLgGK/0pnnq+bNYLQunIfirC+c5Nt3gn5z6Nxyz+lSFxePuFlNjHZ5e+CjuxjjUand9fA4vkZIKNTlBf7lK7SMpTyyu8Gj+BttJiVvBGFdfmGfsNfCvNzNNwigDb7gJmzQFYRBxjAkC3HqI3bURoaIdehTskIIf0qzk6B7PU0w1chBki2J0OO/TzcloA6mGJEEARgSIJEXaFkoJUk8RlgWDacPciRonyzX+nbGXmLcbVOSA16IZXurP8YMbS5gVn9JlyG8m2J0UFaYkOUUwbmNU5hm4qxiG9NLlOTrLeZbm1zlf2iA2Fn/bO83FzhzPPnUWf1WwcDHA3u4jNrbR/UGWcUG2iEScIIMIK3DRBZgsdZl1WnRTF20ERgBK7n9oj9F4pchuH0sIqi97REVJWHHx6ga3neDfaCPavSxrK02zFH6lwLHJ1VOclsVKUKXp+aR2G08IFt0d/vKMjTUoo17bX5veE0ZFXOMYQols9nD6NnZDIqIE0Q+GXpk4y6SxLbAsjCXRFshQIBLwaobcToLa6WTvD8Ldsd8dzL0e2gNIAhELs4SLVcInO/zaiVeYUSlXY48fDpapXalSeUWQu9VB1ocJD3tJIBxqEoUQqMkJ9GSFrScknO7y8fwV5qwWnhA8NZjl+cEx/tVzH0etuUy9BE4nxe4kWL0EGSTITh+RpNkFwVLI2MHfzhNMZJdCT8YU1QBLpqRa4PU0Mtgf4e8bbZWui5ycoPaZY3SPCfrLEfMLW/xcZYvP+K8yoWLyQmaESSfcSHK0tUdT+zgixRMx3+mc5WJ7jte+vczkZQM7TUy3N8wS29+xFpaFcF1kpUz3sQVaJyza51LkWMipmZv8fHmTTxUvcdLepiIjNBAYQ2wSfKnAKPJCsmzX+Q+O/Zj/efmXiM8tYL+sSUfJAAcEE0XQD/AaKe6OxQ+3F3lWZC19V1+ext2WLLyUYvVjrF5C76pHY3aOP/jyJ/n58it80d8kLyRVFdA6BXavQukldddF54eWSAmlMJUivRmL5TOrfLp6mTP2FttJic2gSOmypHQtxNza2K0L9ToMb7Qm1RBGyFYfu59HRIJBbBGkNgU3olmM6U96eDUPb2tYXXskcN2v0ILRYMSui1gAxmhE4CB8B4C4CNFEyicmr/NI/iZP5q6jgchI1qMKlzuTmJs+hRVB6UaEt9lH9AJEnKAKPsYqEuclJgcyFZlHSg8rh9/BjWAU9unP5egsSj5WXeWkt0VkFM82j/HczQWWvhmTu7RNurKKfitBrlSgDSJOENqg8jEz+TZVq0tOjaGNyL4k+WFaRX5IpCmmP4BUU7zkkOZd4rKDUw9QrQFs7WRamVFbECERUkKSYHcTrIFFPcrTNy7aGGwEk1ab/qwhXLHxD86yd4QYZXsGIbLTy7xsaYqJY3QQ3s4UlcMMUaUwSqDtrLabCAVeQ+M0ot2abjqKb3uh3kPfvbtrYBbySqZKtI87/M7Z7/Kl8o8pCIcdnefZziL5FUX5aoTYrGfaylHKNRxuAjWCkJhSgWC2AKd7/PzxK1xwdvCEQAIXBwt8Z+sU5Wc8iqsJhZduH64iyS4FpjfA6CGRcmxEqnEbCVZfkRiJFBpPxGgjSFKF3dfIMCbVd3a/eUeMMr3HKsSLE2z9XMpD52/ye0v/L3MqxRc2trDQSGKT0tMJLa34cXCcWlxkOyqSt0LKasBfr56jtlZm6fsxudUOut48mKSQoZdXFPKkM2PUHrbgiRb//YN/wUPuGnNqFIo09I0hNrCRugAoDFUTUZEGW0imleFXCy/yL+Y+Tm++ythN/3ZW5QHBJAkiDHGaMW5DsbFdRgcWIpDMPm3I3+qhLl7dDaFXikXKizP86JeOMeO2+KK/iS0kRZGQzEV0ay6l99jr8sPgcBIpIbLD+ESFzqLgN6ZeZcZqsZUW+Jv6GV5cmWXplRBvpZVt3u+WGp2miH6A20zxtmza0z5FN2LK79Cr2rSXXHI1G3c9hxgMu1pHZC1Y7vY6GR4eRhuEMFnK/7BGhnFtwnGP9lK2WH5l8TJfGX+aqkzxpeK5KMcLwTH+j6eepHBVcfJvWshWHxqt3awhQ6ZX8ptFTNFH+w7lyyoTm9e7mHZnSFzugKFCZEX8JqtsPa6QF9o85N+ir13+qPYRLj59ivlnUnIvrWbtUd6GREnHRpRLDE5N0Dxpc2xylVP5bfIyxBZpVtjSE+hi7vWVk/cLRmO0RA8CRJIgtxWyZWM1vcwr0w92iX3WtHaoYygVMWMlth/16JxOeKi4SlEO6Bmd3Rq1jfZM1lrksAhX92KPpxejM28bwN6w1hBCWgjXQY+V6E/b9KcFKgC3Zai8UEc02qTd3rsfRvv8HchcDjk5zton8ySfbPPZwkt4Ar7an+B/vPR5Bt+eZPa7Payr67cP03ukxAGQrS/PJZot0T5u88j8NT5TeYWyVNTTlJW0wNfWz7F+ZZKTFwfYtT6is0evZ1nDshTpm4qkyijFaRl+dGMRvSigAt+8eYbBzSLjYQzJ/tQDG3lszLnjdI8XaPz9Lp+Yv8x/Nfk95qwOS5bDSE5RSwfUteJKPM6f1h/judoc2zfGELHEFJLswiYM5Z+4LKyk5J9fwXS66IModzBK4JmcoPvIHDd/zfDEhdf4e9M/5FF3DV8YtrXghXCW73dP8scXH0Otu/jrAm1BXDKMfXSLf7j0fT6bf42KBCXAsxMiXwxF8wcLk6boMMTeaFF2JGE1h902OB1D+eIO1Jqko9psZB4sESV4VkJBBbgiy0iPhcYvBcRF93bdr7uIw0mkhtCOQNsGX4VZyq322eiV0F0buxsMxeFvswmPbrZyzwTZjRYYbJVStgOKbkTD06SuxDgW0rKyooJpOiRR+3Tj2KuVGj6zcW2ikmIwJVger/OR4jUWLLBx6JuYZ/on+drmBYqXFeVrCfLGRlY9eBC8bsMSqR4WQ0xQgTf8PAPDVPM7BiERxQLBXIlwOuH8eB1bJNSTKi/vzJDbEPg3eph25826glEWje8jyiXihXFaxy26S4ZHi3VmnRa2SPBkjKsStA3GsfZlkbwJu97O4T+DEJIUoc0wNJxmOi8hsucblp4wBZ+05BGVQeZjXBmjyG6NMQKNRHua1FEIyz7UtWmMzsLmQOZRyl7M/itkVvaiWCCYKxCMSeKCIbctsHsmq3PW679eT/R22E9CKQQi52WFfqcNT8yuIYXmRpLjj7efoHatyvzlFHujmWUXJiO73+X5dvtacjg8V0KQ5BRxQTDh9hhXXbQxdIzFSjzO5k4Zd0thtcJM76b37CVy1F7KzjyTw7ZVo/3WCgxm0+NKcZyiFdLbypOrSVSoM2/W3TbNdpCVMnphivqDJTonBF8+/WO+UHyehx2FLfIA9HVEaBLWUofr8QTf65zimfVFOmtFcmsWQkOYWGBAaEF+PSV/q49udzAHVTJAyKzUQiFHMKaYmN/hgdI681aD7TTHTlrgb9rnuNic4+rmBPkXPAqrmsJqQGpLwjGLtfkxrkxP8Sn/CqlJSA3EqUJFWc22Q4E0RQxC7HaEV3NwOhqnoxGdPjp4w9mW89B5l4lcg0mrs1unTwqROSb2CYeaSKmBxu5ZXO5PM+n4eDJmfaeMs6OQg+yGk4Ua3vCLe5vHOsPK4KU8YUURTmhmxjosF2ss52q0Yo+bYprUhaTi4bS9bCiUgt1aMDjACX0AACAASURBVHd5AxiFNIzGGIGQClyHYNqneUqR+1iN35z+CU/mrlIQPm0dcDn2+F+/91nmvq5Y+Nub6J066dsscBNnmSVmMGDUYy9L4XfunHt6SISCExOsPely/sx1fmnqJeppgWfqx+k8O870lQS11SAd6dn2FCSVnpsdYqcXaC/7bH7SsHBmg9+afZlP51+jKAN6xmHWbrJc2OFW4ThJ3sZWKms9st8bmzFgUozR6G4P4div0/eIQmFYsyULcRkliWfKBJMO4ZjGy8XU4iI9NwvdBkZhi5TiTIdgfCw7DA6R+HMXQ7sRAj1KK34rQlwpMzg5wa3P2CSlFONoKpcs/I04u9GH4VvPu7cK7+3T+ErXhZlJtj5SxD/T5DcmfsIz/VOZwPhfneLElQj3+evobm8YjnxvBErY1m4xXJOmQz3YwRAqMQy5JnlJVDZUrD4SzbY2vBTO8nT7NM5LOSZeTBC9AFKdPfuwsrcu5jM3RsHPQu8jXaOlMLbEa6RMf1/RaE3wzfky1WdVptfcaGXe77tmWOatUdOTdB+d5+ZvaD7z4EX+0dRTPOIMKAgXJSSp0SSkbKYRm2mOP25+hB/Wllh5cYbCTcn0liZ1DKkH2s5E8ioEfz1EbTRI30sE5G7CtommiwTjgqVim37q8MPBMl/busCrt6Y59n9Z5NZ7nF5dxfRuh52VUni+T/P0eZ4/Pk8wpoCElrbZaRQ49VoP09qfmkvvCGMwOmsppHZsytfsrAVRojOCNapwPkS6PEfzbIGvTP8pT3qrKFEAQBnBoO+SG+zPYx9eIpWmuDsBuU2bb944zanJGsuFGpCl2qa+g3Sd3d5HI02TGDX29VywHagUMXmPwXye7pxETwbM5NtMOl1cGeOpBONptJJZkpAaNggOgn3f5HbDe1JgXJvBpEUwpfnE5BrH7B1cAQMT8WxU5H+6+XkKl2yKV9uYVhsdhO/8vHs8XsIZFrJMktsFO+8AhGMTlS2C6YR5v0VF9bkVVWkEOZyWQEWa3RYUo3Y1jpNpLHI5TM6lP5ujPyVR431m8m3GrCyskCLIi4gZu8lZ3+UvjqfYXZfpFzxEFO1bBds3wQwFxqO2JkPPg4ljRCJ2iZQQAqsxIJdqxl706baLfNU9j7cUc6zUpK9tFJrFSpNLExX0whQyjkibh4xIjfA2c01YNjKfIzgzTeukQ3osyDR/icTpauzGMOz5bvNuRPhhf5qNS4Uol0jKHnFR4FkpPe3yV9sXeOnmLMdvxLgbHcxg2DsPXudp2m27NPQaCM8Fy8o8N8U8WAqiGBlE6EZzV5C/30UcjTYIrbEGGrureL41jysTNtwtftQ9wbPbC6iQ7HJiW1kSDGA8B+M6xBM+RgpUkCBDhRzI7D1SIIMUWxtEaijctLE7NsXVGHcnRHT76PDuZbepYhFRLlF/coHmGckjp6/wC5VXOW518YSLxjDQAR2d0DGCn4bz3Igm+GljgbV6CbsjQEPqCLrHBHHRYGYHmIaDt6Eyb/NhKLKqNQiQCax3Smz38xgj2Hl+kuKqwL+6hWh1SNvtjETpPbrbJCH1DMeLO+RFQgz8OFjCNB2sWgMdHZK9xmTdDITWGCFIigptC0Q6iezHqG4fXfKJqz7rn/ToL8cct2uZkJ6sBVdsNGnHxuruzyMfTiJlDDqKsTYalIsO3RdKvHJe4swnCGHQtiHJWyjfQ3guZpA1ON1t8pvzEIU8xveIZoqEFZv2oqJ3TLM4U+dkocas00ShyVshKh9jVCaYHNVwMqneX1fnsDgnZLU0Us+hNyex5rp8vvoCx+0mnhB0dMK3Ohe48vQS8z8JERcvk4bvQqJGfx+Gm3wW3tPtzu1D4cNiGM4JS5LibJOl3A4V1eeFeIF238NvGFSYgpWJ0Rn2rxoVH03zHmnBoTurGEwbpqttZrw2noiJjMIzkqIMOWY1qcg+C2e2uCWnmPnLfNaj7qCIFHuyLuNklzSZfpLp0yxrqLXTiF4fa9NiulaieKrKmjXOD0rH+WT+EqmR2CLlibGbvDgzT3e5QHm7AK32oQ3vvQmj0gjFIvXzLp0TmocW11hpl2k287j1GLXdJI2id77Vj3qcDQm3DsPdg+CuPbdtQaVEVHGIi4aCSqklRV68Mo9/2SH38k1MZ092MOyWWNltvzTcf7AdRDGPybmkvkMw6ZG6EqedYLdCZBgBfQgzD/S+j2+aYncT3LrFaxuThKnFzXyVF+szbK6MUQ4ZlloZ6qGURJdypAWHwZSDkeC0JHZ/SCLjFDSoXggDieoK7E6clV/ZbCP6QRYKje/eGhVjZeL5KhufSVleXuO/OPZVlqw+k8pFDsXkdZ2wkbqsxON8t32Gq90Jrq5PYBoO+U62+cZFQXR6wNREm09PX+Ubt87Q7VbBgHlDWGnfseezZWSoN/KYloPdkhz/ywBnZYf01trbkj0hBHHB8ETxBlWZ0jGC77VO4W4r9OZ2Jgk5DDBDiUSqMZYgGFNEJUHs57ECg9vw6S44tE9ILvzSa3xp+gectmK8oT5KYwiMxmor3NaeS89dXGeHk0hBJmatN3Bv2Ez+dIpNJ88LYo44sJASuvMORgly8SRq2Kg3WpwgHLdpL1oE41kVcGuQ9XTSlkFUQyrugHqUp5e4tBOXi9uzqKs5Cusp1nYnE2pHcZYJuDedeT+hFGnepns84cxEnSnVQWEIDPwonOFbG6eZ+rHGW22j3wcRGh34utPNMt7uYIaUkCITogJRZHFjME5qJD9tLNDf8RlraUSkMZbK3icVIp/LfjeMUP0AVZdMhRWSVx0Gz03xrcUZ/mLqCU4+dotHq7f4UuUZfJlQsfr81sJP+JpzgXhhCjvV6Furd8yW941RUsLwQmek2BXiGjksp2E0ZnToBgH5QchSe4LL/WX+s0e+xH/90Fc556xz3K7RejTHn+UexG3MYG/X9qUy7wfCnvkz6lMWfewMrRMOxV9b5/HyNucK6/zL+seRax52o4npdt+59teoGbRtZdlho7pddzGcIiwbWcjTPVfNGqAvRASRzdc2z5N/zaF8Te+WRRGWhXQccF3M9Dhp0SWYcrPNvijQDmgbopIhKWgox7h+gBCGYLWAv+4y/QMXd7WF2NpBdzr76+UY3vadlQbVuEx/usDVCZ+rExPogYXsKYyCOCcxtoJEI1KNdixSV4ExiHSY8asBDWIQIaKhmFxrjNao0YV0MEDHCebdyPMHwcirrRTxwjjtEx7FqRbHC3U8kaCBvo7Z1oaVpMS/3vkFXqzPsro+huhZyFDgtCUqAKsHwQREVc3PnbrCA4V1pu0WXzdnyW0KrHbw7l7//YAxGCmwBmDd8HAbArdhsHd6mN7bx7FksYhZmoXJkEe9G3hCcinx+c7VU5Q2DCa6O/UEPzCGUhRne0DrRIn2MjAX4noRJT9gLt/ms8VNPld6kWWrhSuG2e0mpatDmlqSVGMGEw5jljUMcd69PeQQEymDDkJEq41/K493vEy76oHMPDdRSWCFFlavgEjzGEvSOOsxmBT0T8TkJ/rMFHrc2h4jHShEoLDsFI1gJ/TRRtIMc7RaPoUdgdPK6uDoMLq96A9i0cjhBmRLRD6h7A6wRUKKoG/g1WCW7UaRE+sDRKf//v72qMxCFN8Wid7hZ5epIY4sNoMi2gg2WkVUOyvaKKOsnMHuQSplFuIJI0yYpbNaYYRlWXhXLXLbE3TnHV4bm2GQ2Hyu+CLzok1Vah71blIbK/JUdQGr7h98lpvRmGRYjXvveh1Vyt/zbCbK5pjd71M+doZavkj9fAHfTZhR8NnSS/ROurw0/iBuzjuYxszvhj36NiGHvTBLRdqLDu2T8A8XnmXabtJJcwQDB29HIPrh7Q37HewRKmusPTp473bRVWFbCM+jP6kIJqAw1ic1gvVmify2wavF2TyVwz6YY2VMwad3skRYVvSnBeG4IS4nYBmEnWUMzRe7nClvMelk8YVv5M+w4Y7TWXdQQR6724d+/46G198VIw1Ku4MN+Js+QksGyhmWRclKixgFRoldjbwwWcjOCrKxUJFGxlnLKRHFr+uRaPaGiN5UZPUOQ2S945KcIs4LPDvBkik7aZ7YKHwZ81I4y8vBHE/dPEmwkadwUyEjECmocNg5IoXUA1ONeKi4yll3ndhYhLFFsW4Qg/B2csFBYfT9GVCxwWlLnLbB6eiMxO4K/+Xrn1VIRN6nP5+nUOwyo0JsYdHRHmnNxWkfcD/Lt4DRJguF9wK0VSYtxTyxuMK54iafLFxiUnWYUyHesGcpZAVUU1ICo4mxcIoRUdnJLmZ3ud7g4SVSADpFt7vIGxuMj+ewApvWKTC2IRyDcEzSOOcTHg8ZG+/wlZNfZ9Jqk5chgXboaI8/UB9jrV2iu1kg7jq8eHMWncismXEscXYU+XWN3QwwQbCnxcMBTSqdudJFYhB1l5uTY1yfnKSn23R0jj+89jh6NUeai7Ds9z98RhuEPTwE30vW1HvFMJPHbWnsKzleGBxDOCm5Vz2qa4bcSjvL1hpm7BljEJ1haNHsSZ3v9nb1Jk6tzrjrUH1+mta5aX7/H3+Oz0++yN8vvsq06nI+t8a/edhC21Xyl9TBbQZ7P/O93HqGrmvdNpQv94A8Tz95kmPODnPWNqftGr9SfY7vH3+EwplFxHOvHS6v1N5kDstClEukS9O0Tudp/rsDfn75Mo/kbrASj/PV2oPkfuSz8Jc1zNrmO9/qR3/Xc7Pw86hIbRTdvcukVMjJcZLZMRoPGOz5Lp+av8bTt5YJ1vNMbqU49cFuGJqcy8ZnpuicgIc/fYlxt4ctNM04Rzd26cRZwdhUS6b9DnNui/O5VUoyYGGpzvMTx/jW+GnC7xWZsiV2EKDbZn+TCozGDAJQityOJnUV/VMpJhEwUMhQYPc0ItFZ2K7VweoNsKTcc5ibLNQaxZm2Jk13Q6+vI753M0txJIeQAquX4DVs1tYqfLOV57vuMlIajBEkL5TJbcH88wNUt4vsDhBBlEk3XAddzBFO52metzkxV2PZ2UYJzdea50kuFZn45k10bedQEA2TZvMxdSUylsS+IHUV9skqTruI1RxDdvuYRut2CRY/x+DCLDd+Hb609CITMvPe7CQFCtcV/tYh2ltG0Cm630dubGP3phCxZNztccLd5oJdwxbgCEFgDH2TkScATxgkUJEJv33ux/w/6jFE3r/rOtrDTaQgW/RhlKVC1hVBQxEXst5BaV5jl0MeX1jlXHGTJ/1LeCKlry2u6Em2kyLrnSLdho/dVGhLoj0F0iCMQHUldltgDVJkkGS35b0ek4OAvF1DyOoJ2n2PtbgCZHWGLKXRniGo2tjbud1ssfdKIsQo7DQqrHgnYQwyNtg9SFoKYylymwZ/O0F0+1m4dDihjTYYePNGa9LsJSEwvT4iipDrinzB4YWr85TsgF/Ov0yMlemnxjRhSZJXahhCO2Qd2N8JRiPbA7y6y2s7k/yksMRH3A1AUpF94qIhHPfIHVRm4ttgty2F7yPyPvGxCZpnfeoPGh5fXOHj5av0tctPe4v8+OUTzN3SWSXo9xA+GDXwRqlde+/aehyGEdPJMr35HGIqZKbSYdLpIqVGpAIwGEsi/RxppUA4laO7COmxgJOFGp6MacQ+tSDPZqdIq+VjIgmxZK1U4UZ1jPaUx3Fvh6rVZcnb4cRElcuTBfpTDhXPyzR+B+TsEEMyorw0y5twNcGEizCS/C2FBEiS24fQHu+ZSfXtsOto3zyI8JA2qH6E07Lxrzpox86ijibzNFUva7ydGGdlJ/N+ByE6SUBrhO8jbIvUkSSllLPlLSqqRzPN86PtY3g1MSx1cXAazNchTRHdAXbfRcY2cQEiB9A2KrSwAg+nW8atT+xmwMYli/pZm6UTazzgr2ILRWhi+trF6htkOBzTQ7THQLbudRiSX48JLzv8dek8Pxo7xjeq5ynaAQUVMm738FVIUQZMWm0ecLawBUhg1m6S96LdjNO7iUNPpEyaQhBibbUpAEb59GYkg0/0+NixFf6jme9wwW5Rlg4g6ZqUjhY8PzjGMzvHCV6pUNgR5LYNcUESjpFlZCiDv5HFl51WgugNhpWZ35DWvF+TS4jshj9qWJwavB1BeyfHC515+r6LJ2OOl+t0pj1aJ4o4rTy5nSK6Meo59w7u8721bOD1upM7UZBzmLlmDRLstpXpgYDqK32s7Q7pxtb7y74alRcIU9LtHRwpmfrmCZ5OT/HTiTnG1TAdYzZksJlDui5am6za8r2EZpvcTcHKpXH+TDzAF4rPYw/bUoQTKd15C9+xs5pfB0kS94byRlmxs5OEUwU2P+IRPtHj9x7/Ez7qreAL+KPOA/z51Qc4+Qcp7rVN0p36u4+9kMNNT4FtZVk7d7FO2KgKf+NsgeYZyeNLNzmV32bR3cGzE7oCUleQ5m2gSGe5QOO0xH+ozqPTmSZvLajwanOKlRsTuOs2069qnLbG6WT6jN7sNP/2o2VOzNf4yrGnWXJq/OKk5rWlaVrtHOWLeWh3sizhfYQYFv3VliB1oFgYkHcjCk7Idb9Ko56jcsnCasssbJemu95jGHq2h2OzS3T3hu9G8+Vu751DfaKqd8gFMYvrVuZJ6wdZ9fUgyJ5/2IP19V+CRBqDmKgwmFBUZhv83eoPmVMdLoUzNJ6dZPpKkhWePST7iklTzGYN23Wweh69eUjmQqIzBiENlpUStl1Uw8Mog7EN1kTAqZlV/pulP2XZDrBFnq4O6WgPt21QQdYHNNte9hDhgyZVOtv/vadeYv5HHskPF4lKZa5VqkRFQVQW9BY0VCJmp5s8WF3n9NQ3sIVGAVWrS8ENb5ehuYs49ERqBOM5RGWH+nlJuBDxpbM/5dH8Dc7aLYoyM+O12PBU/wL/4urHaVytkl+VjK9orFBjJKSOBJOJzzHZ/wpNxsj3qeruLvaER3Zfsiyw7SGR0lg9g9WweG5zHjVjmHVbTLg9yvkB9WqBcMzCqxQRcYwcFeJ8l0av2QZ4u3DincTIw2UsgYpAxlm1YxEnHy6F3WhMr0/5Sp/2iTx/Vn+ER4orKIYeCwXkPESSHL66S2+FEalVCgYBoiGY+uE4nfUJ/kHvHzFV6XK6so3dUsiELKx0AD0FgSwpQKks3ObYCN9HjxWISx61h336s4aJj27wizOX+Ki3gitgLXX4pz/9LO7FHO71VUyz9d49pkJkh9Yow0uObJd3nEgKy0L4Pt0FSXxywL839SNKMmAnLeCoFGNrtC1JPEWct4h9gdDQvlLh26slnLrC6gm8HcP8jsarBzhbPQgjRBhj1/LkNnMEEz7X1Tj12QJzdoPHc9exnU9l+9AbvW13++I2bJsiqhWSqTKtE4reYsqvL1xmyu5Qtbr8pLDIxdIsUWUSp+UgpMw8Mka/zju4l0y9CftdOqbbgyjKEp+HTcUzrat+h8KpWWZ2PJ6n9rjmczMrHLPaeMIQGBt/TeDuHDIR9lAWINs9CmtFwjGHXs7BjEVIabAsTeonJIBbCsm5MSerNR4qrTFnDfBFdlZ2jGYrKuHVU2QnQL/hM3axX4T4HTDSLNvXt7AdG99zMK5NmrOJxhzigkN7cYavnR3nC595gdP2NtNKU5F9clYMwjrySAEgs+7iUUkRHQ84v7jBl8eeYVppJlSB1GgGJuLFaJ6/rp2n9+wE069oSlc6yCjF2IpgykMmo+rCgBFkHVlM1kxzGG7aN+/TMHwh9jJl20Y4dta0VxusAThtQbueZ/v/b+/NYyzJ8rvezzmx3zXz3tyzlqy9u6a7p6dnsWHGi8wgGVtgQAbMk57ek4z8FxKWQGCBhHj/gIUAiT8AywgQIMN7lmfMjDQj1OPxeJue3mamu6urumvtqszKPfPuN27cGxHnvD8iblZWdy1ZVZlZWdnxkVJVdSvvjfO9cU7EL37nt4wUKFs9imZAxfNZKyVbWnHZw+x4STbNILzzVHHPxaB23Xi6CxWjhUBL0kJ2OomxiOIn257RGtULsOY3yK3keH9jmqIZUDIDEBplJPWo9AFocbAjhvFFQmxdJEZ+vEH+doH1QYHVI3k25vJYbYGMVOKhMYykTtW+jlNseW1kqZik848WCCZcehWTxgsRo0ea/OPT3+Y5a4NTVoG1uMtiNIJ7waPyQUS8tPLwxq7D7Ku0WCQqib8Rw4ufYYDaA+1pUdpgTHN6ep2f85YIteaNvoVlxCCTmnXKkoQFibJAxlBYkBiBoPJhH7PRx1jZRHd9VC8gHt60hUSuW1ieR+78cwzKDr6yceWAk2YTy4ru3J+HZVb2us/g0Ci2bdRIgf64iz+tyM12+LnSJabMJlXZp2J2cGTE64VJYtfEEiK5ZN5jDT/VMIhtKN9H9MRW2MCOPUhSEpYtps+u88XSR4xLQQz0lUVuPcZs9lBP2zPzMfRggO50cVe6uFMWYVESeAZKapQSSENheDFjpS4TuTYvlpZ43lukIk0cYRFrRS222BgUsBt9hJ+WdThIBuM2dJTcn6PFpbteF0LgmhY5zyX/wgmWyXPjz00wY9axRERJBuTMAT1p332f3QMOvCE1TE3e/EyJjc9pfuWlt/nZ4gccNwWOSOoh1VWPdwcl/um3/galG5Ljb7Yx2kGSaZFzUbZBv2zQHxEMyhoZgtUVFOdjvI0BxnrjTt+wj7Pbiyi9mMnKSFKM8uNuRyGIizmUY+K0Y/ptE9k0aQQevZzFqOlzNF9n40SeWlAldguMmRJrvYMMk1gGHcd3smXu0pIGdqvdN6a00hBGGEGE2dUom8QLmLcRPTdpXhs9wUIdxsr5mo1mng+8KUYdn1KhR6fg3anl8zT3+YcGspD37sGWeqKkbaHTYN3kHIXIhSWsTY/pxji940Wat4o4DYXTUggzufl9oq3OXmtJK0WrkSLrnx+lPyIIJjSDaoxV9vnrZy/wYv42n7U3yAuJrwb8oX+Eb9depPJBRP566+42MHfNc7nVbgTYKmIp8vnEmDJk6mUI01/ffY+csEy0axOXIuaKmyhgGKrU7LlYDQOzrzB7Me5GkMSpCTAbvWTrqJkkTsTD4prbb946Dd4eDHBaGqstOOcuM2W0uBWV6DRyVDc1op8kXiTv2btYMITEKJcQpUISWH8uT+eo4ORnF/hi5RYTRhsbRaANVsMRGqFHb0Jgd2zsnIcE1MevJ8OHgaEnOlJPb/3FcTK+R5gkwjCQUxM0T1j8i9Pf4py1SU463IoGXPMnKF1uIpafbhPfe6I1qt1GfhQz3ugwVsgRjnpoS6Klg7IkyhL0KpPcKE/x7peO8Nljt/mpud8nlhqlNf988S/z1qWTnL85n9YTvI/H+IAZkXehNToKUT2wFjbx1nPcDKq84OYwrDYzZo/juRoXJmexBiG0Wns2lINvSLkOVEbwJwX2kQ4v529xyqrjCA+Fwlcxb/Sr/GHzPOWrgtKtEHO1sfV+bRkoJ3HLKwsQGqsjsdok3cqb/aR1Srg/Llwh7xT/i4sucd5CpzFRaI3QEOWT4McwJ4kd0IbGMSMcGVE2fRSCY6U6G+NFer5Df8xGqDyWX4JeD3rB1hbePZ8YU8/UJ2IcnhCtNaIfYwaaIJc8wfvTDp4UWMsuqi8e3xgQcstbpzUoLVBaYMg7hUyfKsOGooV80nonfYpS22q7COtOTzLxsQu/6vUQcYw0DFzHIHILyFgjIp14Toz9bc48rOWkywUGE3k6xwSDEYWqhoyMdpktN3mlcItT1hp5IQnRtNWAtzonuLA2w1gjRHZ7SSUIIUkKDsmtYPJhy5Styu+GBMcB27prO0Hv5YVc6cQj7Rvc6lRYiCxiBGtRiY7vYLYFRqAxghijFSQtqaIYXW+ier20TcyD4hKTwqKRK4hdkgQCbXAhOIpoWLgNlQQ/73H5A2Emnm6mxxlUcnSPuHRnBMFUTNXt4siI+aiCrxzasccPGie50ayiBcSWgJyXtIrZPs5tWyWCMDlPQj41r8bdmYI7mDNCIDwP/+w43VnNeXuTcloZ+8PBODfaVdxmF+U/YomZfULHSVabGAygZmHX8ncaStsW2rVxNj0GIza9SZcruQkaxySBjgkRvLs4i7dgobr+/j6g7TY6Kd2gfR8z0Gz0C/jaAZLyHo6MiPImpmvv6TAOtiElBLI6Sv3zEwSf8/n7n/lDvugsUjGSrIP1OGIpzvHrb/4K1qUcc3+0vlVbSedctGcTTOXpj5j4k0kov92UVD6M8ZYDrJUG2u+hWp39659kGAjHof2ZKu0jBp3jyYVHhgIjEIjhdr4BUUETlUNyYz7nymucy61wxl4hdAxOOOtIoblUnGIzKOONGxSLFs5aD7m0Du3OnXou9/KMkG6HxfGTb5ukn6cHA2Snh7eZo3laEkxFtJ/T5G56HKvPYGzUieuNR1+4acXseLpCMCrIFwKmcy1GrB4XV6YxeiKpgvsULwZDIyo+dxR/2sMIFJYfYV9fS7TKpNkoQsDqRpJ+fq/6Uu0OxppNARiMumhTJsaFldRD2a8SD8K2kaUirbMjtI4ZFH5ynVG3hyEVJwubHPc2OGptUpZJ6vTNyOad4Dhf//ErFC7bWOsbSfzX8OJOGv9kWch87s6BTHOryr02JNq2EoMljasbZontxRaS6veRzRYTb04xXzvOfyn+FI4MudqZQC3kKdzWeKs9ZNNHtDro/oDY97f6lz3wPEgDmfcQI2Ua56D6wjrjRpe3guP8+0s/zegFQemdlbRdzB7G9QmBLBWgOsrSV8foTWriuYBS0Wcm79Poe/xZcIpXw+dY2Sgjb7tYbYEZgNtKHuwGR0cxN11k00m1CbRjJ0U4gz7a7yWhEXtYufy+6DTJ5hGRnoeYnmDlVwN+/sSHWyUBfBXyb299lZsXZji7ceHAGlLD8hM6iiAIUO32J7aGpWHgeR7Hm6dYapd54/wcOdmnEecp/kGe6rtJA+YDk5H4uKTlPOxOzLXGGGuVErCBr8GSU5huXwAAIABJREFUMf6khV3PPfRjnoSHGlJCiKPAfwMmSaKLfltr/W+FEBXg/wPmgJvA39Ra13dzcMIwUIUcnRlJsdDDlSFtbRJEiqW4wJ90nuN7K2exLuUo30hbdKTu+qiSJyxa9KrJE6EZgNEDp6XwlgPMjTa6nVwce1Gbi+p1BgSAYJYTHBNnCPWAC7xOD58BAUKI0SfVmLSSkPQqEn9aUzpTxzZjBpFBMLCIQoMoTJ6MDFNRKfocL9d5PrfMKXuNKbNNoA1CbTLjNdks5VkoJL2ihE7TmZW+E3ehFYH2uajf3KbvJMfEOULd593BH9NTHTxy6Me4ICXHSC9mSiNihRyopJK8F1Mda7PpFrgdVijfKFO41cVc3EgqHveCJB5maMR+/Gk23faR+Rxicoy1V4q0T8f8+Yll5nKbhNogaDkUO+AHNS50XmWgfO53Dj1yMGw4uMsMt6Y6x3LUz0rCokRGJoX5YwgFaMitx9jNCKfZSQyE3scqEYvUQ2MaKNciyhtoQ6ByLjKMCGTA++FrDOjdV+OuzVPHQZcKdGYTY/+zlVXG7Q6mVIxZbcpGD5uYEMlSrPlRb45XN85jrVs4dZ3UgBr2VYSkrIeZNMvGTD1z8o5XShsSpKQ3aHHh9jcYRF2IFUetsxx3PkNf93mv/ye7qpE4Rgd9Ckt9lOXyvfnTFL3EMJRx8jATuyaibyG6O5w2qWZZKqHmpmidyhMfDThZ3uT7vVP8/vvHWfgH/w022lzphRzhBEc59Yl5+thr8WNjQUjU3DTd4wWaLw3IjfaYG2miEISxwbX5SWTHxK5JCk3wNhQy1mgBYV4QeoLOjINdtDB7+SRmD9BCYHZDZKP7iZ6dgfa5yFsPvJ7uyVocesQetL2YBtvrcydonyrw+dnLfKHwEQBNNWAptrl5YYbKBZEG2N/7c3aicdfm6U752Fh1HCeJSEGYhLOImHbssRaWcJoao94lfsADyoHUeC9S3TLUNFo56lGemKSelCViYju9vuwhO/FIRcDf11r/SAhRBH4ohPgO8H8D39Va/6YQ4jeA3wD+0W4OTpgmcdmle1RxppCku6/HebrK4fuds3zj6otYbxeZfHeAu9hJqma7NtFoDn/KIRiVhMXESrc6msJSTP56A1FrJp4o308mm9ac4SVKYpRIh7zJd6noSZa5SYUJ5sRz/In+FjHRk2uUSfp4UBXER3r86unXGDdbhNqkHbu0lctCUKEX2/Rii2m3ySl3jVe8m4zLPkUpaKqItuxxzKnRLTp8lJ8hdgVykBbQU3ESg6PubBfepU9/l4qeYmUwT0VMMid/ipvqA+a5+qSnDJRChgptguWF/PT0NeJpyfzcKO/+8BSjo0XGhEDWO8i0HY8IwzTOwrjbmDLSeLJyif6RUWo/EfL8ySX+2tiPCLTFrf4YRs3CbmjkIOKs+QrFqECkBvc8hzf1h9RYm3pykffAMNB5j+ZJifmFOn/1+EWk0Hx3+Sz90CSMDYI3RijdlNhL3n0Dx4VhoF2bsGAS5iXKEMRFBxFGSMNKNKrifTXu1jwVrkM0kqM9p6icrfGVkWtMmC1yok+gLUJtIoUi1JKb0QivNU/xzkdHKa4IvFrqNTOMpCH10IAabtvFceKhM9P76DC2TQqkEjw38XOUrHGieo0f1H6PauEEi70Pd12jjuPkSfbmBqP+CI0zJdYqCrPaQ8RJq5ewaCKUi2x2E88Z3DuLcOjlNS2k56Inq9SfL7D+BcWXTtzis6XbvLpxnovzFZ6b+UUmbJfBrZu80f82o4x9Yp7uylokmU+t00Xqz0l++ZU3mXXqKC15u3mcq/VxSu/bFBZjSlfqCD8JPNblAnHJZfMzecKCIBgXyIFEhmaSQBInTXPduomnNUbHTzysw2MiHno93dW1qHWSkLGT78MykZ5L7TNFaucF/3D8TV601wCbdWVysT/D9Pc15beXiB7gKdyJxl27ZzwuWic9PsMYocASEeuqyEq/jNOIoNF+YFD+M6FxiFIY/Ziw6bARFlBaY4jEkFJ2EuKzlxEgDzWktNbLwHL697YQ4gNgFvgl4GfTX/uvwB+xm19mup2jTYGIwQ9t1qMiV3pTzPcqvHbtJM41l9ErEXZzABL6xyqEBZNe1UBLEDEUbsdYvsJdC5Knp/XNtF5UtLVN4uBuBa6bwiKni/Tpsc4Sn+dnALCwGdD7q7uiUSS9klTHwlc2tog5a60RI4gRNNwcIQaxlrgiJC/7VGUfV0CgNZvK4VJwhG8uvcSt22NULwgKSyHOR+tJ9lCne2f7AXCEh0PS186UNjlVYiAC1qIFvmD+HGiY5jjXufT4moZ71c02tpRYrRl6fZOq1WXSavKThesc+akGl1+a5PKXJ5GNSXKL05g+WF2NEQ4v0MM6NBCUE0O4dT6kONHhV0/+mNPOKqesdV7vneS6P05+SZBfjbBjG0uX0Tq67zmc5jjXeH/0ic7dA/SLMEJZMJ73eSm3wJTZ5HlviVpUYDUs8b/0S6zMFnA3RnCXTWSnu1WfR7gOIuehjk3SPpGncUoS5ZLvQpk58qs2+c40TquA6vp7P08dm6hgw0zAz85c5Se9G+TTVkWxFoRIGsplKRzl6+uv8NblE4y+bVG6GeLU00rJlonQDphGYmgMy2CkvTGTRIu0/6CVNLJ2DRvXGAOlsSyXvDlC4NdZC2/xeX56dzWmnlRVa2DEisqlPN1pg7bhIsqKxvMCf9LC6loUFl28tRDndgPR66dbWek+vGEgPBftOfgnR+hVTRrnYDAdMnd0nXqQ45vtF6n/6RTT85rqegibdYxIkKN0z3n6RGsxRdg2MpejeVISnfP5cvEqloh4v3eUC2vT9K6VOXIlxF32kY1OWqhXovIOg7JNMCboVzT6aC9pGg2w5GK1JMV5nbSNCWO0unub867rzX6txa2EhqHn4W6PnkgLyOpjM3ROlVj7iwO+cvYaZ9Iq2RtqwL9a/gX+7Mppzn7UQa0/uJL5TjTu6j3jMRGGpD+RZ1BOYvRu6yqtyEEOFDykVMyzohGS+NzhKVdaJG1idFLE2uiTOBj2kEeKkRJCzAGfA94AJlMjC2CFZOvvXu/5NeDXAFwecZ9yWNEZCJWkE7vM9yrMt0eRqw7uBjj1pBq5ci36FYtBXtIfFZh+Uj7ArUVYrQHGwhra7xF3ug+Mb+jpLm0alKkwoI8j0sa6iT27Kxq1Upg9jdExWBuUmLEaOCLGFQpLQFX2iRFYaML0xiWBQMO6crgxmOCd9lHmVyq48zbF2yHucgdVa2z1cUsO9MkaNVv6RJUBAY7w0EJj45LWGn98fVolgfuGxPRB+yYKwYjhc8ZeY8aqUysXeLX0AjfaVa5UJ9FdE6NjYAwSg1mGyYVQS+hXFJQH/NKL7/Jyfp6XnQUsobCEYjUsM98dxalp7Ga4LQPu/ufQxoX7zPknmqeQbKXGCi3AMSIqRoejZouTVov12GYxGmFxZoR3zRk6Rypoo4g3mEQMG73mXFTBpXUyT/uYxD8egakgFgQtCzMwyNvWVtD5ns7TNAA8diSlQofnvGWmjBiLJKg81ooQxY2wwLX+JBeWZ3Bu25RuhbhrPrKzrQ3M9riN1PjQvd6dYOU001ErnWhLDSoAP2zSjjYoyzIDHezJWkTrJE5ECApLfbTh0JswiEoxuhzRN0yivETEEi0t5KCIEXhb7VO0EGAIwhGXsGBSP23Rr2r0qS6TJZ+q2+XK5jjtjTyzH8bkF3uwUUP1AnqqTZv6PefpE69F2Kr/NRjRHBmvM2du0tUWsZb0fAenJrAbSVwjsQLTQDs2/YpDUDXpVzTReMi5qXXMtPbcxd4sIrKQUdLFQAzSbT117/T5p7IW4a4yL8IykY4Dk2P0jhepnzF5+cRH/PLY2xRlcsNdinP8aPko3ocuxsY60SPERu3HPeOxkZKwZBJ5Glcmhn+kDESsH6mzxYHWCHfqscnkzxiIESiS2m9Cc58VtTvs2JASQhSArwG/rrVuba/LoLXWQoh7jlNr/dvAbwOUROXRtKik07VyNbGSLAUj+JFFEJkY6UNtmDMJjjpEnqBfTjLzlAVuDQqLA9wb6+h2h7jZemigbqQj3uMHnONlTGHd9c2nE+aJNerBANodRq72MfoOvz/7WX40cZTVqfc57axwzKxTlCGu0LhC0FUaX5tcDcdYCKv895s/werKCIUPbI5cjcjfqCGW1pOtyrSC7z1vYkN9+jXOyc8l+obajHRL7T7rasf6tEYNQkS7w9iFPlbX5nunzxKMWTxnrzJndnjO6nJufI3+mMHKsRJx6nA10Mh0AHZa2TsnQywUVUNjkTxlrMYWFwdT/M6lLyKv5ph7r4WxVk/c8Fo/+BwKcd/V9ETzFJLCf0GA6cNqu8hCWKUkA05aAePGgJzY4O9M/DGb1QK/+2tfYr49yuW1EXQs0RpKoz5lr8OLo9cYtztM2w2WByPc9Ku8WX8epy6SorFKE+lwb+epkOn2oqSa95mz14m1TkKagLaGTeXwWws/w+VbUxz9hoG72sVarifBx2F4V/kNnbbj2OqruC0uQ1hmciOOIvS27MRIh/y4+W3Omq9gxHfHN+zWWtx6TxQRd7pYF+eprIzi1kepn7XozkqUo4ldTW8y6e3ZPJEjdjXaJG3sq1Em2NNdZiub5EILOzaIlWB1tUztnXGqFzSTS33sywtbD3ORGiRrcQ/nqcjnUZMV9NEef2HyMpNGyGpMcnMRSYkSf9rFzpmYvZj+qE1vzGDji4rCbJP/48R7TFsNZqw661GJ5XCEi1eOYDcFo+/Vkc0uutVJDNHhTXnb9XVf12LaDgW4kxlq22kvyCLhbIWbv+hhnm3zt858n68W3+eo6RNqeCM4ym/d+hnsV0tMf2s+6cKww4SO/bhnPAnCtuhMGUTVASWR1AgbsXusP0LI0EHXCInnLfYMnJGAUSsxgg00jogYFASxayL3sDTHjgwpIYRFYkT9jtb66+nLq0KIaa31shBiGljb9dFphYwVMhAEoUlfGeTMkLIbsF5No0GlRb8MsasBjdEXOA1wazH2po/udNG94KFGlNKK9/gBUxxjQswCYOPQ172tUgu7oVErDXGMVfPJOZLmNY8bzSl+p+fy0vgy5wtLHLVqW08Pa1GJxcEoP6wfY6lVov1hhcK6YPRqRO5WF7lWT7byouiT+93bLi5KK97TrzEl5pgQR9Aq8UL1dQ8Hj77yh4viSQVCHOOsdijaJW5cm6LddzjtrnLSXmPW6JCXmjyKnKxhoLEEWIAhEs+bgcARJiAx0s7egY5oxJIfBsf5Tu084nqO0nUw6m10t3tH4wPOYV/3IIn52320gkFIbk2zfqvM61OnoAQz5jUkUJSKvPSpyIC/VL3A7WKFD8tThCrRd8yrMWp1OeOs4ooQS0Qs9kep9XPYDYHTUknNoijkPfX9PZ+nRDHGQLPSKvJG9zRG/ipSKHzl8LZ/gkvtaa5cPEJhwSB/s4Fs+eh2B8I7dcy2vppY3T+hII6TdkJaJy1hYonSinf632VaHmeS2XSu7oHG7WiF9nsIw8BbtIncIkaQFOLUMokJ0gIQoE2RFD80h69pVCzxQ4vNRoEwMJF1i9yGJLesKd7qYa00Ue0OehCiVPJA86B5uhtrUTg2YdHB8/pMWC1cISnKkGmrwexYg1s9k3poY/YM5AAGJehXNNXjdZ6vrvC8u5Qk+CiPS/4MF5vTOCsmuVWdGFFp6rzW2+o3pTerp7oW07ITslxCl/K0z4zQnjVxnm/wpel5vlK4zJThI4Gl2OO19mluX5jiyHyE2qzdqYT+EPbrnvFECEnsgnRipNAEyqIT2clDs3q4R+rZ0JgkeMSOpJQPKBs9DJKHvpzRJywmJYUcw9izDMWdZO0J4D8BH2it/822//om8H8Bv5n++Y1dH53SGH6Es+nhzzhEyuBors6k08I7H1Lr5Wj4HgUn2cpaXxjFu21SvdjHma+jF1eIg/5Dq9xqrbnE2+Qpclyc3Xp9nBmWucUczxEyYFc0poaGWFwjV+9wrFVlMGLTna7yp89X+cHxOY5X6zhmRKvvstYq0NvIUb5kkltVTFxvIxtd9NIqejBI+kc9yMrWySbBJf02eUocF2e26ZtmKb7BHOdY5hYm1v0/Z8f6krRcdeUjvKU8x+VpNs+P85/NL/PTk9f4c4WrnDRr5KWiKDQ5aZATNpa4d7DosGp9W8VcGszwPxa/xMJbsxz7gz7Oh4vEm/UkNkuph57DZW4BNO55oCeVndZ1qfyojrdZ5Lvlc6zOFXn52DyTxoBxw8HEwDAlZ611FGuoyiUkaWbithtnR/epxTH/rj3JhwtTHL0Y4d3uotodLvp/uvfzVCtEt4e73ie4UuZ/6C+wcqxEpA1u+yO8//5xilcNzrzWxlxtEK+sEQ89UB9vQv2wQw0vbMPq2Qgu6jfIUeCYOL31/3uyFu8aiE5S3Xs9xGaN4nWPkmPDRBXlWcS5NAGgaNAvi6Q2lAfKEGgTom6OdcsjtyQp1zXV9zvIpg+1BrrTJRqEW0kgl3jrofN0N9aizrkEYzZjhRqn7FUK0sESIV/0blKc63FrZoy3zs3RCR2UFkznWsx5m5xzl6kYHUYMn6VolLc6J/jW5ReQNzymX4/wlrqoWgOGSSIfbx0DT2ctap0UrzWSmnPRTIXusRyLfylm7tgi/8+pbzBrdJg2bPpa0taKd4Jj/O9rz3PmvzQQK+vEO9zS29d7xpMgBZEHph1joGnGOVb9EiJ6SPkOniGNQiLyOQZFydnRdWasOoYQuGjGzTa9SUUwauDa9qNVvX8EduKR+jLwfwIXhBDvpK/9YxID6neFEL8K3AL+5q6OLL0hy1aPwmIB/5hNY+Axl9vENUPGnQ45c8CI22O5XaTVzjHynklxKcK5VYN6M9lG28GX1mSTFeYpUOZ1/R0ATvMCxznHBV5nUd8kTmoe/+au6IpjdNeHMMICzE0Hd90jt56jVymwViimT8Gagg/VtsJb7mK2AsRmA90fbLUW2ckNq6k3WOFWok+9mugTLyb69Oss8hEeOZwkbmFXGBoWuSsbWK0y9cYUX5+e5n9Wv4wx5VMuBHxxcp7T3hovuAscNZvk0ziMUEOgDTaVx3o0wtfWP8/l2jidd6vkF2H2owjn5kbyhJ9Wzt7JOUxTrpfvO+gnEpycV7leIxfHVP9ojGtHTvJ3//zf5vNjt/kroz/iRbtOJTWoJAKJgUq94hExsdb4OuSNfpXvNF/gvbdOMXJVkL++iai32AgWWdY3936eao1qtbEWJNOvOfhXS3x34ougwejD9G1FbsnHXNxMSohsS2x4bNd5+r4mG6zodK7qV/dO4wPGMWyUThgitEaaJoZlYaeFDlXORlkG2pZJ6IEh0KZEC7CbPQw/RK5sJut0e+FOdnat2ZW1KATCD/DW+lz9aIL/mPsZRmb+NyMywhWCWbNOXvZhBDqxix/b9JXJUr/Mu81ZuqHDhp+j2cojF1xKH0FhMSZ3vY5odYgHg7u7J2w7/8n15imtRa3AsBCeR/tEntp5yfMnF/iJyk2mjC45ASExKzFcDSf4l2/+PIULDmL1alJJf4fs6z3jSQgjcmua2obHh4NJrnQnWW4XmU7LkzyIZ0WjkAI8l9ATTDot8rKfPJ4KgSUitKNR5lMuf6C1/jO4r5/5L+zucD527DBC+gG5tQizaVIPPEJt4BCRN/s4RkTeGHCzViFq2lQ/CLCXWujby8nFa4eW54gY46v88j3/b5id8Ib+Li1dq+2OMI3q92EQIvp9RBpwm7/pUnAcGFbAVmnl47SDuRpukTysGOBO9Yk7+iDRuGuknbvjG/OYty0mLhdRRyboT3g0T+bpVgt853yOD8YnWR0rcdZdoWomJS4G2qAde3zUH2cxGOEHF86Qu2Vy8tt1ZCMJqo+DfuKCT7+HnZxDgD/Qv7d36Rtao2oNRKfLOFCcLXHbm+A7p/Lkz/YxSheY001y4s42pgJirQlJDMhN5fBHref5zs1zVC4IKhc76IVlVNBnRI3yVbE/81R1ktiX4hshxZxHXM4DIAcRopHUYIvT8iG7GXcwQnXfNN4XrdHhAB0CQZrWv73JuGFgiOTPYVV2HSeeZh1GaK3u6yne6TzdjbWoez3MjQ7ewgQ/LBzjrdETzNkbzJhNLBFTlAFHrRpdw6EW57nVG+NWp8K1lXHilo27YlKowcj1EO92B7laQ7U7qEF47+2v7WtR/I27XruXxr1Yi0lHBwmugz8hGRzv85Xqdb6Qu8FIcnelrxVLcYkf+3OU33YY/bBPvFl7JE/Fvt8zHhMdx7g1hdWQ3BiMs+iX8bsuQkdJbOwDeFY0DmM6lQ0lM8ASd7bvDDTaVAeijtRTQ8cxqtEkd8VkojRFszHJ154vY9oxYctG9gzMjqBySTO1NMC+uID2/cRIOegl77UGHaMHaitST/TTCPo0JZx0q2Rry+Sga7oXKkb1FSKKEN0u3kcWuXddsCx0wUPbOd61X+TH1meT7R2RZFigdZIZpTTPNzcQwQC1WSMOo7sMqIOGDgfoKER+tIC77HDmxiiq4PLj4su8NvETRJ4gzAuUkcTaaJlkKCo7Kdfh1DVuXTG7EmAtzKOaraQP5B64ox8sJPEIxxubSeDuSnLR1UqhHrUx7GFguF4VicG0/f+2t0Y5KPNSa1SzhegFzH1NElbz/NbLv0SUh8gDbWq0QZopK7AbydzLrUWcqveRvTaiGyD6YRJn2u8TD7a1s3lYE+qn8T0Mm2ybJtpKbm06EvjKpq08NuMmC1GZq4Mp/vVbfxH3qsvcq6uwWSc+rHNZa+x2jNUyuNCeZbVdJO6axI7GyOeg3T44c/ZJ0BoZwVK/TMPL4+smoYa2chGhREb67r6fu8yBNqTQSYCtbrbI3y4DHspwUTYUOmD6YLc15atdjLUGcav18E7zB43tLUKe9VL992N7OwMfoPmJXxm6PLeyQdPvRZOksj5TaI0KgsSb0Wwlga+GQakyCp6LKnrJVpAx/BEox0AojbXpJ4HbjSZxt/fUjcZhz8BnaEXtPR8/H/pgztBhrTy5sIxdyzFmzBJ5BlFOokyBNgANMlI49Qi70cdYbWwZTirdsn3k69LTvv5qBUphtzXmpsU7jSN0YocPrBmu++N8WJ/A+9Bl9GoMK+tJSZzDiko6TZg9WPbL9PoWxOJO66lhH8ynfc6eBK0Q/RCro/nx+hFGrB6uHBBryQe9Gay6xOqqOyUS9oADbkjp5Emo30e83qQoBSUzfdIYnvi0xUj0rHpsMu7msJ3D1IgkioiWVz4RlyAAIYYh50mWjErfl5HxxGiNardR7TbG2gYG4MpPRmpopZMtyWd53g3jT4M+otFi7C2b0s08N2snuO6ewG5BbkVRut6lMn+duN4gHu4CHFK01pjdEKfucG1+AvoGMpAoW6JdJ+3fyYF9GNgJOopQK2uMvmuz7o3xey9V+c7Jc9hmzNpamRPfG+As1In3sE/pwTaktqMSt/qOXMsZGQeVe83bZ/gilvEMoYYNoJ/yOPaSoQetFyAbbewwonLZJLYETjPGrgUYK3VUq723zaIPCMIw6I86DIoCw1Yo30yavAOkGY5J+MjTHeeTogcDxGaD0Q88LN/Dv16lZ0ClrXFur0G9uacT/9kxpIZkBlRGRkZGxr0YhhEonfTxNCTFdlpnrtlKSsYc1hCKj5MmRPTGTAYj4LgDepGD1U4MKW2mnQQeocL5QUVHEfHqGmJ1jeLrgiJsBdPvpSdqyLNnSGVkZGRkZDwIFaO1Qkega3WAu8t0fBpI2x+Nvr1OYb5IY7nE+FqEs+lj3t5EdzrEXf/wfSfD+Np93L3KDKmMjIyMjMPH8IZ6yOOgHoSOY/TiCrYfMKrGMDc6UG8RN5pPPZFlz9lHbULv58GEWAe6wMa+HfTxGePucR7XWo8/7E1CiDZwec9Gtbs8ssZn/BzC4de403n6adCYrcWDQ7YW78OnROPhXov7aUgBCCHe1lp/YV8P+hg87jifFX1w+DU+yTgzjQeHwz5P4fBrzObp3r13Pzns8xQeb6x7W+4zIyMjIyMjI+MQkxlSGRkZGRkZGRmPydMwpH77KRzzcXjccT4r+uDwa3yScWYaDw6HfZ7C4deYzdO9e+9+ctjnKTzGWPc9RiojIyMjIyMj47CQbe1lZGRkZGRkZDwmmSGVkZGRkZGRkfGY7JshJYT4eSHEZSHENSHEb+zXcR+GEOKoEOJ7QohLQoiLQoi/l77+z4QQi0KId9KfX9jBZ2UanxK7pfGg6oPDrzGbp5nGj33OodaXvifT+JTYTY1orff8BzCA68BJwAbeBc7vx7F3MLZp4JX070XgCnAe+GfAP8g0fno0HmR9nwaN2TzNNH5a9GUaD49GrfW+eaS+BFzTWt/QWg+A/xf4pX069gPRWi9rrX+U/r0NfADMPsZHZRqfIruk8cDqg8OvMZunj8Rh13jY9UGm8amyixr3zZCaBRa2/fs2jzngvUQIMQd8DngjfenvCiHeE0L8ZyHE6EPenmk8IDyBxmdCHxx+jdk8/dRrPOz6INN4YHhCjVmw+RAhRAH4GvDrWusW8B+AU8DLwDLwr5/i8HaFTGOm8VngsOuDTCOHQONh1weZRnaocb8MqUXg6LZ/H0lfOxAIISySL/J3tNZfB9Bar2qtY621Av4jiYvyQWQanzK7oPFA64PDrzGbp5nGlMOuDzKNT51d0rhvhtRbwBkhxAkhhA38CvDNfTr2AxFCCOA/AR9orf/Nttent/3aXwPef8hHZRqfIruk8cDqg8OvMZunW2QaD78+yDQ+VXZR4/5k7ekkKv4XSKLirwP/ZL+Ou4NxfQXQwHvAO+nPLwD/HbiQvv5NYDrTePg1HlR9nwaN2TzNNH6a9GUaD4/GrEVMRkZGRkZGRsZjkgWbZ2RkZGRkZGQ8JpkhlZGRkZGRkZHxmGTBjeGRAAAAQ0lEQVSGVEZGRkZGRkbGY5IZUhkZGRkZGRkZj0lmSGVkZGRkZGRkPCaZIZWRkZGRkZGR8ZhkhlRGRkZGRkZGxmPy/wPIONdE6Yt3SAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 720x216 with 10 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cMhOXA1V4SP_"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}