bayesopt.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YqbPufOoO5IV"
      },
      "source": [
        "# Programming Task\n",
        "\n",
        "Find the optimal learning rate of a simple ResNet on FashionMNIST. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GuP8Yi8qQJ4g"
      },
      "source": [
        "## Installation\n",
        "\n",
        "Clone the GitHub repository `minimal-bayesopt` and install its dependencies.\n",
        "\n",
        "***IMPORTANT:*** Restart the kernel after installing the dependencies!\n",
        "\n",
        "If possible, use a GPU! (Resources -> Change runtime type -> Hardware Accelerator -> GPU)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "-qC9_l9pOsDY",
        "outputId": "0ed01809-3725-4bd7-ef2b-85c8158c0bb5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "fatal: destination path 'minimal-bayesopt' already exists and is not an empty directory.\n",
            "/content/minimal-bayesopt\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Obtaining file:///content/minimal-bayesopt (from -r requirements.txt (line 1))\n",
            "Collecting pytorch-lightning>=1.5.10\n",
            "  Downloading pytorch_lightning-1.6.4-py3-none-any.whl (585 kB)\n",
            "\u001b[K     |████████████████████████████████| 585 kB 19.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.11.0+cu113)\n",
            "Collecting torchmetrics>=0.8.0\n",
            "  Downloading torchmetrics-0.9.2-py3-none-any.whl (419 kB)\n",
            "\u001b[K     |████████████████████████████████| 419 kB 55.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torchvision>=0.10.0 in /usr/local/lib/python3.7/dist-packages (from minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (0.12.0+cu113)\n",
            "Requirement already satisfied: scikit-learn>=0.24.2 in /usr/local/lib/python3.7/dist-packages (from minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.0.2)\n",
            "Requirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.7/dist-packages (from minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.21.6)\n",
            "Collecting matplotlib>=3.3.4\n",
            "  Downloading matplotlib-3.5.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (11.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 11.2 MB 19.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: seaborn>=0.11.2 in /usr/local/lib/python3.7/dist-packages (from minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (0.11.2)\n",
            "Collecting ConfigSpace>=0.4.0\n",
            "  Downloading ConfigSpace-0.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.7 MB 66.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cython in /usr/local/lib/python3.7/dist-packages (from ConfigSpace>=0.4.0->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (0.29.30)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from ConfigSpace>=0.4.0->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.4.1)\n",
            "Requirement already satisfied: pyparsing in /usr/local/lib/python3.7/dist-packages (from ConfigSpace>=0.4.0->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (3.0.9)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.3.4->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.4.3)\n",
            "Collecting fonttools>=4.22.0\n",
            "  Downloading fonttools-4.34.4-py3-none-any.whl (944 kB)\n",
            "\u001b[K     |████████████████████████████████| 944 kB 51.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.3.4->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (7.1.2)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.3.4->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (2.8.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.3.4->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (21.3)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.3.4->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (0.11.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib>=3.3.4->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (4.1.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7->matplotlib>=3.3.4->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.15.0)\n",
            "Collecting pyDeprecate>=0.3.1\n",
            "  Downloading pyDeprecate-0.3.2-py3-none-any.whl (10 kB)\n",
            "Requirement already satisfied: protobuf<=3.20.1 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (3.17.3)\n",
            "Requirement already satisfied: tensorboard>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (2.8.0)\n",
            "Collecting PyYAML>=5.4\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 32.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.57.0 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (4.64.0)\n",
            "Collecting fsspec[http]!=2021.06.0,>=2021.05.0\n",
            "  Downloading fsspec-2022.5.0-py3-none-any.whl (140 kB)\n",
            "\u001b[K     |████████████████████████████████| 140 kB 63.8 MB/s \n",
            "\u001b[?25hCollecting aiohttp\n",
            "  Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 60.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (2.23.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.24.2->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.1.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.24.2->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (3.1.0)\n",
            "Requirement already satisfied: pandas>=0.23 in /usr/local/lib/python3.7/dist-packages (from seaborn>=0.11.2->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.3.5)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.23->seaborn>=0.11.2->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (2022.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (0.4.6)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (3.3.7)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (0.6.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (0.37.1)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.35.0)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.46.3)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.0.1)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.1.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (57.4.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.8.1)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (4.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (4.2.4)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (4.11.4)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (3.8.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (0.4.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (2022.6.15)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (3.2.0)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (94 kB)\n",
            "\u001b[K     |████████████████████████████████| 94 kB 3.4 MB/s \n",
            "\u001b[?25hCollecting asynctest==0.13.0\n",
            "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
            "Collecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (21.4.0)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |████████████████████████████████| 271 kB 42.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning>=1.5.10->minimal-bayesopt==0.0.0->-r requirements.txt (line 1)) (2.1.0)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (144 kB)\n",
            "\u001b[K     |████████████████████████████████| 144 kB 51.4 MB/s \n",
            "\u001b[?25hInstalling collected packages: multidict, frozenlist, yarl, asynctest, async-timeout, aiosignal, fsspec, fonttools, aiohttp, torchmetrics, PyYAML, pyDeprecate, matplotlib, pytorch-lightning, ConfigSpace, minimal-bayesopt\n",
            "  Attempting uninstall: PyYAML\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Attempting uninstall: matplotlib\n",
            "    Found existing installation: matplotlib 3.2.2\n",
            "    Uninstalling matplotlib-3.2.2:\n",
            "      Successfully uninstalled matplotlib-3.2.2\n",
            "  Running setup.py develop for minimal-bayesopt\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "albumentations 0.1.12 requires imgaug<0.2.7,>=0.2.5, but you have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Successfully installed ConfigSpace-0.5.0 PyYAML-6.0 aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.2 asynctest-0.13.0 fonttools-4.34.4 frozenlist-1.3.0 fsspec-2022.5.0 matplotlib-3.5.2 minimal-bayesopt-0.0.0 multidict-6.0.2 pyDeprecate-0.3.2 pytorch-lightning-1.6.4 torchmetrics-0.9.2 yarl-1.7.2\n"
          ]
        },
        {
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "matplotlib",
                  "mpl_toolkits"
                ]
              }
            }
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "!git clone https://github.com/stwerner97/minimal-bayesopt\n",
        "%cd /content/minimal-bayesopt\n",
        "!pip install -r requirements.txt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IFTzb2XrREeO"
      },
      "source": [
        "## Building the ResNet Architecture\n",
        "The ResNet uses residual blocks as in the [original paper](https://arxiv.org/pdf/1512.03385.pdf), but also uses skip connections inspired by the [NVAE paper](https://arxiv.org/pdf/2007.03898.pdf)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "_KGHrcSLREzV"
      },
      "outputs": [],
      "source": [
        "from typing import Union, Tuple, Dict\n",
        "\n",
        "import torch\n",
        "import pytorch_lightning as pl\n",
        "from torch import nn\n",
        "from torch import Tensor\n",
        "from torch.optim import Optimizer, SGD\n",
        "from torchmetrics import F1Score\n",
        "\n",
        "\n",
        "class ConvBlock(nn.Module):\n",
        "    \"\"\"Basic Conv. Neural Network building block.\"\"\"\n",
        "    def __init__(\n",
        "        self,\n",
        "        in_channels: int,\n",
        "        out_channels: int,\n",
        "        kernel_size: Union[int, Tuple[int]],\n",
        "        stride: Union[int, Tuple[int]],\n",
        "        padding: Union[int, Tuple[int]],\n",
        "        padding_mode: str = \"zeros\",\n",
        "        bias: bool = False,\n",
        "        activation: nn.Module = nn.SiLU,\n",
        "        norm_layer: nn.Module = nn.BatchNorm2d,\n",
        "    ):\n",
        "        super().__init__()\n",
        "\n",
        "        self.conv = nn.Conv2d(\n",
        "            in_channels=in_channels,\n",
        "            out_channels=out_channels,\n",
        "            kernel_size=kernel_size,\n",
        "            stride=stride,\n",
        "            padding=padding,\n",
        "            padding_mode=padding_mode,\n",
        "            bias=bias,\n",
        "        )\n",
        "\n",
        "        self.norm_layer = norm_layer\n",
        "        if self.norm_layer is not None:\n",
        "            self.norm_layer = self.norm_layer(out_channels)\n",
        "\n",
        "        self.activation = activation\n",
        "        if self.activation is not None:\n",
        "            self.activation = activation()\n",
        "\n",
        "    def forward(self, x: Tensor) -> Tensor:\n",
        "        \"\"\"Apply conv. filters, normalization and activation function.\"\"\"\n",
        "        out = self.conv(x)\n",
        "        if self.norm_layer is not None:\n",
        "            out = self.norm_layer(out)\n",
        "\n",
        "        if self.activation is not None:\n",
        "            out = self.activation(out)\n",
        "\n",
        "        return out\n",
        "\n",
        "\n",
        "class ResNetBlock(nn.Module):\n",
        "    \"\"\"Residual learning building block.\n",
        "\n",
        "    Residual learning building block stacking two conv. blocks. Downsampling\n",
        "    is applied at the first conv. block (if stride > 1). Similarly, the\n",
        "    number of channels is expanded / reduced at the first conv. block.\n",
        "    The skip connection is inspired by https://arxiv.org/pdf/2007.03898.pdf.\n",
        "    It downsamples the input using 1x1 convolutions.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    in_channels : int\n",
        "        Number of channels of the input tensor.\n",
        "    out_channels : int\n",
        "        Number of channels of the output tensor. The number of channels is\n",
        "        expanded / reduced at the first conv. layer.\n",
        "    stride : int\n",
        "        Downsampling performed in the first conv. layer.\n",
        "    activation : nn.Module\n",
        "        Nonlinear activation function. Activations are applied within the\n",
        "        first conv. block and after the skip connection and main branch\n",
        "        are accumulated.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        in_channels: int,\n",
        "        out_channels: int,\n",
        "        stride: int,\n",
        "        activation: nn.Module = nn.SiLU,\n",
        "        norm_layer: nn.Module = nn.BatchNorm2d,\n",
        "    ):\n",
        "        super().__init__()\n",
        "\n",
        "        self.convblock1 = ConvBlock(\n",
        "            in_channels=in_channels,\n",
        "            out_channels=out_channels,\n",
        "            kernel_size=3,\n",
        "            stride=stride,\n",
        "            padding=1,\n",
        "            activation=activation,\n",
        "            norm_layer=norm_layer,\n",
        "        )\n",
        "\n",
        "        self.convblock2 = ConvBlock(\n",
        "            in_channels=out_channels,\n",
        "            out_channels=out_channels,\n",
        "            kernel_size=3,\n",
        "            stride=1,\n",
        "            padding=1,\n",
        "            activation=None,\n",
        "            norm_layer=norm_layer,\n",
        "        )\n",
        "\n",
        "        self.skipblock = ConvBlock(\n",
        "            in_channels=in_channels,\n",
        "            out_channels=out_channels,\n",
        "            kernel_size=1,\n",
        "            stride=stride,\n",
        "            padding=0,\n",
        "            activation=None,\n",
        "            norm_layer=norm_layer,\n",
        "        )\n",
        "\n",
        "        self.activation = activation()\n",
        "\n",
        "    def forward(self, x: Tensor) -> Tensor:\n",
        "        \"\"\"Passes input tenor thorugh residual block.\"\"\"\n",
        "        skip = self.skipblock(x)\n",
        "        out = self.convblock1(x)\n",
        "        out = self.convblock2(out)\n",
        "        out = self.activation(out + skip)\n",
        "\n",
        "        return out"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8ORTO87DR1zb"
      },
      "source": [
        "The ResNet stacks four residual blocks after an initial convolution and pooling layer. Similar to the [original paper](https://arxiv.org/pdf/1512.03385.pdf), we use global average pooling before the fully connected output layer and do not use dropouts or any other means to regularize the network."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "jtNXEn8zRNaq"
      },
      "outputs": [],
      "source": [
        "class ResNetModel(nn.Module):\n",
        "    \"\"\"ResNet neural network architecture for FashionMNIST.\n",
        "\n",
        "    The architecture is a lightweight version of the ResNet proposed\n",
        "    in https://arxiv.org/pdf/1512.03385.pdf. As in the paper, the architecture\n",
        "    does not use strong regularization such as dropout or maxout. After an\n",
        "    initial conv. layer and pooling stage, the architecture stacks ResNet\n",
        "    blocks. Finally, logits for each class are computed using global average\n",
        "    pooling and a fully connected layer.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    num_classes : int\n",
        "        Number of possible class labels in the classification task.\n",
        "    activation : nn.Module\n",
        "        Activation function to use throughout the architecture.\n",
        "    norm_layer : nn.Module\n",
        "        Normalization layer to use throughout the architecture.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        num_classes: int = 10,\n",
        "        activation: nn.Module = nn.SiLU,\n",
        "        norm_layer: nn.Module = nn.BatchNorm2d,\n",
        "    ):\n",
        "        super().__init__()\n",
        "\n",
        "        self.conv5x5block_l1 = ConvBlock(\n",
        "            in_channels=1,\n",
        "            out_channels=32,\n",
        "            kernel_size=5,\n",
        "            stride=1,\n",
        "            padding=2,\n",
        "            activation=activation,\n",
        "            norm_layer=norm_layer,\n",
        "        )\n",
        "        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
        "\n",
        "        self.resblock_l2 = ResNetBlock(\n",
        "            in_channels=32,\n",
        "            out_channels=32,\n",
        "            stride=2,\n",
        "            activation=activation,\n",
        "            norm_layer=norm_layer,\n",
        "        )\n",
        "\n",
        "        self.resblock_l3 = ResNetBlock(\n",
        "            in_channels=32,\n",
        "            out_channels=32,\n",
        "            stride=1,\n",
        "            activation=activation,\n",
        "            norm_layer=norm_layer,\n",
        "        )\n",
        "\n",
        "        self.resblock_l4 = ResNetBlock(\n",
        "            in_channels=32,\n",
        "            out_channels=64,\n",
        "            stride=2,\n",
        "            activation=activation,\n",
        "            norm_layer=norm_layer,\n",
        "        )\n",
        "\n",
        "        self.resblock_l5 = ResNetBlock(\n",
        "            in_channels=64,\n",
        "            out_channels=64,\n",
        "            stride=1,\n",
        "            activation=activation,\n",
        "            norm_layer=norm_layer,\n",
        "        )\n",
        "\n",
        "        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n",
        "        self.fclayer = nn.Linear(64, num_classes)\n",
        "\n",
        "    def forward(self, images: Tensor) -> Tensor:\n",
        "        \"\"\"Maps FashionMNIST images to logits for each class.\n",
        "\n",
        "        Parameters\n",
        "        ----------\n",
        "        images : Tensor (Batch size, Channels, Height, Width)\n",
        "            Batch of FashionMNIST images.\n",
        "\n",
        "        Returns\n",
        "        -------\n",
        "        Tensor (Batch size, Num. Classes)\n",
        "            Raw, unnormalized logit scores for each class.\n",
        "        \"\"\"\n",
        "        out = self.conv5x5block_l1(images)\n",
        "        out = self.maxpool(out)\n",
        "\n",
        "        out = self.resblock_l2(out)\n",
        "        out = self.resblock_l3(out)\n",
        "\n",
        "        out = self.resblock_l4(out)\n",
        "        out = self.resblock_l5(out)\n",
        "\n",
        "        out = self.avgpool(out)\n",
        "\n",
        "        out = torch.flatten(out, start_dim=1)\n",
        "        out = self.fclayer(out)\n",
        "\n",
        "        return out"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hkyisiVySYd_"
      },
      "source": [
        "## Training Module\n",
        "\n",
        "We use `pytorch-lightning` as a lightweight and modular training framework. During the training stage, we minimize the cross-entropy loss as an auxiliary to the F1 score that is used as a validation metric."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "FbFhsP-xSYnW"
      },
      "outputs": [],
      "source": [
        "class ResNetFashionMnistModule(pl.LightningModule):\n",
        "    \"\"\"Training module for the FashionMNIST classification task.\n",
        "\n",
        "    During training, minimizes the cross-entropy among predicted and\n",
        "    ground-truth labels as an auxiliary to the F1 metric (used during\n",
        "    validation & testing). Optimizes the NN with a SGD optimizer.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    model : nn.Module\n",
        "        Neural network that maps input images to logits for each class label.\n",
        "    num_classes : int\n",
        "        Number of classes of the classification task.\n",
        "    lr : float\n",
        "        Learning rate of the SGD optimizer.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, model: nn.Module, num_classes: int, lr: float = 1e-3):\n",
        "        super().__init__()\n",
        "\n",
        "        self.model = model\n",
        "        self.lr = lr\n",
        "\n",
        "        self.loss = nn.CrossEntropyLoss()\n",
        "        self.f1 = F1Score(num_classes=num_classes)\n",
        "\n",
        "    def forward(self, *args, **kwargs):\n",
        "        \"\"\"Wraps the forward pass of the inner model.\"\"\"\n",
        "        return self.model(*args, **kwargs)\n",
        "\n",
        "    def training_step(self, batch: Tuple[Tensor], batch_idx: int) -> Dict:\n",
        "        \"\"\"Computes the cross-entropy between predicted and ground-truth labels.\n",
        "\n",
        "        Parameters\n",
        "        ----------\n",
        "        batch : Tuple[Tensor]\n",
        "            Batch of FashionMNIST images and their target labels.\n",
        "        batch_idx : int\n",
        "            Index of batch in training data loader.\n",
        "\n",
        "        Returns\n",
        "        -------\n",
        "        Dict\n",
        "            Cross-entropy training loss.\n",
        "        \"\"\"\n",
        "        images, targets = batch\n",
        "\n",
        "        logits = self(images)\n",
        "        loss = self.loss(logits, targets)\n",
        "        self.log(\"train_cross_entropy\", loss)\n",
        "\n",
        "        return {\"loss\": loss}\n",
        "\n",
        "    def validation_step(self, batch: Tuple[Tensor], batch_idx: int) -> Dict:\n",
        "        \"\"\"Computes the predicted class labels.\n",
        "\n",
        "        Parameters\n",
        "        ----------\n",
        "        batch : Tuple[Tensor]\n",
        "            Batch of FashionMNIST images and their target labels.\n",
        "        batch_idx : int\n",
        "            Index of batch in training data loader.\n",
        "\n",
        "        Returns\n",
        "        -------\n",
        "        Dict\n",
        "            Predicted and ground-truth labels of the images.\n",
        "        \"\"\"\n",
        "        images, targets = batch\n",
        "\n",
        "        logits = self(images)\n",
        "        probs = torch.softmax(logits, dim=1)\n",
        "        preds = torch.argmax(probs, dim=1)\n",
        "\n",
        "        return {\"Targets\": targets, \"Preds.\": preds}\n",
        "\n",
        "    def validation_epoch_end(self, outputs: Dict) -> Dict:\n",
        "        \"\"\"Compares predicted and ground-truth class labels in terms of the F1 score.\n",
        "\n",
        "        Parameters\n",
        "        ----------\n",
        "        batch : Tuple[Tensor]\n",
        "            Batch of FashionMNIST images and their target labels.\n",
        "        batch_idx : int\n",
        "            Index of batch in training data loader.\n",
        "\n",
        "        Returns\n",
        "        -------\n",
        "        Tensor\n",
        "            Dict that stores the F1 score of validation stage.\n",
        "        \"\"\"\n",
        "\n",
        "        targets = torch.cat([output[\"Targets\"] for output in outputs])\n",
        "        preds = torch.cat([output[\"Preds.\"] for output in outputs])\n",
        "\n",
        "        f1_score = self.f1(preds, targets)\n",
        "        self.log(\"val_f1_score\", f1_score)\n",
        "\n",
        "        return f1_score\n",
        "\n",
        "    def configure_optimizers(self) -> Optimizer:\n",
        "        \"\"\"Set up SGD optimizer.\"\"\"\n",
        "        return SGD(self.parameters(), lr=self.lr)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BI14hDmsPKAM"
      },
      "source": [
        "## Defining the Objective Function\n",
        "The `optimize` function first prepares the FashionMNIST data, splits it into training and validation data before running the training loop. The ResNet is trained for at most `max_epochs`, although the early stopping callback may terminate the optimization sooner. The `optimize` function reports back the final F1 score on the validation data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "MkgDnPOFPIZB"
      },
      "outputs": [],
      "source": [
        "import logging\n",
        "import warnings\n",
        "from argparse import Namespace\n",
        "from functools import partial\n",
        "\n",
        "import torch\n",
        "import torchvision\n",
        "import numpy as np\n",
        "import pytorch_lightning as pl\n",
        "import ConfigSpace.hyperparameters as CSH\n",
        "from torch import Generator\n",
        "from torchvision import transforms\n",
        "from sklearn.gaussian_process.kernels import Matern\n",
        "from matplotlib.backends.backend_pdf import PdfPages\n",
        "\n",
        "\n",
        "def optimize(\n",
        "    val_size: float = 0.3,\n",
        "    max_epochs: int = 10,\n",
        "    batch_size: int = 128,\n",
        "    patience: int = 0,\n",
        "    lr: float = 1e-3,\n",
        "    generator: Generator = None,\n",
        ") -> float:\n",
        "    \"\"\"Trains and evaluates a ResNet on the FashionMNIST classification task.\n",
        "\n",
        "    Splits the training data into training and validation sets. Trains the ResNet until the early\n",
        "    stopping criterion is met or the maximum number of epochs is reached. Reports the validation\n",
        "    performance in terms of its F1 score.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    val_size : float\n",
        "        Fraction of the number of training samples used as validation data.\n",
        "    max_epochs : int\n",
        "        Number of epochs the neural network is trained at most.\n",
        "    batch_size : int\n",
        "        Batch size of training & validation data loaders.\n",
        "    patience : int\n",
        "        Maximum number of validation checks with no improvement before the training is stopped early.\n",
        "    lr : float\n",
        "        Learning rate of the SGD optimizer.\n",
        "    generator: Generator\n",
        "        Random number generator used to train / test split.\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    float\n",
        "        Final F1 score of the ResNet on the validation data.\n",
        "    \"\"\"\n",
        "\n",
        "    logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)\n",
        "    warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "    transform = transforms.ToTensor()\n",
        "    data = torchvision.datasets.FashionMNIST(\n",
        "        \"fashion-mnist\", download=True, train=True, transform=transform\n",
        "    )\n",
        "\n",
        "    train = int(np.floor((1.0 - val_size) * len(data)))\n",
        "    val = int(np.ceil(val_size * len(data)))\n",
        "    train, val = torch.utils.data.random_split(data, (train, val), generator=generator)\n",
        "\n",
        "    train = torch.utils.data.DataLoader(train, batch_size=batch_size)\n",
        "    val = torch.utils.data.DataLoader(val, batch_size=batch_size)\n",
        "\n",
        "    model = ResNetModel(num_classes=10)\n",
        "    module = ResNetFashionMnistModule(model, num_classes=10, lr=lr)\n",
        "\n",
        "    early_stopping = pl.callbacks.EarlyStopping(\n",
        "        monitor=\"val_f1_score\", mode=\"max\", patience=patience\n",
        "    )\n",
        "\n",
        "    trainer = pl.Trainer(\n",
        "        gpus=torch.cuda.device_count(),\n",
        "        max_epochs=max_epochs,\n",
        "        callbacks=[early_stopping],\n",
        "        enable_progress_bar=False,\n",
        "        enable_model_summary=False,\n",
        "        logger=None,\n",
        "    )\n",
        "    trainer.validate(module, val, verbose=False)\n",
        "    trainer.fit(module, train, val)\n",
        "\n",
        "    f1_score = trainer.logged_metrics[\"val_f1_score\"].item()\n",
        "    return f1_score"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XrdEkcs5UHlg"
      },
      "source": [
        "## Bayesian Optimization\n",
        "\n",
        "The `BayesOpt` class implements the Bayesian Optimization algorithm. The `search` method uses `scipy.optimize` to find a parameter in the search space that maximizes the acquisition function (here: expected improvement). In contrast to the objective function, it is cheap to evaluate when using a Gaussian Process as a probabilistic surrogate. After evaluating the sample in terms of the objective function, the `update` method is called to re-train the surrogte given the newly acquired data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "TMtoOGBYP3MD"
      },
      "outputs": [],
      "source": [
        "import sys\n",
        "import logging\n",
        "from functools import partial\n",
        "from typing import Callable, Dict, Tuple\n",
        "\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import ConfigSpace.hyperparameters as CSH\n",
        "from scipy.stats import norm\n",
        "from scipy.optimize import minimize, Bounds\n",
        "from sklearn.gaussian_process import GaussianProcessRegressor\n",
        "\n",
        "\n",
        "class BayesOpt:\n",
        "    \"\"\"Bayesian Optimization algorithm.\n",
        "\n",
        "    Implements a Bayesian Optimization algorithm that maximizes the value of an objective function w.r.t. a parameter of the search space.\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    space : CSH.FloatHyperparameter\n",
        "        Search space of the parameter to optimize.\n",
        "    objective : Callable (m,) -> (1)\n",
        "        The objective function to maximize with Bayesian Optimization. Expects a function that maps a 1-D array (the parameter) to a scalar value (the objective value).\n",
        "    acquisition : Dict\n",
        "        Dictionary that specifies the acquisition function to use (key: `acquisition`) and other parameters.\n",
        "    minimizer_kwargs : Dict\n",
        "        Specifies the configuration of the optimizer used to maximize the acquisition function. The parameters are passed to `scipy.optimize`.\n",
        "    surrogate : Dict\n",
        "        Specifies the configuration of the Gaussian Process Regressor. Its parameters are passed to the constructor of the surrogate.\n",
        "    log_freq : int\n",
        "        Specifies the frequency at which `BayesOpt` renders the landscape of the surrogate and acquisition function.\n",
        "    seed : int\n",
        "        The seed of the pseudo random number generator to use.\n",
        "\n",
        "    Attributes\n",
        "    ----------\n",
        "    surrogate : GaussianProcessRegressor\n",
        "        Probabilistic surrogate of the objective function's landscape.\n",
        "    xsamples : np.ndarray\n",
        "        Samples of the search space where the objective function was evaluated.\n",
        "    ysamples : np.ndarray\n",
        "        Objective value of the search space samples in `xsamples`.\n",
        "    \"\"\"\n",
        "\n",
        "    EPS = 1e-8\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        space: CSH.FloatHyperparameter,\n",
        "        objective: Callable,\n",
        "        acquisition: Dict = None,\n",
        "        minimizer_kwargs: Dict = None,\n",
        "        surrogate: Dict = None,\n",
        "        log_freq: int = None,\n",
        "        seed: int = None,\n",
        "    ):\n",
        "        np.random.seed(seed)\n",
        "\n",
        "        self.space = space\n",
        "        self.objective = objective\n",
        "        self.log_freq = log_freq\n",
        "        self.seed = seed\n",
        "\n",
        "        logging.basicConfig(\n",
        "            level=logging.INFO,\n",
        "            format=\"%(asctime)s [%(levelname)s] %(message)s\",\n",
        "            handlers=[logging.StreamHandler(sys.stdout)],\n",
        "        )\n",
        "\n",
        "        if acquisition is None:\n",
        "            acquisition = {\"acquisition\": \"expected_improvement\", \"xi\": 0.01}\n",
        "\n",
        "        # Pass static parameters such as `xi` to the acquisition function.\n",
        "        # Now, the function expects only samples of the search space as an input.\n",
        "        self.acquisition = getattr(self, acquisition.pop(\"acquisition\"))\n",
        "        self.acquisition = partial(self.acquisition, **acquisition)\n",
        "\n",
        "        if surrogate is None:\n",
        "            surrogate = {}\n",
        "\n",
        "        if minimizer_kwargs is None:\n",
        "            minimizer_kwargs = {\"method\": \"L-BFGS-B\", \"nrestarts\": 25}\n",
        "\n",
        "        self.minimizer_kwargs = minimizer_kwargs\n",
        "\n",
        "        # Standardizes target values, to mitigate problems posed by\n",
        "        # ill-conditioned covariance matrices.\n",
        "        self.surrogate = GaussianProcessRegressor(\n",
        "            random_state=self.seed, normalize_y=True, **surrogate\n",
        "        )\n",
        "\n",
        "        self.xsamples = np.empty((0, 1), dtype=np.float32)\n",
        "        self.ysamples = np.empty((0, 1), dtype=np.float32)\n",
        "\n",
        "    def optimize(self, niters: int) -> Tuple[float]:\n",
        "        \"\"\"Run the Bayesian Optimization loop.\n",
        "\n",
        "        Iteratively search the surrogate landscape for a sample of the search space maximizing the acquisition function, evaluate its objective value and refit the surrogate.\n",
        "\n",
        "        Parameters\n",
        "        ----------\n",
        "        niters : int\n",
        "            The number of iterations of the Bayesian Optimization loop. Overall, the objective value is evaluated `niter` times.\n",
        "\n",
        "        Returns\n",
        "        -------\n",
        "        Tuple (float, float)\n",
        "            The parameter that maximized the objective function during the search and its objective value.\n",
        "        \"\"\"\n",
        "        for _ in range(niters):\n",
        "            xsample = self.search()\n",
        "            ysample = self.objective(**{self.space.name: xsample})\n",
        "\n",
        "            xsample = np.asarray(xsample, dtype=np.float32).reshape(1, -1)\n",
        "            ysample = np.asarray(ysample, dtype=np.float32).reshape(1, -1)\n",
        "\n",
        "            self.update(xsample, ysample)\n",
        "\n",
        "            if self.log_freq is not None and self.num_xsamples % self.log_freq == 0:\n",
        "                logging.info(\n",
        "                    f\"Num Evals. {self.num_xsamples}; Max. Obj. Value: {self.yopt:.5f}; Best Sample.: {self.xopt:.5f}\"\n",
        "                )\n",
        "                self.render()\n",
        "\n",
        "        return self.xopt, self.yopt\n",
        "\n",
        "    def search(self, nrestarts: int = 25, minimizer_kwargs: Dict = {}) -> float:\n",
        "        \"\"\"Search for a maximizer of the acquisition function.\n",
        "\n",
        "        Search the surrogate landscape for a point that maximizes the acquisition function using `scipy.optimize`.\n",
        "\n",
        "        Parameters\n",
        "        ----------\n",
        "        nrestarts : int\n",
        "            The number of restarts of the L-BFGS-B optimization algorithm.\n",
        "        minimizer_kwargs : Dict\n",
        "            Parameters passed to `scipy.optimize`.\n",
        "\n",
        "        Returns\n",
        "        -------\n",
        "        float\n",
        "            The sample whose acquisition function value was maximum during the search.\n",
        "        \"\"\"\n",
        "        if self.num_xsamples <= 0:\n",
        "            return self.space.rvs()\n",
        "\n",
        "        # Define an auxiliary function as the negative acquisition function,\n",
        "        # since SciPy expects a minimization rather than a maximization problem.\n",
        "        def auxiliary(xsample):\n",
        "            return -1.0 * self.acquisition(xsamples=xsample.reshape(1, -1))\n",
        "\n",
        "        bounds = Bounds(self.space.lower, self.space.upper)\n",
        "\n",
        "        minimizer = None\n",
        "\n",
        "        # Local optimization algorithms (e.g. L-BFGS-B) are sensitive to their starting conditions.\n",
        "        # Use `nrestarts` random samples of the search space as starting points\n",
        "        # to the optimization.\n",
        "        for _ in range(nrestarts):\n",
        "            x0 = np.random.uniform(self.space.lower, self.space.upper, size=(1,))\n",
        "            result = minimize(auxiliary, x0=x0, bounds=bounds, **minimizer_kwargs)\n",
        "\n",
        "            minimizer = result if minimizer is None else minimizer\n",
        "            minimizer = result if result.fun < minimizer.fun else minimizer\n",
        "\n",
        "        return float(minimizer.x)\n",
        "\n",
        "    def update(self, xsample: np.ndarray, ysample: np.ndarray) -> None:\n",
        "        \"\"\"Update the surrogate given a sample and its objective value.\n",
        "\n",
        "        Refits the Gaussian Process Regressor given the a sample and its objective value.\n",
        "\n",
        "        Parameters\n",
        "        ----------\n",
        "        xsample : np.ndarray (m, 1)\n",
        "            Samples of the search space.\n",
        "        ysample : np.ndarray (m, 1)\n",
        "            Objective values of the samples.\n",
        "        \"\"\"\n",
        "        self.xsamples = np.vstack((self.xsamples, xsample))\n",
        "        self.ysamples = np.vstack((self.ysamples, ysample))\n",
        "\n",
        "        self.surrogate.fit(self.xsamples, self.ysamples)\n",
        "\n",
        "    def expected_improvement(\n",
        "        self, xsamples: np.ndarray, xi: float = 0.01\n",
        "    ) -> np.ndarray:\n",
        "        \"\"\"Compute the Expected Improvement (EI) at the passed sampling points.\n",
        "\n",
        "        Evaluates the EI at sampling points `xsamples` under the probabilistic surrogate model. EI trades-off the predicted probability of improvement and the extent to which the objective value is expected to increase.\n",
        "\n",
        "        Parameters\n",
        "        ----------\n",
        "        xsamples : np.ndarray (m, n)\n",
        "            Sample points at which the EI acquisition function is computed.\n",
        "        xi : float\n",
        "            An exploration parameter. The Bayesian Optimization samples points of higher uncertainty (standard deviation) as `xi` increases.\n",
        "\n",
        "        Returns\n",
        "        -------\n",
        "        np.ndarray (m, 1)\n",
        "            EI scores evaluated at sampling points `xsamples`.\n",
        "        \"\"\"\n",
        "\n",
        "        mean, std = self.surrogate.predict(xsamples, return_std=True)\n",
        "        mean, std = mean.reshape(-1), std.reshape(-1)\n",
        "\n",
        "        impr = mean - self.yopt - xi\n",
        "        Z = impr / (std + self.EPS)\n",
        "        eximpr = impr * norm.cdf(Z) + std * norm.pdf(Z)\n",
        "\n",
        "        return eximpr\n",
        "\n",
        "    def render(self, n_samples: int = 200) -> plt.Figure:\n",
        "        \"\"\"Shows the surrogate and acquisition function landscape.\n",
        "\n",
        "        Parameters\n",
        "        ----------\n",
        "        n_samples : int\n",
        "            Number of functions samples from the Gaussian Process Regressor.\n",
        "\n",
        "        Returns\n",
        "        -------\n",
        "        plt.Figure\n",
        "            Figure that plots the surrogate and acquisition function landscape.\n",
        "        \"\"\"\n",
        "\n",
        "        sns.set_style(\"whitegrid\")\n",
        "        plt.rcParams.update({\"font.size\": 14})\n",
        "\n",
        "        fig, axes = plt.subplots(\n",
        "            nrows=2,\n",
        "            figsize=(10, 4),\n",
        "            gridspec_kw={\"height_ratios\": [3, 1]},\n",
        "            sharex=True,\n",
        "        )\n",
        "\n",
        "        axes[0].set_xlim(self.space.lower, self.space.upper)\n",
        "        axes[0].set_ylabel(\"Obj. Value\")\n",
        "        axes[1].set_xlim(self.space.lower, self.space.upper)\n",
        "        axes[1].set_xlabel(self.space.name)\n",
        "        axes[1].set_ylabel(\"Acq. Func.\")\n",
        "\n",
        "        # Sample & plot functions defined over the search space from the Gaussian Process Regressor.\n",
        "        xsamples = np.linspace(self.space.lower, self.space.upper, 1000).reshape(-1, 1)\n",
        "        mean, std = self.surrogate.predict(xsamples, return_std=True)\n",
        "        mean, std = mean.reshape(-1), std.reshape(-1)\n",
        "        ysamples = self.surrogate.sample_y(xsamples, n_samples=n_samples)\n",
        "        ysamples = np.squeeze(ysamples)\n",
        "\n",
        "        for ysample in ysamples.T:\n",
        "            axes[0].plot(xsamples, ysample, alpha=0.05, color=\"tab:gray\")\n",
        "\n",
        "        # Plot posterior mean & uncertainty estimates.\n",
        "        axes[0].plot(xsamples, mean, alpha=0.8, color=\"black\")\n",
        "        axes[0].fill_between(xsamples.reshape(-1), mean + std, mean - std, alpha=0.4, color=\"darkorange\")\n",
        "\n",
        "        # Plot the search space samples & objective values of prior evaluations.\n",
        "        xsampels = self.xsamples.reshape(-1)\n",
        "        ysamples = self.ysamples.reshape(-1)\n",
        "        axes[0].scatter(xsampels, ysamples, s=40, color=\"black\", zorder=2.5)\n",
        "        \n",
        "        # Plot marker for best sample found so far.\n",
        "        axes[0].scatter(self.xopt, self.yopt, s=100, color=\"maroon\", zorder=2.5, marker=\"X\")\n",
        "\n",
        "        # Plot the acquisition function landscape.\n",
        "        xsamples = np.linspace(self.space.lower, self.space.upper, 1000).reshape(-1, 1)\n",
        "        eximpr = self.acquisition(xsamples=xsamples)\n",
        "        xsamples, eximpr = xsamples.reshape(-1), eximpr.reshape(-1)\n",
        "        axes[1].plot(xsamples, eximpr, color=\"tab:blue\")\n",
        "        axes[1].fill_between(xsamples, eximpr, alpha=0.4, color=\"tab:blue\")\n",
        "\n",
        "\n",
        "        sns.despine()\n",
        "        plt.tight_layout()\n",
        "        return fig\n",
        "\n",
        "    @property\n",
        "    def yopt(self) -> float:\n",
        "        \"\"\"float: Objective value of the best search space sample.\"\"\"\n",
        "        yopt = np.max(self.ysamples, axis=0)\n",
        "        return float(yopt)\n",
        "\n",
        "    @property\n",
        "    def xopt(self) -> float:\n",
        "        \"\"\"float: Parameter of the search space with max. objective value.\"\"\"\n",
        "        xopt = self.xsamples[np.argmax(self.ysamples, axis=0)]\n",
        "        return float(xopt)\n",
        "\n",
        "    @property\n",
        "    def num_xsamples(self) -> int:\n",
        "        \"\"\"int: Number of evaluated samples.\"\"\"\n",
        "        return len(self.xsamples)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lKU3ePzJVnzu"
      },
      "source": [
        "## Run the Optimization Loop\n",
        "\n",
        "First, we define the search space of the learning rate parameter over a log-scale. Then, the optimization is run for 10 steps. At each step, we plot the samples, uncertainty estimate and acquisition function landscape. For the purpose of this demo, the `max_epoch` parameter is set to a low number."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fbnRHB3uVn8R"
      },
      "outputs": [],
      "source": [
        "from sklearn.gaussian_process.kernels import Matern\n",
        "\n",
        "\n",
        "# Define search space of LR over log scale.\n",
        "lr = CSH.UniformFloatHyperparameter(\n",
        "    \"lr\", lower=1e-5, upper=1e-1, log=True\n",
        ")\n",
        "\n",
        "# Define parameters of acquisition function, minimization and surrogate.\n",
        "acquisition = {\"acquisition\": \"expected_improvement\", \"xi\": 0.01}\n",
        "minimizer_kwargs = {\"method\": \"L-BFGS-B\", \"nrestarts\": 25}\n",
        "kernel = 3.0 * Matern(length_scale=1.0, length_scale_bounds=(1e-1, 10.0), nu=1.5)\n",
        "surrogate = {\"n_restarts_optimizer\": 10, \"kernel\": kernel}\n",
        "\n",
        "# Random number generator used to train / test split.\n",
        "generator = Generator()\n",
        "generator.manual_seed(0)\n",
        "\n",
        "# Define parameters of evaluation function except for learning rate.\n",
        "objective = {\n",
        "    \"val_size\": 0.3,\n",
        "    \"max_epochs\": 20,\n",
        "    \"batch_size\": 128,\n",
        "    \"patience\": 0,\n",
        "    \"generator\": generator,\n",
        "}\n",
        "objective = partial(optimize, **objective)\n",
        "\n",
        "bayesopt = BayesOpt(\n",
        "    space=lr,\n",
        "    objective=objective,\n",
        "    acquisition=acquisition,\n",
        "    minimizer_kwargs=minimizer_kwargs,\n",
        "    surrogate=surrogate,\n",
        "    log_freq=1,\n",
        "    seed=0,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jouu0cVcBFue"
      },
      "source": [
        "Each plot shows the parameters sampled so far and their objective value (F1 validation score). The cross highlights the best performing parameter. The black line and orange area around it show the posterior mean and uncertainty estimates, respectively. Additionally, we show functions samples from the posterior in grey. Below, the acquisition function landscape (here: expected improvement) is shown."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "id": "mtH4PJX9WPVS",
        "outputId": "176ed0c9-1615-4ce1-c4b5-1195cb303409"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsAAAAEQCAYAAAC++cJdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAACK/UlEQVR4nOz9d5xkZZn3j7/vEyp2mE7Tk3NgGGaIQ1YWERDUZTEghp8B0cV91BVQFBFRxISy6OKKCdTHBVYBV/EB1xXxp8iA5Dg5586puiuccH//OHXXnK7pGQaYntTXm1e/quvUCXedU0N/6jqf+3MprbVGEARBEARBEMYI1oEegCAIgiAIgiDsT0QAC4IgCIIgCGMKEcCCIAiCIAjCmEIEsCAIgiAIgjCmEAEsCIIgCIIgjClEAAuCIAiCIAhjisNWAK9evfpAD0E4AGzYsOFAD0HYz8g1H5vIdR97yDUX9iWHrQD2ff9AD0E4AOTz+QM9BGE/I9d8bCLXfewh11zYlxy2AlgQBEEQBEEQRkIEsCAIgiAIgjCmEAEsCIIgCIIgjCn2uwC+4447eMMb3sCiRYt429vexpNPPrnH9X/3u99xwQUXcPTRR3Paaafx6U9/mo6Ojv00WkEQBEEYG2itCcOw8uN5HsVikWKxSKlUwvM8giAgDMO92l9QKvGf553Hf553HkGptNtlgnAgcPbnwR544AG+9rWvcd1113H88cdz55138pGPfIT777+fSZMm7bL+U089xVVXXcVnP/tZzjrrLLq6uvjyl7/Mpz/9aX7+85/vz6ELgiAIwmFBGIYEQUCpVCKfzzMwMEBnZyfd3d10dXXR399Pf38/xWIRz/PwPK+yDYDrujiOQ3NzM9OmTWPOnDlMmzaN2tpaLCuqqwWlEne++c1seuQRAH5x3nks+fa3efzKK9n22GMA3PnmN/Oe++/HTiQOzIkQxjT7VQD/9Kc/5cILL+Siiy4C4Nprr+Xhhx/mrrvu4sorr9xl/WeffZYJEybwwQ9+EICpU6fyvve9jxtuuGF/DlsQBEEQDllMNXdoaIiBgQHa29vZvHkz27dvp7Ozk2KxSGdnJz09PfT29tLX10culyOfz1MsFtFaj7hf27apqamp/DQ1NdHS0sKiRYvI/PrXbF26FL+c3LDmoYdYedxx2IBb3n7TI49w1wUX8L7f/37/nAhBiLHfBHCpVOKll17ikksuGbb8tNNO45lnnhlxm+OOO46bb76Zhx56iDPPPJOenh4eeOABXv/61++PIQuCIAjCIYmp8A4MDNDR0cGGDRvYtm0bW7duJZ/Ps2XLFnbs2EF7ezvd3d1oHdkaHMeivj5LfX2WCRPqSKdTJJNJbNvBslxA4ft+xR6Ry+XI5YbYuLGDlStXAIr77ruPCwYGaC4WscvjcdkpfAXhYGC/CeCenh6CIKC5uXnY8qamJpYuXTriNsceeyz/9m//xqc//WmKxSK+73PaaafxzW9+82WPVywWWb58+T4Zu3DoUCgU5LqPMeSaj03kuu+K1hrf98nn83R1ddHR0cG2bdsoFAoVEbxlyxY6O9sATTKZoKWlkWOPncf48Y00NY0jk2lAqRqSwIRmh/pxLSQyWZLJDG4yjXIS2HYka8OghJ8forN9Ky+ta2doaIiOjg46OjaxdNN2Tli1lgmlcETh6wHNixdzzDe+sdfXUa752GTBggWjst/9aoF4paxZs4avfOUr/Mu//Aunn346HR0d3HjjjXzxi1/kxhtv3OO2yWRy1E6acPCyfPlyue5jDLnmYxO57jvxfb8icjdv3syaNWvYuHEjXV1drF27lnXr1jEwMIBS0NrawpIlxzNlyjSam1tQymb2xAzz5i6gZcocahuaSaVSuK6LZVnYto1SquLt3VvygwN86uOfJPjZz0YUwAFw5g9+wJFHH73X+5RrLuxL9psAbmhowLZtOjs7hy3v6uqipaVlxG1++MMfsnjxYi699FIAjjjiCNLpNO9973u54oormDBhwqiPWxAEQRAORozFYfv27axatYp169bR1dXFmjVrWLFiBV1dXVgWTJ48meOOW8yMGdNJJmuYVA9HLjyGSbMX0tDYTCKRwHEcHMepVHdfKwk3yckbN7J2N6/bwO3nnsuZP/4xb/7Hf9wnxxSEV8J+E8CJRIKFCxeydOlSzjvvvMrypUuXcs4554y4TaFQ2OUfY+XWy17GsAiCIAjC4YKxOfT19bFlyxZefPFF1qxZQ1tbG8uWLWPdunUEgU9zczOnn34ys2bNJJWqZUZrkiMXHc+kGUdSW1tLMpmsCF6l1D4f510XXMC2xx7bre/XBWhv53fvfje/fPvbueyyyzj11FP3+TgEYXfsVwvEhz70Ia666ioWL17Mcccdx1133UV7ezsXX3wxAFdddRVAxd5w5plncu2113LnnXfyute9jvb2dr72ta+xcOHCEWPTBEEQBOFwJC58161bx7Jly1i/fj0bNmzg+eefp62tDdd1mDdvLgsWzKW5eSITa2HhMccz84jjqKmpIZVKYdv2qInePeER2R7iKRC2bYNSPPjggzz33HOceuqpfPGLX2TixIn7dWzC2GS/CuDzzz+fnp4ebr31Vtrb25k3bx4/+tGPmDx5MgDbt28ftv7b3vY2BgcHueOOO/jmN79JbW0tJ510Ep/5zGf257AFQRAE4YDh+z69vb0Vsbt27VpWrVrFCy+8QH9/H7W1tZx66knMnz+HrJvl2EXTmHPUKbS0TiSZTOK67n4Xve/+7W+H5QBPP+WUXXKAp512Gudf90X6v38rTzzxBL/97W9ZsWIFJ554Il/5yldISD6wMIoovbuAv0McMcuPTeS6jz3kmo9NxsJ1D4KA/v5+Nm3axBNPPMH69etZtmwZzz//PPn8EK2t41m0aCEzZ86iKeFw4qmnMmPBYrLZbKXa+0onr+3T8ZdK3HXBBUAkiO1EYsRlWmtu/tY3+OOf/8oLL7yA67osWbKED3zgA5x//vkV4T4Wrrmw/xABLBxWyHUfe8g1H5scztc9DEOGhobYvHkzjz/+OKtWrRomfKdMmcRxxx3NhAnTmN3icPTJZzF5xjzS6TTJZBLLsva7xWFfUCgU+NznPscjjzzC1q1bmThxImeccQbXXnstDQ0Nh/U1F/Y/B3UMmiAIgiCMFbTWeJ7Htm3bePzxx3nhhRdYtmwZzz77LPn8EJMnT+SEE/6B1tapHDWjlmNOPY/G5glkMhkSicQBrfbuC1KpFDfffDN/+9vD/Md/fJ/HHnuUe+65h40bN3LBBRdwwgknHOghCocRIoAFQRAE4QDj+z5dXV0888wzPProo6xdu5YnnniCgYF+Jk+ewPHHn8GECdM4/ojJLD7xH2hobiWdTuO6h1d/NaUUr3vd6znppJO55ppr+Otf/8qjjy5l3bp1nHnmmXz1q18lnU4f6GEKhwEigAVBEAThAKG1ZnBwkJdeeok//vGPbNy4kccff5yOjg6amuo5//w3MmXKbJbMn8TiU95IQ/N40un0PsvrPVhJJBLceOONPPbYo9xyy/dYuvQR7r33V7S1tfGxj32M008//UAPUTjEEQEsCIIgCAcA3/fZvHkz//u//8tLL73E3//+dzZv3kxNTZozzzyJOXMWc+zsJo5//VtobGmtTGwbKyilOOWUUzn22OO45ppr+POf/5c///khNm/ezOmnn84NN9xwyNs+hAOHCGBBEARB2I+EYUhfXx9Lly7l4Ycf5sknn2T58uUkEjYnn7yIhQuXcNzMBpa84Z9oap1Umdg2VkmlUnz729/m3759I395+BGeeupJ2tra6O7u5vrrr2f8+PEHeojCIYgIYEEQBEHYTxSLRVatWsXdd9/N8uXLefLJJymViixcOJPjjz+ZBa21vOEf30PzpGmHxcS2fYVSivPf8o985J8/xmc/+1n+/Oc/87vf/Y4tW7bwvve9r9JQSxD2FhHAgiAIgjDKaK3p6enh17/+NY8//jhLly6lu7ubyZObOPXUs5jZMIV/uvifaJ0+f8xXfPdEXV0d//Ef/8E3v/YV/uePD/H000+xdetWnnjiCW666aYDPTzhEEIEsCAIgiCMIqVSiSeffJJ7772Xxx57jPXr11Nbm+Tss09ixozFvPufzmL6EUtIpVKHZH7v/sayLK7+wnW88+L38qUvfYmHH36Ye++9l46ODr797W+LJULYK0QAC4IgCMIo0d7ezr//+7/z3HPP8eyzz6KU5oQTjmDx4lP5x384loUnn0smmxXh+yqYM2cOP/3pT/nsZz/LQw89xIMPPsjFF1/MW9/6Vi6//PIDPTzhIEcEsCAIgiDsY3zf5w9/+AO/+tWveOyxxxgYGGD27EmcdNJpnH38XE5/8/tI19SJ1eE14rouN910E3f8359z5y9/xXPPPceOHTtYt24dt9xyy4EennAQIwJYEARBEPYh27Zt41vf+haPPPIIW7ZsobGxlre+9Q0smT+Hi97/z9Q0torw3YcopXjfBz7I2y96F5/85Cf585//zG9+8xu2b9/Od77zHaZMmXKghygchIgAFgRBEIR9QBAEfPWrX+Wxxx7jhRdewLZtTjllMQsXnsgnPvxumqbOF+E7iqTTaX70ox/xuc99jr/85S8sXbqUd7zjHZxxxhl885vfPNDDEw4yRAALgiAIwmtk1apVfPWrX+XRRx8ll8sxd+40Tjrp9fzzxecy85gzRfjuJ5RSfPOb3+SlF1/gui99mb///e/ce++9dHd38+Mf/5hcLse2bduYNGkSNTU1B3q4wgFEBLAgCIIgvAKqRdSnPvUpHnnkEbZu3UpTUyNveMNb+OcLT+Xos96D5SQO9HDHJAuPWsTdd9/NJz/5Sf70pz9x//33M336dNrb23EchyAI+PCHP8zNN9+M44gUGovIVRcEQRCEvcD3fS6//HJuu+02bNumWCwyceJEfN/HdV1OPfVUzj7ySC79/GewU7UHerhjHqUUt9xyC9dccw0///nP2bRp07DXb7/9dgCZLDdGkXsygiAIgrAXXH755dx+++3k83lyuRye51VE1bve9S5+9fNb+efrrxfxe5Bx9dVX093dvcvyoaEhbrvtNnK53AEYlXCgEQEsCIIgCC9DLpfjJz/5CUNDQ7u81tPdzbXXXouVbTkAIxNejm3btmHb9oiv+b7Ptm3b9vOIhIMBEcCCIAiCsAfCMOTKK6/E87wRX7cdR0TUQcykSZMIgmDE13zf55prrpHrNwYRASwIgiAII+D7Pr/61a9417vexR/+8IfdiqggCJg0adJ+Hp2wt9TU1PDhD3+YTCYzbLnjODQ1NfHII49w4YUX8ulPf3q311g4/BABLAiCIAgxPM/joYce4mMf+xjXX389jzzyCE1NTSxatIhMOj1s3Uwmw4c//GGJ1DrIufnmm7nkkktIp9PU1NSQTqe57LLLePivf+XUU09l+/bt3H333Vx00UX84he/QGt9oIcsjDJ7nQLxl7/8hTvvvJPNmzdz2223MXHiRO6++26mTJnCKaecMppjFARBEIRRRWtNPp/nueeeq7Qv3rhxI3V1dZx99tlc9d5TmH/mJVx+xZWVFIggCLjkkku4+eabD/TwhZfBcRxuueUWvv71r++SA3zPPfdwxRVX8Le//Y1HH32UlStX8sgjj3DppZdy/PHHo5Q6wKMXRoO9EsD33Xcf1113He985zt59NFH8X0fiG77/OQnPxEBLAiCIByShGFIPp/nhRde4L777uPpp5/mxRdfxLZtTjxxMScuWsLnPn8lVnocwG5FlHBoUFNTw7x583ZZ/m//9m+8+OILfOtb32bp0qXcd999rFixgpNOOolLL72UOXPmiBA+zNgrAfyTn/yEG264gTe/+c3cfffdleXHHHMM//7v/z5qgxMEQRCE0SAMQ/r7+1m5ciX3338/K1as4KmnnqJQKDBv3hSWLDmDz/7zO2mavWSXbXcnooRDm6OOWsSPf/xjvv7lL/PY00/zwgvPsnHjRlavXs3ChQu55JJLmDFjhgjhw4S9EsAbN27kmGOO2WV5JpOR/DxBEAThkMHzPHp7e1m+fDl/+tOf2LBhA48//jh9fX1MnNjAKaecwQWnL+YNb78M200d6OEK+5lEIsF1X/0qa9es4d9vuYVHH32Uxx57hGXLlrFx40aOPPJI3v3udzN16lRpb32Is1cCePz48WzYsIHJkycPW/7EE08wbdq0URmYIAiCIOwrCoUCXV1dLFu2jEcffZTt27fz2GOP0dbWRkNDhnPPPYWjph3FBy99D41T5kmVb4wze84cbrzxRu7/3W944H/+yGOP/Y0HH/wDL7zwAhs3bmT+/Pmcf/75zJo1S1opH6Ls1VW76KKLuOGGG7jhhhsA2L59O08++STf+ta3+MQnPjGqAxQEQRCEV0MQBAwNDdHW1saqVat4+umn6erq4vHHH2f9+vVkMkle//pjmTfveC78hxNY9LrzcFz3QA9bOEhIJpNc+PaLOO11/8CPbv0+L61YyZNPLuW3v/0106fPYs2aNcyaNYszzjiDRYsWkUgkDvSQhVfAXgngj3zkI+RyOS655BKKxSLvf//7SSQSXHLJJbz3ve8d7TEKgiAIwl6htcb3ffr6+ujo6GD58uUsW7aMgYEBnnrqKZYvX47jWJxwwtEsWrSIJXPG87rz303j+ElS9RV2QSlFa2srV139eZ55+ml+9/9m8+KLL/L880/x61+vY8aM2WzYsIEJEyZw2mmnceyxx1JbWyufpUOAva7bX3755Vx22WWsWbMGrTWzZ88mm82O5tgEQRAEYa8waQ49PT0Vq8PatWsZGhrihRde4PnnnycIPI48ciHHHXck0+obOPvNb2L6EceSTCYP9PCFg5xkMsmJJ53EkQsX8qc//oHHnzySF198gRdeeIZ77lnHrFlzaGtr489//jPHHHMMS5YsoaWlRarCBzGvyLiSTqdZtGjRaI1FEARBEPaaMAwJgoCBgQF6enro6OhgzZo1rF27Fs/zeOmll3juuecoFPLMnDmLE088lob6Zl5/wjyOPvVN1I8bJxOZhL3Gsizq6up46wUXcvySk/jzQ3/kqKMW8fzzz/Diiy+wbt0apk+fyY4dO3j22WeZPHkyS5YsYdasWdTU1Mhn7SBjrwTwZZddtsfXf/CDH+yTwQiCIAjCntBaEwQBhUKBzs5OOjs76ejoYNOmTWzYsIFSqcSKFSt45plnGBoaZMqUqeVqXAMLJ9dz7OvfzLSZc6QyJ7xqHMdh6tSpvPs9/z/Wrl3L//+hVhYtOoaXXnqOF198kY0bN9DaOoGjjz6aLVu2UFNTw1FHHcWCBQuYMGECruti2/aBfhtjnr0SwA0NDcOee57HypUr2b59O2efffaoDEwQBEEQIKr0hmGI53kUCgX6+/vZtm0bXV1dbN68mc2bN+P7PqtXr+bpp59iYCDHhAkTOOuss5g4sYHJ2RRHn3IG8xcdR21trVTihNeMUopEIsH8+fOZMmUKK1euZOL4BhYvPo5Vq5bx/PMv8r//+wfGjavnqKMW09vby9///ncmT57MwoULmTlzJo2NjTiOg2VZ4hk+AOyVAP76178+4vJvfOMb0gVHEARB2KdorSui1/d9PM8jn8+Ty+Xo7e1lx44drFy5koGBAcIwZN26dTz11BP09vbT0tLC6ae/jilTxtNoJznymPnMO/pUJk6cKFVfYZ9jWRa1tbUce+yxzJ49mxUrVvDk3xtYsGAR69ev4rnnXuJvf/sbf//7Y8ybN58jjzySrVu34rouc+bMYe7cuUyePJn6+npc1xUxvB95TeF173rXu3jPe97Dxz/+8X01HkEQBGEMYgSvEb3mMZfLkcvl6O/vZ/v27WzdupX29na01mzcuJGnnnqCrq4eGhoaOPvss5kxYyo1ymHWjAbmHvt6ps+YJVVfYdSxbZtx48axZMkS5s2bx6pVq3juqWZmzZpPe/s2li1bwYoVK3jppReZNGkiRx55FIVCgeXLl1NTU8OMGTOYPXs2kyZNora2lkQigWVZ8rkdRV6TAF6/fv2+GocgCIIwhohXeat/isViRfj29PTQ3t5esTxordmwYQNPP/0oXV056urqOPPMM5kzZw6u8pjRmmX6kScyc/Z8WltbcSXXV9hPKKVwHIfGxkaWLFnC/PnzWb16NS8++yStrZM5+eQ8K1cuY/nyVTz44IOkUknmzJnLvHnzGBgY4MUXX6SxsZHW1lZmzpzJxIkTKzYJ27axbVuqw/uQvRLApgGGQWtNR0cHf/3rX3n7298+KgMTBEEQDi/iItcIYPNo/L2m2tvV1UVnZyfbtm0jl8uhtWbdurU888zTdHf3Ul9fzz/8wz8wZ84cHMtjar1i0pwlTJ0deTJl1r1woDBCuKGhgeOOO445c+awefNmnn/+eWrSaY4++jg2b17PqlXrWL58OS+++CJNTQ3MnTufuXPn0t3dzfLlyxk3bhytra1MmTKFlpYWmpubyWazFTEsE+leG3slgFeuXDnsuWVZNDY2cvXVV4sAFgRBEEYkLnDDMASiAor5CYKAYrHI4OBgpdrb3d1NR0cH3d1dDA4OobVm7dqVPPPMC/T09DBu3LhyxXcKlrJoTVlMnHUE46fOY8aMGbS0tEhrWuGgQCmF67o0NjZSW1vL1KlT2b59O6tXrSKbcZk2bRbFYo41a9azatUaHnvsMf7+978zcWIrs2bNYebMmfT29rJy5UpqampoaWlh0qRJtLS00NjYSF1dHclkUgTxq0RprfWBHsRosHz5chYsWHCghyHsZ+S6jz3kmh9cBEEwrLobJz6pzQjfvr4+ent76ezspLu7m97uboqeRxiGFeHb29tbqabNmjULRwU0kGPy3AWMmzSXqVOnMXHiRDKZjNwiPow51P+tmy6F/f39dHd3s379etauXEFbVzcAPT1trFmziXXr1tHX14dSiokTW5k9ey4zZswgnU4DkMlkqKuro6WlpVIZrquro6amhkQiUZlMJxPq9ox8TRYEQRBeNdWWBoP54xsEAZ7nUSqVhtkcenp66OnpYWBgIGpb3N6OBoLAY82aVTz77Iv09fXR2NjIG9/4RmbOnImrNE22pnVKC3l3HtOOWMjUqVNpamqS6pdw0GMqwk1NTdTV1dHa2sq8efPYvn07mzdvYtOGBA0NrZxwwrF0d3eybt1m1q1bx8MPP8zf/vY3xo9vZtq0qUybFkWo7dixg0QiQSaTob6+nqamJhobGxk3bhy1tbXU1dWRSCQqHmKZVDec3Qrgl2t+EUcaYQiCIIwdRqryKqWwbbuyvFAoVESvEb69vb309PTQ19dXEb4DfX2EgOcNsnz5Wl544QUGBwdpbm4upzrMwFaKyckSTRPrSbUsoKa+kVKpxMKFC0mlUgf2ZAjCq8B1XVzXpaamhubmZmbMmEHHEQvYtm0b27dtIZNwaGqawAknHE13dw8bNmxh06ZNPPHE0zzxxNNksxmmTZvCtGkzmTRpEr29vWzcuJFkMkltbS01NTXU19czbtw4xo0bR01NTWV5fFLdWBbEuxXA1c0vBEEQhLHJnqq85g+o7/vDRK/v+5RKpZ1Cd2Cg8vtgbxcDBQ+AfL6PF19cxbJlyygWi0yaNIkzzjiDyZMnk1CKyakijS1pUq1LSNY20tjYyOTJk9m+fbuIX+GQx7IsampqyGazNDU1MXXqVHp65tHZ2UlbWxsd2zeSdhM0NbVy/PGLGRoaZPPmNjZt2sSaNetZvnwVlmXR0tLE5MmTmDRpKq2trXR2dgKQTCbJZDJks1lqa2upr6+vVIeNKM5kMhXbxFi6k7JbAby75heCIAjC4c+eqrwGI3bjlV5T/e3r66O/v5/BwUH6+/vJDQzQ391JMQQoMTAwxPPPv8TKlSsJgoAZM2Zw9NFHM378eBLA5GSR8a1Jkq0noFPRLd0JEyYwfvx4UqkUbW1tB+S8CMJoYDrLJRKJij1ixowZ9PbOr7T87urcQeeObWQy45g/fw5B4LFjRxdbt25l27ZtPPPM8zz99HPYts2ECeOZOHECra2TGD9+fCUO0HEc0uk02WyWdDpNXV1dRRzHq8TZbLaSRXy4+oj3uwf4jjvu4LbbbqOjo4O5c+fy+c9/nhNOOGHEdT/3uc/x3//937ssT6fTPPvss6M8UkEQhLHDy3l5lVIjWht830cphe/7DAwM0N/fz8DAALlcjqGhIQb6exjo6Seq95bo7s7x3HPPsWbNGpRSzJ07l6OPPppx48aRBiYmi7ROSOK2Hg2pSOw2NjYyYcIE6urqxvQtW2FsYNs22WyWTCZDY2MjEydOrEwY7e7upqenh872Nno6dpC0U0yePBmIvohu3x5FB27dupUnn3wGeAalFI2NUaRaa+sEWlsnUFtbWxG2xkecTCaHVYXr6+sr1eP6+nrS6XTFunE4sNcC+LHHHuP+++9n27ZteJ437LX/+3//717t44EHHuBrX/sa1113Hccffzx33nknH/nIR7j//vuZNGnSLutfc801XHnllcOWvfvd72bJkiV7O2xBEARhN1Tn8gIVsauUqsxaN/YGE1sWX980rTA2h1wuR26gj8GeTgoe+JTQ2mLr1h08//zzbNmyBdd1Oeqoo1i8eHFUfQJaEiVaxydItC5CZVux3agS1tzcTEtLi7QxFsYcJk/YVGXjYtj8W+vt7aWrq4u+zjZ6ejtJJLJMnz4N8CkWi7S399DW1kZbWxurV69l2bIVAKRSSZqaGmlubqa5eTxNTU3U19dXRLHjOGQyGdLpNJlMhpqamkq1OJvNRl9Y02lqa2txHOeQFMV7JYB//etfc91113H22Wfz+OOPc9ZZZ7Fhwwa2bNnCP/7jP+71wX76059y4YUXctFFFwFw7bXX8vDDD3PXXXftInQBamtrqa2trTx/6qmn2Lx5MzfeeONeH1MQBEHYSRiGBEEwrMoLVP7wmddKpVLFx2usDUYYB0FAPp8f5u/t7+9naKCXwb4cRQBKBIHN6tXreOGFKMM3k8lwwgknsHDhQpLJJHVAS6rE+KYEmQlHENRMQVl2xQ/Z0tJCJpORqq8w5qm2SLS0tFAsFhkaGqJYLNLb21tJV+nt6aavu4OB/l6SyRqmTp0K+Ggd0N2do62trWKrePHFZQTBCwC4btTFrrm5kYaGZhoaGipC15BIJKitrSWdTlcm2tXV1VFbW0tDQwM1NTUkk0lSqdRB/+92rwTw7bffzhe/+EXe+c53cuyxx3LllVcydepUrr/+ejKZzF4dqFQq8dJLL3HJJZcMW37aaafxzDPP7NU+7r77bubOnctxxx23V+sLgiCMdapbDgOVR1PlNU0pTEavEb/xSq9ZZ2hoiP7+/kqMWW6gn2J/D54PJUpoEuTzeZYtW8ayZcvI5/M0NTXxD//wD8yePRvbtqkDxqc9xje6ZCYeQZCdTGg5ZNJp6uvraWxspL6+Htd1D1v/oSC8FhzHqVRpgyCgqamJYrFYsSgZ7313dzf9/dHE077+bpIqSVNTE+ABFkHg0ds7uNNn3NXFqlVr8bwVlWOlUkkaGqI0iYaGJhoaGiqT6OI2imw2S0NDw7AvsHV1ddTX15NMJkkmkwfmZO2GvRLAmzdv5pRTTgGiNzk4OAjAe9/7Xt7//vfz6U9/+mX30dPTQxAENDc3D1ve1NTE0qVLX3b7gYEBfv/733PFFVfszZAFQRDGLNWi1wjZuOCFqNobBAGFQqFS5fV9f1i6g2lTbNoT9/T0MNDfx+BgN8FgQKns7lW49HTneO6FF1izZg1BEDBt2jQWLVrEpEmTIh+iguasT0u9Q6p1Dqp+OtpyScX+eDY0NBwS1SNBOBgwNgnHcSoeXvPv2HyRzefzDA4OUiwWK41nent6KOT6GBzqJ2WnaGpqYv78uUD0/4rBwRy9vYOVL7o9PT2sW7eBYtF0BtZYllVOlIiqwNU/ZsJsJpOhtbW1XF1uZurUqTQ0NJBMJl+2a2Mul2Pbtm3Mmzdvn5+7vRLA48aNq4je1tZWVq9ezRFHHEFvby+FQmGfD2ok7rvvPsIw5IILLtir9YvFIsuXLx/lUQkHG4VCQa77GEOu+fD2wtXJDfFubHFBHBe8wLCJbkDlD6fxGg7mcniFHKEPJbqI/rQ5hLqWDZs289JLL7FlyxYcx2HevHksWrSIcePGAVCDT0uyl9ok2A1TGaiZTm/RxursqfgMbduuZATvTdVXrvvYQ6753lP9/wST6pLNZnFdl7q6usqXXyOQvVKJYn6AfKGfpGqgpqaBKVMmA92VfRYKRXp6+ujvH6S/P09f3xADAzna2rZRKpWAnf+/SadT1NTUU1NTV0mWqKmpqfyk02mUUsyePZvm5mYmTJhAMpmsTKr9xje+wb333ovrugwMDOzzc7RXAviEE07gkUceYf78+Zx33nnccMMNLF26lEcffZTTTjttrw7U0NCAbduVbDpDV1cXLS0tL7v9r371K84555zK/1BfjmQyeUi3TBReHYd6q0zhlTNWr3l1dTducTDVXuP31VpXqkGe51X+KJrcT7M8DEMGBwd3egl7exkc6KU0NIAugcIjgUtKNZHPF3h65UpeXLaMgYEBMpkMS5YsYcGCBaRSKdJAXRoakwENaUXNhGOxmuai3CRBEJDJZKitra2E9Jtw/r1lrF73sYxc89eG+X9F/Euy8fybya6maY2pGJs0l0Kuj6HcAINDQ/TU9VDw4lI3QOuQQqFIf/8A/f1mQmyeXK6f3t4utmzZgOcFgPlyq7AsqyKKTdqEeXzmmWdYuXIlvh9N5hsN9iiAN27cyPTp07n22msrA/jnf/5nbNvm6aef5rzzzuNjH/vYXh0okUiwcOFCli5dynnnnVdZvnTpUs4555w9bvv888+zYsUKPv/5z+/VsQRBEA43qsVuPK4s/hj/w2b+oAVBAETC2HGcYa8PDg5WqkCdnZ10d3czmOvHy/cR5DVQwsXBsi2wXTZu6+TZl15iVdnmMHHiRE488URmzpwZhfoDdVkY5/o0Zi3S46fhNM/HTmYq1eaWlhbq6+upqanBdV0cxxGvryCMMiPZiqorxSPdSTK/xxNhisVi5f8dhUKBwlAf+dwAAwO9DOYGGRwYouCBFztOqVQil4sm60WP/eRygwwN5eno6GXDhgJBEB1rf+R871EAn3vuuSxZsoR3vvOdnHvuuUB0Aj/60Y++qoN96EMf4qqrrmLx4sUcd9xx3HXXXbS3t3PxxRcDcNVVVwHskvLwy1/+khkzZnDSSSe9quMKgiAcalRXdqutDTC8WYWp4FanPMS7OxlbQz6fp1Qqkc/n6evri5pW9HRTKgwSFotYfgkHi6TtENgJ/ELA86tX88yyZbS1tVVsDgsXLqSxsZEkkHEi4VujfMZlbdKts0i2zMVOZitVZ9OataamhkQi8YqrvoIg7FuM9Wkk4vMF4v/fif8eX6/abmG+fMcb5OgwpDDYT6k0RFCKKs35XD+dPZ1s39xHT6nE1q1bueuuuypfmEeLPQrgH/7wh9x77718/vOf54YbbuAtb3kL73jHOzjyyCNf1cHOP/98enp6uPXWW2lvb2fevHn86Ec/Koc4w/bt23fZJpfL8cADD/Av//Ivr+qYgiAIhwK7m7hm/thUv+b7fkXohmGIZVmVR6VU5XmxWKz8mNub/f395HI5+nu7KOUHCQp5KHk4Fli2i0on8APo7M7x7PLlPL9iBfl8nvr6ek455RTmzZtHMpkkDWQTUJeBhPYYV+NSO3EW6fFHYCUylT+A8bikRCKBbdtS9RWEg5w9iePRJJfLcc8994y6AFa6WsqPQHd3N7/97W/59a9/zZo1azjiiCN45zvfyVvf+tZhOb0HE+IVGpvIdR97HMrXPC5oTdWkuopiMnnNMtgZX2aqp5ZlDfP6GrFrHoeGhio/g7k+iv19+MVBKPjYToiyEtgO+CHkSyFr12/i2WXL2bhlCwDTpk1j4cKFTJ48mYRSZG3IpiCVgIxVIpNKUDdpBjUTF6KcZEWYu65bEb4m2WFfVX0P5esuvDrkmo8dPvGJT3D77bczNDQE7Fp13hfs1SS4xsZGPvShD/GhD32I5557jnvuuYebbrqJG2+8kXPOOUcaUwiCIOwF1VVeIxTNcuPLNRm8pqJrqrpmhrSp7prM3mKxSKlUIggCBgcHKZVKleWFQoGhgX4KA31orx9/yMe1NSnHRWUdAqAUQFv7AM8tX8GLK1cyODRENpvl2GOPZf78+TTU1pIEajOQdsG1wFUlatIJaifOJztxIW4yVYlVsyyrInwzmQy2bVfEr1R9BUF4OW6++WYAbrvttlGzSe1VBbgarTW///3v+fKXv0x/f/9BGUsi3xTHJnLdxx4H+zUfybZQ/WPEL1ARu3GPrEl0MILWeHiNYDb+OvNasVgkyOcoDg3gDfXgl0pYgOMkoknYGgoaSsWAlas38tzy5WzauhWUYtq0aRxxxBFMnzqVpGVR40A2AwkbHAscimSSSeqmzSPTPI9EKjUsZzibzVJbW0smk6kI3tHw+h7s113Y98g1H3sc8Bxgw8aNG7n33nv5zW9+Q1dXF6eddhrvfOc79/mgBEEQDmWqo4XindXi1V9T3XUch1Qqheu6JBIJgGF+XSNqfd8fNiPbiF7P8wh8n0K+j2CwhzA/gFf0UCHYjouVTGBbkcWhWIT2jj6eXb6CZatWMZTPk62p4fgTToiqvdksSQtq0pBxIeECAdiqSCadZNy048i2zMZ2E5XbkrZtk0qlyGQylWQHU6mWqq8gCK+WmpqaURG/sBcCuFAo8Pvf/557772Xp556ismTJ3PxxRfz9re/ndbW1lEZlCAIwqFEtdj1PG9YKgPsnFBi23alumsiwIBKdbenp4ehoaGKaI6nPJiKsZnUViqV8Ar9MNRFUBwi9DwCX6EcB9txCRXYFgx5MNBdYvnatby4YhXb2tpQSjF9+nSOWLCAmVOmkLQU6STUJCNvr22DXwIrLJHKpmicdhJ1rTNRll2p9pr3kkgkqK2tJZlMVqKWJOFBEISDmT0K4C984Qv8/ve/x/M83vjGN3L77bdXWiILgiCMNeKZmCbep1rwApXqp2VZu1gZbNuuCGbTdMJMVjM2CCN04w0sjKc3DHzCfC/BUCcUClihh+fbBI6Ftl20AlTk6y0MadZv2sqLK1exZv16/CCgoaGBk086iflz59KcyWCnIOVATQqSduSQ8LwQwoBxDQ00TF5EzfgoqceyLLTWlSSHZDJJOp0mlUpV3p9UfQVBOBTYowB+/vnn+dd//VcuuOAC6uvr99eYBEEQDjjVIfDxTmqm2msqocbnGhd/JqnB2ARMxdaEyJvtzT6MADY+4WKxuNM7PJTDL/ZgFzpRgwW0Dgh1Ah/wlIu2wNLgBeD70NHZy7PLVrFs9Wpyg4Okkknmz5/PUfPnM2F8M9mkwnIg7UDCiSq+KoCS52Epm/qWJpqmHUVNw8TK+IygTyaTFfFrbBvV50AQBOFgZ48C+L777ttf4xAEQTggVAtdU901Fd543i7stDIYIWhaChtBWyqVACrbB0EwzM4Qr4zGK7vx6m8YBFDswx/shqEOdMlDaSjgUtQ2obbxdSRavRACoK+3yMq163hp1Sp2GIvD1KmcccopzJ4+nVTSJpkC14aEBckUJB3wihCUSigrwbiJ02iYciQ19U2VcRpRb+waruuSSqWGWRzE7iAIwqHGK5oEJwiCcKgyktCtTmiojiYDKpVP07wBdloU4lXc+LZx/6+pnpoxGP9ufFvLsrC1hz/Uje7fjj+QQ+PhBRa+ZeOHLkUftAU6gMCHUMHQkM+q9RtZsWoN6zdvJgxDmhobed3JJ3PUvDmMq8/gJsEGHDt6TCTAdSAsQqlYxE0lqRu/kPrJc8nU1FaEr1KqMinPvH/j943bOczvgiAIhxIigAVBOOyIi924n9Y8H6n5BOwMWze5tcbGEI8aAyrV2moBXf2aaQdq7A9m3aiaaqGK/Xj9bZT6tuJ7Hp4HluXiKyiFLoUACCJrQxiA1lAohqzfvJXlq9ewdv0GPM8jm81y3OKFHDFnDtMnNmO7CscG5YITRDYHxwELCL2A0A9J1NZSM2ERdc3TyWSzlcq2iWBLJpMAlefGyyw+X0EQDgdEAAuCcEhTXdGtjhurbi0c9+bGkxkcx6ksj+f1mkqtiS2L7w8iG4PJ6TWC12T0mu0ty8J1XXRxEErdeJ1bKQ0M4Acevq/AdQiUi2dDsQTFACh7egMf/FCzfXsby1auYdX69eTzeZLJJEfMnc2C+XOYPWUiqYTCtsFSEBJVfNMJUDral/I9wCXdNIGaCXOoHTeedDpd6SgXn7BnKtamAmwqvaYSLD5fQRAOdUQAC4JwUFNdoa3umjY0NERfX98uldd4ddLYDEz1svq2vcnUNZPczPGMhUEpVRHF8eVmDPEECDM+27bRoY/j96AG2in276DkefgehLaLUuC7LqUAinkohpEfN7SBINpPW2cXy1avY8WqtQwMDOA4NnNmTeeoBXOYPWVq5Ot1wbIj0WurKLfXAlw3sjloXcJ2Erjj51HTNI3acY24rls5B8bX67pu5X2b59XpFeLzFQThcEEEsCAIB4S4YKwWuOZ381jdNS2+npmoFq/mmipvvGppjmkmqpnoMiNsjSA2Ynl3x4p7fU0sGESisVTMExa7CXJdlPq3EQwN4QGhdvFDUJZLUUVJDUUfPB8KHlDux6m1pqu9k5dWrWPlmvX09vVjKcWMaVM467QTOGLuDDJJF6vclQ0Fyo4Eb9qKrA5+AFYYEOZD3Gwt7rgjqW2dRk1NTUXE2rZdEbiO41TOX7XwBZngJgjC4clrEsAf/OAH2bJlCw8++OC+Go8gCIcB1eJ2JIG7O6p9umZf8QllRuwCFX8q7GwjbLJ2jViNV3eNRcKMJT6xLW6ViFebTbUXqFRNAQK/hDfQgTfYhZdro9Q/QBgEaGWhHZsAlyCIRKmvoJSPEhsKBaLAXcBSmh0dHaxcvY7lq9fT1z+AshQzpkzmdScdzcJ5M0in0lgKXAuUAtuJNnftyNurNAQBhMUSkMBpmEimcQbZxhbS6XRF6Bqrh/H3xj3JyWRyWKSZTHATBOFw5jUJ4EWLFjFx4sR9NRZBEA4h4iLRCMU9idv4JKu4qKr26VZPTDNV1vgterPcCDXXdYdtm8vlKlm9pmo7ki0iHlUW9wTHjxsXhH6pRDHXQSHXSbF3G95QHu37hIDlumBZBMoi9MHzIv+u50NJAz4EIQROJHq3tbezcu06lq9cT38uh21ZTJ86mTNPOY7586dTk0hVhC6UJ7FZZZuDrdBowhDCQojSPmRSZCYsJN04jVQ6M0z4uq5byfA15ynapzMsy9e8b5ngJgjC4c5rEsBXXnnlvhqHIAgHKdUV0d1Vcaurs/E4LWNJMPuKR44ZMWaI+3XN8+pJa/EJb/l8nr6+voqloVrwmrHG/bvG8mCEbTzxwbzmeR6B5+ENdpHv7yDft51gaIgwDLA0WK6D5SpC2wUFfhhVYT0viiorAl4BQgd0CFqHbN6xnTVrN7By7Qb6BwaxbZuZ0ydz1utOYMHs6aSTSRRR3JmtIKEiL6+bAIWFDkNCDUGgoVRCOwmS41pJNk2jpqG10pktXu2Nf0Ew/maT5Wsq5+Y6SSMLQRDGCuIBFgQBYJhA3ZPQNbm4u6voxqluKhG3GFQfu9rzGxfBZhJavFprfLwDAwN0dXVV1jP7N4IvPu64/7X62J7nEXol8gMdeAOdlHI7KA3mCXWAHQCug+MocBwUNqVA43shIeVKb1kAlzzQCgINBa/EhrWbWbNuA+s2bqZQKOG4DjOnTeaNrz+ReTOnU1OTQIUQqCin1waSSUglQYWglUXghYQ6RPshhD5OJoU9/kjqmqfhJFKk02nS6XSlkmsmuRkxWygUKgI3nU6TSCQqgl8muAmCMBbZrQC+4YYbuOKKK8hkMtxwww173Ekmk2Hu3Lmcf/758j9RQTgEqG4KUS1IjajdW6Eb32+8na/5HRhW6a2u6MYnr5n9GL+uEcTG0mDGa/ZXKpUolUqV92LGb9r1xiub5n2a8fnFIbyBLgoDbQT92yjli/gh2JYmwMVyFZZy0Bq0ZREEIb4HQRjgl+0Nnga/BNqOBHDfQI516zaydsMGNm7ZThiGZDNpjpgzk/lzZjBnxmRc26lMYrM0KAvSqag1sassPB2ifUUx1Gg/xApLKDuB0ziBmoZpuLWNZLPZin0h3q3NnEfTgQ4in3QmkyGZTA675jLBTRCEscpuBfDKlSsrf7hWrly5x514nscdd9zB3/72N775zW/u2xEKgvCq2Rv7QlzoVk8w2xPxdIZq+0HcU2uIWw2MxzQuwk274OpOamZymtlnvGWweY9m/6ZFr3kv8VzfoaEhvHw/erCLYm4HpZ52PN/DCwAscGws28FOgLIsbK0IghDtawLbIiyFkbc3hKIHfjmnN9Sajs5u1q7dwLoNG9ne0YkFNDXUc8pxi5g7ZzpTJ7ViWyoSvYBtlSexuZBJWjiWBmXj+QGDXkhQhDD0SDgaJ1OLO24+2cbJJMve3lQqNSzhwnh9zfkwVgcjfBOJxLBzJxPcBEEY6+xWAP/iF78Y8ffd8cILL/DBD35wnwxKEIRXzqup6u6N37N6kppp9FCdmVt9Oz0uqs3xq8WusTTEq8bxjm3xqm58UpxJLbBtm/7+fsaNG1cZr+m8FnhF/Hw33kAX4WAbpf5eip6HF4JSLtoGbbnYCYWFRmkLrS0CrdEaCAMUUXpDvhBS9KDkR9aG/FCJLdu2sm7DJtZv2sLg4CDKUkydPJ43nXkic2fMoKFhXGRpsCM/b6Aj4es6kEwpUjakLIei75PLa4qhT1CEpO2RTLqkGmeRbppOpr6xktxgrAum4hsX+/FqeyKRqFgdYOeXBZngJgiCEPGKPcCDg4MAZLPZYcvnz58v1V9B2A/sjVfXiKKRJqVVUx1ZFq+8xnNy4/YDQ3yiFTBM7Fbvw7QCjotdy7KGTVqLj8mM33QiMxm1xucaF8mDg4MEQ334Qz34g+3ooU68oQEKgYZQUVTlOAXLRTtgKUUChbYUygaNQochOvQJNZQKUT5vqfzjh5ruvh7Wb9jMuo2b2LK9DXRIOpVg/qwpzJsxlRnTppHOpAnLQte2otQGy4oaVdQ6NikXkrZNCU2h4NFd8PBKYFMinUyQndhATes8MuMmYVVl9Zrza5aZ58YPbSq+qVSKVCo1rOIrHdwEQRCGs9cC+Gc/+xk/+9nPaGtrA2D8+PF86EMf4gMf+EClV/wb3/jGURuoIIxFRsqkraZa6FaLnPiksGrBHLcVGFEatx2Ydc0+za10c/s87tk1ebvVrYDjlVwzfrM/sy8jak312FQuTZXXeFo9zyOfz+MXc5RyXfiDnQRbVtG7AwLfx/MgtGwCrdCOG4lR18IONHYCrFBjOzY2Cl9rSn5AqaAJwkjoFjzQOrI4FEseG7ZuZf3GzazfuJmBgRyWBZPGN3H2aYuZPWMqE8e34tsWhNF2lg2pcjavq0A5NhnXwbE1vg7xPJ/ewYBiEdAeaUfT0FxP/cRjqW2dhZtM43lexSZivmAY4Vv9hcZ8KUkkEiSTSVKpFEClEizJDoIgCCOzVwL4xhtv5Fe/+hUf/vCHOeaYYwB49tln+Y//+A/a29u56qqrRnOMgnDY80qqutU2huoWvEZwmmXmcaQGE9XNJmDn7fKRPMFmH0bgVld3480qTIUXdlaKTRxXvA1vfJ24UDPtib1iHn+wGz/XSWmgjXConaDo4YUK34ehoUGoqyewUmhXYasQS0VZu3YSFApLO4RhQImAQt6P2g4HEHhRNm8pAD8M2d7WycbNW9m4aQtbdrShw5BU0mX+rCnMn3Uc82ZNJZHOEgTlTF+iqLJUBtzyZDbHsdFYuC6oMKRQKlLyoVgEzw9IWAEN2QRNU46kfsqRpGubKhVcz/MqE9kSiUTl/MTPkYloM18estlsZXJb3HstwlcQBGH37JUAvueee7jhhht405veVFl2yimnMHPmTK677joRwIKwF8Rv87+SqDHY6Z+NR3aZ/VQTXy/ePrjaIzyS3zYueuNjM95fI37j1Wgz1rhYi1ej48/j1eMwDHEcp3IMz/PwvSJerosw34M/2EGxfxu6UEQHimIY4hNZGDTpqP9vSoHn4CYs0koR2BoC0FiEgYVX8ij5PqXApxREcWVhGE1gCwNNT66fDeu3sH7zVjZs3kaxVEIBk1qbOfOkRRwxZyqTJreiAhsvjDoWBwEk3Kj1sG1HP8qy0cpCBSGW0qA98oVI9PqeRimfTNKlcfoMmmcsJF0/qWJdyOVyleq3SWowFgdzvsxzYxcxE/6M8K2u+EqygyAIwp7ZawvE/PnzR1w20i1ZQRhrVE8+q/7drBOnuitZ/Hl8cll8nfi28SYPu6vqxq0HcaFbvU8jmGzbJggCisViZRzxphKu61ZaDxtrxEiNLIzINeI37uk1gs33PIa6tuAP9eANdcBgO15uEB/wA01gRVm9vpUCR2GHIVgaW4OTVJEa9T0KFgTaxgsVpVJIWBb9JR0JXnRUrdUa8oN51m7exrpNW1i3aSt9/TkU0FBfy3FHzmLuzMnMnDqJbDZNSFQl1gGEKsrlTVrglD29IYowsAiVxtEhEBCGipKvKRRBaY9U0mX8hAk0TzqS2kkzcRyXYrHI0NBQ5ZyYKDPTnthgKubmC4+pqGez2cr5N8JXPL6CIAivjL0SwBdccAF33HEHX/jCF4Ytv+uuu7jgggtGZWCCsL8ZSayOJF7j4ra6bW+c3U06i/tm48vMPnZXPY0L6ngmbtyzW90BzVgS4p5eU200YzDeXbOOOa5pnWtSA+ITz+K+3vj+4scGdiYT+EW8oR68wV70UAdhoQOGhtCARlMKXQIbApXCVwrLUigdolxNwtdYtoW2LcLQJ9AW+YLG83wCrcgVNYHlEYZRx7VCAJaCUMPQYJENW7azcdt2Nmzaxo6OqGFGOpVgzvRJzD/tGGbPmExLUx2WUoSALmf7Kg0ZtzyJTRFVnEOFrxQqhKStCJyAQEcWirAEQVAi4bpMmNRC3cQ5NE6ci5NIVb5UDA4OVSq42Wx22IQ2oGJ9SCQSletirokRvvFWxiJ8BUEQXh17bIRh8H2f++67j7/97W8VD/Bzzz1He3s7b33rW0d9kIKwr6i2HuzJhmAwQrU6jivewMEQtyxUrxtfZ6RWwdVtfk2r3/jkNLMsPuaRBGe8MmwqtXHbQrFYHCaGHcepZOg6jjMswcFElsHO1AFzmz1uybAsizAI0KUcYXGAcKgHP9eOzrej/RK2irLAQuUSWuAlUnihUx53iOeXUCpE6ZBAWyhlEXoKHWq8gkegNUoBKsAq2xHCUFPywQ2iau1AocCm9dtZv3U7G7dsp62jG9C4ts20KeN5y1lLmDdjMtMmN2Mri3JIBGEAvk+0fyuayGbp6LkGPK2wA41lKxJ2CKFFEAaERQCPhHJINTVRO3EWdeNnk8rWAlGTjkIuVznvxtdr4swMpgKcSCTKQnmwInwzmUzFO22ug+T4CoIgvDb22AgjzsKFCwHYunUrAM3NzTQ3N7Nu3bpRHJ4gvHJeqcit9r3CcLFrqmsjTQir9srG97m7n+oxVndNi1dr4/YCU201v8erv0BFhFaLY7PMWBiMeI0LdzORLZfLVbqHxSerVVejAULfg1IfYbEfXejCH2hD5/sIfR9zNizbIVQKP1XDYAl8dDSBzAsIAx8/8FFW5Ml1bBs8ja8AQiwVojVQLpQnAJ+oslsKom36B/KsXLODrW3b2LBlO+2d3dG6rsPMKa0sOep45s2cxMypLVg4hFYkmsNyEwtUVPENFSg7qvRaOrJLhChUqLEcSKJxXRsVBigflC4CLk5LI5mWWWQappLK1FSsCaZyW/0FIx4TZ4SwSbooFov09/dXvNHpdLryZSRemRd/ryAIwmtnrxphCMLBxu4ErqlYVhMXufF9mMfdedmrby2/XDpDdQxZtQ93JKEbrzAboWpsC9UT2cx61UkPZhIa7KwOjihaR4gri6c3VKcHRI8aSnmUPwCF3iiFYbCdsFgoTwrTWJYTdXtwEgQqiRdqCiF4BfADKAbFqMpathZUzi+gtAKl8YMAxwJbl19AYYea0I48vHlf097dz8atbazf1MbGLTvo6O4hDCGdcpg5tZWTjl7CglkTmT6xBeUm8D3QTkgQakKt8f1oEltYPkZQAteODmdbYJU7VigXbGXhOpAiGrj2ClHjjMYWEo3TSdZNxk0kK5YF3/cr3mlz3uO5xbDT15tMJiv+3kKhUKnIm4lwcUtJ/FoKgiAI+4a9ngQ3MDDAhg0bAJg+fTp1dXWjNSZBAF59JTdeIauu0r7SSZvxY+0ubzc+rrjIjYvd3Qlds70RtPFWwtURZ2Z9kwlb3XFtpHNWKpUqiREmsmykCnEllkwFhMVuVGkAhrrKXt0BNAE6VIShJlAOVsJGJ+oJtY+nIstCyQsp5TV+GOAFAb4XVWwhmkiGijSyshSOpbGdcktgrdGRFCYMNVghSoFf8lm/rZPVG3awYUsbGza3MZjPA5BKJpg1tZVTj5/H5OYJHDW/GcuxCQMbH4Uf+oReADry5vqaSnQZOsrpTViQTiscNKZlm8IikYS08sAroYIQbBe7aTKqfiputgUnEbUhTqfTlXNuKrTxz4XrRpP4TM6xEb2O4+B5HoODg5XrbYSx2Ua8vYIgCKPLywrgbdu2cf311/PXv/512B/N17/+9Vx77bVMnjx51AcpHP68XBtfQ3W1tVokVtsG4ttVWxAMe7IvVFsj4mM0M/Pjk9CqG0iYsZl9xLupGYEcP6b53QghgxG9ZgyWZQ3zCI+U42v8onHvaEVQBUWUP4TlD6KKPViFLnS+Dx16gCLQGh+X0AGtUgRY4ETegCBQeEWN55fwghA/CCJLQrmy6vtRSgIaEnY0icxNQ9IGrcCxHBSaoGy+DVRkPejP5Vi7oY21G9pYs2UHW7d34pffS1NDHfPnTGHu9AnMm97KxJYGHEehcejr9/GwCUoBFgFhGE1i017UxlgDKgTXhVQiGocT9a5AaRtl26Qtja1KJHQhEs2Oi9U8H1ITsWrGoZRVsSuY62CuqfkMxq9bIpGoCFpjdQAq/l5zzVzXrXivjV2i+k6FIAiCsO/ZowBua2vjoosuwrIsPvnJTzJnzhwAVq9ezZ133snFF1/MPffcQ2tr634ZrHDoUy1y98ZSEK+qjRTtVU21R3J31d9qkVu9vpkEVp36UJ2pa8ZliFd/489huKXCVP1MlJjZX3UihBlT3FcaT2CIe4NNhdeIcx3koTiIFQxhlfrRxS6swgBKe+gwsh5oyyWvIbBdApWqTC4DCAuawCq3CdYhvhdQ1CFBCbwQSn7UYRiiJhCWBdk0JN1I1KZcFxVCaGk0ITpUKG2RKxbYsr2DjVs62LiljQ1bO+js6Su3EbaYMrGF045fyKzprcybPoGmuij3N8QiDK3IYlEM0Nqn6IPlBQRhOeM3jCrNIZHYTSXBtRSOgsC2UNhYWGSUTwKfBEMoy4J0PV5mOjo7HjsZTWIzAtUIWXN94paUeNMKU+mNd3ALw5B8Pk+xWMT3/YpQNuvFUzcEQRCE/cMeBfD3vvc9pkyZws9+9rNKi02AN77xjXzwgx/kkksu4T/+4z+4/vrrR32gwqFHtcgdyX6wpza+caEbTzao3raakSrAI1XVRsrMjY+z2k5QbaUYqdWvqbRWT54zk6Pi58D4gOPrV8/sr26eYR6NRaLSPCI/QFgahOIAutSHVehBeQUUHpZWBGh8HTWRCLVD6KSiiV4hkbVBaVQAytLoMMAPgqhZhA+er6MmEuWKKmX/rAW4TrnK60CtY+EkHGyl0IQoDaFSlDzN5u2drN/awcYt7Wzc2sa2Hd0E5XNRW5Nl6qTxnHD0fGZNncCMyS0kUlG7Yq0ioT3oawINWofYhGg7ci2Uyl3cLD+KQLMtqElBwrVIWNHEtTAEmwRYAWmKpLRH2glQtotXMwE/MxUvMQ6toi8jiZhdwXxBMZ8r3/crfmtjQTHi2NhSzGfN8zyKxWLF5mAyf5PJpFgcBEEQDjB7FMB/+ctf+Pa3vz1M/BrS6TSf+tSn+MxnPjNqgxMOHeKV2b2dKDaSIK22EMS3j9sEYNfJa/FtRvIM765iXC1u4xXgeCXYCFxjfzCiPC5i4rex41XravGvtR4msKori9UVYd/38Ys5iv29+KUcXqEPVegjKPWhiz6OU04vIKroAuC6BCTRIWhlgdKEXoBjKXTBR9sarSN7QAD4fmRvCIKAYlBOXPB3jtmyyo0gXEg5imTCImFbJJzI7xCgCAJNZ/cAazZtZ23Zt7t5eyfFoocGkskEUya2cPqJRzNlcguTx7cwrj4LKtp3lEMWTVbz0BCCbatI3CrQVmRt8Hwo6chSYVtQk1S4LmRcC9uysGyHgKgTRtr2SZDHtYBkkrBmHkOpFgK3DoiuV7J8LUyHtfikQZOuEU9yMBXe+KOp5JtJbabaaxIdqgWyIAiCcODYowDu7u5m2rRpu319+vTpdHd37/NBCQc/1Z3G4hgROFJVd6Tt92RPMOxOzFYLWlN1M/swj/GUA3OsuMiMt/mNpyMYwRv3e8Zb/5qf+G3seJJDfPzV1e1SqVQRSsViMRK+pTx+MUdYGKJUHEDn+yh5A6iiTxD4UdtdQGGBbeHYYKXSlJTCCjRYFoEO0KGOZn7pEpZS+FqDH8WQeVgEWqELmoAQ36McDwblzhRlry44CUgmIWk7JBxFKqmwlY2lFEXfZ+uOHtZsbWP95jY2b+1k844u8oVo0p1lWUya0MQxC+cxacJ4pk1qoaGuHpRCE6Uv2BbYTjmKzAJbaVA2Vhhg2wqliarXYTSBzQsAO7Ijp13IpBxKQz6NjUnQkVXDpoTy8jhKY9suVv14gvRUislGfBV9OXAch0zZp2uquNV+XtMIJF4FjnfMM9tCOe+3nKwRBEFF9BprhFgcBEEQDi72KICbmprYuHEjEyZMGPH1DRs20NTUNCoDEw4e4pXd6ipqdUbsSLFh1WJ3TxPQqm/zx48XF5TVPtj4sUxl1lTvYKfgro4gMwI0nrxg3ktc5JgWwEbMmElLpuoXt2oYYW0mqPm+XxFGnudRyg8ReIMQFPELeUIvhy4OEBZLUXlTRSLa1hrLcbEdIKlwVRLLiry2OgQVhoTKwQsBQoo6QHs6EsihRYAm1ApLRxVeQh11DzZfNswEMSuqrtoO2Da4lkXSUaQTDgknKsp6JY+N27tZu7mN9Zs72bi9ky07uvC9KPPXdRwmjm/kmCNnM2VCM60tzbS2NmIruzy+nY0lHLecmKbAsRUoRegobEthAUEY7bPkabRSeEG0XdK1qE0qMmmHGldjuzZg01Xsxi6BDj1spbAyGZyWI/CS4yklasoJE5HoTcVykA3mS4upzsazkuNffOKxZubzaXKTjTXCdV1qaqI8YBG9giAIBy97FMCvf/3r+c53vsPPf/7zyixmQ7FY5Lvf/S5nnHHGqA5Q2L9Ue3ZHsiLEmyNUx2/Fq6oj3fYfqZHD7iwCpmpaLZzj+45Xc8MwZHBwkJ6enmHHAIZZGOIZuuZ9GIFrBG8ymay8V/O+LMvC87xKpa9YLFZEbVysh2FI6BXAL+F7Q2ivQFjMEZT6oVTC0lHJ1bI1hBrbcaOkhBpQVgrHVpjOu1pBGFporfACH41FqRSgUQSAHUS/hypKRNBhgLIhDMJKY4dAAyoSukopEpaKurJZOhKijkXSVaRdF0VIe2cfa7d0smlbD5u2d7J5exfb23sIy+cslXSZ3NrM6ccvYPKkZqZPaIlaCTsWXgEKOqrWhkFU1XVUlMBgo0i6oFAoy8J2bJSxvegAPwBP2wShItQ6qkA7Ng1ZRTZtkbYtHNuOrmGpiDfkEQKBX8Cqn4VVNxmdbgQ7hWc81bFkBvNZinezM4K22p4Q/4xUT1YcHBysXHvzGamtrR32BUkQBEE4uNmjAP74xz/O29/+ds4++2ze+973MmvWLADWrl3LnXfeSRAEfOc739kf4xRGgeqq7EiWgvit/WqxW50pG3+tWujGq6Rm4peZGATDK7rmNXM7Or7/kawT8TEYC4OZjGZSHMxtaTNjvzpzNX77OwgChoaGhlWLjeAx1ohoBpkPQR7CIpTyWKUBwmIfoeehAo9AR1EElqNxlIvtQpgGx8lEFgMUlmOhIkMDqKhS6/khYaApBQGBVtiE+FphYRHqcoYwkR9WWdHEMAU42OACWFhJK6ruYpVtE2FkN1AWrquwLJuBvhybO7rZsr2LTdu62LSji607uskXSpVObuPqskwc38CRc6czbVIzUyc109Jch/bDcpUW8j70F0EPRWLbtaP0BTelSNkWjhOJXktFXmStQ0LA93y8IERhEajoPCQdcJQdCd6ki+NY2L6HVypQKnrkfIXCQWebcZomYWebGdrRQ/PE6SjLworlGsdzmk2TEHPdTbJDvJ1z/LMZ93GHYcjQ0FDlc2WWZ7NZksmkiF5BEIRDkD0K4NbWVv7rv/6LL3/5y9x8883Dqmann346X/ziFyUC7RDh5XJ2qyenVVde40kHuxOh8WXx5g/VUV3xCWfxiltciMbFcPw41Y0h4ikKxmqQLzdLMELI3O42tgWzv0KhUJlgFt9/xbLge2hviMAroIIClAZR3iB2qRcCTRAGKBRhuVevj4tyQNsWdrIWFYZR1dVW2Fpj2xbKUuXGDxBqjecHeAHlhIOocguakBBbRx5Y3448D45loy2FhYNlRU5gpXRU3VUKmyhjVyuNUtHxAj8gly+yva2PrR09bGnrZvP2Lra2ddHbN1i5/qlkIrIwLJzNlAlNTGxuYOqkJrKZZDSBLgzwfU3Rh/6hkJIHBFFCg7E0JDKQdmxSriJhRQkQkfSFINQEIQR+1BEjUJH/IuE6JG1IuIqkZeEmXCzlowt5gsEhhnxNERfSdej6VuxsM1ZqHI69M4UhmcpXJqdVx+eZSWiJRIJUKlWxJsTtMOYzZawt5jXzpcd8dh3Hoa6ujkQiIQkOgiAIhzgv2whjypQp/PjHP6avr4+NGzcCMG3aNMaNGzfaYxNeBdVCdKSqbrVvtzoBYcRb+rHKbHx5vJo7UsJCvHuZeR5fZvYT99Ka2fPxsRuxG09JMPYDc3ylVGVi2Uhi3hzb3LrWYYD28oTeEH4hR1AaROcH8Eq9hKUA7XvYtoqyadEo5RDailBBaLtoOxVNQlPgqChT11Yaq2wvoBwzFpYFnx+EBEXwtSYMNIHSKK2xykm6oaWwlVXxwirCSLwrjW3ZWOX1saJrYFvlGXGBhU9Id98g23f0sLW9my1t3bR19bKjo4f+gSF0uabr2DatLQ3MnjqRSUsamdzayKQJjTTV1UXXRSlCHRDq6DwN5EMCP4waS4RRtm55/h3pcmOJVMIm5dhopaJUCYAQfB2gA0Wow6j1sFY4lo3tOiRtH0vZKHR0Dv0iFD28EhQCFz9RC7WthIkG3HQdiWSqIjyNHcGI1WQyWenKZj43ptKbTCYrsWPmc2PsMPEvZvG7GfHPXDwOTUSvIAjC4cNet0Kur69n8eLFozkW4RUykg1gdxaGuPCMV7/MT7zKa9YBhi2LvxbPuDU+yOoqb3yMRoDExWt8Mpp5D9Ui1ywz61ZPhDMTlkx113EcstlsWeCGWEExigrzC/iFfkpD3YSFPkpDBQLPw9eK0HhOHUA5WE7ki7VStVElVUURYBaAXf7yEIaE5bgEraI2u0rZ6FCT98rnU5tQBU0Q6nItNDqvDpCyLQId4loKZVvYWNiOQusQq7ymRUiobTx8Al8x5AW0d3azva2Ptu5I4LZ1dNPW2Uux5GG8C8lUgtamccybOZkJ4xuY2DSO1vGNtDSPw7bK9ogwBAWhDin6AYEXRYuZFOUwiAS8siNbQyIZxYylXJuka+MojbaA0EIrCz/w0aHCx2QxW6AgYUXnVKkwmmwX5sHT2NqLJrlph1K6CS/diHLqsLP12FbUatgIUBM3Fv8cGmuD+ZJjPgeZTKbyu/k8GetK/CfehS/ua48LXhG9giAIhyd7LYCFA8+esnZN5av6j3Vc5MbFZXwb8/tIQnqkiq15hF3jyeLpDfGKbqlUGnbLuXqS3EgpE0bg2HYkhszktMpkNQWUcoRBAVXqR5cGKQ29hLN5FcXcACXPJwih4EPgazwUoXaiJg42OIkUtlKkHBvHsrAtcFwFoUbpkNBSkWjVFqHWBDok8FVZnEa39wMdorSORKMOoyqpiWkgsjMoIOO6hGgcy8FSFlpHk9UIIwGtgpBi4KFLDvlSie1dfbR39tHe1UtXdx8d3f109PTRP1C2LZQvfX1dlvFN4zhh8TxaW8YxoaWJ1qY6xtXXYNlRtpkialGswxA/DPBKAejIduGXrReVyi3lJAhH4SpNOmGRdF2SjoXjKByl0JZFEISEIZSCAIIQX/kEPlhaoezofFo22PhYuoTj+4Shjr4suAnC7CRyiXHg1KMSUe5uNpUilUpRU1OD67q7fGHzPK8iSI3v1rZtMpkMjY2Nwzy9nueRy+WGWXEqlf+qCXBxz7D5qe4mKAiCIBxeiAA+iNmd4I0/xlvgxjuTGbEJw0Vs3BoQF6Gm2hqPfYovr/bmxoWumRwUhmElEioutOOTkeKdzqp/N9Vcy7JwHYeUo7HDYlQx9AsE+R50fw+lXDd+cZBCwcPTFp4HJR1S8C36B/voG2gkVIDlRpOvLLCTipRSOA4kEi5uuaSrzHcIpSJbgh8QAKFW6ECjQ40mIAxCLFuBVgRao8qWjQDQfuTZDYkmvWkVYltWVMFVNhrwwpAwCCkGAb7W5EsBnZ39dPXlaO/qob2rj86ePrp6+hkYGCQ2LDLpFOOb6pkzfRLNTXWMbxwX/TSPI5NJQhhWEiXCiqANKXma6ESUX4u0PWE5jowwEvwuUUc3N2mRBBw3QdKxSdoKxwYUeIHG0+AHIcV8iVApin4I2kJrwLJwVIhLCScAjYcOFMpyCJK1FGtaCBPj8Jwsth1NPkyVfbnJZJJMJjNsQlq8BbX5bJh14tnL5stRIpGgWCxWPn9me/P5M5MfR5q0Fm9eIg0qBEEQxgYigA8iqv21I1VXzXqmqls9KS1+OzheRTOTxOK2griQHSmFId7m14jc6upuXBybn8rkpHKUmBlHvCGACj1sXYKggOUXUEEeBrux8v0EhX60X2IIje9FgtPzNYFyKAQKX0eTrzSpqMGDBmUpXEeB3UgyGWXnOoDj7qzuKqWgPAGtGJTQHhRDAEW5kImyjJAMIkuDDrG0wg8DgqBcMSV6JNJ/WDpqywsa5UQT13KlAl29A/R0D9DZN0B370D0vCdHd/8AxWIpKrmWK6812TRNjXXMnzWJ5oZ6xjfW09hYR1NDHTWZJAQK5US2DhV5IwgBLwywNHilSJRTrvRGCRAaX2tcBdpWJO2yXcWChBP5jROOTcKxcGyFbYe4dtQa2Q9DSoEilw+jc68VXhDgB5FtwrIUjvZwNSTwUDry/1rJNEFqPJ7bSGBnCO1URWw6jkNNWfQam4L53MXvDhiBW53WUG2xMekMAwMDdHZ2Vr7wxe0Q5nNoKrrxuybVHfsEQRCEscN+F8B33HEHt912Gx0dHcydO5fPf/7znHDCCbtdv1Qqceutt/Lb3/6W9vZ2mpubueSSS3j/+9+/H0c9OlSnKsT9sLCz+hoXo3EREPffxn2+ZgZ7dVW4upIc9wLH/bdxUWxin8z6QEWQGFuCESsV0aEDCEpYwSAEBXSpB1XqIejvIxjshVKJMPDAsgg1BEFIgMLHQQMFoKQT+H5IECo833idLUBjWZQnnAWkHBU1VHBtLGURFHyymciLG7XUDfEJKAVQCgNCHbX8VWEk2LQKo0SFQBNSLpiGURtgpf0omcF4fIm6o6E1+XyegcFBenpz9OdydPXm6O3P0dU9QHdfjqFCIRKjRNXWhOvSUF9DU0Mts6dPYFx9LY0NtTTU1dIwro50spyzHTVCi+wIOvIe+0GU9mDpKNYstCIFbKsoFk0pcFSkyJWysFwby1JYSuG4Do6CZMLCscAiwLYi1e4QTdjzgpBSoCl4NgUvwPchJMTzyoLSBnSAG0Aaj6SjcCyNcl10agKeM46inSW00mi183NoPhMmfSE+kSz+ObQsq5Kha2wJZoKb+XdgJq6N5Ak3CSLZbHaXCrH53FZXg0X0CoIgjG32qwB+4IEH+NrXvsZ1113H8ccfz5133slHPvIR7r//fiZNmjTiNldccQU7duzgK1/5CtOnT6erq4tCobA/h73PiIvZeNJCXOACu1R0TVXX/HGP59VWdzKLV4XNvuJCeqSYMnPrOV5dMwIinU4Pm3VvqxA7KKDCAgQ5wmIeneshLPThB/34gz6B9ghCi7CcPeAHGj900VZUMS1phQ5T+DrE9zRa2/ihQoeackoWygoil60VeWhdBZYbRnaGEJRjRUJO2YRaU/AVYeAzmAfdV8Qvx24Zf6vWkd2hfLeeUGnKug5laQILrDBKaOgfytPXO0huIEffwCC5XI7egUH6+3P0DgzROzAYxZUpMGVc27ZpGFdLU30tkyeOp6G+loaGWhrra2mor8V1kzjlRAmlo/NglzWYZTqxlePSLNuKBK8VVWOjfFsLq/wmdPmwtm0RBgrLjloTO5bGsS0UGtdS2EqjVGT1CAIdpVGENkVfR7aRir1DE+iAQHu4YfRZsYGM8qJmFArspItVM4kgMQ7fqaGgUnjaqnxmXdclUbYqGJuB8WvH0ziqH+NRdUaUmhzmuG83/m8jOmcWqVSq4gGura0dJnqrLTji6xUEQRDi7FcB/NOf/pQLL7yQiy66CIBrr72Whx9+mLvuuosrr7xyl/X/9re/8eijj/LHP/6RxsZGIIplO1QwotN4Ek3VqtJMASrVK0Nc7BqhWj2xLD6xzVSE41XbeDU5LhziQtdU3Yb5cLWPCooQFlB+idAfRBX7CYI8XrGfgq8JvIAgVGg0fhC1qi3hoFRUKY0qpqloQpiOfKPap5x7C35Qvg2tg50TrxQopVE6moBl2eDYptoaPY/ydnUlmSDwQ7QK8UN/p6gMoRBCOBR5Ya2y3omqqZqBoTy5oSEGBobo6x+ifyjPwOAQA4ND5HJD9A8MMZAb3Dk5r2wHdWyb+tosdXVZpk+dwOLaLLU1WepqaqjPRsvT6WhCnUlQsEIIVPn9RPPlcK1I7CYcC8uOBG2U12tjK4XSCmVH3d8cM6lRRbFrOlBR6+IwEsvK0tiWhZ1U2I5ChSFaK3w0GkXei5IngsCvfBEhUIQEUTMKNMr3UCFYoU/SjmwNlqOwMmncTCs6VU9JZfHdLMWQymfRwsKxHVLlfN145JixHRjbS/yLVfwzbirEphocn6gZt/yYeDwjms1x4sLZHD/+eR+pgYsgCIIgGPabAC6VSrz00ktccsklw5afdtppPPPMMyNu8+CDD7Jo0SJ+9rOf8Zvf/IZUKsXrX/96Lr/8crLZ7P4Y9isi/ofciF7TmSyevGAwAqFYLA7z+8YbMsSrw/EoMSMs4vml1S1+TcVYBz42JdzARwclVOgRFPoI/KjJg18qojw/muxE1KAgashgEVg2uux1DS0FoYOyyhPvgkjB+uVEAcrtb4OyX0CV2/CqmBjVYSRuLbscLWZF/tlyjC5alQMOQijPsUJ7RAkC5aKrr82+NPlikaHBPEPFArmhPD1deQp+WdDmhsgNDdHfP8RgvrBLRBxAOpWkrjZDTU2aWdNaqa+roaEuS21NDeNqs9TUZUlnUqggskxEaQ9EFVgi0V92FKAsSNkK21I4SpNM2lhEwtK2y15TFbX4Lfd+w7FtKHuIA1uj/DDq9qYsSkHkP3YU+E5UlcVWWEbQaiiUomzhQOuy7cMMLyQMddRWOfRRaLQXYusAx7JQVghJFzvRgE6Nw041YiUy+E6aQEfpDr7vYykLWyssS1FXVzfM+mIErLk7EP9sm8/xSN5wI3rNl8H4l7T4HQ+gsk180ls8vSR+d0NEryAIgrC37DcB3NPTQxAENDc3D1ve1NTE0qVLR9xm8+bNPPXUUyQSCW655Rb6+/u54YYbaG9v59///d/3eLxiscjy5cv32fhHIl7hrXQPq/LcmqprPHC/+g94dT5pvEFEvKpbHemkwwBHhQRBEcLIRWt5eZQfRjPzdR4VDkUxWOWKpGUsAQDKxrIThLiE2o29ryjTNmSnbcAURaNqa5Rxi9lXWexGGxvPKCTsaGJZQkEQ9XPAtsHzoKyzUcpk5YLvhwwVCgwO5hksFBgaKjBUyDOUj37P5/MMDhXIFwoM5qPHalFrqoa12Qw1mQw1NVlam1qoyWSorclQV5umNpshm8pQm81EzSSIhK1SUdavH5ZbDEcpZpSGiKLTysdwypYFx4JEuR2xq8B2wFKR4rc0EPgQlJcHleQyhsLyccod4EINJb8cVUZ0bspF4+i86qiKHKrofIZBJLpV+ZpG11Nj6yI2BVToY1karRWeUgRWksCtIXTqCZ0s2klgOVm0iqq1YTGEogf0oVQ/sDOVA6h4akulEpZl0dfXN2K+dHUHtuqubCNNnozfkYhvV/1jrm11/J9lWXiex7p1617JP13hMKBQKIz6/+OFgwu55mOTBQsWjMp+D+oUCPPH8aabbqK2thaIbBMf/vCH6ezs3EVMx0kmk/v8pJk/4IVCgWKxWHk0wjeTyVSqWTCyuB3J1mAqxCYvNwwC/FKBICwRmp6zoQ9eAT/0okpoCbT28G2FYyusUEcdt5RDqKPUA00apdKEdiS4jICz1E6xF5Q9tabxApRFLmXBWu7+ZdmUO6IR3c43lc9yIoFSkRC0Qk3B88gXinQPFckXihQKRYbyRfLFIoV8kaFitLxYiB7NT6kUdXWriGmTkqAUmVSSbCZFNpuiftw4sqno99p0mkwmRV1tmmxNCjvI0FifQjmqEnPmhVQqttqkgpXfs7IjoevaO0VnujwGS0X/QBIuYNvRedYKy7GxiCwbJvXBUgpllbvOlcvhkWUjahZR8sHXIWGgogl/ZQ2nbB15dl2N64ClNQnHjqrtQWRjsFCUdAChxgpMVzVQ2otOeqDBVmg3hUq0QqIO5dZCMoVysqB2JnIYf3c89xkYVsmNpyPEK7DxFI94I5TqLn/m3211XF4897naB1x97HgGcHUudfUYli9fPmr/gxQOXuS6jz3kmgv7kv0mgBsaGrBtm87OzmHLu7q6aGlpGXGblpYWWltbK+IXYPbs2QBs27ZtjwLYCMs93RLN5XJs27aNSZMmUVNTs9t95PN5BgcHKRQKDA4ODqv0mopuXMyaH/PcrOd5HoFfhCDA90vo0CcMPXTgE/pEZcgAQl0ukWKqhpHvUmFHCQhlX6mNiwqixALbAl22FbjlW/TlbrxRw4iy6EOV2/OWBdhO0asp+QHFoodXLFH0SgSeR9EvRe2FSx6lYomSF/1eLJUoFuO/R4+FYjG6Fa92NlaooKJb/plUkkw6SSaVoKWhhkyqkXQqRTKZoK42RTaTpiadIptMk0inSKdTWLZCheVqsRP5ii3KqQ1B2X8cQrEQvY6p4JYrtUqV0xUCUG6UqmDGZ+nyObLBCRVWeQOlI9VsEVkAQEcV69AHSxHaFjrUqCAEpdCWQoUBfjkrWAc6iivT5dbGloVrhSRUgGtZJBwLbWkUNgSaUhglYhRKJUIvmrimtbljUBbljoOVSKNT9VhuHVYyg3LTKCeNE7MkGMsB7GxgYsSo+WwbMTxSkodJZYh3SRvJz2uotuDEq7XmbkZ8f9XiOr6fuDA34493ahMEQRCE18p+E8CJRIKFCxeydOlSzjvvvMrypUuXcs4554y4zXHHHcf//M//MDg4WPH8btiwAYDJkyfv8Xi5XI5HH32U2tpa6uvrqa2tJZPJkE6nCcOQyy+/nNtuuw3btgmCgA9/+MPcdNNNFItFBgYG6O/vp7e3tyJ+8/l8xatbKkWi0PyxLhXzeL4HYWRL8L2d4yhrsCh9AK/SQUyX/7OIFK2RFVHfAReL6OIoF8IgyqX1/CKlQoDWsYYXYYDne3i+j+eVrRglj5LnUypGy33fL79eXs+PJuT5gdkmWl/rcJhorYQc7HzAti3SyQSpVIJkIkE65VKfrSGVckklEmQySbLpFDXpFMmkSzaVJFUWvOlkEtuxsVxlmqRFSQ3lp8rYKsreX2OPcIyadiLRaikqObgWoBI7B5zTUJeKfLcajWsqiiiUjjqRuYqoKm5ZZe9ulJxglX0OFhpLmX1Ej5Hy1oRKgVZYWkcVXiDUNrbeOQlQl/3AtgoJwgDbirbRqpyxWwoolgLyOfB8j9BW+GE5Wiy0sRIKK12Lk6jFSdagkilsK3q0nMSwKigw4pe86lg7IyJNMkP8x4hgk7VrtjeP8cpt/LXK56QsjuPL4/aG+P7jSSYjedfN/iSjVxAEQRhN9qsF4kMf+hBXXXUVixcv5rjjjuOuu+6ivb2diy++GICrrroKgBtvvBGAt7zlLXz/+9/n6quv5hOf+AT9/f189atf5dxzz6WpqWmPx+rr6+N3v/vdsD/M5ve//e1vrFixohKcD/CDH/yAhx56iFNOOWWX9c3v1Y0qoh+fMAwIQ1O5MsuJLRue91sdfVY92c08r+6oBuwqUEegslwpXNfBdRwSblThS7gOqYSDm02RcKPlyaRLIulEgjaZIOm6pFMJ0qkEyYRLJh0tSyYS2I5p+UDFW2zybrU2+bllD225amlE4aAfVW4pltfZOcyKhYIgEsRWOQ7NssBTUeXV+Jdty1RD7cjOUc6zBXDsoFwJVRBqnCgCAtuxUFiRTQGFraLJZLZrl7220SQzVTbXRrfwNUprlI4SKJRlRXYHDV45Y1jrkFBblMKAIAwhCCqeae37UWvkEEqmCYdy0DbYbhpVk8ZNj8Oxk6h0hkSiBm0lsMsV0bi31lRAjaCMT4SMV1kNJuXDpCaY300sGeyMFIt/7qrj8OJdA+P50/FjxY8dH2O1beLlBG/8eIIgCIIwmuxXAXz++efT09PDrbfeSnt7O/PmzeNHP/pRpZq7ffv2Yetns1l++tOfcsMNN/COd7yDuro63vjGN44YmVZNd3c39977S0yPLiO3wjCkra1tl/V932fZsmV0dXWVBYIett2uz/dM9SSekSb3xGe2m0pc3Atp2zZO+cct/zi2TcIsT9jYrk3CcUgkXFKug5soC92Eg+NE+bCasmg0w9exCWhGpJbfoSl2xm24YRglL4RB5AnWaqedwtgSVGziVlieBGeXt1VWOS2hLHort/PLE8u0Lv+uomquJtpG6bJ9AVBEFoMoOiwag60j64GyNE75/CadgIRrR7YFW6GtSNjaZQsE5fdoqXKFkiCyL5S7pqkwwPeD6AsM5SYdfkDgR7nCQQihHxCl6iq0LjepQKMsF5UA5WZRiTShk0a7KWzLJe1mcdwkOIlhHfriMXTxSDsjHmFnhTUuPON+2/jnyTSbMNFg1Y0nqkVudWOI6k5rcUayOZhjx9tYVwveeDyZ2bcIXkEQBOFAovRI2VCHAb///e/5r//6EUrVlidVRcKpv3+I3/72vmHVX4PrurztbW+joaEB8FHKKlcn47eazQQeF8uydxG11ULilWJEpF1+tBguRmFnrqwyopFyZm55pchvujNFwIovK69vxYQwVfs3Ytj8rog8wwGR4DVJD8Zba6mdghUdO4YFlhPtQDM8RaHyXlXZX2tZO98foKyoMYZFFLmgiCb7oXX5fStcq7weOmr6APT2FmioT5azdO1oKprWaNMJjpAgCCPfcFSwxQ/DyLoSQohCB6r8PaEcKGbbOC7YdgrlpnBTWWw3g5NIYidSYCexnBRa7Uz8MOLQVEPjk7iAYZ8T84WHyvkYngu9y2ekLCBNDFm8e1p1l7P4JLd4JTdeOa5ukhLfNj6hrTrOLB59Zh5HskzEv/zFUyP2FTIxZmwi133sIddc2Jcc1CkQr4X6ulouPO8t1NY2UDeulvrG8dSPa8FyXO6//4ERBbDjONx6660opejv76evu5P+3i56+3opDA6SKwzhDxUo+SV8v0DolSuhIVFHMSCqDfqYKVZhOfHVTGZTlVcUChtdft0IXct4X8sawTYvlJ9blCOxdFTRtMuvmdvuVnkl2xy0vF0l+qxchfXLrylFOcVg5zFNpVgRCVsndmytdgp0bY5RFr8m81apssgqt+O1VLkCi4piwmJVZ8syE86i+LCKKIaoKYNdPmAYRK2JlY7sBSGUyjorshkEhDoglxsgCOojry6aMFCR41pHNVtLOYQWaMdF6yTYDiqZwHKSOHYClXBwrCROKoVTFrtG6BnROFK6QRSJHL3vRCKxi+gz1VnK7y/+e/wHdiYfGNFpRG48jcH8bmL24tVZ87gnUWqIR/LF48pgeFKDEb1mf/FzUb2/6jsegiAIgnCwcdgK4Nq6eo5809uHtfI1XHrppdx+++0MDQ1VlmUyGS655JJy9Rfq6+uZPHlyRVCYiDLf9ykUChQKBTzPo1CI8ml9369EooVB2btbyhMEITr0CfDB8wl8Hx0GBKGP0pGHGO2h/UhBap+KTcECdBgSqqhrml2O24ra4+5MOQDKgjXqgKbKolyV7QKRrSEy5ioUYRgp30hMRxYJ06zCUlFWL8S8uZSFucnCLQtvi1jShGUEYBhNNiOaTEYYRi14ywG3ITs9wVEsWdmiYYMpDEZiNfoyEQ3ErpS0ta2wLBfsBNqxcRwXZUeTw7RyINVLurk1WkdZaMtFOS6aaF3KIs90wQMqAtdUakfyehs/thmfebRtm1QqNcy2ErcVxMWgsTbA8Ii8ynsuZ0DHrTBm/XgSg7EyeJ63i5e2WnhWZ+xWe3dH2t5YNKon1VVvW+1NFrErCIIgHCoctgLYcRzGjRs34ms333wzwLAUiEsuuaSyHHbeZjb7SiQSpFKpYaIoHupvxLGZwGZaH5s4NFOVi3duM1VoI67is/bN8uh4AYQBge8R6ihjlqAsXnQIVjmTVpW7fwFlvVvWjToSqFpXRHG0PMQmqtiG5a4WDsbjG/kZlBVVlp1yeHC5Vghh5KMNlIWtLEKrLG7LtWxL2QSWDdg7K8CKqOKqy3Vw5ZZ9E9E2Aao80UxFObjYhEqVrShlIVe+PnFhCZEgc2wb7e3Arhk/rPpprmN8O3P+lVIVIRmvpMbFphHIRjSbRyN6415cE4lnnnueVxkHRB0R48Iybp0xwrO62xkwTHxWC+zqFIi40I23xK727lZXqaupzt6NC13x7QqCIAiHMoetAN4TjuNwyy238PWvf32POcBx4oLYEK8SxsVxvLoXrxwagQtUxFe82xvsFB3xamNcdJtc4bhYG8m/acbnOM6w/cePUT3L39xON5OtzLLqpIH47e+EbeP7Pja7xnEZwRQ/fnVLaFP1jE8Gi78+UsRW9Xuofj1elYyvY8YWv0YmUSGemRv3t8bvHhjRGa/axq9pfBxxW0N1Nm+8yYSp8MYjzaptBPHX48K4+stY3J4Rp7oSHT+H1UJ2JLErCIIgCIcbY1IAG2pqapg3b96r3j4ujAxxUVw9yWh3UWhx4VQdb2UEZ7VABEbcr6lsWpY1rOlBPE6tWizFBVx8wlN1XFZcwMXHYvysZh9mXfM8nnpQXUE01oDqqmjchxqfjDXSceLHSqVS1NTUVMR8XMjFz3O10BzJOlAt/Ksr+EZUx9M84ueoupFE9Xjj4rL6PFRfK/P+qkV2fJ/V3uLq6xBHqrqCIAjCWGZMC+DRYE+iuLpCV33bvdp3Wr2NqZaabc1xqmfYx0V1XPhUVy+rq6imYm1eH0nMV/tIR0ocqKZaWJuKqxGQ8XWqq47Vk7Ti5y4ubOP7rRbkuxOgZvxxoVstOKsFazymrlrcjhQlVr2P+HmKn+u4UK0Wr9XV5OrzHt/vSNFl1Z8PqeoKgiAIYx0RwPuBlxMrcbE6ko0hXqmNbzdS16543mp1BbdaHMWzWKtF0kgWgmphWD0pKp6sEV/XHHskEVhd8Tb7Hdb8Y4TK90hVUGBY84WR3sPuqpzVInkk20L1/szjSDaCana3zkjneqTH6s9I/DrH9ylVXUEQBEF4eUQAHyCMsBqJ3VWJzXPDSFVlszwuUOMV3LgtIu5Trj5etQAzv48k/KrXMVXX+Gsj3Yqv9i6PJJDj76daxFaPwbIsstksjY2Nw8YTf91gKscjWSGqxzISr/b1kQTzSF+Adufrjr+XkSrDgiAIgiC8PCKAD0JeTsyMJIp3J5INu6tYjrSPeNXZvFYtyF7ueLtjJEuB+X13VdFqX+6e3k82m6Wurm6v3//+oPrcVi8bieoqbvU5EQRBEATh1SMC+BBkb4RQdVWxWoCNtM+RqqC78/i+nOgdqaobfz5aQi7esGF/sacvJC8ncHf3IwiCIAjC6CEC+DBlT0JqJIG2NwJ5T9XXw0247a6qvrvHakTgCoIgCMLBiwjgMcjLibCRKsfx5S+375Ee9/T77vbxcstHEqHV9o3qdXe37JXYOfZkzxCBKwiCIAgHPyKAhV14JRaL3f2+N2J5NAiCoNJ9bW+oFuUjRZmZRxG2giAIgnB4IAJYeFW8UkG4t9aB3W23p3HEqc4Bjq8jAlYQBEEQBBABLOwn9pcIjSdLCIIgCIIgjIQoBUEQBEEQBGFMIQJYEARBEARBGFOIABYEQRAEQRDGFCKABUEQBEEQhDGF0gciq2o/8Oyzz5JMJg/0MARBEARBEIRXieM4zJ07d5/v97AVwIIgCIIgCIIwEmKBEARBEARBEMYUIoAFQRAEQRCEMYUIYEEQBEEQBGFMIQJYEARBEARBGFOIABYEQRAEQRDGFCKABUEQBEEQhDGFCGBBEARBEARhTCECWBAEQRAEQRhTiAAWBEEQBEEQxhQigAVBEARBEIQxhQhgQRAEQRAEYUwhAlgQBEEQBEEYU4gAFgRBEARBEMYUIoAFQRAEQRCEMYUIYEEQBEEQBGFMcdgL4CeeeILLLruM173udcyfP59f//rXo3q89vZ2rrzySt70pjexYMECPve5z+2yzurVq/nkJz/JWWedxfz587nllltGdUyCIAiCIAjCTg57ATw0NMS8efO45pprSKVSo368UqlEQ0MDH/3oRzn66KNHXCefzzN58mQ+9alPMWXKlFEfkyAIgiAIgrAT50Ac9Oabb6ajo4Ovfe1ro36sM844gzPOOAOAq6++epfXS6US3/3ud/nd735HX18fc+bM4VOf+hSve93rXtXxpkyZwhe+8AUA/vCHP4y4zuLFi1m8eDEAP/zhD1/VcQRBEARBEIRXxwERwM8++yxbtmw5EIfehauvvprNmzdz0003MWHCBP7yl7/wsY99jHvuuYcjjjjiQA9PEARBEARB2MccEAH885///EAcdhc2bdrE/fffz0MPPcSkSZMAeN/73sfSpUv5r//6L770pS8d2AEKgiAIgiAI+5wDIoAPFl566SW01rz5zW8etrxUKnHyyScD4Ps+Cxcu3ON+TjzxRH7xi1+M2jgFQRAEQRCEfceoCuCbb76ZCRMm8O53v3vY8rvuuou2tjY+9alPjebhXxatNUop7rnnHhxn+KkwE+Ycx+GBBx7Y437S6fSojVEQBEEQBEHYt4yqAP7tb3/Ld7/73V2WL1y4kB/96EcHXAAvWLAArTUdHR2Viu9IzJ49ez+OShAEQRAEQRhNRlUAd3V10djYuMvyhoYGOjs7R/PQFQYHB9m0aRMAYRiybds2li9fTn19PTNnzuStb30rV199NZ/97GdZuHAhvb29PP7440ydOpVzzjnnVR1z+fLlAORyOZRSLF++HNd1mTNnDhBZLNauXQtAsViko6OD5cuXk8lkmD59+j5414IgCIIgCMLuUFprPVo7P/fcc7nsssu48MILhy3/9a9/za233sof//jH0Tp0hb///e+8//3v32X5hRdeyDe+8Q08z+MHP/gBv/nNb2hra6O+vp5Fixbx8Y9/nKOOOupVHXP+/Pm7LJs8eTIPPfQQAFu2bOGss87aZR3xEguCIAiCIIw+oyqAb7/9dn7wgx/w6U9/umIxePTRR/m3f/s3Lr30Uj7ykY+M1qEFQRAEQRAEYURGVQAD3HTTTfz85z/H8zwAXNfl/e9/P5/+9KdH87CCIAiCIAiCMCKjLoAhake8Zs0aIJpQls1mR/uQrF69mrlz5476cYSDiw0bNjBjxowDPQxhPyLXfGwi133sIddc2JfslxzgTCZTaf27v/B9f78eTzg4yOfzB3oIwn5GrvnYRK776NIzWKIhmzjQwxiGXHNhXzKqArhYLPLzn/+cxx57jK6uLsIwHPb67373u9E8vCAIgiAIr5Af/XUtX3tgBd9/z7Gcv3jSgR6OIIwKoyqAv/SlL/Hggw/ypje9iWOPPRal1GgeThAEQRCE18htD68H4O6ntogAFg5bRlUA/+lPf+K73/0up5566mgeRhAEQRCEfUDBC2gfKAKwoWvoAI9GEEYPazR3nkqlmDBhwmgeQhAEQRCEfcS6jkE04NqK7X15Cl5woIckCKPCqArgSy+9lJ/97Gfsh6AJQRAEQRBeI2s7cgDMaMpS8ELaBwoHeESCMDqMqgVi6dKlPPnkkzz88MPMnj0bxxl+uB/84AejeXhBEARBEF4Bm3si28PUhjSr23Ns6ykwrXH0o0sFYX8zqgK4oaGBs88++zXt45ZbbuF73/vesGXNzc088sgjr2m/giAIgiAMp2OgSMKxaKlNArCtT6LHhMOTURXAX//61/fJfmbOnMkvfvGLynPbtvfJfgVBEARB2EnHQJFswmZcJsoA3tYrAlg4PNkvjTBeK47j0NLScqCHIQiCIAiHNR0DRdKuTYMRwH3iARYOT0ZVAL/1rW/d4+t72whj8+bNnH766SQSCY4++miuuOIKpk6dui+GKAiCIAhCmY5ckXTCpiblYClo7xcBLByejKoAPvfcc4c99zyPFStW8PTTT/Oe97xnr/axePFivv71rzNr1iy6u7u59dZbufjii/l//+//0dDQsNvtisUiy5cvf03jFw49CoWCXPcxhlzzsYlc99GhrTfPpDqHtrYdpBzFls6+g+Y8yzUfmyxYsGBU9juqAvjjH//4iMt/8pOfsG3btr3axxlnnDHs+dFHH80b3/hGfvOb3/ChD31ot9slk8lRO2nCwcvy5cvluo8x5JqPTeS673sKXsCgt46muhomTphIbTqHr5yD5jzLNRf2JaOaA7w7zjnnnL22P1STzWaZM2cOGzZs2LeDEgRBEIQxTGcu6gCXSUS1sWzSZrDoH8ghCcKocUAE8BNPPEEqlXpV2xaLRdavXy+T4gRBEARhH9JRboGcTkTSoCbpMFQK8ILwQA5LEEaFUbVAXHbZZbssa29vZ/ny5fyf//N/9mof3/zmNznzzDOZOHEi3d3dfP/732doaIgLL7xwXw9XEARBEMYsnbkSANlkJA0yCYeCF1D0Q1z7gNTLBGHUGPVGGHGUUsyZM4crrriC008/fa/2sWPHDq644gp6e3tpaGjgmGOO4Ve/+hWTJ08ejSELgiAIwpjEVIDrUi4A2YRN0Q8ZKvnUJA+J1FRB2GtG5ROdy+WoqanZJ40wbr755n0wIkEQBEEQ9oQRwPXpSABnyqK3Y6DI+NpXZ1sUhIOVUbmnsWTJErq6uirPv/SlL9Hd3T0ahxIEQRAEYR/QkSuQSdgk3ajbajYRPbb3Fw/ksARhVBgVAay1Hvb8vvvuY3BwcDQOJQiCIAjCPqBjoEgmYePaCtjpBTbpEIJwOLFfXO3VglgQBEEQhIOL9nIbZMeKpEGmXAEWASwcjoyaAFZKjdauBUEQBEHYx7T3F0gnbGyrXAEu5wF3DogAFg4/Rm1a50033UQ6nQaiFsjf+973qK2tHbbOF77whdE6vCAIgiAIe4nWmvaBIkfFJrtVKsCDpQM1LEEYNUZFAC9ZsoRNmzZVnh977LG7tD6WCrEgCIIgHBz05T28QFd8vwCObZFwLHpEAAuHIaMigH/xi1+Mxm4FQRAEQRgF2ss2h2zSHrY8k7Dpy3sHYkiCMKpIaxdBEARBGOOYqLO0O7wulk3YDBT8AzEkQRhVRAALgiAIwhinfaAAjFABTjrkiiKAhcMPEcCCIAiCMMZpr+oCZ6hNOgyVAoJQ4kyFwwsRwIIgCIIwxmnvL5JwLGqSwy0QmYRDwQso+sEBGpkgjA4igAVBEARhjNM+UKAm6ZBwhsuCbMLGDzU9QzIRTji8EAEsCIIgCGMc0wXOtYfLgky5ItzWVzgQwxKEUeOACOAPfvCDvPGNbzwQhxYEQRAEoYr2/gJp19pFAJtucGaSnCAcLoxaJ7g9sWjRIiZOnHggDi0IgiAIQgytNW39Rea11uzymkmF6ByQZhjC4cUBEcBXXnnlgTisIAiCIAhVdOZK5L2AuqoECIgmwYFUgIXDD/EAC4IgCMIYZmPXIAD16V1rYtlEVAE2MWmCcLgwqhXgq6++eq/X/frXvz6KIxEEQRAEYSQ2dg0BUJ9O7PJaOmHjWIod/VIBFg4vRlUA9/T08MQTT2BZFvPmzQNg9erVhGHICSecMJqHFgRBEARhL9jYNYhSML42tctrSinq0y47JAVCOMwYVQF87LHHkkwm+frXv04mkwFgaGiIa665hnnz5vGxj31sNA8vCIIgCMLLsGLHAI2ZRMXuUE1jNkH3YAmtNUqp/Tw6QRgdRtUD/Itf/IJPfOITFfELkMlk+Jd/+Rf+8z//czQPLQiCIAjCXvDC1j6asglSuxHADdkEfXmPvCfd4ITDh1EVwIODg7S3t++yvKOjg3w+P5qHFgRBEAThZejKFdneV6C5Nom1m+puYyZB0Q/FBiEcVoyqAD733HO5+uqruf/++9myZQtbtmzh/vvv55prruGcc855Rfu64447eMMb3sCiRYt429vexpNPPjlKoxYEQRCEscFj67oBmFCX3O06rXWRN/jpjT37ZUyCsD8YVQ/wl770Jb7xjW/wuc99Dt/3AbBtm3e84x189rOf3ev9PPDAA3zta1/juuuu4/jjj+fOO+/kIx/5CPfffz+TJk0areELgiAIwmHN/y7bQTZpM6N51yYYhmmNkY3xsfXdvOOEqftraIIwqoyqAE6lUnzpS1/iqquuYtOmTQBMmzZtmCd4b/jpT3/KhRdeyEUXXQTAtddey8MPP8xdd90lTTUEQRAE4VWwrTfPAy9sZ/6EWhoyu0agGdIJm6ZsgqekAiwcRoyqAO7o6CAIAiZMmMARRxxRWb5jxw4cx6G5ufll91EqlXjppZe45JJLhi0/7bTTeOaZZ3a7XRBqtve9Np+x1q9p82gfr30X6H0wkH3xXg6GMeiXOaPb+j0y5VD30R3HPtjHvriu+2Qc+2An+2Akr3YcG3tLOG0D+2gU++dzuj/Gcbi8j92xvqtIsLVvv47jYDkf++KU9uU9vvWHFQAcO3UctrXndIfjpjfwx2Vt3Pn3jbxubguOve/TIBR73mfnoC8+5DFG2rWpz+zaoXBfMKoC+DOf+Qznn39+pXJrePjhh/n973/P7bff/rL76OnpIQiCXcRyU1MTS5cu3e12y7b3c9UvHnp1AxcOcTYf6AEI+50tB3oAwgFh64EewCFNyrU4fU4zDdkkXbk9d3qb31rD4+u7+fx/v7ifRrc7Nh3g4wv7E9tS/P3zZ9Fcs3uP+qtlVAXwiy++yBe/+MVdlp9wwgl861vfGs1D05xxeMOs3Xua9icHS2ziQTGMfTCIPe0iCAJse+Qon33NQXE+4bDK5Xw17yQIfGw79r+yUf6MHWq85vdykJ7PV/NvfZ+M4yD59/ZaRqG1xlKK5ozDpDoLO+yHcM/b1AD/Z0kDz+7IU/DCfX4e9qaqHQYB1n76/7twcJB0FDv6CoeeAA6CgFKptMvyYrE44vKRaGhowLZtOjs7hy3v6uqipaVlt9tNbMhy+0el29xYY/ny5SxYsOBAD0PYj8g1H5vIdR97yDUX9iWjKoAXL17MXXfdxZe//OVhy++8804WLVq0V/tIJBIsXLiQpUuXct5551WWL126dI9RasVikeXLl7+6gQuHNHLdxx5yzccmct3HHnLNxx6O4zB37tx9v999vscYl19+OR/4wAdYuXIlJ598MgCPPfYYy5cv56c//ele7+dDH/oQV111FYsXL+a4447jrrvuor29nYsvvni32xxzzDGvdfiCIAiCIAjCYYjS+2Iq+h5YsWIFP/nJTyrf2hYsWMCll146LBVib7jjjju47bbbaG9vZ968eVx99dUsWbJkNIYsCIIgCIIgHMaMugDeHUuXLuXUU089EIcWBEEQBEEQxjCjaoGopq2tjXvvvZd7772Xbdu2iZdHEARBEARB2O+MugAOgoA//elP3H333SxdupT58+dz8cUX86Y3vWm0Dy0IgiAIgiAIuzBqAnjdunXcfffd/Pa3vyWdTvOWt7yFpUuXcuONNzJnzpzROqwgCIIgCIIg7JFREcDvec97WL16Neeccw7f+c53OPHEEwH4yU9+MhqHEwRBEARBEIS9xhqNnT777LNccMEFfPCDH6yIX0F4pdxxxx284Q1vYNGiRbztbW/jySef3OP6jz/+OG9729tYtGgRZ511Fnfdddew13/4wx/y9re/neOOO46TTz6Zyy67jFWrVo3mWxBeBfv6usf54Q9/yPz587n++uv39bCF18BoXPP29nY++9nPcvLJJ7No0SLOP/98Hn/88dF6C8KrYF9f9yAI+M53vlPZ5xve8AZuvvlmfN8fzbchvAJeyTVvb2/nyiuv5E1vehMLFizgc5/73Ijr/eEPf+D888/nqKOO4vzzz+ePf/zj3g1GjwIvvfSS/tKXvqRPOOEEfcEFF+if/vSnur29XR955JF69erVo3FI4TDj/vvv10ceeaT+5S9/qdesWaOvv/56fcwxx+itW7eOuP6mTZv00Ucfra+//nq9Zs0a/ctf/lIfeeSR+n/+538q61xyySX6nnvu0StXrtQrVqzQ//Iv/6JPPfVU3dPTs5/elfByjMZ1NzzzzDP6zDPP1G9961v1l7/85dF+K8JeMhrXvK+vT7/hDW/Qn/nMZ/Rzzz2nN23apJcuXarXrFmzv96W8DKMxnW/9dZb9ZIlS/Sf/vQnvXnzZv3ggw/qE044QX/ve9/bX29L2AOv9Jpv3rxZf+UrX9H33nuvfte73qU/+9nP7rLO008/rRcsWKC///3v6zVr1ujvf//7esGCBfrZZ5992fGMigA2FAoF/d///d/6fe97nz7yyCP1EUccoX/84x/r3t7e0TyscBjwjne8Q19zzTXDlp199tn629/+9ojr33jjjfrss88etuzzn/+8vuiii3Z7jFwup4844gj9pz/96bUPWNgnjNZ17+/v12eddZZ+9NFH9fve9z4RwAcRo3HNb7rpJv2ud71r3w9W2GeMxnX/6Ec/qq+66qph61x11VX6ox/96D4atfBaeKXXPM5HP/rREQXwv/7rv+oPfvCDw5Z94AMf0JdffvnL7nNULBCGZDLJP/3TP/GLX/yCBx54gA9/+MP87Gc/47TTTuPSSy8dzUMLhzClUomXXnqJ0047bdjy0047jWeeeWbEbZ599tld1j/99NN58cUX8TxvxG0GBwcJw5C6urp9M3DhNTGa1/3aa6/l3HPPrXSkFA4ORuuaP/jggxx99NF86lOf4pRTTuGCCy7gP//zP9EHJvZeqGK0rvvxxx/P3//+d9auXfv/tXc3oU1sYRjHn5Ja0V1p6qJtEOpHaZJFWxBXugpuRtCNiNa4EEoXKiharBsRrA0ILXUjKnRRNFUCYlAaRMQPkCBKQTdxYbDVFoU6JZsuRKznrm64sfGa5mYy5s7/B1n0ZHLmPbwJeTKZTCVJ2WxWL1680M6dOx1YBVajnJ6X4lfPi1LmdDQA/9PGjRt1+vRpPXv2TGNjY1qzZk21do0ak8vltLy8LL/fXzDe1NSkL1++FH2MbdtqamoqGPP7/fr+/btyuVzRx1y8eFGdnZ3q7u6uTOH4T5zqeyKR0MePH3XixAlH6kb5nOr53NycJicnFQgEND4+rsOHD2tkZETxeNyZhWBVnOp7X1+f9uzZI8uyFAqFZFmW9u7dq97eXmcWgpKV0/NS2La9Yk6/31/SnFX9RxiS5PP5FIlEFIlEqr1rIC8Wi2l6elq3bt2Sz+dzuxw45P379xodHdXk5CQfuj3EGKNwOKxTp05JkoLBoD58+KB4PK5Dhw65XB2ckkqllEwmNTIyos2bN+vt27caHh5WW1ub9u3b53Z5+MNUPQADv9PY2CifzyfbtgvGFxcX1dzcXPQxfr9fi4uLBWO2bau+vl6NjY0F48PDw0qlUpqYmFAgEKhs8SibE31//vy5crmcdu/enb9/eXlZr1690u3bt/X69Ws1NDRUfjEoiVOv9ebmZm3atKlgm/b2dn3+/LmC1aNcTvX90qVLOnLkiCzLkiR1dHTo06dPun79OgHYZeX0vBR+v3/FnLZtlzRn1U6BAErV0NCgUCikdDpdMJ5Op395ukJXV1fR7cPhcMGRv6GhIU1NTWliYmLFGyTc5UTfI5GI7t+/r2Qymb+Fw2FZlqVkMslRYZc59Vrv6enRzMxMwTazs7NqaWmpYPUol1N9//r164pv9Hw+n378+FHB6lGOcnpeil89L0qa87c/kwNcMDU1ZUKhkEkkEiabzZoLFy6Yrq4uMz8/b4wxZmBgwAwMDOS3//sSOUNDQyabzZpEImFCoVDBJXLOnz9vuru7TTqdNgsLC/nb0tJS1deH4pzo+8+4CsSfxYmev3nzxgSDQXPlyhUzOztrUqmU6enpMTdv3qz6+lCcE30/c+aM2bFjh3ny5ImZm5szDx8+NNu3bzexWKzq68NKq+25McZkMhmTyWTMwYMHTX9/v8lkMgWX052enjadnZ3m2rVrJpvNmqtXr5pgMFjSZdDqjOFnsfgzxeNxjY+Pa2FhQVu3btXZs2e1bds2SVI0GpUk3bhxI7/9y5cvFYvF9O7dO23YsEF9fX06cOBA/v6Ojo6i+zl27JiOHz/u4EqwGpXu+8+i0ai2bNmic+fOObsQlMyJnj99+lSjo6OamZlRS0uLent7FY1GVVdXV72F4V9Vuu9LS0u6fPmyHj16lP9q3bIsHT16VGvXrq3u4lDUante7H27tbVVjx8/zv/94MEDjY2NaX5+XoFAQCdPntSuXbt+WwsBGAAAAJ7COcAAAADwFAIwAAAAPIUADAAAAE8hAAMAAMBTCMAAAADwFAIwAAAAPIUADAA1YnBwUP39/W6XAQA1jwAMAAAATyEAA8D/wLdv39wuAQBqRr3bBQAAVi8ajaq9vV3r16/X3bt31draqjt37rhdFgDUBAIwANSoe/fuaf/+/YrH426XAgA1hQAMADWqra1Ng4ODbpcBADWHc4ABoEaFw2G3SwCAmkQABoAatW7dOrdLAICaRAAGAACApxCAAQAA4CkEYAAAAHhKnTHGuF0EAAAAUC0cAQYAAICnEIABAADgKQRgAAAAeAoBGAAAAJ5CAAYAAICnEIABAADgKQRgAAAAeAoBGAAAAJ5CAAYAAICn/AV6geCct+QPBQAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 720x288 with 2 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "%matplotlib inline\n",
        "import pylab as pyl\n",
        "from IPython import display\n",
        "\n",
        "\n",
        "bayesopt.optimize(1)\n",
        "\n",
        "for idx in range(9):\n",
        "    xopt, yopt = bayesopt.optimize(1)\n",
        "    bayesopt.render()\n",
        "\n",
        "    display.clear_output(wait=True)\n",
        "    display.display(pyl.gcf())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-hSVo6BJvrMT"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "Minimal-BayesOpt.ipynb",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}