IMP_parities.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Rohith-Rongali/IMP/blob/main/IMP_DT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sTt4FBrCswwi"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "TkKwoTDlskqo"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Dataset\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "import torch.nn.utils.prune as prune\n",
        "\n",
        "from copy import deepcopy\n",
        "\n",
        "from tqdm import tqdm\n",
        "import os\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from types import SimpleNamespace"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yci_s3iDs1Mm"
      },
      "source": [
        "# Data functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "wgzl99bdtK7G"
      },
      "outputs": [],
      "source": [
        "def generate_data(data_config):\n",
        "    # n = 128\n",
        "    # k = 7\n",
        "    num_points = data_config.num_points\n",
        "    n = data_config.n\n",
        "    k = data_config.k\n",
        "    alpha = data_config.alpha\n",
        "    bias = data_config.bias\n",
        "\n",
        "    data = torch.zeros(num_points, n)\n",
        "    \n",
        "    # Generate alpha fraction of points with {-1, 1} coordinates\n",
        "    alpha_points = int(num_points * alpha)\n",
        "    if alpha > 0.0:\n",
        "        probabilities = torch.tensor([1-bias, bias])\n",
        "        alpha_data = torch.multinomial(probabilities, alpha_points * n, replacement=True).view(alpha_points, n).type(torch.FloatTensor)\n",
        "        alpha_data = alpha_data * 2 - 1\n",
        "        data[:alpha_points] = alpha_data\n",
        "    \n",
        "    # Generate (1-alpha) fraction of points with {-1, 1} coordinates\n",
        "    uniform_points = num_points - alpha_points\n",
        "    uniform_data = torch.randint(0, 2, size=(uniform_points, n)).type(torch.FloatTensor)\n",
        "    uniform_data = uniform_data * 2 - 1\n",
        "    data[alpha_points:] = uniform_data\n",
        "\n",
        "    indices = torch.randint(0, n, (k,))\n",
        "    selected_columns = data.index_select(1, indices)\n",
        "    labels = torch.prod(selected_columns, dim=1)\n",
        "    labels = (labels + 1) / 2\n",
        "    \n",
        "    return data,labels.long()\n",
        "\n",
        "class CustomDataset(Dataset):\n",
        "    def __init__(self, x, y, transform=None):\n",
        "        self.x = x\n",
        "        self.y = y\n",
        "        self.transform = transform\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.x)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        sample_x = self.x[idx]\n",
        "        sample_y = self.y[idx]\n",
        "\n",
        "        if self.transform:\n",
        "            sample_x, sample_y = self.transform(sample_x, sample_y)\n",
        "\n",
        "        return sample_x, sample_y"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3ikUnEJVwRei"
      },
      "source": [
        "# Models\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "LzvqA-FRwVVY"
      },
      "outputs": [],
      "source": [
        "class DNN(nn.Module):\n",
        "    def __init__(self, dim_in, dim_out, width, depth):\n",
        "        super(DNN, self).__init__()\n",
        "        self.depth = depth\n",
        "        self.layers = nn.ModuleList([nn.Linear(dim_in if i == 0 else width, width) for i in range(self.depth)])\n",
        "        self.output_layer = nn.Linear(width, dim_out)\n",
        "        self.relu = nn.ReLU()\n",
        "\n",
        "    def forward(self, x):\n",
        "        for layer in self.layers:\n",
        "            x = self.relu(layer(x))\n",
        "        x = self.output_layer(x)\n",
        "        return x"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dMpRYgpoxEJf"
      },
      "source": [
        "# Configs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "Nz3C2i92xG4c"
      },
      "outputs": [],
      "source": [
        "\n",
        "model_config = SimpleNamespace(\n",
        "    width=64,\n",
        "    depth=2\n",
        ")\n",
        "\n",
        "data_config = SimpleNamespace(\n",
        "    num_points=10000,\n",
        "    n=40,\n",
        "    k=3,\n",
        "    alpha=0.0,\n",
        "    bias=0.5,\n",
        "    dim_out=2\n",
        ")    \n",
        "\n",
        "train_config = SimpleNamespace(\n",
        "    batch_size=32,\n",
        "    lr=0.01,\n",
        "    epochs=100\n",
        ")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7BEZbB2Ps18N"
      },
      "source": [
        "# Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "2VUhFr590ybb"
      },
      "outputs": [],
      "source": [
        "device = torch.device(\"cuda:1\" if torch.cuda.is_available() else \"cpu\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "cMMdrJJitsUq"
      },
      "outputs": [],
      "source": [
        "x_train,y_train= generate_data(data_config)\n",
        "x_test,y_test= generate_data(data_config)\n",
        "data = [x_train, y_train, x_test, y_test]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "XrkZpOuWvJwj"
      },
      "outputs": [],
      "source": [
        "# Create DataLoaders for train and test datasets\n",
        "batch_size = 32\n",
        "\n",
        "# Assuming you have a custom Dataset class, you can create instances like this:\n",
        "train_dataset = CustomDataset(x_train, y_train)\n",
        "test_dataset = CustomDataset(x_test, y_test)\n",
        "\n",
        "# Create DataLoaders\n",
        "train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
        "test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "diQWbC5aw32C"
      },
      "outputs": [],
      "source": [
        "model = DNN(dim_in=data_config.n, dim_out=data_config.dim_out, width=model_config.width, depth=model_config.depth)\n",
        "loss_fn = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=train_config.lr)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "HkxSrBb4ziwR"
      },
      "outputs": [],
      "source": [
        "def train(model,loss_fn,optimizer,train_dataloader,\n",
        "          data,num_epochs=train_config.epochs,log_epochs=10,log_acc=10,thresh=1e-5,print_std=True):\n",
        "  [x_train,y_train,x_test,y_test]=data\n",
        "  model.to(device)\n",
        "  Train_losses=[]\n",
        "  acc_dict = {'train':[],'test':[]}\n",
        "  for epoch in range(num_epochs):\n",
        "      model.train()\n",
        "      for x_batch, y_batch in train_dataloader:\n",
        "          x_batch = x_batch.to(device)\n",
        "          y_batch = y_batch.to(device)\n",
        "          pred = model(x_batch)\n",
        "          loss = loss_fn(pred, y_batch)\n",
        "          loss.backward()\n",
        "          optimizer.step()\n",
        "          optimizer.zero_grad()\n",
        "\n",
        "\n",
        "      if epoch % log_epochs == 0:\n",
        "          loss_full = loss_fn(model(x_train.to(device)),y_train.to(device))\n",
        "          Train_losses.append(loss_full.item())\n",
        "          print(f'Epoch {epoch} Loss {loss_full.item():.4f}')\n",
        "      if log_acc!=0:\n",
        "        if epoch % log_acc == 0:\n",
        "          train_pred = model(x_train.to(device))\n",
        "          pred = torch.argmax(model(x_train.to(device)),dim=1)\n",
        "          zero_mask = (pred-y_train.to(device) == 0.0)\n",
        "          train_acc = zero_mask.sum().item()/len(y_train)\n",
        "          acc_dict['train'].append(train_acc)\n",
        "\n",
        "          test_pred = model(x_test.to(device))[:,0]\n",
        "          pred = torch.argmax(model(x_test.to(device)),dim=1)\n",
        "          zero_mask = (pred-y_test.to(device) == 0.0)\n",
        "          test_acc = zero_mask.sum().item()/len(y_test)\n",
        "          acc_dict['test'].append(test_acc)\n",
        "\n",
        "          if print_std:\n",
        "            print(f'Epoch {epoch} train_acc {train_acc} test_acc {test_acc}')\n",
        "      # if loss_full.item() < thresh:\n",
        "      #     print(f'Early stopping at epoch {epoch} because loss is below 0.01')\n",
        "      #     break\n",
        "\n",
        "  return Train_losses,acc_dict"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YVP7CC6l1Il0",
        "outputId": "cbc5c26e-520e-47fa-be90-a112c01d7803"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.0003\n",
            "Epoch 0 train_acc 1.0 test_acc 0.4888\n",
            "Epoch 10 Loss 0.0000\n",
            "Epoch 10 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 20 Loss 0.0000\n",
            "Epoch 20 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 30 Loss 0.0000\n",
            "Epoch 30 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 40 Loss 0.0000\n",
            "Epoch 40 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 50 Loss 0.0000\n",
            "Epoch 50 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 60 Loss 0.0000\n",
            "Epoch 60 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 70 Loss 0.0000\n",
            "Epoch 70 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 80 Loss 0.0000\n",
            "Epoch 80 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 90 Loss 0.0000\n",
            "Epoch 90 train_acc 1.0 test_acc 0.4889\n"
          ]
        }
      ],
      "source": [
        "losses, acc_dict = train(model,loss_fn,optimizer,train_dataloader,data)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6qunqIvXVyp9"
      },
      "source": [
        "## Pruning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Oaqx3cYxV1Ps",
        "outputId": "1753670f-5a54-44ea-90b1-961af83ee189"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  0%|          | 0/15 [00:00<?, ?it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.0006\n",
            "Epoch 0 train_acc 0.9999 test_acc 0.489\n",
            "Epoch 10 Loss 0.0000\n",
            "Epoch 10 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 20 Loss 0.0000\n",
            "Epoch 20 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 30 Loss 0.0000\n",
            "Epoch 30 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 40 Loss 0.0000\n",
            "Epoch 40 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 50 Loss 0.0000\n",
            "Epoch 50 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 60 Loss 0.0000\n",
            "Epoch 60 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 70 Loss 0.0000\n",
            "Epoch 70 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 80 Loss 0.0000\n",
            "Epoch 80 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 90 Loss 0.0000\n",
            "Epoch 90 train_acc 1.0 test_acc 0.4889\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  7%|▋         | 1/15 [00:31<07:23, 31.64s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.4898\n",
            "Epoch 0 train_acc 0.7889 test_acc 0.4945\n",
            "Epoch 10 Loss 0.0004\n",
            "Epoch 10 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 20 Loss 0.0001\n",
            "Epoch 20 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 30 Loss 0.0000\n",
            "Epoch 30 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 40 Loss 0.0000\n",
            "Epoch 40 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 50 Loss 0.0000\n",
            "Epoch 50 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 60 Loss 0.0000\n",
            "Epoch 60 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 70 Loss 0.0000\n",
            "Epoch 70 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 80 Loss 0.0000\n",
            "Epoch 80 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 90 Loss 0.0000\n",
            "Epoch 90 train_acc 1.0 test_acc 0.4889\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 13%|█▎        | 2/15 [01:07<07:26, 34.37s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.2237\n",
            "Epoch 0 train_acc 0.9636 test_acc 0.4919\n",
            "Epoch 10 Loss 0.0003\n",
            "Epoch 10 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 20 Loss 0.0001\n",
            "Epoch 20 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 30 Loss 0.0000\n",
            "Epoch 30 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 40 Loss 0.0000\n",
            "Epoch 40 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 50 Loss 0.0000\n",
            "Epoch 50 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 60 Loss 0.0000\n",
            "Epoch 60 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 70 Loss 0.0000\n",
            "Epoch 70 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 80 Loss 0.0000\n",
            "Epoch 80 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 90 Loss 0.0000\n",
            "Epoch 90 train_acc 1.0 test_acc 0.4889\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 20%|██        | 3/15 [01:44<07:03, 35.29s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.0000\n",
            "Epoch 0 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 10 Loss 0.0000\n",
            "Epoch 10 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 20 Loss 0.0000\n",
            "Epoch 20 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 30 Loss 0.0000\n",
            "Epoch 30 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 40 Loss 0.0000\n",
            "Epoch 40 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 50 Loss 0.0000\n",
            "Epoch 50 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 60 Loss 0.0000\n",
            "Epoch 60 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 70 Loss 0.0000\n",
            "Epoch 70 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 80 Loss 0.0000\n",
            "Epoch 80 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 90 Loss 0.0000\n",
            "Epoch 90 train_acc 1.0 test_acc 0.4889\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 27%|██▋       | 4/15 [02:21<06:38, 36.19s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.0000\n",
            "Epoch 0 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 10 Loss 0.0000\n",
            "Epoch 10 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 20 Loss 0.0000\n",
            "Epoch 20 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 30 Loss 0.0000\n",
            "Epoch 30 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 40 Loss 0.0000\n",
            "Epoch 40 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 50 Loss 0.0000\n",
            "Epoch 50 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 60 Loss 0.0000\n",
            "Epoch 60 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 70 Loss 0.0000\n",
            "Epoch 70 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 80 Loss 0.0000\n",
            "Epoch 80 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 90 Loss 0.0000\n",
            "Epoch 90 train_acc 1.0 test_acc 0.4889\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 33%|███▎      | 5/15 [02:59<06:05, 36.53s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.0000\n",
            "Epoch 0 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 10 Loss 0.0000\n",
            "Epoch 10 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 20 Loss 0.0000\n",
            "Epoch 20 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 30 Loss 0.0000\n",
            "Epoch 30 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 40 Loss 0.0000\n",
            "Epoch 40 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 50 Loss 0.0000\n",
            "Epoch 50 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 60 Loss 0.0000\n",
            "Epoch 60 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 70 Loss 0.0000\n",
            "Epoch 70 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 80 Loss 0.0000\n",
            "Epoch 80 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 90 Loss 0.0000\n",
            "Epoch 90 train_acc 1.0 test_acc 0.4889\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 40%|████      | 6/15 [03:35<05:28, 36.53s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.0000\n",
            "Epoch 0 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 10 Loss 0.0000\n",
            "Epoch 10 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 20 Loss 0.0000\n",
            "Epoch 20 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 30 Loss 0.0000\n",
            "Epoch 30 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 40 Loss 0.0000\n",
            "Epoch 40 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 50 Loss 0.0000\n",
            "Epoch 50 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 60 Loss 0.0000\n",
            "Epoch 60 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 70 Loss 0.0000\n",
            "Epoch 70 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 80 Loss 0.0000\n",
            "Epoch 80 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 90 Loss 0.0000\n",
            "Epoch 90 train_acc 1.0 test_acc 0.4889\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 47%|████▋     | 7/15 [04:11<04:50, 36.25s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.0033\n",
            "Epoch 0 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 10 Loss 0.0001\n",
            "Epoch 10 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 20 Loss 0.0000\n",
            "Epoch 20 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 30 Loss 0.0000\n",
            "Epoch 30 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 40 Loss 0.0000\n",
            "Epoch 40 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 50 Loss 0.0000\n",
            "Epoch 50 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 60 Loss 0.0000\n",
            "Epoch 60 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 70 Loss 0.0000\n",
            "Epoch 70 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 80 Loss 0.0000\n",
            "Epoch 80 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 90 Loss 0.0000\n",
            "Epoch 90 train_acc 1.0 test_acc 0.4889\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 53%|█████▎    | 8/15 [04:47<04:13, 36.28s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.0003\n",
            "Epoch 0 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 10 Loss 0.0000\n",
            "Epoch 10 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 20 Loss 0.0000\n",
            "Epoch 20 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 30 Loss 0.0000\n",
            "Epoch 30 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 40 Loss 0.0000\n",
            "Epoch 40 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 50 Loss 0.0000\n",
            "Epoch 50 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 60 Loss 0.0000\n",
            "Epoch 60 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 70 Loss 0.0000\n",
            "Epoch 70 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 80 Loss 0.0000\n",
            "Epoch 80 train_acc 1.0 test_acc 0.4889\n",
            "Epoch 90 Loss 0.0000\n",
            "Epoch 90 train_acc 1.0 test_acc 0.4889\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 60%|██████    | 9/15 [05:23<03:36, 36.10s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.5618\n",
            "Epoch 0 train_acc 0.7513 test_acc 0.4969\n",
            "Epoch 10 Loss 0.5614\n",
            "Epoch 10 train_acc 0.7513 test_acc 0.4969\n",
            "Epoch 20 Loss 0.5616\n",
            "Epoch 20 train_acc 0.7513 test_acc 0.4969\n",
            "Epoch 30 Loss 0.5613\n",
            "Epoch 30 train_acc 0.7513 test_acc 0.4969\n",
            "Epoch 40 Loss 0.5611\n",
            "Epoch 40 train_acc 0.7513 test_acc 0.4969\n",
            "Epoch 50 Loss 0.5622\n",
            "Epoch 50 train_acc 0.7513 test_acc 0.4969\n",
            "Epoch 60 Loss 0.5613\n",
            "Epoch 60 train_acc 0.7513 test_acc 0.4969\n",
            "Epoch 70 Loss 0.5616\n",
            "Epoch 70 train_acc 0.7513 test_acc 0.4969\n",
            "Epoch 80 Loss 0.5617\n",
            "Epoch 80 train_acc 0.7513 test_acc 0.4969\n",
            "Epoch 90 Loss 0.5614\n",
            "Epoch 90 train_acc 0.7513 test_acc 0.4969\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 67%|██████▋   | 10/15 [06:00<03:01, 36.31s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.6684\n",
            "Epoch 0 train_acc 0.6247 test_acc 0.4964\n",
            "Epoch 10 Loss 0.6597\n",
            "Epoch 10 train_acc 0.6247 test_acc 0.4964\n",
            "Epoch 20 Loss 0.6595\n",
            "Epoch 20 train_acc 0.6247 test_acc 0.4964\n",
            "Epoch 30 Loss 0.6610\n",
            "Epoch 30 train_acc 0.6247 test_acc 0.4964\n",
            "Epoch 40 Loss 0.6600\n",
            "Epoch 40 train_acc 0.6247 test_acc 0.4964\n",
            "Epoch 50 Loss 0.6595\n",
            "Epoch 50 train_acc 0.6247 test_acc 0.4964\n",
            "Epoch 60 Loss 0.6601\n",
            "Epoch 60 train_acc 0.6247 test_acc 0.4964\n",
            "Epoch 70 Loss 0.6600\n",
            "Epoch 70 train_acc 0.6247 test_acc 0.4964\n",
            "Epoch 80 Loss 0.6596\n",
            "Epoch 80 train_acc 0.6247 test_acc 0.4964\n",
            "Epoch 90 Loss 0.6595\n",
            "Epoch 90 train_acc 0.6247 test_acc 0.4964\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 73%|███████▎  | 11/15 [06:36<02:25, 36.41s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.6932\n",
            "Epoch 0 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 10 Loss 0.6948\n",
            "Epoch 10 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 20 Loss 0.6942\n",
            "Epoch 20 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 30 Loss 0.6932\n",
            "Epoch 30 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 40 Loss 0.6933\n",
            "Epoch 40 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 50 Loss 0.6934\n",
            "Epoch 50 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 60 Loss 0.6932\n",
            "Epoch 60 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 70 Loss 0.6932\n",
            "Epoch 70 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 80 Loss 0.6950\n",
            "Epoch 80 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 90 Loss 0.6933\n",
            "Epoch 90 train_acc 0.4997 test_acc 0.4969\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 80%|████████  | 12/15 [07:12<01:49, 36.35s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.6932\n",
            "Epoch 0 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 10 Loss 0.6938\n",
            "Epoch 10 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 20 Loss 0.6939\n",
            "Epoch 20 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 30 Loss 0.6936\n",
            "Epoch 30 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 40 Loss 0.6931\n",
            "Epoch 40 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 50 Loss 0.6945\n",
            "Epoch 50 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 60 Loss 0.6932\n",
            "Epoch 60 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 70 Loss 0.6934\n",
            "Epoch 70 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 80 Loss 0.6934\n",
            "Epoch 80 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 90 Loss 0.6932\n",
            "Epoch 90 train_acc 0.5003 test_acc 0.5031\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 87%|████████▋ | 13/15 [07:49<01:12, 36.41s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.6933\n",
            "Epoch 0 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 10 Loss 0.6932\n",
            "Epoch 10 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 20 Loss 0.6935\n",
            "Epoch 20 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 30 Loss 0.6933\n",
            "Epoch 30 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 40 Loss 0.6933\n",
            "Epoch 40 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 50 Loss 0.6938\n",
            "Epoch 50 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 60 Loss 0.6931\n",
            "Epoch 60 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 70 Loss 0.6932\n",
            "Epoch 70 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 80 Loss 0.6932\n",
            "Epoch 80 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 90 Loss 0.6937\n",
            "Epoch 90 train_acc 0.4997 test_acc 0.4969\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            " 93%|█████████▎| 14/15 [08:25<00:36, 36.23s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 Loss 0.6935\n",
            "Epoch 0 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 10 Loss 0.6932\n",
            "Epoch 10 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 20 Loss 0.6935\n",
            "Epoch 20 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 30 Loss 0.6934\n",
            "Epoch 30 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 40 Loss 0.6931\n",
            "Epoch 40 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 50 Loss 0.6936\n",
            "Epoch 50 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 60 Loss 0.6935\n",
            "Epoch 60 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 70 Loss 0.6938\n",
            "Epoch 70 train_acc 0.5003 test_acc 0.5031\n",
            "Epoch 80 Loss 0.6937\n",
            "Epoch 80 train_acc 0.4997 test_acc 0.4969\n",
            "Epoch 90 Loss 0.6935\n",
            "Epoch 90 train_acc 0.4997 test_acc 0.4969\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "                                               \r"
          ]
        }
      ],
      "source": [
        "INITIAL_WEIGHTS = deepcopy(model.state_dict())\n",
        "prune_epochs=15\n",
        "p = 0.2\n",
        "losses = {}\n",
        "acc_dict = {}\n",
        "for n in tqdm(range(1, prune_epochs + 1), total=prune_epochs, leave=False):\n",
        "\n",
        "  losses[n], acc_dict[n] = train(model,loss_fn,optimizer,train_dataloader,data)\n",
        "\n",
        "  pruning_rate = p ** (1 / n**0.5)\n",
        "\n",
        "  # Prune the model\n",
        "  prune.l1_unstructured(model.layers[0], name=\"weight\", amount=pruning_rate)\n",
        "  prune.l1_unstructured(model.layers[1], name=\"weight\", amount=pruning_rate)\n",
        "  prune.l1_unstructured(model.output_layer, name=\"weight\", amount=pruning_rate/2)\n",
        "\n",
        "  # Reset weights\n",
        "  reseted_weights = deepcopy(model.state_dict())\n",
        "  for param in reseted_weights.keys():\n",
        "      if param.split(\".\")[-1].replace(\"_orig\", \"\") in (\"bias\", \"weight\"):\n",
        "          reseted_weights[param] = deepcopy(INITIAL_WEIGHTS[param.replace(\"_orig\", \"\")])\n",
        "  if n< prune_epochs-1:\n",
        "    model.load_state_dict(reseted_weights)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "scyMB0gHdgRf"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yyyfoYEXVvK3"
      },
      "source": [
        "## Probing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 435
        },
        "id": "lpxcyy_6GEQC",
        "outputId": "f3422a4d-fcd4-42eb-ce29-43a1288698c9"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAGiCAYAAADgJq2oAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/SrBM8AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA76ElEQVR4nO3df1xUZb4H8M+MwowKM4gKAwmKN69Q+SMxxrF2S5kNqu3Klb2rLl1/xJXdFizB3dLWxDU3MFPxB8l180dusRhb2aotRZB6W0fUUbbViFtdC1IHLBZGsRl+zLl/GGcbGYRh5sSR+bxfr+eVc85znnkeevHlme955jkKQRAEEBGRz1D2dQeIiOj7xcBPRORjGPiJiHwMAz8RkY9h4Cci8jEM/EREPoaBn4jIxzDwExH5GAZ+IiIfw8BPRORjGPiJiL6Vn5+P0aNHQ61WQ6/X4/jx413WPXv2LJKTkzF69GgoFArk5eX1qk2bzYb09HQMGzYMAQEBSE5ORl1dnTeH1QkDPxERgL179yIrKwvZ2dk4deoUJk6ciISEBNTX17usf/XqVYwZMwa5ubnQ6XS9bjMzMxP79+9HcXExDh8+jAsXLmDWrFmSjFEkEBGREBcXJ6Snp4uv29vbhfDwcCEnJ6fba0eNGiVs3LjR7TYbGxsFPz8/obi4WKxTVVUlABBMJpMHo7mxgVL9QcnPz8e6detgsVgwceJEbNmyBXFxcd1e53A4cOHCBQQGBkKhUEjVPSJykyAIuHz5MsLDw6FUSpcssNlsaGlp8UpbgiB0iiMqlQoqlcrpWEtLC8xmM5YvXy4eUyqVMBqNMJlMvXrvnrRpNpvR2toKo9Eo1omOjkZkZCRMJhOmTp3aq/fujiSBv+PjTUFBAfR6PfLy8pCQkIDq6mqEhITc8NoLFy4gIiJCim4RkRfU1tZi5MiRkrRts9kQNSoAlvp2r7QXEBCAK1euOB3Lzs7GqlWrnI599dVXaG9vR2hoqNPx0NBQfPzxx7167560abFY4O/vj6CgoE51LBZLr963JyQJ/Bs2bMCiRYuwcOFCAEBBQQEOHjyInTt3YtmyZTe8NjAwEAAwctUKKNVqKbpHRL3gsNnw5ao14u+oFFpaWmCpb8c58yhoAj37VGG97EBU7Beora2FRqMRj18/2/dFXg/87n5kstvtsNvt4uvLly9fu0atZuAnkqHvIwWrCVR6HPjFtjQap8DvyvDhwzFgwIBOq2nq6uq6vHHbnZ60qdPp0NLSgsbGRqdZvyfv2xNeT9Td6OONq48uOTk50Gq1YmGah4jaBYdXSk/5+/sjNjYWZWVl4jGHw4GysjIYDIZejaEnbcbGxsLPz8+pTnV1NWpqanr9vj0h2c3dnlq+fDmysrLE11arlcGfyMc5IMABz54K6+71WVlZmD9/PqZMmYK4uDjk5eWhublZTFnPmzcPt9xyC3JycgBcy2589NFH4r/Pnz+PyspKBAQE4NZbb+1Rm1qtFqmpqcjKykJwcDA0Gg0WL14Mg8Eg2Y1dQILA7+5HJld32InItzngQM/n61234Y7Zs2fj0qVLWLlyJSwWCyZNmoSSkhIxe1FTU+O0munChQu48847xdcvvPACXnjhBdx77704dOhQj9oEgI0bN0KpVCI5ORl2ux0JCQl48cUXPRh59xSC4P2Hrev1esTFxWHLli0Arn28iYyMREZGRrc3d61WK7RaLSJz1zDHTyQjDpsNNctWoKmpqduceW91/P5fqB7plZu74eO+lLS/NytJUj3dfbwhIrqRdkFAu4dzUk+v788kCfw9+XhDRNSVvsjx+xLJbu5mZGQgIyNDquaJiKiX+nxVDxHR9RwQ0M4Zv2QY+IlIdpjqkRa3ZSYi8jGc8ROR7HBVj7QY+IlIdhzfFk/bINeY6iEi8jGc8ROR7LR7YVWPp9f3Zwz8RCQ77cK14mkb5BoDPxHJDnP80mKOn4jIx3DGT0Sy44AC7fDsSV8OD6/vzxj4iUh2HMK14mkb5BpTPUREPoYzfiKSnXYvpHo8vb4/Y+AnItlh4JcWUz1ERD6GM34ikh2HoIBD8HBVj4fX92cM/EQkO0z1SIupHiIiH8MZPxHJTjuUaPdwXtrupb70Rwz8RCQ7ghdy/AJz/F1i4Cci2WGOX1rM8RMR+RjO+IlIdtoFJdoFD3P83KunSwz8RCQ7Dijg8DAh4eATuLrEVA8RkY/hjJ+IZIc3d6XFwE9EsuOdHD9TPV1hqoeIyMdwxk9EsnPt5i4fvSgVBn4ikh2HF7Zs4KqerjHVQ0TkYxj4iUh2Om7uelrclZ+fj9GjR0OtVkOv1+P48eM3rF9cXIzo6Gio1WqMHz8eb7/9ttN5hULhsqxbt06sM3r06E7nc3Nz3e67Oxj4iUh2HFB6pbhj7969yMrKQnZ2Nk6dOoWJEyciISEB9fX1LusfPXoUc+fORWpqKk6fPo2kpCQkJSXhzJkzYp2LFy86lZ07d0KhUCA5OdmprdWrVzvVW7x4sfs/NDcw8BOR7LQLCq8Ud2zYsAGLFi3CwoULcdttt6GgoACDBw/Gzp07XdbftGkTEhMT8etf/xoxMTF49tlnMXnyZGzdulWso9PpnMpbb72F6dOnY8yYMU5tBQYGOtUbMmSI+z80NzDwE1G/ZrVanYrdbu9Up6WlBWazGUajUTymVCphNBphMplctmsymZzqA0BCQkKX9evq6nDw4EGkpqZ2Opebm4thw4bhzjvvxLp169DW1ubOEN3GVT1EJDveeRDLtVU9ERERTsezs7OxatUqp2NfffUV2tvbERoa6nQ8NDQUH3/8scv2LRaLy/oWi8Vl/ZdffhmBgYGYNWuW0/HHH38ckydPRnBwMI4ePYrly5fj4sWL2LBhQ7dj7C0GfiKSHYeghMPDb+46vv3mbm1tLTQajXhcpVJ51G5v7dy5EykpKVCr1U7Hs7KyxH9PmDAB/v7++PnPf46cnBzJ+srAT0T9mkajcQr8rgwfPhwDBgxAXV2d0/G6ujrodDqX1+h0uh7X/5//+R9UV1dj79693fZXr9ejra0Nn3/+OcaNG9dt/d5gjp+IZKcj1eNp6Sl/f3/ExsairKxMPOZwOFBWVgaDweDyGoPB4FQfAEpLS13W37FjB2JjYzFx4sRu+1JZWQmlUomQkJAe999dnPETkew4ALdX5bhqwx1ZWVmYP38+pkyZgri4OOTl5aG5uRkLFy4EAMybNw+33HILcnJyAABPPPEE7r33Xqxfvx4PPfQQioqKcPLkSWzfvt2pXavViuLiYqxfv77Te5pMJlRUVGD69OkIDAyEyWRCZmYmHnnkEQwdOrRX4+4JBn4iIgCzZ8/GpUuXsHLlSlgsFkyaNAklJSXiDdyamhoolf/8FDFt2jQUFhZixYoVePrppzF27Fjs27cPd9xxh1O7RUVFEAQBc+fO7fSeKpUKRUVFWLVqFex2O6KiopCZmemU95eCQhDktXep1WqFVqtFZO4aKK+7CUJEfcdhs6Fm2Qo0NTV1mzPvrY7f/22n7sKgAM/mpd9cacNjk09I2t+bFWf8RCQ73tmPn7cwu8KfDBGRj+GMn4hkh/vxS4uBn4hkh6keabn9kzly5AgefvhhhIeHQ6FQYN++fU7nBUHAypUrERYWhkGDBsFoNOKTTz7xVn+JyAd83+v4fY3bP5nm5mZMnDgR+fn5Ls8///zz2Lx5MwoKClBRUYEhQ4YgISEBNpvN484SEZHn3E71PPDAA3jggQdcnhMEAXl5eVixYgVmzpwJANizZw9CQ0Oxb98+zJkzp9M1drvdabc8q9XqbpeIqJ9xCAo4PP0Cl4fX92de/Sx07tw5WCwWp61KtVot9Hp9l1uV5uTkQKvViuX6nfSIyPc4vJDmcfdBLL7Eqz+Zju1I3dmqdPny5WhqahJLbW2tN7tERETX6fNVPSqVqs+2SSUiefLOtsyc8XfFqz+Zju1I3dnalIjoeu1QeKWQa14N/FFRUdDpdE5blVqtVlRUVHS5tSkREX2/3E71XLlyBZ9++qn4+ty5c6isrERwcDAiIyOxZMkSrFmzBmPHjkVUVBSeeeYZhIeHIykpyZv9JqJ+jKkeabkd+E+ePInp06eLrzu2D50/fz52796NJ598Es3NzUhLS0NjYyPuuecelJSUdHrcGBFRV9oBj1M17d7pSr/kduC/7777cKOdnBUKBVavXo3Vq1d71DEiIpJGn6/qISK6HlM90mLgJyLZ4SZt0mLgJyLZEbywLbPA5Zxd4p9EIiIfwxk/EckOUz3SYuAnItnh7pzS4p9EIiIfwxk/EcmON56gxSdwdY2Bn4hkh6keafFPIhGRj+GMn4hkx+GFJ2jxCVxdY+AnItlpFxRo9zBV4+n1/Rn/JBIR+RjO+IlIdnhzV1oM/EQkO4IXducU+M3dLjHwE5HseOOZuXzmbtf4J5GIyMdwxk9EsuMQPM/RO7p+UKDPY+AnItnhE7ikxZ8MEZGPYeAnItlxfPsELk+Lu/Lz8zF69Gio1Wro9XocP378hvWLi4sRHR0NtVqN8ePH4+2333Y6v2DBAigUCqeSmJjoVKehoQEpKSnQaDQICgpCamoqrly54nbf3cHAT0Sy0/HNXU+LO/bu3YusrCxkZ2fj1KlTmDhxIhISElBfX++y/tGjRzF37lykpqbi9OnTSEpKQlJSEs6cOeNULzExERcvXhTLH//4R6fzKSkpOHv2LEpLS3HgwAEcOXIEaWlp7v3A3MTAT0QEYMOGDVi0aBEWLlyI2267DQUFBRg8eDB27tzpsv6mTZuQmJiIX//614iJicGzzz6LyZMnY+vWrU71VCoVdDqdWIYOHSqeq6qqQklJCV566SXo9Xrcc8892LJlC4qKinDhwgXJxsrAT0Sy03Fz19MCAFar1anY7fZO79fS0gKz2Qyj0SgeUyqVMBqNMJlMLvtoMpmc6gNAQkJCp/qHDh1CSEgIxo0bh8ceewxff/21UxtBQUGYMmWKeMxoNEKpVKKiosL9H1wPMfATkew4oBC3beh1+TbHHxERAa1WK5acnJxO7/fVV1+hvb0doaGhTsdDQ0NhsVhc9tFisXRbPzExEXv27EFZWRnWrl2Lw4cP44EHHkB7e7vYRkhIiFMbAwcORHBwcJfv6w1czklE/VptbS00Go34WqVSfW/vPWfOHPHf48ePx4QJE/Av//IvOHToEOLj47+3flyPM34ikh3BCyt6hG9n/BqNxqm4CvzDhw/HgAEDUFdX53S8rq4OOp3OZR91Op1b9QFgzJgxGD58OD799FOxjetvHre1taGhoeGG7XiKgZ+IZMfjNI+bu3v6+/sjNjYWZWVl/+yDw4GysjIYDAaX1xgMBqf6AFBaWtplfQD48ssv8fXXXyMsLExso7GxEWazWaxTXl4Oh8MBvV7f4/67i6keIpKdvvjmblZWFubPn48pU6YgLi4OeXl5aG5uxsKFCwEA8+bNwy233CLeI3jiiSdw7733Yv369XjooYdQVFSEkydPYvv27QCAK1eu4Le//S2Sk5Oh0+nw2Wef4cknn8Stt96KhIQEAEBMTAwSExOxaNEiFBQUoLW1FRkZGZgzZw7Cw8M9Gv+NMPATEQGYPXs2Ll26hJUrV8JisWDSpEkoKSkRb+DW1NRAqfznH5Np06ahsLAQK1aswNNPP42xY8di3759uOOOOwAAAwYMwIcffoiXX34ZjY2NCA8Px/33349nn33WKd306quvIiMjA/Hx8VAqlUhOTsbmzZslHatCEARZbWVktVqh1WoRmbsGSrW6r7tDRN9y2GyoWbYCTU1NTjdLvanj93/mu4/Cb4i/R221Nrfgrft3StrfmxVn/EQkO73dcuH6Nsg13twlIvIxnPETkezwmbvSYuAnItlh4JcWUz1ERD6GM34ikh3O+KXFwE9EssPALy2meoiIfAxn/EQkOwI8X4cvq2+mygwDPxHJDlM90mLgJyLZYeCXFnP8REQ+hjN+IpIdzvilxcBPRLLDwC8tpnqIiHyMW4E/JycHd911FwIDAxESEoKkpCRUV1c71bHZbEhPT8ewYcMQEBCA5OTkTs+lJCK6EUFQeKWQa24F/sOHDyM9PR3Hjh1DaWkpWltbcf/996O5uVmsk5mZif3796O4uBiHDx/GhQsXMGvWLK93nIj6L08ftO6N/fz7M7dy/CUlJU6vd+/ejZCQEJjNZvzwhz9EU1MTduzYgcLCQsyYMQMAsGvXLsTExODYsWOYOnWq93pORES94lGOv6mpCQAQHBwMADCbzWhtbYXRaBTrREdHIzIyEiaTyWUbdrsdVqvVqRCRb+u4uetpIdd6HfgdDgeWLFmCu+++W3y4sMVigb+/P4KCgpzqhoaGwmKxuGwnJycHWq1WLBEREb3tEhH1E8zxS6vXgT89PR1nzpxBUVGRRx1Yvnw5mpqaxFJbW+tRe0REdGO9WsefkZGBAwcO4MiRIxg5cqR4XKfToaWlBY2NjU6z/rq6Ouh0OpdtqVQqqFSq3nSDiPopruOXllszfkEQkJGRgTfffBPl5eWIiopyOh8bGws/Pz+UlZWJx6qrq1FTUwODweCdHhNRv8dUj7TcmvGnp6ejsLAQb731FgIDA8W8vVarxaBBg6DVapGamoqsrCwEBwdDo9Fg8eLFMBgMXNFDRD0meGHGz8DfNbcC/7Zt2wAA9913n9PxXbt2YcGCBQCAjRs3QqlUIjk5GXa7HQkJCXjxxRe90lkiIvKcW4FfELp/tIFarUZ+fj7y8/N73Ski8m0CgB6Em27bINe4SRsRyY4DCig8/OYtv7nbNW7SRkTkYzjjJyLZ8caqHN7c7RoDPxHJjkNQQMF1/JJhqoeIyMdwxk9EsiMIXljVw2U9XWLgJyLZYY5fWkz1EBH5GM74iUh2OOOXFgM/EckOV/VIi6keIpKdjpu7nhZ35efnY/To0VCr1dDr9Th+/PgN6xcXFyM6OhpqtRrjx4/H22+/LZ5rbW3FU089hfHjx2PIkCEIDw/HvHnzcOHCBac2Ro8eDYVC4VRyc3Pd77wbGPiJiADs3bsXWVlZyM7OxqlTpzBx4kQkJCSgvr7eZf2jR49i7ty5SE1NxenTp5GUlISkpCScOXMGAHD16lWcOnUKzzzzDE6dOoU33ngD1dXV+Ld/+7dOba1evRoXL14Uy+LFiyUdK1M9RCQ712bsnub4r/33+ud4d/Xwpw0bNmDRokVYuHAhAKCgoAAHDx7Ezp07sWzZsk71N23ahMTERPz6178GADz77LMoLS3F1q1bUVBQAK1Wi9LSUqdrtm7diri4ONTU1CAyMlI8HhgY2OXDqqTAGT8RyY43H8QSERHh9FzvnJycTu/X0tICs9kMo9EoHlMqlTAajTCZTC77aDKZnOoDQEJCQpf1AaCpqQkKhaLTc8lzc3MxbNgw3HnnnVi3bh3a2tp6+qPqFc74iahfq62thUajEV+7mu1/9dVXaG9vR2hoqNPx0NBQfPzxxy7btVgsLut3PKDqejabDU899RTmzp3r1J/HH38ckydPRnBwMI4ePYrly5fj4sWL2LBhQ4/H6C4GfiKSHQGe76ffcb1Go3EKtH2htbUVP/3pTyEIgvhAqw5ZWVnivydMmAB/f3/8/Oc/R05OjmTPI2eqh4hk5/t+5u7w4cMxYMAA1NXVOR2vq6vrMveu0+l6VL8j6H/xxRcoLS3t9o+QXq9HW1sbPv/88x73310M/ETk8/z9/REbG4uysjLxmMPhQFlZGQwGg8trDAaDU30AKC0tdarfEfQ/+eQTvPfeexg2bFi3famsrIRSqURISEgvR9M9pnqISH68mevpoaysLMyfPx9TpkxBXFwc8vLy0NzcLK7ymTdvHm655Rbx5vATTzyBe++9F+vXr8dDDz2EoqIinDx5Etu3bwdwLej/5Cc/walTp3DgwAG0t7eL+f/g4GD4+/vDZDKhoqIC06dPR2BgIEwmEzIzM/HII49g6NChHv4AusbAT0Ty44UtG+Dm9bNnz8alS5ewcuVKWCwWTJo0CSUlJeIN3JqaGiiV/0ySTJs2DYWFhVixYgWefvppjB07Fvv27cMdd9wBADh//jz+/Oc/AwAmTZrk9F7vv/8+7rvvPqhUKhQVFWHVqlWw2+2IiopCZmamU95fCgqhJ09Q/x5ZrVZotVpE5q6BUq3u6+4Q0bccNhtqlq1AU1OTZDdLO37/o3b9BsrBnv3+O67acG7h7yTt782KOX4iIh/DVA8RyQ5355QWAz8RyY+gcDtH77INcompHiIiH8MZPxHJDp+5Ky0GfiKSnz5Yx+9LmOohIvIxnPETkexwVY+0GPiJSJ6YqpEMUz1ERD6GM34ikh2meqTFwE9E8sNVPZJi4CciGVJ8Wzxtg1xhjp+IyMdwxk9E8sNUj6QY+IlIfhj4JcVUDxGRj+GMn4jkh9syS4qBn4hkh7tzSoupHiIiH8MZPxHJD2/uSoqBn4jkhzl+STHVQ0TkYzjjJyLZUQjXiqdtkGsM/EQkP8zxS4qBn4jkhzl+SbmV49+2bRsmTJgAjUYDjUYDg8GAv/zlL+J5m82G9PR0DBs2DAEBAUhOTkZdXZ3XO01ERL3nVuAfOXIkcnNzYTabcfLkScyYMQMzZ87E2bNnAQCZmZnYv38/iouLcfjwYVy4cAGzZs2SpONE1I8JXirkklupnocfftjp9e9+9zts27YNx44dw8iRI7Fjxw4UFhZixowZAIBdu3YhJiYGx44dw9SpU73XayLq35jjl1Svl3O2t7ejqKgIzc3NMBgMMJvNaG1thdFoFOtER0cjMjISJpOpy3bsdjusVqtTISIi6bgd+P/+978jICAAKpUKv/jFL/Dmm2/itttug8Vigb+/P4KCgpzqh4aGwmKxdNleTk4OtFqtWCIiItweBBH1M0z1SMrtwD9u3DhUVlaioqICjz32GObPn4+PPvqo1x1Yvnw5mpqaxFJbW9vrtoion+hY1eNpIZfcXs7p7++PW2+9FQAQGxuLEydOYNOmTZg9ezZaWlrQ2NjoNOuvq6uDTqfrsj2VSgWVSuV+z4mIqFc83rLB4XDAbrcjNjYWfn5+KCsrE89VV1ejpqYGBoPB07chIh/S8c1dTwu55taMf/ny5XjggQcQGRmJy5cvo7CwEIcOHcI777wDrVaL1NRUZGVlITg4GBqNBosXL4bBYOCKHiJyD1f1SMqtGX99fT3mzZuHcePGIT4+HidOnMA777yDH/3oRwCAjRs34sc//jGSk5Pxwx/+EDqdDm+88YYkHSci8rb8/HyMHj0aarUaer0ex48fv2H94uJiREdHQ61WY/z48Xj77bedzguCgJUrVyIsLAyDBg2C0WjEJ5984lSnoaEBKSkp0Gg0CAoKQmpqKq5cueL1sX2XW4F/x44d+Pzzz2G321FfX4/33ntPDPoAoFarkZ+fj4aGBjQ3N+ONN964YX6fiEgu9u7di6ysLGRnZ+PUqVOYOHEiEhISUF9f77L+0aNHMXfuXKSmpuL06dNISkpCUlISzpw5I9Z5/vnnsXnzZhQUFKCiogJDhgxBQkICbDabWCclJQVnz55FaWkpDhw4gCNHjiAtLU3SsSoEQV4PKLNardBqtYjMXQOlWt3X3SGibzlsNtQsW4GmpiZoNBpJ3qPj93/UWs9//x02G754agVqa2ud+tvVghK9Xo+77roLW7duvXa9w4GIiAgsXrwYy5Yt61R/9uzZaG5uxoEDB8RjU6dOxaRJk1BQUABBEBAeHo6lS5fiV7/6FQCgqakJoaGh2L17N+bMmYOqqircdtttOHHiBKZMmQIAKCkpwYMPPogvv/wS4eHhHv0MusL9+IlIfry4nDMiIsLpu0I5OTmd3q6lpQVms9npC6hKpRJGo7HLL6CaTCan+gCQkJAg1j937hwsFotTHa1WC71eL9YxmUwICgoSgz4AGI1GKJVKVFRU9PKH1z3uzklE/ZqrGf/1vvrqK7S3tyM0NNTpeGhoKD7++GOX7VosFpf1O76w2vHf7uqEhIQ4nR84cCCCg4Nv+MVXTzHwE5H8eHFVT8duwvRPTPUQkfx8z1s2DB8+HAMGDOi0jfyNvoCq0+luWL/jv93Vuf7mcVtbGxoaGiRdGMPAT0Q+z9/fH7GxsU5fQHU4HCgrK+vyC6gGg8GpPgCUlpaK9aOioqDT6ZzqWK1WVFRUiHUMBgMaGxthNpvFOuXl5XA4HNDr9V4b3/WY6iEi2emLZ+5mZWVh/vz5mDJlCuLi4pCXl4fm5mYsXLgQADBv3jzccsst4s3hJ554Avfeey/Wr1+Phx56CEVFRTh58iS2b99+7f0VCixZsgRr1qzB2LFjERUVhWeeeQbh4eFISkoCAMTExCAxMRGLFi1CQUEBWltbkZGRgTlz5ki2ogdg4CciOeqDb+7Onj0bly5dwsqVK2GxWDBp0iSUlJSIN2dramqgVP4zSTJt2jQUFhZixYoVePrppzF27Fjs27cPd9xxh1jnySefRHNzM9LS0tDY2Ih77rkHJSUlUH9nqeqrr76KjIwMxMfHQ6lUIjk5GZs3b/Zs7N3gOn4i6pHvcx3/6DW/88o6/s9X/EbS/t6sOOMnIvnhXj2SYuAnItnpixy/L+GqHiIiH8MZPxHJjzeeoMUncHWJgZ+I5Ic5fkkx8BOR7DDHLy3m+ImIfAxn/EQkP0z1SIqBn4jkxxsPS2fg7xJTPUREPoYzfiKSH6Z6JMXAT0Tyw8AvKaZ6iIh8DGf8RCQ7XMcvLc74iYh8DAM/EZGPYaqHiOSHN3clxcBPRLLDHL+0GPiJSJ4YuCXDHD8RkY/hjJ+I5Ic5fkkx8BOR7DDHLy2meoiIfAxn/EQkP0z1SIqBn4hkh6keaTHVQ0TkYzjjJyL5YapHUgz8RCQ/DPySYqqHiMjHcMZPRLLDm7vSYuAnIvlhqkdSDPxEJD8M/JJijp+IyMdwxk9EssMcv7QY+IlIfpjqkRRTPUREbmpoaEBKSgo0Gg2CgoKQmpqKK1eu3PAam82G9PR0DBs2DAEBAUhOTkZdXZ14/m9/+xvmzp2LiIgIDBo0CDExMdi0aZNTG4cOHYJCoehULBaLW/3njJ+IZEfuqZ6UlBRcvHgRpaWlaG1txcKFC5GWlobCwsIur8nMzMTBgwdRXFwMrVaLjIwMzJo1C3/9618BAGazGSEhIXjllVcQERGBo0ePIi0tDQMGDEBGRoZTW9XV1dBoNOLrkJAQt/rPwE9E8iPjVE9VVRVKSkpw4sQJTJkyBQCwZcsWPPjgg3jhhRcQHh7e6Zqmpibs2LEDhYWFmDFjBgBg165diImJwbFjxzB16lQ8+uijTteMGTMGJpMJb7zxRqfAHxISgqCgoF6PwaNUT25uLhQKBZYsWSIe6+7jDBHR98lqtToVu93uUXsmkwlBQUFi0AcAo9EIpVKJiooKl9eYzWa0trbCaDSKx6KjoxEZGQmTydTlezU1NSE4OLjT8UmTJiEsLAw/+tGPxE8M7uh14D9x4gT++7//GxMmTHA6npmZif3796O4uBiHDx/GhQsXMGvWrN6+DRH5IsFLBUBERAS0Wq1YcnJyPOqaxWLplFoZOHAggoODu8y1WywW+Pv7d5qlh4aGdnnN0aNHsXfvXqSlpYnHwsLCUFBQgNdffx2vv/46IiIicN999+HUqVNujaFXqZ4rV64gJSUFv//977FmzRrxeE8+zhARdUfxbfG0DQCora11yoerVCqX9ZctW4a1a9fesM2qqioPe9UzZ86cwcyZM5GdnY37779fPD5u3DiMGzdOfD1t2jR89tln2LhxI/7whz/0uP1eBf709HQ89NBDMBqNToG/u48zrgK/3W53+uhltVp70yUiIpc0Go1T4O/K0qVLsWDBghvWGTNmDHQ6Herr652Ot7W1oaGhATqdzuV1Op0OLS0taGxsdJr119XVdbrmo48+Qnx8PNLS0rBixYpu+x0XF4cPPvig23rf5XbgLyoqwqlTp3DixIlO53rzcSYnJwe//e1v3e0GEfVnfXBzd8SIERgxYkS39QwGAxobG2E2mxEbGwsAKC8vh8PhgF6vd3lNbGws/Pz8UFZWhuTkZADXVubU1NTAYDCI9c6ePYsZM2Zg/vz5+N3vftejfldWViIsLKxHdTu4Ffhra2vxxBNPoLS0FGq12q036sry5cuRlZUlvrZarYiIiPBK20R0c5Lzcs6YmBgkJiZi0aJFKCgoQGtrKzIyMjBnzhxxRc/58+cRHx+PPXv2IC4uDlqtFqmpqcjKykJwcDA0Gg0WL14Mg8EgZkLOnDmDGTNmICEhAVlZWeJkecCAAeIfpLy8PERFReH222+HzWbDSy+9hPLycrz77rtujcGtwG82m1FfX4/JkyeLx9rb23HkyBFs3boV77zzTo8/znRQqVRd5tyIyEfJeDknALz66qvIyMhAfHw8lEolkpOTsXnzZvF8a2srqqurcfXqVfHYxo0bxbp2ux0JCQl48cUXxfN/+tOfcOnSJbzyyit45ZVXxOOjRo3C559/DgBoaWnB0qVLcf78eQwePBgTJkzAe++9h+nTp7vVf4UgCD3+8Vy+fBlffPGF07GFCxciOjoaTz31FCIiIjBixAj88Y9/dPo4Ex0d3WWO/3pWqxVarRaRuWug9NKnCiLynMNmQ82yFWhqaupRzrw3On7/b//5cxig8uz3v91uw9n/flrS/t6s3JrxBwYG4o477nA6NmTIEAwbNkw83t3HGSKiHuFeO5Lx+jd3u/s4Q0TUHTnn+PsDjwP/oUOHnF6r1Wrk5+cjPz/f06aJiEgC3KuHiORH5jd3b3YM/EQkO0z1SIv78RMR+RjO+IlIfpjqkRQDPxHJDlM90mKqh4jIx3DGT0Tyw1SPpBj4iUh+GPglxcBPRLLDHL+0mOMnIvIxnPETkfww1SMpBn4ikh2FIEDR8x3ju2yDXGOqh4jIx3DGT0Tyw1SPpBj4iUh2uKpHWkz1EBH5GM74iUh+mOqRFAM/EckOUz3SYqqHiMjHcMZPRPLDVI+kGPiJSHaY6pEWAz8RyQ9n/JJijp+IyMdwxk9EssRUjXQY+IlIfgThWvG0DXKJqR4iIh/DGT8RyQ5X9UiLgZ+I5IereiTFVA8RkY/hjJ+IZEfhuFY8bYNcY+AnIvlhqkdSTPUQEfkYBn4ikp2OVT2eFqk0NDQgJSUFGo0GQUFBSE1NxZUrV254jc1mQ3p6OoYNG4aAgAAkJyejrq7OedwKRadSVFTkVOfQoUOYPHkyVCoVbr31Vuzevdvt/jPwE5H8dHyBy9MikZSUFJw9exalpaU4cOAAjhw5grS0tBtek5mZif3796O4uBiHDx/GhQsXMGvWrE71du3ahYsXL4olKSlJPHfu3Dk89NBDmD59OiorK7FkyRL813/9F9555x23+s8cPxHJjpzX8VdVVaGkpAQnTpzAlClTAABbtmzBgw8+iBdeeAHh4eGdrmlqasKOHTtQWFiIGTNmALgW4GNiYnDs2DFMnTpVrBsUFASdTufyvQsKChAVFYX169cDAGJiYvDBBx9g48aNSEhI6PEYOOMnon7NarU6Fbvd7lF7JpMJQUFBYtAHAKPRCKVSiYqKCpfXmM1mtLa2wmg0iseio6MRGRkJk8nkVDc9PR3Dhw9HXFwcdu7cCeE7n1xMJpNTGwCQkJDQqY3uMPATkfwIXioAIiIioNVqxZKTk+NR1ywWC0JCQpyODRw4EMHBwbBYLF1e4+/vj6CgIKfjoaGhTtesXr0ar732GkpLS5GcnIxf/vKX2LJli1M7oaGhndqwWq345ptvejwGpnqISHa8meqpra2FRqMRj6tUKpf1ly1bhrVr196wzaqqKs861Y1nnnlG/Pedd96J5uZmrFu3Do8//rhX34eBn4j6NY1G4xT4u7J06VIsWLDghnXGjBkDnU6H+vp6p+NtbW1oaGjoMjev0+nQ0tKCxsZGp1l/XV1dl9cAgF6vx7PPPgu73Q6VSgWdTtdpJVBdXR00Gg0GDRp04wF+BwM/EclPH2zLPGLECIwYMaLbegaDAY2NjTCbzYiNjQUAlJeXw+FwQK/Xu7wmNjYWfn5+KCsrQ3JyMgCguroaNTU1MBgMXb5XZWUlhg4dKn5KMRgMePvtt53qlJaW3rANVxj4iUh25LyqJyYmBomJiVi0aBEKCgrQ2tqKjIwMzJkzR1zRc/78ecTHx2PPnj2Ii4uDVqtFamoqsrKyEBwcDI1Gg8WLF8NgMIgrevbv34+6ujpMnToVarUapaWleO655/CrX/1KfO9f/OIX2Lp1K5588kk8+uijKC8vx2uvvYaDBw+6NQYGfiIiN7366qvIyMhAfHw8lEolkpOTsXnzZvF8a2srqqurcfXqVfHYxo0bxbp2ux0JCQl48cUXxfN+fn7Iz89HZmYmBEHArbfeig0bNmDRokVinaioKBw8eBCZmZnYtGkTRo4ciZdeesmtpZwAoBAEeT2mxmq1QqvVIjJ3DZRqdV93h4i+5bDZULNsBZqamnqUM++Njt9/Q+JqDPTz7Pe/rdUGU8lKSft7s+KMn4hkR86pnv6A6/iJiHwMZ/xEJD8O4VrxtA1yiYGfiOSH+/FLyq1Uz6pVqzptGRodHS2e78m2o0RE3VHAC9sy9/UgZMztHP/tt9/utGXoBx98IJ7r6bajRETUd9xO9QwcONDlV4zd2Xb0u+x2u9NueVar1d0uEVF/0wff3PUlbs/4P/nkE4SHh2PMmDFISUlBTU0NAPe2Hf2unJwcp53zIiIiejEMIupP5P4ErpudW4Ffr9dj9+7dKCkpwbZt23Du3Dn84Ac/wOXLl3u87ej1li9fjqamJrHU1tb2aiBERNQzbqV6HnjgAfHfEyZMgF6vx6hRo/Daa6+5tTPcd6lUqi63SSUiH8VVPZLy6AtcQUFB+Nd//Vd8+umnTtuOfld3244SEV1PIQheKeSaR4H/ypUr+OyzzxAWFua07WiHnmw7SkRE3y+3Uj2/+tWv8PDDD2PUqFG4cOECsrOzMWDAAMydO7dH244SEfWI49viaRvkkluB/8svv8TcuXPx9ddfY8SIEbjnnntw7Ngx8eEF3W07SkTUE95I1TDV0zW3An9RUdENz6vVauTn5yM/P9+jThERkXS4Vw8RyQ9X9UiKgZ+I5Iff3JUUAz8RyQ4fxCItPoiFiMjHcMZPRPLDVI+kGPiJSHYUjmvF0zbINaZ6iIh8DGf8RCQ/TPVIioGfiOSH6/glxVQPEZGP4YyfiGSHe/VIi4GfiOSHOX5JMdVDRORjOOMnIvkR4Pl++pzwd4mBn4hkhzl+aTHwE5H8CPBCjt8rPemXmOMnIvIxnPETkfxwVY+kGPiJSH4cABReaINcYqqHiMjHMPATkex0rOrxtEiloaEBKSkp0Gg0CAoKQmpqKq5cuXLDa2w2G9LT0zFs2DAEBAQgOTkZdXV14vndu3dDoVC4LPX19QCAQ4cOuTxvsVjc6j9TPUQkPzLP8aekpODixYsoLS1Fa2srFi5ciLS0NBQWFnZ5TWZmJg4ePIji4mJotVpkZGRg1qxZ+Otf/woAmD17NhITE52uWbBgAWw2G0JCQpyOV1dXQ6PRiK+vP98dBn4iIjdUVVWhpKQEJ06cwJQpUwAAW7ZswYMPPogXXngB4eHhna5pamrCjh07UFhYiBkzZgAAdu3ahZiYGBw7dgxTp07FoEGDMGjQIPGaS5cuoby8HDt27OjUXkhICIKCgno9BqZ6iEh+Omb8nhYAVqvVqdjtdo+6ZjKZEBQUJAZ9ADAajVAqlaioqHB5jdlsRmtrK4xGo3gsOjoakZGRMJlMLq/Zs2cPBg8ejJ/85Cedzk2aNAlhYWH40Y9+JH5icAcDPxHJjxcDf0REBLRarVhycnI86prFYumUWhk4cCCCg4O7zLVbLBb4+/t3mqWHhoZ2ec2OHTvws5/9zOlTQFhYGAoKCvD666/j9ddfR0REBO677z6cOnXKrTEw1UNE/Vptba1TPlylUrmst2zZMqxdu/aGbVVVVXm1b10xmUyoqqrCH/7wB6fj48aNw7hx48TX06ZNw2effYaNGzd2qnsjDPxEJD9eXMev0WicAn9Xli5digULFtywzpgxY6DT6cRVNh3a2trQ0NAAnU7n8jqdToeWlhY0NjY6zfrr6upcXvPSSy9h0qRJiI2N7bbfcXFx+OCDD7qt910M/EQkO32xSduIESMwYsSIbusZDAY0NjbCbDaLgbm8vBwOhwN6vd7lNbGxsfDz80NZWRmSk5MBXFuZU1NTA4PB4FT3ypUreO2113qckqqsrERYWFiP6nZg4Cci+ZHxcs6YmBgkJiZi0aJFKCgoQGtrKzIyMjBnzhxxRc/58+cRHx+PPXv2IC4uDlqtFqmpqcjKykJwcDA0Gg0WL14Mg8GAqVOnOrW/d+9etLW14ZFHHun03nl5eYiKisLtt98Om82Gl156CeXl5Xj33XfdGgMDPxGRm1599VVkZGQgPj4eSqUSycnJ2Lx5s3i+tbUV1dXVuHr1qnhs48aNYl273Y6EhAS8+OKLndresWMHZs2a5XK5ZktLC5YuXYrz589j8ODBmDBhAt577z1Mnz7drf4rBEFeOxlZrVZotVpE5q6BUq3u6+4Q0bccNhtqlq1AU1NTj3LmvdHx+2/8lyUYOMD1Tdieamu3473P8iTt782KM34ikh8Zp3r6A67jJyLyMZzxE5EMeWHGz0dwdYmBn4jkh6keSTHVQ0TkYzjjJyL5cQjwOFXj4Iy/Kwz8RCQ/guNa8bQNcompHiIiH8MZPxHJD2/uSoqBn4jkhzl+STHwE5H8cMYvKeb4iYh8DGf8RCQ/Arww4/dKT/olBn4ikh+meiTFVA8RkY9xO/CfP38ejzzyCIYNG4ZBgwZh/PjxOHnypHheEASsXLkSYWFhGDRoEIxGIz755BOvdpqI+jmHwzuFXHIr8P/jH//A3XffDT8/P/zlL3/BRx99hPXr12Po0KFineeffx6bN29GQUEBKioqMGTIECQkJMBms3m980TUT3Wkejwt5JJbOf61a9ciIiICu3btEo9FRUWJ/xYEAXl5eVixYgVmzpwJANizZw9CQ0Oxb98+zJkzx0vdJiKi3nJrxv/nP/8ZU6ZMwX/8x38gJCQEd955J37/+9+L58+dOweLxQKj0Sge02q10Ov1MJlMLtu02+2wWq1OhYh8HGf8knIr8P/f//0ftm3bhrFjx+Kdd97BY489hscffxwvv/wyAMBisQAAQkNDna4LDQ0Vz10vJycHWq1WLBEREb0ZBxH1Jw7BO4VccivwOxwOTJ48Gc899xzuvPNOpKWlYdGiRSgoKOh1B5YvX46mpiax1NbW9rotIiLqnluBPywsDLfddpvTsZiYGNTU1AAAdDodAKCurs6pTl1dnXjueiqVChqNxqkQkW8TBIdXCrnmVuC/++67UV1d7XTsf//3fzFq1CgA12706nQ6lJWVieetVisqKipgMBi80F0i8gmCF9I8zPF3ya1VPZmZmZg2bRqee+45/PSnP8Xx48exfft2bN++HQCgUCiwZMkSrFmzBmPHjkVUVBSeeeYZhIeHIykpSYr+E1F/JHhhd04G/i65FfjvuusuvPnmm1i+fDlWr16NqKgo5OXlISUlRazz5JNPorm5GWlpaWhsbMQ999yDkpISqNVqr3eeiIjc5/ZePT/+8Y/x4x//uMvzCoUCq1evxurVqz3qGBH5MIcDUPDRi1LhJm1EJD9M9UiKm7QREfkYzviJSHYEhwOCh6keLufsGgM/EckPUz2SYqqHiMjHcMZPRPLjEAAFZ/xSYeAnIvkRBACeLudk4O8KUz1ERD6GM34ikh3BIUDwMNUjcMbfJQZ+IpIfwQHPUz1cztkVpnqISHYEh+CVIpWGhgakpKRAo9EgKCgIqampuHLlyg2v2b59O+677z5oNBooFAo0Njb2qt0PP/wQP/jBD6BWqxEREYHnn3/e7f4z8BMRuSklJQVnz55FaWkpDhw4gCNHjiAtLe2G11y9ehWJiYl4+umne92u1WrF/fffj1GjRsFsNmPdunVYtWqVuENyT8ku1dORl3PYbH3cEyL6ro7fye8jd94m2D1O1bShFQA6PcdbpVJBpVL1ut2qqiqUlJTgxIkTmDJlCgBgy5YtePDBB/HCCy8gPDzc5XVLliwBABw6dKjX7b766qtoaWnBzp074e/vj9tvvx2VlZXYsGFDt394nAgyU1tb2/GVPRYWFhmW2tpayX7/v/nmG0Gn03mtrwEBAZ2OZWdne9THHTt2CEFBQU7HWltbhQEDBghvvPFGt9e///77AgDhH//4h9vt/ud//qcwc+ZMpzrl5eUCAKGhoaHHY5DdjD88PBy1tbUIDAzE5cuXERERgdra2n79SEar1cpx9hP9eYyCIODy5ctdzmi9Qa1W49y5c2hpafFKe4IgQKFQOB3zZLYPABaLBSEhIU7HBg4ciODgYFgsFknbtVgsiIqKcqoTGhoqnhs6dGiP3kt2gV+pVGLkyJEAIP4P85Vn8XKc/Ud/HaNWq5X8PdRqdZ88uGnZsmVYu3btDetUVVV9T72RluwCPxFRX1i6dCkWLFhwwzpjxoyBTqdDfX290/G2tjY0NDRAp9P1+v170q5Op0NdXZ1TnY7X7rw3Az8REYARI0ZgxIgR3dYzGAxobGyE2WxGbGwsAKC8vBwOhwN6vb7X79+Tdg0GA37zm9+gtbUVfn5+AIDS0lKMGzeux2keQObLOVUqFbKzsz3Oyckdx9l/+MIYfV1MTAwSExOxaNEiHD9+HH/961+RkZGBOXPmiPc/zp8/j+joaBw/fly8zmKxoLKyEp9++ikA4O9//zsqKyvR0NDQ43Z/9rOfwd/fH6mpqTh79iz27t2LTZs2ISsry71B9Pg2MBERCYIgCF9//bUwd+5cISAgQNBoNMLChQuFy5cvi+fPnTsnABDef/998Vh2drbLlUe7du3qcbuCIAh/+9vfhHvuuUdQqVTCLbfcIuTm5rrdf4UgcEMLIiJfIutUDxEReR8DPxGRj2HgJyLyMQz8REQ+RtaBPz8/H6NHj4ZarYZer3daGnUzOnLkCB5++GGEh4dDoVBg3759TucFQcDKlSsRFhaGQYMGwWg04pNPPumbzvZSTk4O7rrrLgQGBiIkJARJSUmorq52qmOz2ZCeno5hw4YhICAAycnJnb6UInfbtm3DhAkTxG/oGgwG/OUvfxHP94cxUv8l28C/d+9eZGVlITs7G6dOncLEiRORkJDQ6ZttN5Pm5mZMnDgR+fn5Ls8///zz2Lx5MwoKClBRUYEhQ4YgISEBtptop9LDhw8jPT0dx44dQ2lpKVpbW3H//fejublZrJOZmYn9+/ejuLgYhw8fxoULFzBr1qw+7LX7Ro4cidzcXJjNZpw8eRIzZszAzJkzcfbsWQD9Y4zUj7m9APR7EhcXJ6Snp4uv29vbhfDwcCEnJ6cPe+U9AIQ333xTfO1wOASdTiesW7dOPNbY2CioVCrhj3/8Yx/00Dvq6+sFAMLhw4cFQbg2Jj8/P6G4uFisU1VVJQAQTCZTX3XTK4YOHSq89NJL/XqM1D/Icsbf0tICs9kMo9EoHlMqlTAajTCZTH3YM+mcO3cOFovFacxarRZ6vf6mHnNTUxMAIDg4GABgNpvR2trqNM7o6GhERkbetONsb29HUVERmpubYTAY+uUYqX+R5V49X331Fdrb28XtRjuEhobi448/7qNeSatj21VXY/Zkq9e+5HA4sGTJEtx999244447AFwbp7+/P4KCgpzq3ozj/Pvf/w6DwQCbzYaAgAC8+eabuO2221BZWdlvxkj9kywDP/UP6enpOHPmDD744IO+7ookxo0bh8rKSjQ1NeFPf/oT5s+fj8OHD/d1t4i6JctUz/DhwzFgwACX2496su2pnHWMq7+MOSMjAwcOHMD7778vPl8BuDbOlpaWTg+avhnH6e/vj1tvvRWxsbHIycnBxIkTsWnTpn41RuqfZBn4/f39ERsbi7KyMvGYw+FAWVkZDAZDH/ZMOlFRUdDpdE5jtlqtqKiouKnGLAgCMjIy8Oabb6K8vLzT04JiY2Ph5+fnNM7q6mrU1NTcVON0xeFwwG639+sxUj/R13eXu1JUVCSoVCph9+7dwkcffSSkpaUJQUFBgsVi6euu9drly5eF06dPC6dPnxYACBs2bBBOnz4tfPHFF4IgCEJubq4QFBQkvPXWW8KHH34ozJw5U4iKihK++eabPu55zz322GOCVqsVDh06JFy8eFEsV69eFev84he/ECIjI4Xy8nLh5MmTgsFgEAwGQx/22n3Lli0TDh8+LJw7d0748MMPhWXLlgkKhUJ49913BUHoH2Ok/ku2gV8QBGHLli1CZGSk4O/vL8TFxQnHjh3r6y55pOMhy9eX+fPnC4JwbUnnM888I4SGhgoqlUqIj48Xqqur+7bTbnI1Ply39ew333wj/PKXvxSGDh0qDB48WPj3f/934eLFi33X6V549NFHhVGjRgn+/v7CiBEjhPj4eDHoC0L/GCP1X9yWmYjIx8gyx09ERNJh4Cci8jEM/EREPoaBn4jIxzDwExH5GAZ+IiIfw8BPRORjGPiJiHwMAz8RkY9h4Cci8jEM/EREPub/Ac43DVhZ1NYCAAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "#normalise weights1 row-wise\n",
        "wt_mat = model.layers[0].weight.data.detach().cpu()\n",
        "w1_normed=F.normalize(wt_mat,p=2,dim=1)\n",
        "#visualise w1_normed as heat-map just in absolute values\n",
        "plt.imshow(w1_normed.abs().numpy())\n",
        "plt.colorbar()\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 435
        },
        "id": "uQUG66BWHeOd",
        "outputId": "a2dd0e78-e1ca-4898-f8ad-d6bd84ed44a1"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgkAAAGiCAYAAABtUVVIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/SrBM8AAAACXBIWXMAAA9hAAAPYQGoP6dpAABAvUlEQVR4nO3de3hU1b3/8c8EyASFmRAgGSIBwpGSKPcgYdRWhdSg1iOHtAUaj0jzg2oJCrFV8CBYtAYv3ImkVC5apUGq0oIaG4PAUcItSC2IKVpsUmCCSpNANBcy+/cHzRzHTEImkxF2eL+eZz2Svdfas1Z4TL6s71p7WQzDMAQAAPANIRe6AwAA4OJEkAAAAHwiSAAAAD4RJAAAAJ8IEgAAgE8ECQAAwCeCBAAA4BNBAgAA8IkgAQAA+ESQAAAAfCJIAADAT1lZWerTp4/CwsKUmJioPXv2NFr30KFDSklJUZ8+fWSxWLRkyZIWPbOqqkrTpk1T165d1alTJ6WkpKi0tLQ1h9UAQQIAAH7YsGGDMjIyNG/ePO3fv1+DBw9WcnKyTp486bP+l19+qb59+2rBggVyOBwtfubMmTO1efNmbdy4Udu3b9fx48c1bty4oIyxnoUDngAAaL7ExERdc801WrFihSTJ7XYrJiZG06dP16xZs5ps26dPH82YMUMzZszw65nl5eXq3r271q9frx/+8IeSpI8++kjx8fEqKCjQyJEjW3+gktoH5ak6N23y9NNPy+VyafDgwVq+fLlGjBhx3nZut1vHjx9X586dZbFYgtU9AECQGIah06dPKzo6WiEhwZuwrqqqUk1NTcDPMQyjwe8bq9Uqq9XaoG5NTY0KCws1e/Zsz7WQkBAlJSWpoKCgRZ/fnGcWFhaqtrZWSUlJnjpxcXHq1auX+YKE+mmT7OxsJSYmasmSJUpOTlZRUZEiIyObbHv8+HHFxMQEo1sAgG9RSUmJevbsGZRnV1VVKbZ3J7lO1gX8rE6dOunMmTNe1+bNm6dHH320Qd3PP/9cdXV1ioqK8roeFRWljz76qEWf35xnulwuhYaGKjw8vEEdl8vVos9tjqAECYsWLdKUKVM0efJkSVJ2drZef/11rVmz5rxTMZ07d5Yk9Xx0jkLCwoLRPQBAELmrqvTPRx/3/DwPhpqaGrlO1uloYW/ZOrd8tqLitFuxCf9QSUmJbDab57qvWYRLUasHCf5OxVRXV6u6utrz9enTp8+1CQsjSAAAE/s2Usa2ziEBBQme59hsXkFCY7p166Z27do12FVQWlra6KLE1nimw+FQTU2NysrKvGYTAvnc5mj1ZFFT0ya+pkQyMzNlt9s9hVQDAKC56gx3wMUfoaGhSkhIUH5+vuea2+1Wfn6+nE5ni8bQnGcmJCSoQ4cOXnWKiopUXFzc4s9tjqAtXGyu2bNnKyMjw/N1RUUFgQIAoFncMuRWyzfptaRtRkaGJk2apOHDh2vEiBFasmSJKisrPSn2u+66S1dccYUyMzMlnZth//DDDz1/PnbsmA4cOKBOnTrpyiuvbNYz7Xa70tLSlJGRoYiICNlsNk2fPl1OpzNoixalIAQJ/k7FNLaCFACA83HLLf/mAhq299f48eP12Wefae7cuXK5XBoyZIhyc3M9M+jFxcVeuzqOHz+uoUOHer5+5pln9Mwzz+iGG27Qtm3bmvVMSVq8eLFCQkKUkpKi6upqJScn69lnn23hyJsnKO9JSExM1IgRI7R8+XJJ56ZNevXqpfT09PMuXKyoqJDdblevBY+zJgEATMhdVaXiWXNUXl7erDx/S9T/rjhe1DPghYvR/f8Z1L6aWVDSDeebNgEAoDXUGYbqAvi3biBtLwVBCRKaM20CAECgLsSahEtJ0BYupqenKz09PViPBwAAQXbBdzcAANBSbhmqYyYhaAgSAACmRbohuDgqGgAA+MRMAgDAtNjdEFwECQAA03L/uwTSHo0j3QAAAHxiJgEAYFp1Ae5uCKTtpYAgAQBgWnXGuRJIezSOIAEAYFqsSQgu1iQAAACfmEkAAJiWWxbVyRJQezSOIAEAYFpu41wJpD0aR7oBAAD4xEwCAMC06gJMNwTS9lJAkAAAMC2ChOAi3QAAAHxiJgEAYFpuwyK3EcDuhgDaXgoIEgAApkW6IbhINwAAAJ+YSQAAmFadQlQXwL9361qxL20RQQIAwLSMANckGKxJaBJBAgDAtFiTEFysSQAAAD4xkwAAMK06I0R1RgBrEji7oUkECQAA03LLIncAk+JuESU0hXQDAADwiZkEAIBpsXAxuAgSAACmFfiaBNINTSHdAAAAfGImAQBgWucWLgZwwBPphiYRJAAATMsd4GuZ2d3QNNINAADAJ4IEAIBp1S9cDKS0RFZWlvr06aOwsDAlJiZqz549TdbfuHGj4uLiFBYWpoEDB+qNN97wum+xWHyWp59+2lOnT58+De4vWLCgRf1vLoIEAIBpuRUScPHXhg0blJGRoXnz5mn//v0aPHiwkpOTdfLkSZ/1d+7cqYkTJyotLU3vv/++xo4dq7Fjx+rgwYOeOidOnPAqa9askcViUUpKitez5s+f71Vv+vTpfvffHwQJAADTqjMsARd/LVq0SFOmTNHkyZN11VVXKTs7W5dddpnWrFnjs/7SpUs1ZswY/fKXv1R8fLwee+wxDRs2TCtWrPDUcTgcXuWPf/yjbrrpJvXt29frWZ07d/aqd/nll/vdf38QJAAALnkVFRVepbq62me9mpoaFRYWKikpyXMtJCRESUlJKigo8NmmoKDAq74kJScnN1q/tLRUr7/+utLS0hrcW7Bggbp27aqhQ4fq6aef1tmzZ5s7xBZhdwMAwLTqAtzdUPfv3Q0xMTFe1+fNm6dHH320Qf3PP/9cdXV1ioqK8roeFRWljz76yOdnuFwun/VdLpfP+s8//7w6d+6scePGeV2/7777NGzYMEVERGjnzp2aPXu2Tpw4oUWLFjU5xkAQJAAATMtthMgdwBsX3f9+42JJSYlsNpvnutVqDbhvLbVmzRqlpqYqLCzM63pGRobnz4MGDVJoaKh+9rOfKTMzM2j9JUgAAFzybDabV5DQmG7duqldu3YqLS31ul5aWiqHw+GzjcPhaHb9//3f/1VRUZE2bNhw3r4kJibq7Nmz+vTTT9W/f//z1m8J1iQAAEyrPt0QSPFHaGioEhISlJ+f77nmdruVn58vp9Pps43T6fSqL0l5eXk+669evVoJCQkaPHjwefty4MABhYSEKDIy0q8x+IOZBACAabmlFu1Q+Hp7f2VkZGjSpEkaPny4RowYoSVLlqiyslKTJ0+WJN1111264oorlJmZKUm6//77dcMNN2jhwoW67bbblJOTo3379mnVqlVez62oqNDGjRu1cOHCBp9ZUFCg3bt366abblLnzp1VUFCgmTNn6s4771SXLl1aMIrmIUgAAMAP48eP12effaa5c+fK5XJpyJAhys3N9SxOLC4uVkjI/81QXHvttVq/fr3mzJmjhx9+WP369dOmTZs0YMAAr+fm5OTIMAxNnDixwWdarVbl5OTo0UcfVXV1tWJjYzVz5kyvdQrBYDGMi+uczIqKCtntdvVa8LhCvrFoAwBw8XNXVal41hyVl5c3K8/fEvW/K1buv0YdO7X837tfnTmre4ftDWpfzYyZBACAaQXyauX69mgc3x0AAOATMwkAANNyyyK3Alm42PK2lwKCBACAaZFuCC6/vzs7duzQ7bffrujoaFksFm3atMnrvmEYmjt3rnr06KGOHTsqKSlJR44caa3+AgDg8W2/J+FS4/d3p7KyUoMHD1ZWVpbP+0899ZSWLVum7Oxs7d69W5dffrmSk5NVVVUVcGcBAMC3x+90wy233KJbbrnF5z3DMLRkyRLNmTNHd9xxhyTphRdeUFRUlDZt2qQJEyY0aFNdXe112lZFRYW/XQIAXKLchkXuQF6mFEDbS0GrzrMcPXpULpfL60hMu92uxMTERo/EzMzMlN1u95RvnsQFAEBj3AGmGtykG5rUqt+d+mMv/TkSc/bs2SovL/eUkpKS1uwSAABooQu+u8FqtV7QIzkBAOYV+FHRzCQ0pVW/O/XHXvpzhCYAAC1VJ0vABY1r1SAhNjZWDofD60jMiooK7d69u9EjNAEAwMXJ73TDmTNn9PHHH3u+Pnr0qA4cOKCIiAj16tVLM2bM0OOPP65+/fopNjZWjzzyiKKjozV27NjW7DcAAKQbgszvIGHfvn266aabPF/XH1M5adIkrVu3Tg8++KAqKys1depUlZWV6frrr1dubq7CONERANDK6qSAUgZ1rdeVNsnvIOHGG29UU6dLWywWzZ8/X/Pnzw+oYwAA4MK64LsbAABoKdINwUWQAAAwLQ54Ci6CBACAaRkBHhVtsAWySYRQAADAJ2YSAACmRbohuAgSAACmxSmQwUUIBQAAfGImAQBgWvVHPgfSHo0jSAAAmBbphuAihAIAAD4xkwAAMC23QuQO4N+7gbS9FBAkAABMq86wqC6AlEEgbS8FhFAAAMAnZhIAAKbFwsXgIkgAAJiWEeApkAZvXGwSQQIAwLTqZFFdAIc0BdL2UkAIBQAAfGImAQBgWm4jsHUFbqMVO9MGESQAAEzLHeCahEDaXgr47gAAAJ8IEgAApuWWJeDSEllZWerTp4/CwsKUmJioPXv2NFl/48aNiouLU1hYmAYOHKg33njD6/7dd98ti8XiVcaMGeNV59SpU0pNTZXNZlN4eLjS0tJ05syZFvW/uQgSAACmVf/GxUCKvzZs2KCMjAzNmzdP+/fv1+DBg5WcnKyTJ0/6rL9z505NnDhRaWlpev/99zV27FiNHTtWBw8e9Ko3ZswYnThxwlN+//vfe91PTU3VoUOHlJeXpy1btmjHjh2aOnWq3/33B0ECAAB+WLRokaZMmaLJkyfrqquuUnZ2ti677DKtWbPGZ/2lS5dqzJgx+uUvf6n4+Hg99thjGjZsmFasWOFVz2q1yuFweEqXLl089w4fPqzc3Fw999xzSkxM1PXXX6/ly5crJydHx48fD9pYCRIAAKZVv3AxkCJJFRUVXqW6utrn59XU1KiwsFBJSUmeayEhIUpKSlJBQYHPNgUFBV71JSk5OblB/W3btikyMlL9+/fXvffeqy+++MLrGeHh4Ro+fLjnWlJSkkJCQrR7927/vml+IEgAAJiWWxbPq5lbVP69JiEmJkZ2u91TMjMzfX7e559/rrq6OkVFRXldj4qKksvl8tnG5XKdt/6YMWP0wgsvKD8/X08++aS2b9+uW265RXV1dZ5nREZGej2jffv2ioiIaPRzWwNbIAEAl7ySkhLZbDbP11ar9Vv9/AkTJnj+PHDgQA0aNEj/8R//oW3btmn06NHfal++jpkEAIBpGQHubDD+PZNgs9m8SmNBQrdu3dSuXTuVlpZ6XS8tLZXD4fDZxuFw+FVfkvr27atu3brp448/9jzjmwsjz549q1OnTjX5nEARJAAATCugVEMLTpAMDQ1VQkKC8vPz/68Pbrfy8/PldDp9tnE6nV71JSkvL6/R+pL0z3/+U1988YV69OjheUZZWZkKCws9dbZu3Sq3263ExES/xuAP0g0AANO6EG9czMjI0KRJkzR8+HCNGDFCS5YsUWVlpSZPnixJuuuuu3TFFVd41jXcf//9uuGGG7Rw4ULddtttysnJ0b59+7Rq1SpJ0pkzZ/SrX/1KKSkpcjgc+uSTT/Tggw/qyiuvVHJysiQpPj5eY8aM0ZQpU5Sdna3a2lqlp6drwoQJio6ObvH4z4cgAQAAP4wfP16fffaZ5s6dK5fLpSFDhig3N9ezOLG4uFghIf8XfFx77bVav3695syZo4cfflj9+vXTpk2bNGDAAElSu3bt9MEHH+j5559XWVmZoqOjdfPNN+uxxx7zSnu89NJLSk9P1+jRoxUSEqKUlBQtW7YsqGO1GIZxUR1vUVFRIbvdrl4LHldIWNiF7g4AwE/uqioVz5qj8vJyr8WAran+d8Udf/6pOlwe2uLn1FbW6I83rwlqX82MmQQAgGkF8mrl+vZoHAsXAQCAT8wkAABMqyU7FL7ZHo0jSAAAmBZBQnCRbgAAAD4xkwAAMC1mEoKLIAEAYFoECcFFugEAAPjETAIAwLQMBfaug4vqbYIXIYIEAIBpkW4ILoIEAIBpESQEF2sSAACAT8wkAABMi5mE4CJIAACYFkFCcJFuAAAAPvkVJGRmZuqaa65R586dFRkZqbFjx6qoqMirTlVVlaZNm6auXbuqU6dOSklJUWlpaat2GgAASTIMS8AFjfMrSNi+fbumTZumXbt2KS8vT7W1tbr55ptVWVnpqTNz5kxt3rxZGzdu1Pbt23X8+HGNGzeu1TsOAIBbloALGufXmoTc3Fyvr9etW6fIyEgVFhbqe9/7nsrLy7V69WqtX79eo0aNkiStXbtW8fHx2rVrl0aOHNl6PQcAAEEV0JqE8vJySVJERIQkqbCwULW1tUpKSvLUiYuLU69evVRQUODzGdXV1aqoqPAqAAA0R/3CxUAKGtfiIMHtdmvGjBm67rrrNGDAAEmSy+VSaGiowsPDvepGRUXJ5XL5fE5mZqbsdrunxMTEtLRLAIBLDGsSgqvFQcK0adN08OBB5eTkBNSB2bNnq7y83FNKSkoCeh4AAGgdLXpPQnp6urZs2aIdO3aoZ8+enusOh0M1NTUqKyvzmk0oLS2Vw+Hw+Syr1Sqr1dqSbgAALnG8JyG4/JpJMAxD6enpeu2117R161bFxsZ63U9ISFCHDh2Un5/vuVZUVKTi4mI5nc7W6TEAAP9GuiG4/JpJmDZtmtavX68//vGP6ty5s2edgd1uV8eOHWW325WWlqaMjAxFRETIZrNp+vTpcjqd7GwAALQ6I8CZBIKEpvkVJKxcuVKSdOONN3pdX7t2re6++25J0uLFixUSEqKUlBRVV1crOTlZzz77bKt0FgAAfHv8ChIMwzhvnbCwMGVlZSkrK6vFnQIAoDkMSc341dRkezSOA54AAKbllkWWAN6ayBsXm8YBTwAAwCdmEgAAphXoDgUWLjaNIAEAYFpuwyIL70kIGtINAADAJ2YSAACmZRgB7m5ge0OTCBIAAKbFmoTgIt0AAAB8YiYBAGBazCQEF0ECAMC02N0QXKQbAACmVb9wMZDSEllZWerTp4/CwsKUmJioPXv2NFl/48aNiouLU1hYmAYOHKg33njDc6+2tlYPPfSQBg4cqMsvv1zR0dG66667dPz4ca9n9OnTRxaLxassWLCgZQNoJoIEAAD8sGHDBmVkZGjevHnav3+/Bg8erOTkZJ08edJn/Z07d2rixIlKS0vT+++/r7Fjx2rs2LE6ePCgJOnLL7/U/v379cgjj2j//v169dVXVVRUpP/8z/9s8Kz58+frxIkTnjJ9+vSgjpV0AwDAtM7NBgSyJuHcfysqKryuW61WWa1Wn20WLVqkKVOmaPLkyZKk7Oxsvf7661qzZo1mzZrVoP7SpUs1ZswY/fKXv5QkPfbYY8rLy9OKFSuUnZ0tu92uvLw8rzYrVqzQiBEjVFxcrF69enmud+7cWQ6Ho8Xj9RczCQAA06pfuBhIkaSYmBjZ7XZPyczM9Pl5NTU1KiwsVFJSkudaSEiIkpKSVFBQ4LNNQUGBV31JSk5ObrS+JJWXl8tisSg8PNzr+oIFC9S1a1cNHTpUTz/9tM6ePducb1OLMZMAALjklZSUyGazeb5ubBbh888/V11dnaKioryuR0VF6aOPPvLZxuVy+azvcrl81q+qqtJDDz2kiRMnevXpvvvu07BhwxQREaGdO3dq9uzZOnHihBYtWtSsMbYEQQIAwLSMf5dA2kuSzWbz+oV8odTW1urHP/6xDMPQypUrve5lZGR4/jxo0CCFhobqZz/7mTIzMxsNagJFugEAYFqtlW5orm7duqldu3YqLS31ul5aWtroWgGHw9Gs+vUBwj/+8Q/l5eWdN2hJTEzU2bNn9emnn/o1Bn8QJAAA0EyhoaFKSEhQfn6+55rb7VZ+fr6cTqfPNk6n06u+JOXl5XnVrw8Qjhw5orfffltdu3Y9b18OHDigkJAQRUZGtnA050e6AQBgXq2Vb/BDRkaGJk2apOHDh2vEiBFasmSJKisrPbsd7rrrLl1xxRWexY/333+/brjhBi1cuFC33XabcnJytG/fPq1atUrSuQDhhz/8ofbv368tW7aorq7Os14hIiJCoaGhKigo0O7du3XTTTepc+fOKigo0MyZM3XnnXeqS5cuAXwDmkaQAAAwrwBfy6wWtB0/frw+++wzzZ07Vy6XS0OGDFFubq5ncWJxcbFCQv5vov7aa6/V+vXrNWfOHD388MPq16+fNm3apAEDBkiSjh07pj/96U+SpCFDhnh91jvvvKMbb7xRVqtVOTk5evTRR1VdXa3Y2FjNnDnTa51CMFgM4+I6KLOiokJ2u129FjyukLCwC90dAICf3FVVKp41R+Xl5UFbDFj/uyJ27f8o5LKW/65wf1mlo5N/HdS+mhlrEgAAgE+kGwAApsUpkMFFkAAAMC/D0qJ1BV7t0SjSDQAAwCdmEgAAphXIcc/17dE4ggQAgHldgPckXEpINwAAAJ+YSQAAmBa7G4KLIAEAYG6kDIKGdAMAAPCJmQQAgGmRbgguggQAgHmxuyGoCBIAACZm+XcJpD0aw5oEAADgEzMJAADzIt0QVAQJAADzIkgIKtINAADAJ2YSAADmxVHRQUWQAAAwLU6BDC7SDQAAwCdmEgAA5sXCxaAiSAAAmBdrEoKKdAMAAPCJmQQAgGlZjHMlkPZoHEECAMC8WJMQVAQJAADzYk1CUPm1JmHlypUaNGiQbDabbDabnE6n3nzzTc/9qqoqTZs2TV27dlWnTp2UkpKi0tLSVu80AAAIPr+ChJ49e2rBggUqLCzUvn37NGrUKN1xxx06dOiQJGnmzJnavHmzNm7cqO3bt+v48eMaN25cUDoOAIAn3RBIQaP8SjfcfvvtXl//+te/1sqVK7Vr1y717NlTq1ev1vr16zVq1ChJ0tq1axUfH69du3Zp5MiRrddrAAAk1iQEWYu3QNbV1SknJ0eVlZVyOp0qLCxUbW2tkpKSPHXi4uLUq1cvFRQUNPqc6upqVVRUeBUAAHDh+R0k/PWvf1WnTp1ktVp1zz336LXXXtNVV10ll8ul0NBQhYeHe9WPioqSy+Vq9HmZmZmy2+2eEhMT4/cgAACXKNINQeV3kNC/f38dOHBAu3fv1r333qtJkybpww8/bHEHZs+erfLyck8pKSlp8bMAAJeY+t0NgRQ0yu8tkKGhobryyislSQkJCdq7d6+WLl2q8ePHq6amRmVlZV6zCaWlpXI4HI0+z2q1ymq1+t9zAAAQVAG/ltntdqu6uloJCQnq0KGD8vPzPfeKiopUXFwsp9MZ6McAANBA/RsXAylonF8zCbNnz9Ytt9yiXr166fTp01q/fr22bdumt956S3a7XWlpacrIyFBERIRsNpumT58up9PJzgYAQHCwuyGo/JpJOHnypO666y71799fo0eP1t69e/XWW2/p+9//viRp8eLF+sEPfqCUlBR973vfk8Ph0KuvvhqUjgMAcKFkZWWpT58+CgsLU2Jiovbs2dNk/Y0bNyouLk5hYWEaOHCg3njjDa/7hmFo7ty56tGjhzp27KikpCQdOXLEq86pU6eUmpoqm82m8PBwpaWl6cyZM60+tq/zK0hYvXq1Pv30U1VXV+vkyZN6++23PQGCJIWFhSkrK0unTp1SZWWlXn311SbXIwAAYDYbNmxQRkaG5s2bp/3792vw4MFKTk7WyZMnfdbfuXOnJk6cqLS0NL3//vsaO3asxo4dq4MHD3rqPPXUU1q2bJmys7O1e/duXX755UpOTlZVVZWnTmpqqg4dOqS8vDxt2bJFO3bs0NSpU4M6VothGBfVZEtFRYXsdrt6LXhcIWFhF7o7AAA/uauqVDxrjsrLy2Wz2YLyGfW/K3o/GdjvCndVlf7x0ByVlJR49bWpRfWJiYm65pprtGLFinPPcLsVExOj6dOna9asWQ3qjx8/XpWVldqyZYvn2siRIzVkyBBlZ2fLMAxFR0frgQce0C9+8QtJUnl5uaKiorRu3TpNmDBBhw8f1lVXXaW9e/dq+PDhkqTc3Fzdeuut+uc//6no6OgWfw+aEvDCRQAALphW2gIZExPj9c6ezMxMnx9XU1OjwsJCrxcHhoSEKCkpqdEXBxYUFHjVl6Tk5GRP/aNHj8rlcnnVsdvtSkxM9NQpKChQeHi4J0CQpKSkJIWEhGj37t0t+MY1D6dAAgAueb5mEnz5/PPPVVdXp6ioKK/rUVFR+uijj3y2cblcPuvXv2iw/r/nqxMZGel1v3379oqIiGjyhYWBIkgAAJhXK+1uqD/dGN5INwAAzOtbfi1zt27d1K5dO5WWlnpdb+rFgQ6Ho8n69f89X51vLow8e/asTp06FdQNAgQJAAA0U2hoqBISErxeHOh2u5Wfn9/oiwOdTqdXfUnKy8vz1I+NjZXD4fCqU1FRod27d3vqOJ1OlZWVqbCw0FNn69atcrvdSkxMbLXxfRPpBgCAaQX61sSWtM3IyNCkSZM0fPhwjRgxQkuWLFFlZaUmT54sSbrrrrt0xRVXeBY/3n///brhhhu0cOFC3XbbbcrJydG+ffu0atWqc32wWDRjxgw9/vjj6tevn2JjY/XII48oOjpaY8eOlSTFx8drzJgxmjJlirKzs1VbW6v09HRNmDAhaDsbJIIEAICZXYA3Lo4fP16fffaZ5s6dK5fLpSFDhig3N9ez8LC4uFghIf83UX/ttddq/fr1mjNnjh5++GH169dPmzZt0oABAzx1HnzwQVVWVmrq1KkqKyvT9ddfr9zcXIV9bXvnSy+9pPT0dI0ePVohISFKSUnRsmXLWj72ZuA9CQCAVvVtviehz+O/Dvg9CZ/O+Z+g9tXMmEkAAJgXZzcEFUECAMC0LsSahEsJuxsAAIBPzCQAAMzra69WbnF7NIogAQBgXqxJCCqCBACAabEmIbhYkwAAAHxiJgEAYF6kG4KKIAEAYF4BphsIEppGugEAAPjETAIAwLxINwQVQQIAwLwIEoKKdAMAAPCJmQQAgGnxnoTgYiYBAAD4RJAAAAB8It0AADAvFi4GFUECAMC0WJMQXAQJAABz4xd90LAmAQAA+MRMAgDAvFiTEFQECQAA02JNQnCRbgAAAD4xkwAAMC/SDUFFkAAAMC3SDcFFugEAAPjETAIAwLxINwQVQQIAwLwIEoKKdAMAAPCJmQQAgGmxcDG4CBIAAOZFuiGoCBIAAOZFkBBUrEkAAAA+MZMAADAt1iQEF0ECAMC8SDcEFekGAACC4NSpU0pNTZXNZlN4eLjS0tJ05syZJttUVVVp2rRp6tq1qzp16qSUlBSVlpZ67v/lL3/RxIkTFRMTo44dOyo+Pl5Lly71esa2bdtksVgaFJfL5fcYmEkAAJjWxZxuSE1N1YkTJ5SXl6fa2lpNnjxZU6dO1fr16xttM3PmTL3++uvauHGj7Ha70tPTNW7cOL333nuSpMLCQkVGRurFF19UTEyMdu7cqalTp6pdu3ZKT0/3elZRUZFsNpvn68jISL/HQJAAADCvizTdcPjwYeXm5mrv3r0aPny4JGn58uW69dZb9cwzzyg6OrpBm/Lycq1evVrr16/XqFGjJElr165VfHy8du3apZEjR+qnP/2pV5u+ffuqoKBAr776aoMgITIyUuHh4QGNI6B0w4IFC2SxWDRjxgzPtfNNlQAAcLGpqKjwKtXV1QE9r6CgQOHh4Z4AQZKSkpIUEhKi3bt3+2xTWFio2tpaJSUlea7FxcWpV69eKigoaPSzysvLFRER0eD6kCFD1KNHD33/+9/3zET4q8VBwt69e/Wb3/xGgwYN8ro+c+ZMbd68WRs3btT27dt1/PhxjRs3rqUfAwBA44xWKJJiYmJkt9s9JTMzM6BuuVyuBtP77du3V0RERKNrA1wul0JDQxv86z8qKqrRNjt37tSGDRs0depUz7UePXooOztbr7zyil555RXFxMToxhtv1P79+/0eR4vSDWfOnFFqaqp++9vf6vHHH/dcb85UCQAArcXy7xJIe0kqKSnxyt9brVaf9WfNmqUnn3yyyWcePnw4gB4138GDB3XHHXdo3rx5uvnmmz3X+/fvr/79+3u+vvbaa/XJJ59o8eLF+t3vfufXZ7QoSJg2bZpuu+02JSUleQUJ55sq8RUkVFdXe03rVFRUtKRLAAC0mM1m8woSGvPAAw/o7rvvbrJO37595XA4dPLkSa/rZ8+e1alTp+RwOHy2czgcqqmpUVlZmddsQmlpaYM2H374oUaPHq2pU6dqzpw55+33iBEj9O6775633jf5HSTk5ORo//792rt3b4N7LZkqyczM1K9+9St/uwEAwLe+cLF79+7q3r37ees5nU6VlZWpsLBQCQkJkqStW7fK7XYrMTHRZ5uEhAR16NBB+fn5SklJkXRuh0JxcbGcTqen3qFDhzRq1ChNmjRJv/71r5vV7wMHDqhHjx7Nqvt1fgUJJSUluv/++5WXl6ewsDC/P8yX2bNnKyMjw/N1RUWFYmJiWuXZAIC27WLdAhkfH68xY8ZoypQpys7OVm1trdLT0zVhwgTPzoZjx45p9OjReuGFFzRixAjZ7XalpaUpIyNDERERstlsmj59upxOp2cm/uDBgxo1apSSk5OVkZHh+Qd4u3btPMHLkiVLFBsbq6uvvlpVVVV67rnntHXrVv35z3/2exx+BQmFhYU6efKkhg0b5rlWV1enHTt2aMWKFXrrrbeaPVVSz2q1Npr7AQCgSRfpFkhJeumll5Senq7Ro0crJCREKSkpWrZsmed+bW2tioqK9OWXX3quLV682FO3urpaycnJevbZZz33//CHP+izzz7Tiy++qBdffNFzvXfv3vr0008lSTU1NXrggQd07NgxXXbZZRo0aJDefvtt3XTTTX6PwWIYRrO/RadPn9Y//vEPr2uTJ09WXFycHnroIcXExKh79+76/e9/7zVVEhcX1+iahG+qqKiQ3W5XrwWPK6SVZisAAN8ed1WVimfNUXl5ebPy/C1R/7vi6p89oXbWlv+uqKuu0qHfPBzUvpqZXzMJnTt31oABA7yuXX755eratavn+vmmSgAAaFWcvxA0rf7GxfNNlQAA0Fou1jUJbUXAQcK2bdu8vg4LC1NWVpaysrICfTQAALiAOLsBAGBeF/HCxbaAIAEAYFqkG4IroAOeAABA28VMAgDAvEg3BBVBAgDAtEg3BBfpBgAA4BMzCQAA8yLdEFQECQAA8yJICCqCBACAabEmIbhYkwAAAHxiJgEAYF6kG4KKIAEAYFoWw5DFaPlv+kDaXgpINwAAAJ+YSQAAmBfphqAiSAAAmBa7G4KLdAMAAPCJmQQAgHmRbggqggQAgGmRbggu0g0AAMAnZhIAAOZFuiGoCBIAAKZFuiG4CBIAAObFTEJQsSYBAAD4xEwCAMDUSBkED0ECAMC8DONcCaQ9GkW6AQAA+MRMAgDAtNjdEFwECQAA82J3Q1CRbgAAAD4xkwAAMC2L+1wJpD0aR5AAADAv0g1BRboBAAD4RJAAADCt+t0NgZRgOXXqlFJTU2Wz2RQeHq60tDSdOXOmyTZVVVWaNm2aunbtqk6dOiklJUWlpaXeY7ZYGpScnByvOtu2bdOwYcNktVp15ZVXat26dS0aA0ECAMC86l+mFEgJktTUVB06dEh5eXnasmWLduzYoalTpzbZZubMmdq8ebM2btyo7du36/jx4xo3blyDemvXrtWJEyc8ZezYsZ57R48e1W233aabbrpJBw4c0IwZM/T//t//01tvveX3GFiTAAAwrYv1PQmHDx9Wbm6u9u7dq+HDh0uSli9frltvvVXPPPOMoqOjG7QpLy/X6tWrtX79eo0aNUrSuWAgPj5eu3bt0siRIz11w8PD5XA4fH52dna2YmNjtXDhQklSfHy83n33XS1evFjJycl+jYOZBADAJa+iosKrVFdXB/S8goIChYeHewIESUpKSlJISIh2797ts01hYaFqa2uVlJTkuRYXF6devXqpoKDAq+60adPUrVs3jRgxQmvWrJHxtRmRgoICr2dIUnJycoNnNAdBAgDAvIxWKJJiYmJkt9s9JTMzM6BuuVwuRUZGel1r3769IiIi5HK5Gm0TGhqq8PBwr+tRUVFebebPn6+XX35ZeXl5SklJ0c9//nMtX77c6zlRUVENnlFRUaGvvvrKr3GQbgAAmFZrpRtKSkpks9k8161Wq8/6s2bN0pNPPtnkMw8fPtzyDjXDI4884vnz0KFDVVlZqaefflr33Xdfq38WQQIA4JJns9m8goTGPPDAA7r77rubrNO3b185HA6dPHnS6/rZs2d16tSpRtcSOBwO1dTUqKyszGs2obS0tNE2kpSYmKjHHntM1dXVslqtcjgcDXZElJaWymazqWPHjk0P8BsIEgAA5vUtHxXdvXt3de/e/bz1nE6nysrKVFhYqISEBEnS1q1b5Xa7lZiY6LNNQkKCOnTooPz8fKWkpEiSioqKVFxcLKfT2ehnHThwQF26dPHMfjidTr3xxhtedfLy8pp8RmMIEgAApnWx7m6Ij4/XmDFjNGXKFGVnZ6u2tlbp6emaMGGCZ2fDsWPHNHr0aL3wwgsaMWKE7Ha70tLSlJGRoYiICNlsNk2fPl1Op9Ozs2Hz5s0qLS3VyJEjFRYWpry8PD3xxBP6xS9+4fnse+65RytWrNCDDz6on/70p9q6datefvllvf76636PgyABAIAgeOmll5Senq7Ro0crJCREKSkpWrZsmed+bW2tioqK9OWXX3quLV682FO3urpaycnJevbZZz33O3TooKysLM2cOVOGYejKK6/UokWLNGXKFE+d2NhYvf7665o5c6aWLl2qnj176rnnnvN7+6MkWQwjiG+SaIGKigrZ7Xb1WvC4QsLCLnR3AAB+cldVqXjWHJWXlzcrz98S9b8rnGPmq32Hlv+uOFtbpYLcuUHtq5kxkwAAMK2LNd3QVvCeBAAA4BMzCQAA83Ib50og7dEoggQAgHl97a2JLW6PRvmVbnj00UcbHE8ZFxfnud+cIy4BAGgtFgV4VPSFHsBFzu81CVdffbXX8ZTvvvuu515zj7gEAAAXP7/TDe3bt/f5ekh/jrj8uurqaq/TtioqKvztEgDgUvUtv3HxUuP3TMKRI0cUHR2tvn37KjU1VcXFxZL8O+Ly6zIzM71O3oqJiWnBMAAAl6KAUg0Bbp+8FPgVJCQmJmrdunXKzc3VypUrdfToUX33u9/V6dOnm33E5TfNnj1b5eXlnlJSUtKigQAAgNblV7rhlltu8fx50KBBSkxMVO/evfXyyy/7fbJUPavV2uiRnAAANIndDUEV0MuUwsPD9Z3vfEcff/yx1xGXX3e+Iy4BAGgpi2EEXNC4gIKEM2fO6JNPPlGPHj28jris15wjLgEAwMXJr3TDL37xC91+++3q3bu3jh8/rnnz5qldu3aaOHFis464BACgVbn/XQJpj0b5FST885//1MSJE/XFF1+oe/fuuv7667Vr1y51795d0vmPuAQAoDUFmjIg3dA0v4KEnJycJu+HhYUpKytLWVlZAXUKAABceJzdAAAwL3Y3BBVBAgDAvHjjYlARJAAATCvQtybyxsWmBbQFEgAAtF3MJAAAzIt0Q1ARJAAATMviPlcCaY/GkW4AAAA+MZMAADAv0g1BRZAAADAv3pMQVKQbAACAT8wkAABMi7MbgosgAQBgXqxJCCrSDQAAwCdmEgAA5mVICuRdB0wkNIkgAQBgWqxJCC6CBACAeRkKcE1Cq/WkTWJNAgAA8ImZBACAebG7IagIEgAA5uWWZAmwPRpFugEAAPhEkAAAMK363Q2BlGA5deqUUlNTZbPZFB4errS0NJ05c6bJNlVVVZo2bZq6du2qTp06KSUlRaWlpZ7769atk8Vi8VlOnjwpSdq2bZvP+y6Xy+8xkG4AAJjXRbwmITU1VSdOnFBeXp5qa2s1efJkTZ06VevXr2+0zcyZM/X6669r48aNstvtSk9P17hx4/Tee+9JksaPH68xY8Z4tbn77rtVVVWlyMhIr+tFRUWy2Wyer795vzkIEgAAaGWHDx9Wbm6u9u7dq+HDh0uSli9frltvvVXPPPOMoqOjG7QpLy/X6tWrtX79eo0aNUqStHbtWsXHx2vXrl0aOXKkOnbsqI4dO3rafPbZZ9q6datWr17d4HmRkZEKDw8PaBykGwAA5lU/kxBIkVRRUeFVqqurA+pWQUGBwsPDPQGCJCUlJSkkJES7d+/22aawsFC1tbVKSkryXIuLi1OvXr1UUFDgs80LL7ygyy67TD/84Q8b3BsyZIh69Oih73//+56ZCH8RJAAAzKuVgoSYmBjZ7XZPyczMDKhbLperwfR++/btFRER0ejaAJfLpdDQ0Ab/+o+Kimq0zerVq/WTn/zEa3ahR48eys7O1iuvvKJXXnlFMTExuvHGG7V//36/x0G6AQBwySspKfHK31utVp/1Zs2apSeffLLJZx0+fLhV+9aYgoICHT58WL/73e+8rvfv31/9+/f3fH3ttdfqk08+0eLFixvUPR+CBACAebXSexJsNptXkNCYBx54QHfffXeTdfr27SuHw+HZbVDv7NmzOnXqlBwOh892DodDNTU1Kisr85pNKC0t9dnmueee05AhQ5SQkHDefo8YMULvvvvueet9E0ECAMC0vu0Dnrp3767u3buft57T6VRZWZkKCws9v8S3bt0qt9utxMREn20SEhLUoUMH5efnKyUlRdK5HQrFxcVyOp1edc+cOaOXX3652WmRAwcOqEePHs2q+3UECQAA87pIt0DGx8drzJgxmjJlirKzs1VbW6v09HRNmDDBs7Ph2LFjGj16tF544QWNGDFCdrtdaWlpysjIUEREhGw2m6ZPny6n06mRI0d6PX/Dhg06e/as7rzzzgafvWTJEsXGxurqq69WVVWVnnvuOW3dulV//vOf/R4HQQIAAEHw0ksvKT09XaNHj1ZISIhSUlK0bNkyz/3a2loVFRXpyy+/9FxbvHixp251dbWSk5P17LPPNnj26tWrNW7cOJ9bHGtqavTAAw/o2LFjuuyyyzRo0CC9/fbbuummm/weg8UwLq7TLSoqKmS329VrweMKCQu70N0BAPjJXVWl4llzVF5e3qw8f0vU/65I+o8Zat/O9yLD5jhbV623P1kS1L6aGTMJAADzukjTDW0F70kAAAA+MZMAADCxAGcSxExCUwgSAADmRbohqEg3AAAAn5hJAACYl9tQQCkDNzMJTSFIAACYl+E+VwJpj0aRbgAAAD4xkwAAMC8WLgYVQQIAwLxYkxBUBAkAAPNiJiGoWJMAAAB8YiYBAGBehgKcSWi1nrRJBAkAAPMi3RBUpBsAAIBPfgcJx44d05133qmuXbuqY8eOGjhwoPbt2+e5bxiG5s6dqx49eqhjx45KSkrSkSNHWrXTAABIktzuwAsa5VeQ8K9//UvXXXedOnTooDfffFMffvihFi5cqC5dunjqPPXUU1q2bJmys7O1e/duXX755UpOTlZVVVWrdx4AcImrTzcEUtAov9YkPPnkk4qJidHatWs912JjYz1/NgxDS5Ys0Zw5c3THHXdIkl544QVFRUVp06ZNmjBhQit1GwAABJtfMwl/+tOfNHz4cP3oRz9SZGSkhg4dqt/+9ree+0ePHpXL5VJSUpLnmt1uV2JiogoKCnw+s7q6WhUVFV4FAIBmYSYhqPwKEv7+979r5cqV6tevn9566y3de++9uu+++/T8889LklwulyQpKirKq11UVJTn3jdlZmbKbrd7SkxMTEvGAQC4FLmNwAsa5VeQ4Ha7NWzYMD3xxBMaOnSopk6dqilTpig7O7vFHZg9e7bKy8s9paSkpMXPAgAArcevIKFHjx666qqrvK7Fx8eruLhYkuRwOCRJpaWlXnVKS0s9977JarXKZrN5FQAAmsMw3AEXNM6vIOG6665TUVGR17W//e1v6t27t6RzixgdDofy8/M99ysqKrR79245nc5W6C4AAF9jBJhqYE1Ck/za3TBz5kxde+21euKJJ/TjH/9Ye/bs0apVq7Rq1SpJksVi0YwZM/T444+rX79+io2N1SOPPKLo6GiNHTs2GP0HAFzKjABPgSRIaJJfQcI111yj1157TbNnz9b8+fMVGxurJUuWKDU11VPnwQcfVGVlpaZOnaqysjJdf/31ys3NVVhYWKt3HgAABI/fZzf84Ac/0A9+8ING71ssFs2fP1/z588PqGMAAJyX2y1ZAlhXwJqEJnHAEwDAvEg3BBUHPAEAAJ+YSQAAmJbhdssIIN3AFsimESQAAMyLdENQkW4AAAA+MZMAADAvtyFZmEkIFoIEAIB5GYakQLZAEiQ0hXQDAADwiZkEAIBpGW5DRgDpBoOZhCYRJAAAzMtwK7B0A1sgm0K6AQBgWobbCLgEy6lTp5Samiqbzabw8HClpaXpzJkzTbZZtWqVbrzxRtlsNlksFpWVlbXouR988IG++93vKiwsTDExMXrqqadaNAaCBAAAgiA1NVWHDh1SXl6etmzZoh07dmjq1KlNtvnyyy81ZswYPfzwwy1+bkVFhW6++Wb17t1bhYWFevrpp/Xoo496Tmz2x0WXbqjPD7mrqi5wTwAALVH/8/vbyPefNaoDShmcVa2kc79Yv85qtcpqtbb4uYcPH1Zubq727t2r4cOHS5KWL1+uW2+9Vc8884yio6N9tpsxY4Ykadu2bS1+7ksvvaSamhqtWbNGoaGhuvrqq3XgwAEtWrTovEFKA8ZFpqSkpP71WRQKhUIxcSkpKQna74qvvvrKcDgcrdLPTp06Nbg2b968gPq3evVqIzw83OtabW2t0a5dO+PVV189b/t33nnHkGT861//8vu5//3f/23ccccdXnW2bt1qSDJOnTrl1zguupmE6OholZSUqHPnzjp9+rRiYmJUUlIim812obsWNBUVFYyzjbgUxigxzramtcdpGIZOnz7d6L+WW0NYWJiOHj2qmpqagJ9lGIYsFovXtUBmESTJ5XIpMjLS61r79u0VEREhl8sV1Oe6XC7FxsZ61YmKivLc69KlS7M/76ILEkJCQtSzZ09J8vyl2Wy2Nv0/aD3G2XZcCmOUGGdb05rjtNvtrfKcpoSFhSksLCzon/N1s2bN0pNPPtlkncOHD39LvQm+iy5IAADgYvXAAw/o7rvvbrJO37595XA4dPLkSa/rZ8+e1alTp+RwOFr8+c15rsPhUGlpqVed+q/9/WyCBAAAmql79+7q3r37ees5nU6VlZWpsLBQCQkJkqStW7fK7XYrMTGxxZ/fnOc6nU79z//8j2pra9WhQwdJUl5envr37+9XqkG6yLdAWq1WzZs3L+Dc0MWOcbYdl8IYJcbZ1lwq4/w2xcfHa8yYMZoyZYr27Nmj9957T+np6ZowYYJnrcaxY8cUFxenPXv2eNq5XC4dOHBAH3/8sSTpr3/9qw4cOKBTp041+7k/+clPFBoaqrS0NB06dEgbNmzQ0qVLlZGR4f9A/FrmCAAAmuWLL74wJk6caHTq1Mmw2WzG5MmTjdOnT3vuHz161JBkvPPOO55r8+bN87kDY+3atc1+rmEYxl/+8hfj+uuvN6xWq3HFFVcYCxYsaNEYLIbBi6sBAEBDF3W6AQAAXDgECQAAwCeCBAAA4BNBAgAA8OmiDhKysrLUp08fhYWFKTEx0WubiBnt2LFDt99+u6Kjo2WxWLRp0yav+4ZhaO7cuerRo4c6duyopKQkHTly5MJ0toUyMzN1zTXXqHPnzoqMjNTYsWNVVFTkVaeqqkrTpk1T165d1alTJ6WkpDR48cfFbuXKlRo0aJDnDXVOp1Nvvvmm535bGOM3LViwQBaLxXMAjdQ2xvnoo4/KYrF4lbi4OM/9tjDGeseOHdOdd96prl27qmPHjho4cKD27dvnud8WfgahdV20QcKGDRuUkZGhefPmaf/+/Ro8eLCSk5MbvGnKTCorKzV48GBlZWX5vP/UU09p2bJlys7O1u7du3X55ZcrOTlZVSY6EXP79u2aNm2adu3apby8PNXW1urmm29WZWWlp87MmTO1efNmbdy4Udu3b9fx48c1bty4C9hr//Xs2VMLFixQYWGh9u3bp1GjRumOO+7QoUOHJLWNMX7d3r179Zvf/EaDBg3yut5Wxnn11VfrxIkTnvLuu+967rWVMf7rX//Sddddpw4dOujNN9/Uhx9+qIULF3q9XKct/AxCK2vRxslvwYgRI4xp06Z5vq6rqzOio6ONzMzMC9ir1iPJeO211zxfu91uw+FwGE8//bTnWllZmWG1Wo3f//73F6CHrePkyZOGJGP79u2GYZwbU4cOHYyNGzd66hw+fNiQZBQUFFyobraKLl26GM8991ybG+Pp06eNfv36GXl5ecYNN9xg3H///YZhtJ2/y3nz5hmDBw/2ea+tjNEwDOOhhx4yrr/++kbvt9WfQQjMRTmTUFNTo8LCQiUlJXmuhYSEKCkpSQUFBRewZ8Fz9OhRuVwurzHb7XYlJiaaeszl5eWSpIiICElSYWGhamtrvcYZFxenXr16mXacdXV1ysnJUWVlpZxOZ5sb47Rp03Tbbbd5jUdqW3+XR44cUXR0tPr27avU1FQVFxdLaltj/NOf/qThw4frRz/6kSIjIzV06FD99re/9dxvqz+DEJiLMkj4/PPPVVdX5znasl5UVFRAR2xezOrH1ZbG7Ha7NWPGDF133XUaMGCApHPjDA0NVXh4uFddM47zr3/9qzp16iSr1ap77rlHr732mq666qo2NcacnBzt379fmZmZDe61lXEmJiZq3bp1ys3N1cqVK3X06FF997vf1enTp9vMGCXp73//u1auXKl+/frprbfe0r333qv77rtPzz//vKS2+TMIgeOAJwTNtGnTdPDgQa/8blvSv39/HThwQOXl5frDH/6gSZMmafv27Re6W62mpKRE999/v/Ly8r7143i/Tbfccovnz4MGDVJiYqJ69+6tl19+WR07dryAPWtdbrdbw4cP1xNPPCFJGjp0qA4ePKjs7GxNmjTpAvcOF6uLciahW7duateunc+jLgM5YvNiVj+utjLm9PR0bdmyRe+884569uzpue5wOFRTU6OysjKv+mYcZ2hoqK688kolJCQoMzNTgwcP1tKlS9vMGAsLC3Xy5EkNGzZM7du3V/v27bV9+3YtW7ZM7du3V1RUVJsY5zeFh4frO9/5jj7++OM283cpST169NBVV13ldS0+Pt6TWmlrP4PQOi7KICE0NFQJCQnKz8/3XHO73crPz5fT6byAPQue2NhYORwOrzFXVFRo9+7dphqzYRhKT0/Xa6+9pq1btyo2NtbrfkJCgjp06OA1zqKiIhUXF5tqnL643W5VV1e3mTGOHj3acwJdfRk+fLhSU1M9f24L4/ymM2fO6JNPPlGPHj3azN+lJF133XUNtiP/7W9/U+/evSW1nZ9BaGUXeuVkY3Jycgyr1WqsW7fO+PDDD42pU6ca4eHhhsvlutBda7HTp08b77//vvH+++8bkoxFixYZ77//vvGPf/zDMAzDWLBggREeHm788Y9/ND744APjjjvuMGJjY42vvvrqAve8+e69917Dbrcb27ZtM06cOOEpX375pafOPffcY/Tq1cvYunWrsW/fPsPpdBpOp/MC9tp/s2bNMrZv324cPXrU+OCDD4xZs2YZFovF+POf/2wYRtsYoy9f391gGG1jnA888ICxbds24+jRo8Z7771nJCUlGd26dTNOnjxpGEbbGKNhGMaePXuM9u3bG7/+9a+NI0eOGC+99JJx2WWXGS+++KKnTlv4GYTWddEGCYZhGMuXLzd69eplhIaGGiNGjDB27dp1obsUkHfeecfnEaCTJk0yDOPcFqRHHnnEiIqKMqxWqzF69GijqKjownbaT77Gp28cc/rVV18ZP//5z40uXboYl112mfFf//VfxokTJy5cp1vgpz/9qdG7d28jNDTU6N69uzF69GhPgGAYbWOMvnwzSGgL4xw/frzRo0cPIzQ01LjiiiuM8ePHGx9//LHnflsYY73NmzcbAwYMMKxWqxEXF2esWrXK635b+BmE1sVR0QAAwKeLck0CAAC48AgSAACATwQJAADAJ4IEAADgE0ECAADwiSABAAD4RJAAAAB8IkgAAAA+ESQAAACfCBIAAIBPBAkAAMCn/w9wAvgyJl34AwAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "#normalise weights1 row-wise\n",
        "wt_mat = model.layers[1].weight.data.detach().cpu()\n",
        "w1_normed=F.normalize(wt_mat,p=2,dim=1)\n",
        "#visualise w1_normed as heat-map just in absolute values\n",
        "plt.imshow(w1_normed.abs().numpy())\n",
        "plt.colorbar()\n",
        "plt.show()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "authorship_tag": "ABX9TyN1gu3P2Bvdjy5Le6WrC1E7",
      "include_colab_link": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}