Training_Pipeline.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/quasi-deus/MemSum/blob/main/Training_Pipeline.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "65a0fab8-62db-474d-8fd8-6f8a230e8778",
      "metadata": {
        "id": "65a0fab8-62db-474d-8fd8-6f8a230e8778"
      },
      "source": [
        "# Preparation"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/quasi-deus/MemSum.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pooqn_g37E0y",
        "outputId": "88b4a0ec-9199-4ea8-d0ff-58fa27b4392e"
      },
      "id": "Pooqn_g37E0y",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'MemSum'...\n",
            "remote: Enumerating objects: 363, done.\u001b[K\n",
            "remote: Counting objects: 100% (97/97), done.\u001b[K\n",
            "remote: Compressing objects: 100% (89/89), done.\u001b[K\n",
            "remote: Total 363 (delta 40), reused 21 (delta 6), pack-reused 266\u001b[K\n",
            "Receiving objects: 100% (363/363), 82.37 MiB | 14.20 MiB/s, done.\n",
            "Resolving deltas: 100% (137/137), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "change work directory to MemSum"
      ],
      "metadata": {
        "id": "b4d1PM0q7K2l"
      },
      "id": "b4d1PM0q7K2l"
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.chdir(\"MemSum\")"
      ],
      "metadata": {
        "id": "ks-waJ8Y7KEe"
      },
      "id": "ks-waJ8Y7KEe",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "id": "f842d787-7479-4fee-9c21-b595f2e5b75f",
      "metadata": {
        "id": "f842d787-7479-4fee-9c21-b595f2e5b75f"
      },
      "source": [
        "## Set Up Environment"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Install dependencies via pip"
      ],
      "metadata": {
        "id": "cNjpJB8-7X6l"
      },
      "id": "cNjpJB8-7X6l"
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -r requirements.txt --quiet"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-9s9HQEi7aeT",
        "outputId": "6bde72ca-63ed-49d8-944b-7666df07b151"
      },
      "id": "-9s9HQEi7aeT",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/60.5 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.5/60.5 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.4/7.4 MB\u001b[0m \u001b[31m66.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.2/9.2 MB\u001b[0m \u001b[31m104.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m492.4/492.4 kB\u001b[0m \u001b[31m47.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m27.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m115.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m81.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.0/66.0 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m375.1/375.1 kB\u001b[0m \u001b[31m37.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.3/57.3 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m14.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.5/212.5 kB\u001b[0m \u001b[31m22.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m86.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m103.2/103.2 kB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m68.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.0/81.0 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.6/62.6 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m72.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h\u001b[33mWARNING: jsonschema 4.3.3 does not provide the extra 'format-nongpl'\u001b[0m\u001b[33m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.4/66.4 kB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for rouge_score (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for pyrouge (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "google-colab 1.0.0 requires requests==2.27.1, but you have requests 2.31.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "id": "6da1d147-6668-4710-a987-bbd6388111fd",
      "metadata": {
        "id": "6da1d147-6668-4710-a987-bbd6388111fd"
      },
      "source": [
        "# Preprocessing Custom data\n",
        "\n",
        "Suppose that you have already splitted the training / validation and  test set:\n",
        "\n",
        "The training data is now stored in a .jsonl file that contains a list of json info, one line for one training instance. Each json (or dictonary) contains two keys:\n",
        "\n",
        "1. \"text\": the value for which is a python list of sentences, this represents the document you want to summarize;\n",
        "2. \"summary\": the value is also a list of sentences. If represent the ground-truth summary. Because the summary can contain multiple sentences, so we store them as a list.\n",
        "\n",
        "The same for the validation file and the testing file.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4feeaa6b-c69c-4aaa-8c6b-b3ad02516cd8",
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4feeaa6b-c69c-4aaa-8c6b-b3ad02516cd8",
        "outputId": "266f8714-8441-4056-bcba-c82bc8f4e245"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "100\n",
            "dict_keys(['text', 'summary'])\n",
            "['a recent systematic analysis showed that in 2011 , 314 ( 296 - 331 ) million children younger than 5 years were mildly , moderately or severely stunted and 258 ( 240 - 274 ) million were mildly , moderately or severely underweight in the developing countries .', 'in iran a study among 752 high school girls in sistan and baluchestan showed prevalence of 16.2% , 8.6% and 1.5% , for underweight , overweight and obesity , respectively .', 'the prevalence of malnutrition among elementary school aged children in tehran varied from 6% to 16% .']\n",
            "['background : the present study was carried out to assess the effects of community nutrition intervention based on advocacy approach on malnutrition status among school - aged children in shiraz , iran.materials and methods : this case - control nutritional intervention has been done between 2008 and 2009 on 2897 primary and secondary school boys and girls ( 7 - 13 years old ) based on advocacy approach in shiraz , iran .', 'the project provided nutritious snacks in public schools over a 2-year period along with advocacy oriented actions in order to implement and promote nutritional intervention . for evaluation of effectiveness of the intervention growth monitoring indices of pre- and post - intervention were statistically compared.results:the frequency of subjects with body mass index lower than 5% decreased significantly after intervention among girls ( p = 0.02 ) .', 'however , there were no significant changes among boys or total population .']\n"
          ]
        }
      ],
      "source": [
        "import json\n",
        "train_corpus = [ json.loads(line) for line in open(\"data/custom_data/train_raw_without_high_rouge_indices_and_scores.jsonl\") ]\n",
        "\n",
        "## as an example, we have 100 instances for training\n",
        "print(len(train_corpus))\n",
        "print(train_corpus[0].keys())\n",
        "print(train_corpus[0][\"text\"][:3])\n",
        "print(train_corpus[0][\"summary\"][:3])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "56cb0ec6-3ede-4687-9565-7cb1b4c0872e",
      "metadata": {
        "id": "56cb0ec6-3ede-4687-9565-7cb1b4c0872e"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "id": "c4c8ff7e-8b6e-4250-aaf8-bd73f828ad78",
      "metadata": {
        "id": "c4c8ff7e-8b6e-4250-aaf8-bd73f828ad78"
      },
      "source": [
        "If you have your own data, process them into the same structure then put them into the data/ folder\n",
        "\n",
        "The next thing we need to do is to create high-ROUGE episodes for the training set, as introduced in the paper: https://aclanthology.org/2022.acl-long.450/:\n",
        "\n",
        "Note:\n",
        "* The '!' in the following commands is only needed when running shell command in a jupyter notebook\n",
        "* Thie script run multi-process tasks. You can change -n_processes to a large value on a machine with powerful multi-core CPUs."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "25b178ac-1fda-4f2a-a1c5-1c290d5ee810",
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "25b178ac-1fda-4f2a-a1c5-1c290d5ee810",
        "outputId": "4dea3112-c312-484c-c48f-f54ed8cc71bf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "70it [01:28,  1.26s/it]\n",
            "finished!\n",
            "30it [01:35,  3.18s/it]\n",
            "finished!\n",
            "90it [01:36,  1.08s/it]\n",
            "finished!\n",
            "50it [01:36,  1.93s/it]\n",
            "finished!\n",
            "10it [01:39,  9.99s/it]\n",
            "finished!\n",
            "20it [01:44,  5.21s/it]\n",
            "finished!\n",
            "40it [01:44,  2.62s/it]\n",
            "finished!\n",
            "80it [01:45,  1.32s/it]\n",
            "finished!\n",
            "60it [01:45,  1.75s/it]\n",
            "finished!\n",
            "100it [01:46,  1.06s/it]\n",
            "finished!\n"
          ]
        }
      ],
      "source": [
        "!cd data_preprocessing; python get_high_rouge_episodes_mp.py -input_corpus_file_name ../data/custom_data/train_raw_without_high_rouge_indices_and_scores.jsonl -output_corpus_file_name ../data/custom_data/train.jsonl -beamsearch_size 2 -n_processes 10\n"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "2c18aae5-d2cc-4724-81ee-04e472d6b887",
      "metadata": {
        "id": "2c18aae5-d2cc-4724-81ee-04e472d6b887"
      },
      "source": [
        "# Download pretrained word embedding"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b35d5711-603b-4198-8531-3fa52d2d2b56",
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196,
          "referenced_widgets": [
            "e384cc0b2a5c497cb0f397b35223ba25",
            "c569a91fb4814f78adc41289012ee4f5",
            "4705e883a16640f2b20eba1a9c2d6db8",
            "2db478a99f5b424d9552c1fdecac4767",
            "5b1e48c3fe91427bb2de6a39ee7798b3",
            "6800dee8b67743f890db8fec615bf5e2",
            "dc3f61194e8440f68600ebcaeff4c9c9",
            "bb1e5151181346f381c41437231483f1",
            "1726268b7d5d44df9f46760889d01c48",
            "3527d2e283ba4b529e33b423002ff8cd",
            "ef7fd05f7c854a02be625ac741603e20",
            "41b8abea73394d1db908b381d119acbd",
            "e1e6bd2cd73547149b2810507854b36e",
            "45d7b088cce9455ab9d16fe65cbaffd8",
            "a4644ebc58c84490b94dcc63f690aa84",
            "02c8e95c747a45738e00d94b64f4acdc",
            "606b4f585a9a454684f9768df988448d",
            "3eee4728e99f4353bb4b758d2c61f537",
            "f1937434682646c0ae931bb9e1a78c15",
            "b39b8ed46f164187a32309eae7b3d6e7",
            "72b6e45a41b44b42ab0893d11e1f9925",
            "1d6a27c3fc92490781e6c72a1854b1ff",
            "aff2038e9a3f417eb81ea2cc76c2e859",
            "0d4abd15f85c456ebb19bb6636c37933",
            "6f2ed96674c24d0fb81b2b18b53a9199",
            "f1a429babe6e47d5808d006ef77144a7",
            "d4578906199e44df8db251d67dae7d1f",
            "7128e2d2ed0b4c5d95b52303c0634723",
            "2be66fca6b684d71baaff69c9566da80",
            "b2cce04be9d043979d63cc3564632cda",
            "0614df6eabf54875a93c89b4535e7d58",
            "9b84274f8cb94ef784391ef509cb81ba",
            "4f8fcbce0b214090b3673da2781c869b",
            "7c59f34146d745e1a9279c7d726cf02b",
            "45b24e318bf34372bfec1bf467b4cf3f",
            "a0edc790c0cd454d82145e37a866c96a",
            "53fd965c57fa4ed89b290cac1dbe5869",
            "1bf89c47d3de426fa65c2b937a176aed",
            "42d02b064f7b49618eb99bc0aca20e9d",
            "69a46d498bc44bdc96e4289411efdb04",
            "d570de483a374703ad527e712841afe7",
            "8d530bbaff014f8f9cb70199b552aaae",
            "4afb942e3c77413792110a2c86f570f5",
            "de823d8135724a3587197c44b9efbabd",
            "fd8458902095426cb8592278859e4afa",
            "dcfd7570d646409b8af0cbdd9a1cd702",
            "86ca5615a5e5417eb8ae2117e4fab137",
            "bae5b20575e6415d8d76186b28900fef",
            "38b3f47006204d06b4ce98e694d4bf98",
            "94170e8f001744ffadbd2fb5213822d2",
            "3a356946afcc476a88a760d3d51d81a1",
            "4c371f3b8fe84c32a3f9b8e5988a02e3",
            "5d221e87cb1d435b8530ba24829469a4",
            "27020e6123e449b5960aad769514536f",
            "920759c000614280a0c0de732223b2cd"
          ]
        },
        "id": "b35d5711-603b-4198-8531-3fa52d2d2b56",
        "outputId": "a7c19c78-cd57-47d4-fc40-58247ecb83b6"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e384cc0b2a5c497cb0f397b35223ba25"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)3b269febd6/README.md:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "41b8abea73394d1db908b381d119acbd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)febd6/.gitattributes:   0%|          | 0.00/1.52k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "aff2038e9a3f417eb81ea2cc76c2e859"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)mbeddings_200dim.pkl:   0%|          | 0.00/320M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7c59f34146d745e1a9279c7d726cf02b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading vocabulary_200dim.pkl:   0%|          | 0.00/4.16M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fd8458902095426cb8592278859e4afa"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/MemSum/model/word_embedding'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "from huggingface_hub import snapshot_download\n",
        "## download the pretrained glove word embedding (200 dimension)\n",
        "snapshot_download('nianlong/memsum-word-embedding', local_dir = \"model/word_embedding\" )"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "fb5a94a2-d7ec-4ca4-804b-e8acf630bb48",
      "metadata": {
        "id": "fb5a94a2-d7ec-4ca4-804b-e8acf630bb48"
      },
      "source": [
        "# Start Training"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "0a904552-16a6-4c6b-ba57-322c9f142b56",
      "metadata": {
        "id": "0a904552-16a6-4c6b-ba57-322c9f142b56"
      },
      "source": [
        "Note:\n",
        "1. you need to switch to the folder src/MemSum_Full;\n",
        "2. You can specify the path to training and validation set, the model_folder (where you want to store model checkpoints) and the log_folder (where you want to store the log info), and other parameters.\n",
        "3. You can provide the absolute path, or relative path, as shown in the example code below.\n",
        "4. n_device means the number of available GPUs\n",
        "5. set save_every and validate_every to 0 will make the training script save and evaluate the model only at the end of each epoch."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b043ab53-ba52-4b49-bf19-8ad6952796b7",
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b043ab53-ba52-4b49-bf19-8ad6952796b7",
        "outputId": "d97fb813-1ca6-41d1-e02e-030676c2a8cc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r0it [00:00, ?it/s]\r100it [00:00, 12944.58it/s]\n",
            "\r0it [00:00, ?it/s]\r100it [00:00, 17219.41it/s]\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py:560: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "24it [00:26,  1.04s/it]Starting validation ...\n",
            "[current_batch: 00025] val: 0.3182, 0.0907, 0.2875\n",
            "25it [00:39,  1.59s/it]\n",
            "24it [00:26,  1.09s/it]Starting validation ...\n",
            "[current_batch: 00050] val: 0.3201, 0.0906, 0.2884\n",
            "25it [00:40,  1.63s/it]\n",
            "24it [00:27,  1.13s/it]Starting validation ...\n",
            "[current_batch: 00075] val: 0.3233, 0.0923, 0.2899\n",
            "25it [00:41,  1.68s/it]\n",
            "24it [00:26,  1.12s/it][current_batch: 00100] loss: 0.509, learning rate: 0.000100\n",
            "Starting validation ...\n",
            "[current_batch: 00100] val: 0.3337, 0.0962, 0.2986\n",
            "25it [00:41,  1.67s/it]\n",
            "24it [00:27,  1.13s/it]Starting validation ...\n",
            "[current_batch: 00125] val: 0.3481, 0.1088, 0.3118\n",
            "25it [00:42,  1.69s/it]\n",
            "24it [00:27,  1.13s/it]Starting validation ...\n",
            "[current_batch: 00150] val: 0.3574, 0.1116, 0.3169\n",
            "25it [00:41,  1.68s/it]\n",
            "24it [00:27,  1.12s/it]Starting validation ...\n",
            "[current_batch: 00175] val: 0.3599, 0.1136, 0.3213\n",
            "25it [00:42,  1.68s/it]\n",
            "24it [00:27,  1.13s/it][current_batch: 00200] loss: 0.497, learning rate: 0.000100\n",
            "Starting validation ...\n",
            "[current_batch: 00200] val: 0.3681, 0.1188, 0.3294\n",
            "25it [00:42,  1.69s/it]\n",
            "24it [00:27,  1.13s/it]Starting validation ...\n",
            "[current_batch: 00225] val: 0.3721, 0.1236, 0.3327\n",
            "25it [00:42,  1.69s/it]\n",
            "24it [00:27,  1.13s/it]Starting validation ...\n",
            "[current_batch: 00250] val: 0.3702, 0.1221, 0.3310\n",
            "25it [00:42,  1.69s/it]\n"
          ]
        }
      ],
      "source": [
        "!cd src; python train.py -training_corpus_file_name ../data/custom_data/train.jsonl -validation_corpus_file_name ../data/custom_data/val.jsonl -model_folder ../model/memsum-custom-data -log_folder ../log/memsum-custom-data -vocabulary_file_name ../model/word_embedding/vocabulary_200dim.pkl -pretrained_unigram_embeddings_file_name ../model/word_embedding/unigram_embeddings_200dim.pkl -max_seq_len 100 -max_doc_len 500 -num_of_epochs 10 -save_every 0 -validate_every 0 -n_device 1 -batch_size_per_device 4 -max_extracted_sentences_per_document 7 -moving_average_decay 0.999 -p_stop_thres 0.6\n"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "c587187c-6de9-4141-a64f-ce4dfeb45754",
      "metadata": {
        "id": "c587187c-6de9-4141-a64f-ce4dfeb45754"
      },
      "source": [
        "# Determine the best checkpoint based on validation loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ed467c81-cec8-461d-9f9c-f885ad367b95",
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ed467c81-cec8-461d-9f9c-f885ad367b95",
        "outputId": "e3e3953a-1020-4aa8-c576-fc2166306e69"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "batch: 225 scores: ('0.3721', '0.1236', '0.3327')\n"
          ]
        }
      ],
      "source": [
        "! cd src; python get_optimal_batch.py -log_file_name ../log/memsum-custom-data/val.log"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "6106b953-9da2-4802-8ddb-242fd36b889e",
      "metadata": {
        "id": "6106b953-9da2-4802-8ddb-242fd36b889e"
      },
      "source": [
        "So the best checkpoint is model_batch_200.pt"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "150d02ea-f540-4bc7-b1d1-bc6c6043fa88",
      "metadata": {
        "id": "150d02ea-f540-4bc7-b1d1-bc6c6043fa88"
      },
      "source": [
        "# Testing trained model on custom dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ef65c109-2f50-4284-ab86-6ca2c5471c14",
      "metadata": {
        "id": "ef65c109-2f50-4284-ab86-6ca2c5471c14"
      },
      "outputs": [],
      "source": [
        "from src.summarizer import MemSum\n",
        "from tqdm import tqdm\n",
        "from rouge_score import rouge_scorer\n",
        "import json\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f6a2b5ad-e37e-4ab5-9176-b1eca64bcb10",
      "metadata": {
        "tags": [],
        "id": "f6a2b5ad-e37e-4ab5-9176-b1eca64bcb10"
      },
      "outputs": [],
      "source": [
        "rouge_cal = rouge_scorer.RougeScorer(['rouge1','rouge2', 'rougeLsum'], use_stemmer=True)\n",
        "\n",
        "memsum_custom_data = MemSum(  \"model/memsum-custom-data/model_batch_200.pt\",\n",
        "                  \"model/word_embedding/vocabulary_200dim.pkl\",\n",
        "                  gpu = 0 ,  max_doc_len = 500  )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a365fbe7-c196-4789-8c5c-177f39cd88c2",
      "metadata": {
        "tags": [],
        "id": "a365fbe7-c196-4789-8c5c-177f39cd88c2"
      },
      "outputs": [],
      "source": [
        "test_corpus_custom_data = [ json.loads(line) for line in open(\"data/custom_data/test.jsonl\")]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7d796f58-0d37-47fb-b13c-985b944a0073",
      "metadata": {
        "tags": [],
        "id": "7d796f58-0d37-47fb-b13c-985b944a0073"
      },
      "outputs": [],
      "source": [
        "def evaluate( model, corpus, p_stop, max_extracted_sentences, rouge_cal ):\n",
        "    scores = []\n",
        "    for data in tqdm(corpus):\n",
        "        gold_summary = data[\"summary\"]\n",
        "        extracted_summary = model.extract( [data[\"text\"]], p_stop_thres = p_stop, max_extracted_sentences_per_document = max_extracted_sentences )[0]\n",
        "\n",
        "        score = rouge_cal.score( \"\\n\".join( gold_summary ), \"\\n\".join(extracted_summary)  )\n",
        "        scores.append( [score[\"rouge1\"].fmeasure, score[\"rouge2\"].fmeasure, score[\"rougeLsum\"].fmeasure ] )\n",
        "\n",
        "    return np.asarray(scores).mean(axis = 0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "878d74d5-c096-40ef-a7e9-3f5db267ef11",
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "878d74d5-c096-40ef-a7e9-3f5db267ef11",
        "outputId": "049778a1-9b35-4af7-ac58-0cc5b114041e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 100/100 [00:16<00:00,  6.01it/s]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.37647216, 0.12130708, 0.335058  ])"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "evaluate( memsum_custom_data, test_corpus_custom_data, 0.6, 7, rouge_cal )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c779a6df-8387-4c88-a380-579feb42e81e",
      "metadata": {
        "id": "c779a6df-8387-4c88-a380-579feb42e81e"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c0b9b7f7-1b29-4fd7-af68-4a3bc3d9844d",
      "metadata": {
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c0b9b7f7-1b29-4fd7-af68-4a3bc3d9844d",
        "outputId": "c8c39675-2a0e-4a00-bda3-ef972e27b35f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['ten patients in this study ( 5 pd with anxiety ; 5 pd without anxiety ) were taking psychotropic drugs ( i.e. , benzodiazepine or selective serotonin reuptake inhibitor ) .',\n",
              " \"anxiety affects quality of life in those living with parkinson 's disease ( pd ) more so than overall cognitive status , motor deficits , apathy , and depression [ 13 ] .\",\n",
              " 'selective serotonin reuptake inhibitor , citalopram , was shown to improve response inhibition deficits in pd , while noradrenaline reuptake blocker , atomoxetine , has been recently reported to have promising effects on cognition in pd [ 45 , 46 ] .',\n",
              " 'overall , neuropsychiatric symptoms in pd have been shown to be negatively associated with cognitive performance .',\n",
              " 'although anxiety and depression are often related and coexist in pd patients , recent research suggests that anxiety rather than depression is the most prominent and prevalent mood disorder in pd [ 5 , 6 ] . yet ,',\n",
              " 'our current understanding of anxiety and its impact on cognition in pd , as well as its neural basis and best treatment practices , remains meager and lags far behind that of depression .',\n",
              " 'future studies should also examine whether treating neuropsychiatric symptoms might impact the progression of cognitive decline and improve cognitive impairments in pd patients .']"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ],
      "source": [
        "document = test_corpus_custom_data[0][\"text\"]\n",
        "extracted_summary = memsum_custom_data.extract( [ document ],\n",
        "                                   p_stop_thres = 0.6,\n",
        "                                   max_extracted_sentences_per_document = 7\n",
        "                                  )[0]\n",
        "extracted_summary"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e4cdccce-4f3c-4c6f-a771-ee8fdd92f02b",
      "metadata": {
        "id": "e4cdccce-4f3c-4c6f-a771-ee8fdd92f02b"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4d96e5c7-9b2b-40ce-b764-7c9b819e810f",
      "metadata": {
        "id": "4d96e5c7-9b2b-40ce-b764-7c9b819e810f"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python [conda env:memsum]",
      "language": "python",
      "name": "conda-env-memsum-py"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.12"
    },
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e384cc0b2a5c497cb0f397b35223ba25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c569a91fb4814f78adc41289012ee4f5",
              "IPY_MODEL_4705e883a16640f2b20eba1a9c2d6db8",
              "IPY_MODEL_2db478a99f5b424d9552c1fdecac4767"
            ],
            "layout": "IPY_MODEL_5b1e48c3fe91427bb2de6a39ee7798b3"
          }
        },
        "c569a91fb4814f78adc41289012ee4f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6800dee8b67743f890db8fec615bf5e2",
            "placeholder": "​",
            "style": "IPY_MODEL_dc3f61194e8440f68600ebcaeff4c9c9",
            "value": "Fetching 4 files: 100%"
          }
        },
        "4705e883a16640f2b20eba1a9c2d6db8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb1e5151181346f381c41437231483f1",
            "max": 4,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1726268b7d5d44df9f46760889d01c48",
            "value": 4
          }
        },
        "2db478a99f5b424d9552c1fdecac4767": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3527d2e283ba4b529e33b423002ff8cd",
            "placeholder": "​",
            "style": "IPY_MODEL_ef7fd05f7c854a02be625ac741603e20",
            "value": " 4/4 [00:22&lt;00:00,  8.42s/it]"
          }
        },
        "5b1e48c3fe91427bb2de6a39ee7798b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6800dee8b67743f890db8fec615bf5e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dc3f61194e8440f68600ebcaeff4c9c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bb1e5151181346f381c41437231483f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1726268b7d5d44df9f46760889d01c48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3527d2e283ba4b529e33b423002ff8cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ef7fd05f7c854a02be625ac741603e20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "41b8abea73394d1db908b381d119acbd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e1e6bd2cd73547149b2810507854b36e",
              "IPY_MODEL_45d7b088cce9455ab9d16fe65cbaffd8",
              "IPY_MODEL_a4644ebc58c84490b94dcc63f690aa84"
            ],
            "layout": "IPY_MODEL_02c8e95c747a45738e00d94b64f4acdc"
          }
        },
        "e1e6bd2cd73547149b2810507854b36e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_606b4f585a9a454684f9768df988448d",
            "placeholder": "​",
            "style": "IPY_MODEL_3eee4728e99f4353bb4b758d2c61f537",
            "value": "Downloading (…)3b269febd6/README.md: 100%"
          }
        },
        "45d7b088cce9455ab9d16fe65cbaffd8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f1937434682646c0ae931bb9e1a78c15",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b39b8ed46f164187a32309eae7b3d6e7",
            "value": 28
          }
        },
        "a4644ebc58c84490b94dcc63f690aa84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_72b6e45a41b44b42ab0893d11e1f9925",
            "placeholder": "​",
            "style": "IPY_MODEL_1d6a27c3fc92490781e6c72a1854b1ff",
            "value": " 28.0/28.0 [00:00&lt;00:00, 1.59kB/s]"
          }
        },
        "02c8e95c747a45738e00d94b64f4acdc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "606b4f585a9a454684f9768df988448d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3eee4728e99f4353bb4b758d2c61f537": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f1937434682646c0ae931bb9e1a78c15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b39b8ed46f164187a32309eae7b3d6e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "72b6e45a41b44b42ab0893d11e1f9925": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d6a27c3fc92490781e6c72a1854b1ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aff2038e9a3f417eb81ea2cc76c2e859": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0d4abd15f85c456ebb19bb6636c37933",
              "IPY_MODEL_6f2ed96674c24d0fb81b2b18b53a9199",
              "IPY_MODEL_f1a429babe6e47d5808d006ef77144a7"
            ],
            "layout": "IPY_MODEL_d4578906199e44df8db251d67dae7d1f"
          }
        },
        "0d4abd15f85c456ebb19bb6636c37933": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7128e2d2ed0b4c5d95b52303c0634723",
            "placeholder": "​",
            "style": "IPY_MODEL_2be66fca6b684d71baaff69c9566da80",
            "value": "Downloading (…)febd6/.gitattributes: 100%"
          }
        },
        "6f2ed96674c24d0fb81b2b18b53a9199": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b2cce04be9d043979d63cc3564632cda",
            "max": 1519,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0614df6eabf54875a93c89b4535e7d58",
            "value": 1519
          }
        },
        "f1a429babe6e47d5808d006ef77144a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9b84274f8cb94ef784391ef509cb81ba",
            "placeholder": "​",
            "style": "IPY_MODEL_4f8fcbce0b214090b3673da2781c869b",
            "value": " 1.52k/1.52k [00:00&lt;00:00, 61.8kB/s]"
          }
        },
        "d4578906199e44df8db251d67dae7d1f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7128e2d2ed0b4c5d95b52303c0634723": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2be66fca6b684d71baaff69c9566da80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b2cce04be9d043979d63cc3564632cda": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0614df6eabf54875a93c89b4535e7d58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9b84274f8cb94ef784391ef509cb81ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4f8fcbce0b214090b3673da2781c869b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7c59f34146d745e1a9279c7d726cf02b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_45b24e318bf34372bfec1bf467b4cf3f",
              "IPY_MODEL_a0edc790c0cd454d82145e37a866c96a",
              "IPY_MODEL_53fd965c57fa4ed89b290cac1dbe5869"
            ],
            "layout": "IPY_MODEL_1bf89c47d3de426fa65c2b937a176aed"
          }
        },
        "45b24e318bf34372bfec1bf467b4cf3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_42d02b064f7b49618eb99bc0aca20e9d",
            "placeholder": "​",
            "style": "IPY_MODEL_69a46d498bc44bdc96e4289411efdb04",
            "value": "Downloading (…)mbeddings_200dim.pkl: 100%"
          }
        },
        "a0edc790c0cd454d82145e37a866c96a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d570de483a374703ad527e712841afe7",
            "max": 320002555,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8d530bbaff014f8f9cb70199b552aaae",
            "value": 320002555
          }
        },
        "53fd965c57fa4ed89b290cac1dbe5869": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4afb942e3c77413792110a2c86f570f5",
            "placeholder": "​",
            "style": "IPY_MODEL_de823d8135724a3587197c44b9efbabd",
            "value": " 320M/320M [00:21&lt;00:00, 14.1MB/s]"
          }
        },
        "1bf89c47d3de426fa65c2b937a176aed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "42d02b064f7b49618eb99bc0aca20e9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "69a46d498bc44bdc96e4289411efdb04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d570de483a374703ad527e712841afe7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d530bbaff014f8f9cb70199b552aaae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4afb942e3c77413792110a2c86f570f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de823d8135724a3587197c44b9efbabd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fd8458902095426cb8592278859e4afa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dcfd7570d646409b8af0cbdd9a1cd702",
              "IPY_MODEL_86ca5615a5e5417eb8ae2117e4fab137",
              "IPY_MODEL_bae5b20575e6415d8d76186b28900fef"
            ],
            "layout": "IPY_MODEL_38b3f47006204d06b4ce98e694d4bf98"
          }
        },
        "dcfd7570d646409b8af0cbdd9a1cd702": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_94170e8f001744ffadbd2fb5213822d2",
            "placeholder": "​",
            "style": "IPY_MODEL_3a356946afcc476a88a760d3d51d81a1",
            "value": "Downloading vocabulary_200dim.pkl: 100%"
          }
        },
        "86ca5615a5e5417eb8ae2117e4fab137": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4c371f3b8fe84c32a3f9b8e5988a02e3",
            "max": 4157881,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5d221e87cb1d435b8530ba24829469a4",
            "value": 4157881
          }
        },
        "bae5b20575e6415d8d76186b28900fef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_27020e6123e449b5960aad769514536f",
            "placeholder": "​",
            "style": "IPY_MODEL_920759c000614280a0c0de732223b2cd",
            "value": " 4.16M/4.16M [00:01&lt;00:00, 2.70MB/s]"
          }
        },
        "38b3f47006204d06b4ce98e694d4bf98": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "94170e8f001744ffadbd2fb5213822d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3a356946afcc476a88a760d3d51d81a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4c371f3b8fe84c32a3f9b8e5988a02e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5d221e87cb1d435b8530ba24829469a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "27020e6123e449b5960aad769514536f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "920759c000614280a0c0de732223b2cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}