02. Training and testing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "-BfAc2rYa8Mm",
   "metadata": {
    "id": "-BfAc2rYa8Mm",
    "tags": []
   },
   "source": [
    "# Installing the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "wS5tG478R_Gl",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "wS5tG478R_Gl",
    "outputId": "634886a6-b80d-4721-a1ca-d91faf208257",
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install transformers\n",
    "#!apt-get install git-lfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb9aecb1-33c7-4fbe-91f6-3f98ed06525f",
   "metadata": {},
   "outputs": [],
   "source": [
    "!python --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "730d9991-111a-416e-8584-2a91391609be",
   "metadata": {
    "id": "730d9991-111a-416e-8584-2a91391609be",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn # for the layers\n",
    "from transformers import  EncoderDecoderModel, AutoTokenizer\n",
    "from torch.optim import AdamW\n",
    "from torch.utils.data import DataLoader, TensorDataset, random_split\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from typing import List, Dict, Union\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "MuhHDnZxco-a",
   "metadata": {
    "id": "MuhHDnZxco-a"
   },
   "source": [
    "# Get the model and its tokeniser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e28aef68-0a52-4136-816f-a11b556927cb",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 471,
     "referenced_widgets": [
      "670fe4f30bc24de49f1a2fe190d1eec5",
      "940fea167ed943388491cfc93dece6fc",
      "bfa08d0d71db4d388b1d5bb68721cab0",
      "8e3265154f424b62afba111807f1fbd2",
      "60a5888afffe4e59b42c81c194351227",
      "09d7b971131844a9bf500b10328093da",
      "fa556c4c30954072ac24226474e6a329",
      "9e6ca257372d47249528831586095146",
      "080057c1221d4438b17971a1800866a7",
      "35cb5929195c4f1eb362058096eef3d9",
      "b5de5ad2c50146e7a3083dab9f7bf7d9",
      "5520d584cfbb4891accbcdde78801843",
      "b97962aaf3c14133bf0d117649b39863",
      "fb206ec77cfd4ae990e4eea0d6a1dcfb",
      "84d22e86bb304f4785ea2056174c8273",
      "9d09c0953a49436489fd76f62e34049f",
      "15f7d8aa46ca4a5d836dcf2b5e47a260",
      "9c94236cef8c44598a82093083b1f40f",
      "92e96c2282b74512838dd3f76e129102",
      "54edebef257e4f9e973aa76ce5bd69f7",
      "0f98592e23ee46078d8cc2b013b16ba5",
      "beab4a0f4f184a86a3c5f20d06fe89ff",
      "b00c011595eb4313b608903aa0a052ad",
      "ed64c3de037e431ca18e453da0b89a57",
      "da4a3cc21e4c4399bd66dab213beec9d",
      "39c2c386ccd04f4a98ec446d75789fce",
      "c73bf20728b54837a6b0d9b73a671967",
      "b35b1a46fc144c31a0544a9b640dc1a6",
      "a178dc61af344b19b966f2a51d873082",
      "2b8d377685af4943a115acf3148f1c86",
      "a0e618e875dc4f5ca37e1a36faa4ed85",
      "4e9fa2844179489283a66f496f1c43f6",
      "2430ea839e404d5b9e4e50c0bd0bc7c5",
      "15f3365f22044513a7f8d8e86f25b1b1",
      "9230ce814b2044a2b62af01a006d74e8",
      "497551327ea946a5b214db56dc90cc86",
      "8b61d119adec45949791072972532752",
      "316aab12af17448698aa1c31903c8e35",
      "3e800ed4c82b4a9da78fe0a334c6551f",
      "cf7a5e2d83dd454d822cecafbb790ef7",
      "096e1f4ac9214213b8fb6d3bc47abbd8",
      "3e9ad3ef0d27413b9b90519affebb282",
      "57d3913b6701469c89bb27bcd64bc16d",
      "4ac3743f2e2241769229d16be80c0503"
     ]
    },
    "id": "e28aef68-0a52-4136-816f-a11b556927cb",
    "outputId": "0f2906c9-b4de-494e-9ed4-71d93c36fedd"
   },
   "outputs": [],
   "source": [
    "roberta_tokenizer = AutoTokenizer.from_pretrained(\"pstroe/roberta-base-latin-cased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "uJILR_cqhWDw",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 254,
     "referenced_widgets": [
      "c6f6cbec5fe04e7da578a8eabf0c09c1",
      "465dc10743e94007bbd00f740609ffd9",
      "0b61184b3c404d4299b4ebba471469f1",
      "9e5c4d90d54244e7881c4b92b39d0dc7",
      "769c790efbd44ad0878b33412aaca217",
      "40d2f828cfd6433d89e0cd5b086d2b40",
      "4760db56862c4d65bb919fd4c5d4bbb3",
      "7d2e926592224586bf84688292fb3e0f",
      "92de1c811d014861a7323157502c4377",
      "d694867018dd44bf9dda36f2bc50b2ff",
      "8c0e77a3839d47dd823775207951b3e4"
     ]
    },
    "id": "uJILR_cqhWDw",
    "outputId": "83db7699-cc5f-48d9-b729-ee10ae7d8363",
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = EncoderDecoderModel.from_encoder_decoder_pretrained(\"pstroe/roberta-base-latin-cased\", \"pstroe/roberta-base-latin-cased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d826a3d4-47fc-453f-80df-b88c9ebd725a",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "d826a3d4-47fc-453f-80df-b88c9ebd725a",
    "outputId": "1a5cad1b-b3c9-490d-df0a-7887566677c8",
    "tags": []
   },
   "outputs": [],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06568934-5054-467b-b712-0b901c65c93e",
   "metadata": {
    "id": "06568934-5054-467b-b712-0b901c65c93e"
   },
   "outputs": [],
   "source": [
    "# # freezing a part of layers\n",
    "\n",
    "# for param in model.parameters():\n",
    "#     param.requires_grad = False\n",
    "\n",
    "# # unfreezing cross attention, pooler and the head\n",
    "\n",
    "# for i in range(12):\n",
    "#     for param in model.decoder.roberta.encoder.layer[i].crossattention.parameters():\n",
    "#         param.requires_grad = True\n",
    "\n",
    "# for param in model.encoder.pooler.parameters():\n",
    "#     param.requires_grad = True\n",
    "\n",
    "# for param in model.decoder.lm_head.parameters():\n",
    "#     param.requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efdb6f56-f339-4768-8510-c0315db8d93c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# unfreezing it all\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = True"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cb3d545-f45d-4419-8165-4ac2a2a7dfda",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Get all data from pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e7c54fd-f1f6-4884-aa7a-911f6b84a39f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "\n",
    "with open('/pbs/home/s/syatsyk/syatsyk/tokenized_data.pkl', 'rb') as f:\n",
    "    data_new = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ba732f4-85fd-45a8-8eac-a0e6ba76f5e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_new.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "_CNUqt17ROY9",
   "metadata": {
    "id": "_CNUqt17ROY9",
    "tags": []
   },
   "source": [
    "# Preparing for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c3690d4-14c3-40fe-bb6e-e5f7c95cc5f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "class PairsDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, x, y):\n",
    "        self.x = x\n",
    "        self.y = y\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        assert idx < len(self.x['input_ids'])\n",
    "        item = {key: val[idx] for key, val in self.x.items()}\n",
    "        item['decoder_attention_mask'] = self.y['attention_mask'][idx]\n",
    "        item['labels'] = self.y['input_ids'][idx]\n",
    "        return item\n",
    "\n",
    "    @property\n",
    "    def n(self):\n",
    "        return len(self.x['input_ids'])\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26a68b7c-c797-4b51-af05-9d03e93f9cad",
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List, Dict, Union\n",
    "\n",
    "class DataCollatorWithPadding:\n",
    "    def __init__(self, tokenizer):\n",
    "        self.tokenizer = tokenizer\n",
    "\n",
    "    def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:\n",
    "        batch = self.tokenizer.pad(\n",
    "            features,\n",
    "            padding=True,\n",
    "        )\n",
    "        ybatch = self.tokenizer.pad(\n",
    "            {'input_ids': batch['labels'], 'attention_mask': batch['decoder_attention_mask']},\n",
    "            padding=True,\n",
    "        )\n",
    "        batch['labels'] = ybatch['input_ids']\n",
    "        batch['decoder_attention_mask'] = ybatch['attention_mask']\n",
    "\n",
    "        return {k: torch.tensor(v) for k, v in batch.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b4667dd-61d5-4855-aee1-17fe4dfc2139",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_new.head(14700)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa286226-e89e-48a4-9c65-f1d9624b514c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 20 000 lines for validation\n",
    "df_val = data_new.sample(20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d20d7473-6850-4d3d-9eb8-17d09b84245f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_new['is_train'] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19dcc9e6-78fb-4ed8-94e4-5b0f4964ef18",
   "metadata": {},
   "outputs": [],
   "source": [
    "# mark the validation set as non-trainable data\n",
    "data_new.loc[df_val.index, 'is_train'] = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5c00e5b-aab6-4a9d-9dff-304dc7258ffd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2da076d7-9172-4cac-a1f7-988f67854450",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = data_new[data_new.is_train]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5c12dd8-b56b-4a16-9bf4-fa73cd96cb60",
   "metadata": {
    "id": "hCqBC-rLUVIg",
    "tags": []
   },
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "hHS7YQt9UdjE",
   "metadata": {
    "id": "hHS7YQt9UdjE",
    "tags": []
   },
   "outputs": [],
   "source": [
    "batch_size = 3\n",
    "\n",
    "#x1, x2, y1, y2 = train_test_split(data_new['input'].tolist(), data_new['target'].tolist(), test_size=0.2, random_state=42)\n",
    "train_dataset = PairsDataset(roberta_tokenizer(df_train['input'].tolist()), roberta_tokenizer(df_train['target'].tolist()))\n",
    "test_dataset = PairsDataset(roberta_tokenizer(df_val['input'].tolist()), roberta_tokenizer(df_val['target'].tolist()))\n",
    "\n",
    "data_collator = DataCollatorWithPadding(tokenizer=roberta_tokenizer)\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=batch_size, drop_last=False, shuffle=True, collate_fn=data_collator)\n",
    "val_dataloader = DataLoader(test_dataset, batch_size=batch_size, drop_last=False, shuffle=True, collate_fn=data_collator)\n",
    "\n",
    "# it takes 3 min for batch = 6, it takes 17 for batch = 24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9frJ1MyqXfNh",
   "metadata": {
    "id": "9frJ1MyqXfNh"
   },
   "outputs": [],
   "source": [
    "loss_function = nn.CrossEntropyLoss(reduction='none')\n",
    "\n",
    "optimizer = AdamW(model.parameters(), lr=5e-2)\n",
    "lr_sched = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=4, T_mult=2, eta_min=1e-6, last_epoch=-1)\n",
    "\n",
    "num_epochs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f0195d9-d0b0-498f-9e3b-4231a3a9558e",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_sched.get_last_lr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b72922f1-90fa-41ec-ab6d-eceaf7710b75",
   "metadata": {},
   "outputs": [],
   "source": [
    "def error_fixing(text, model, n=None, max_length='auto', temperature=1.0, beams=3):\n",
    "    texts = [text] if isinstance(text, str) else text\n",
    "    inputs = roberta_tokenizer(texts, return_tensors='pt', padding=True)['input_ids'].to(model.device)\n",
    "    if max_length == 'auto':\n",
    "        max_length = int(inputs.shape[1] * 1.2) + 10\n",
    "    result = model.generate(\n",
    "        inputs,\n",
    "        num_return_sequences=n or 1,\n",
    "        do_sample=True,\n",
    "        temperature=temperature,\n",
    "        repetition_penalty=3.0,\n",
    "        max_length=max_length,\n",
    "        bad_words_ids=None,  # unk\n",
    "        num_beams=beams,\n",
    "    )\n",
    "    texts = [roberta_tokenizer.decode(r, skip_special_tokens=True) for r in result]\n",
    "    if not n and isinstance(texts, str):\n",
    "        return texts\n",
    "    return texts[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "865d5bd4-694a-403f-b6c3-fc24235dc3e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = df_val.sample(3)['input'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ddf9a73e-d4e4-4444-8d25-8f91205e05c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "\n",
    "def cleanup():\n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()\n",
    "\n",
    "cleanup()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2272226f-6977-479b-9a76-283e2dba1103",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for parallel training\n",
    "\n",
    "from torch.nn import DataParallel\n",
    "\n",
    "# checkpoint path\n",
    "checkpoint_path = \"model_checkpoints\"\n",
    "os.makedirs(checkpoint_path, exist_ok=True)\n",
    "\n",
    "checkpoint_path = \"model_checkpoints/model_epoch_3.pt\" # start from the last saved epoch\n",
    "\n",
    "# initialise the model on all available GPUs\n",
    "model = DataParallel(model)\n",
    "model.to('cuda')\n",
    "\n",
    "# check whether there is a checkpoint file\n",
    "if os.path.exists(checkpoint_path):\n",
    "    print(f\"Loading checkpoint '{checkpoint_path}'\")\n",
    "    checkpoint = torch.load(checkpoint_path)\n",
    "    model.module.load_state_dict(checkpoint['model_state_dict']) # for multiple GPUs\n",
    "    optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n",
    "    start_epoch = checkpoint['epoch'] + 1  # start at the last epoch\n",
    "    train_loss = checkpoint['train_loss']  # get the loss history for train_set\n",
    "    val_loss = checkpoint['val_loss']  # get the loss history for train_set\n",
    "else:\n",
    "    print(\"No checkpoint found at '{checkpoint_path}'. Starting from scratch.\")\n",
    "    start_epoch = 0\n",
    "    train_loss = []\n",
    "    val_loss = []\n",
    "\n",
    "accumulation_steps = 250\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    model.train()\n",
    "    total_train_loss = 0.0\n",
    "    total_train_step = 0\n",
    "    progress_bar = tqdm(train_dataloader)\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    for step, batch in enumerate(progress_bar):\n",
    "        input_ids = batch['input_ids'].to('cuda')\n",
    "        input_attention_mask = batch['attention_mask'].to('cuda')\n",
    "        target_ids = batch['labels'].to('cuda')\n",
    "        target_attention_mask = batch['decoder_attention_mask'].to('cuda')\n",
    "\n",
    "        try:\n",
    "            output = model(input_ids=input_ids,\n",
    "                           attention_mask=input_attention_mask,\n",
    "                           decoder_input_ids=target_ids,\n",
    "                           decoder_attention_mask=target_attention_mask)\n",
    "\n",
    "            max_target_length = min(target_ids.size(1), output.logits.size(1))\n",
    "            loss = loss_function(output.logits[:, :max_target_length].contiguous().view(-1, roberta_tokenizer.vocab_size), target_ids.view(-1))\n",
    "            loss = loss * target_attention_mask.view(-1)\n",
    "            loss = loss.mean() / accumulation_steps\n",
    "            loss.backward()\n",
    "\n",
    "            if (step + 1) % accumulation_steps == 0 or (step + 1) == len(progress_bar):\n",
    "                optimizer.step()\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "            total_train_loss += loss.item() * accumulation_steps\n",
    "            total_train_step += 1\n",
    "\n",
    "            if total_train_step == 0:\n",
    "                continue\n",
    "            progress_bar.set_description(f\"Epoch {epoch} - Avg Train Loss: {total_train_loss / total_train_step:.4f}\")\n",
    "\n",
    "        except RuntimeError as e:\n",
    "            print(f\"RuntimeError: {e}\")\n",
    "            print(f\"Logits shape: {output.logits.shape}\")\n",
    "            print(f\"Target shape: {target_ids.shape}\")\n",
    "            print(f\"Max target length: {max_target_length}\")\n",
    "            continue\n",
    "\n",
    "    train_loss.append(total_train_loss / total_train_step)\n",
    "\n",
    "    # save the model and the optimiser's state\n",
    "    checkpoint_path = \"model_checkpoints\"\n",
    "    os.makedirs(checkpoint_path, exist_ok=True)\n",
    "    model_save_path = os.path.join(checkpoint_path, f\"model_epoch_{epoch}.pt\")\n",
    "    torch.save({\n",
    "        'epoch': epoch,\n",
    "        'model_state_dict': model.module.state_dict(),  # for multiple GPUs, DataParallel\n",
    "        'optimizer_state_dict': optimizer.state_dict(),\n",
    "        'train_loss': train_loss,\n",
    "        'val_loss': val_loss,\n",
    "    }, model_save_path)\n",
    "    print(f\"Model saved to {model_save_path}\")\n",
    "\n",
    "    model.eval()\n",
    "    total_val_loss = 0\n",
    "    total_val_step = 0\n",
    "    val_progress_bar = tqdm(val_dataloader, desc=f\"Epoch {epoch + 1} Validation\")\n",
    "    with torch.no_grad():\n",
    "        for batch in val_progress_bar:\n",
    "            input_ids = batch['input_ids'].to('cuda')\n",
    "            input_attention_mask = batch['attention_mask'].to('cuda')\n",
    "            target_ids = batch['labels'].to('cuda')\n",
    "            target_attention_mask = batch['decoder_attention_mask'].to('cuda')\n",
    "\n",
    "            output = model(input_ids=input_ids,\n",
    "                           attention_mask=input_attention_mask,\n",
    "                           decoder_input_ids=target_ids,\n",
    "                           labels=target_ids,\n",
    "                           decoder_attention_mask=target_attention_mask)\n",
    "\n",
    "            valid_loss = output.loss\n",
    "            total_val_loss += valid_loss.item()\n",
    "            total_val_step += 1\n",
    "            val_progress_bar.set_description(f'Current Val loss (epoch: {epoch}): {total_val_loss / total_val_step:.4f}')\n",
    "\n",
    "    val_loss.append(total_val_loss / total_val_step)\n",
    "\n",
    "    lr_sched.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "NICFktRXfjj3",
   "metadata": {
    "id": "NICFktRXfjj3"
   },
   "source": [
    "# Results visualisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "348d2c67-388e-4577-b715-fb645914de26",
   "metadata": {
    "id": "348d2c67-388e-4577-b715-fb645914de26"
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d60c99b7-0576-4720-a1cd-c87843984c40",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 430
    },
    "id": "d60c99b7-0576-4720-a1cd-c87843984c40",
    "outputId": "9393e231-f0dd-4a28-8526-b6434cb2dacc"
   },
   "outputs": [],
   "source": [
    "plt.plot(train_loss[1:], label='Training Loss')\n",
    "plt.plot(val_loss[1:], label='Validation Loss')\n",
    "\n",
    "#plt.gca().xaxis.set_major_locator(MaxNLocator(integer=True)) # чтобы были целые числа по осям\n",
    "\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3575c304-6a05-4e19-9c49-eafc9248777e",
   "metadata": {},
   "source": [
    "# Check results for a model from checkpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3194222c-9d5a-4b16-b21f-2dd57597787e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "checkpoint_path = \"/pbs/home/s/syatsyk/syatsyk/biblissima_spellchecker/model_checkpoints/model_best.pt\"\n",
    "\n",
    "# get the model and the tokenizer\n",
    "#model = EncoderDecoderModel.from_encoder_decoder_pretrained(\"pstroe/roberta-base-latin-cased\", \"pstroe/roberta-base-latin-cased\")\n",
    "roberta_tokenizer = AutoTokenizer.from_pretrained(\"pstroe/roberta-base-latin-cased\")\n",
    "\n",
    "# get the checkpoint\n",
    "checkpoint = torch.load(checkpoint_path, map_location='cuda')\n",
    "model.load_state_dict(checkpoint['model_state_dict']) #, strict=False\n",
    "\n",
    "# wrap it in DataParallel\n",
    "model = torch.nn.DataParallel(model).to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b9f4909-586c-4029-abb3-a1c0c0140c5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def error_fixing(text, model, roberta_tokenizer, n=None, max_length='auto', temperature=1.0, beams=3, device='cuda'):\n",
    "    texts = [text] if isinstance(text, str) else text\n",
    "    inputs = roberta_tokenizer(texts, return_tensors='pt', padding=True)['input_ids'].to(device)\n",
    "    if max_length == 'auto':\n",
    "        max_length = int(inputs.shape[1] * 1.2) + 10\n",
    "\n",
    "    model_to_generate = model.module if isinstance(model, torch.nn.DataParallel) else model\n",
    "\n",
    "    result = model_to_generate.generate(\n",
    "        inputs,\n",
    "        num_return_sequences=n or 1,\n",
    "        do_sample=True,\n",
    "        temperature=temperature,\n",
    "        repetition_penalty=3.0,\n",
    "        max_length=max_length,\n",
    "        bad_words_ids=None,\n",
    "        num_beams=beams,\n",
    "    )\n",
    "\n",
    "    texts = [roberta_tokenizer.decode(r, skip_special_tokens=True) for r in result]\n",
    "    return texts[0] if n is None else texts\n",
    "\n",
    "input_texts = data_new['input'].head(2).tolist()\n",
    "target_texts = data_new['target'].head(2).tolist()\n",
    "\n",
    "problematic_batch = []  # Для сбора проблемных входных данных\n",
    "\n",
    "for inp, trg in zip(input_texts, target_texts):\n",
    "    try:\n",
    "        res = error_fixing(inp, model, roberta_tokenizer, temperature=2.0, beams=10, device='cuda')\n",
    "        print(f\"\\n| true: {trg}\\n|  inp: {inp}\\n| pred: {res}\")\n",
    "    except Exception as e:\n",
    "        print(f\"Error processing input: {inp} with error: {e}\")\n",
    "        problematic_batch.append(inp)\n",
    "        continue"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "machine_shape": "hm",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "python3.9",
   "language": "python",
   "name": "python3.9"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "080057c1221d4438b17971a1800866a7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "096e1f4ac9214213b8fb6d3bc47abbd8": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "09d7b971131844a9bf500b10328093da": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0b61184b3c404d4299b4ebba471469f1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7d2e926592224586bf84688292fb3e0f",
      "max": 498875371,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_92de1c811d014861a7323157502c4377",
      "value": 498875371
     }
    },
    "0f98592e23ee46078d8cc2b013b16ba5": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "15f3365f22044513a7f8d8e86f25b1b1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_9230ce814b2044a2b62af01a006d74e8",
       "IPY_MODEL_497551327ea946a5b214db56dc90cc86",
       "IPY_MODEL_8b61d119adec45949791072972532752"
      ],
      "layout": "IPY_MODEL_316aab12af17448698aa1c31903c8e35"
     }
    },
    "15f7d8aa46ca4a5d836dcf2b5e47a260": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2430ea839e404d5b9e4e50c0bd0bc7c5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "2b8d377685af4943a115acf3148f1c86": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "316aab12af17448698aa1c31903c8e35": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "35cb5929195c4f1eb362058096eef3d9": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "39c2c386ccd04f4a98ec446d75789fce": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4e9fa2844179489283a66f496f1c43f6",
      "placeholder": "​",
      "style": "IPY_MODEL_2430ea839e404d5b9e4e50c0bd0bc7c5",
      "value": " 857k/857k [00:00&lt;00:00, 23.7MB/s]"
     }
    },
    "3e800ed4c82b4a9da78fe0a334c6551f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3e9ad3ef0d27413b9b90519affebb282": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "40d2f828cfd6433d89e0cd5b086d2b40": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "465dc10743e94007bbd00f740609ffd9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_40d2f828cfd6433d89e0cd5b086d2b40",
      "placeholder": "​",
      "style": "IPY_MODEL_4760db56862c4d65bb919fd4c5d4bbb3",
      "value": "pytorch_model.bin: 100%"
     }
    },
    "4760db56862c4d65bb919fd4c5d4bbb3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "497551327ea946a5b214db56dc90cc86": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_096e1f4ac9214213b8fb6d3bc47abbd8",
      "max": 515052,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_3e9ad3ef0d27413b9b90519affebb282",
      "value": 515052
     }
    },
    "4ac3743f2e2241769229d16be80c0503": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "4e9fa2844179489283a66f496f1c43f6": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "54edebef257e4f9e973aa76ce5bd69f7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "5520d584cfbb4891accbcdde78801843": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b97962aaf3c14133bf0d117649b39863",
       "IPY_MODEL_fb206ec77cfd4ae990e4eea0d6a1dcfb",
       "IPY_MODEL_84d22e86bb304f4785ea2056174c8273"
      ],
      "layout": "IPY_MODEL_9d09c0953a49436489fd76f62e34049f"
     }
    },
    "57d3913b6701469c89bb27bcd64bc16d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "60a5888afffe4e59b42c81c194351227": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "670fe4f30bc24de49f1a2fe190d1eec5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_940fea167ed943388491cfc93dece6fc",
       "IPY_MODEL_bfa08d0d71db4d388b1d5bb68721cab0",
       "IPY_MODEL_8e3265154f424b62afba111807f1fbd2"
      ],
      "layout": "IPY_MODEL_60a5888afffe4e59b42c81c194351227"
     }
    },
    "769c790efbd44ad0878b33412aaca217": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7d2e926592224586bf84688292fb3e0f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "84d22e86bb304f4785ea2056174c8273": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0f98592e23ee46078d8cc2b013b16ba5",
      "placeholder": "​",
      "style": "IPY_MODEL_beab4a0f4f184a86a3c5f20d06fe89ff",
      "value": " 637/637 [00:00&lt;00:00, 22.9kB/s]"
     }
    },
    "8b61d119adec45949791072972532752": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_57d3913b6701469c89bb27bcd64bc16d",
      "placeholder": "​",
      "style": "IPY_MODEL_4ac3743f2e2241769229d16be80c0503",
      "value": " 515k/515k [00:00&lt;00:00, 12.1MB/s]"
     }
    },
    "8c0e77a3839d47dd823775207951b3e4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "8e3265154f424b62afba111807f1fbd2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_35cb5929195c4f1eb362058096eef3d9",
      "placeholder": "​",
      "style": "IPY_MODEL_b5de5ad2c50146e7a3083dab9f7bf7d9",
      "value": " 1.47M/1.47M [00:00&lt;00:00, 16.7MB/s]"
     }
    },
    "9230ce814b2044a2b62af01a006d74e8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3e800ed4c82b4a9da78fe0a334c6551f",
      "placeholder": "​",
      "style": "IPY_MODEL_cf7a5e2d83dd454d822cecafbb790ef7",
      "value": "merges.txt: 100%"
     }
    },
    "92de1c811d014861a7323157502c4377": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "92e96c2282b74512838dd3f76e129102": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "940fea167ed943388491cfc93dece6fc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_09d7b971131844a9bf500b10328093da",
      "placeholder": "​",
      "style": "IPY_MODEL_fa556c4c30954072ac24226474e6a329",
      "value": "tokenizer_config.json: 100%"
     }
    },
    "9c94236cef8c44598a82093083b1f40f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9d09c0953a49436489fd76f62e34049f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9e5c4d90d54244e7881c4b92b39d0dc7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d694867018dd44bf9dda36f2bc50b2ff",
      "placeholder": "​",
      "style": "IPY_MODEL_8c0e77a3839d47dd823775207951b3e4",
      "value": " 499M/499M [00:05&lt;00:00, 77.8MB/s]"
     }
    },
    "9e6ca257372d47249528831586095146": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a0e618e875dc4f5ca37e1a36faa4ed85": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "a178dc61af344b19b966f2a51d873082": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b00c011595eb4313b608903aa0a052ad": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_ed64c3de037e431ca18e453da0b89a57",
       "IPY_MODEL_da4a3cc21e4c4399bd66dab213beec9d",
       "IPY_MODEL_39c2c386ccd04f4a98ec446d75789fce"
      ],
      "layout": "IPY_MODEL_c73bf20728b54837a6b0d9b73a671967"
     }
    },
    "b35b1a46fc144c31a0544a9b640dc1a6": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b5de5ad2c50146e7a3083dab9f7bf7d9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b97962aaf3c14133bf0d117649b39863": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_15f7d8aa46ca4a5d836dcf2b5e47a260",
      "placeholder": "​",
      "style": "IPY_MODEL_9c94236cef8c44598a82093083b1f40f",
      "value": "config.json: 100%"
     }
    },
    "beab4a0f4f184a86a3c5f20d06fe89ff": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "bfa08d0d71db4d388b1d5bb68721cab0": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9e6ca257372d47249528831586095146",
      "max": 1472750,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_080057c1221d4438b17971a1800866a7",
      "value": 1472750
     }
    },
    "c6f6cbec5fe04e7da578a8eabf0c09c1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_465dc10743e94007bbd00f740609ffd9",
       "IPY_MODEL_0b61184b3c404d4299b4ebba471469f1",
       "IPY_MODEL_9e5c4d90d54244e7881c4b92b39d0dc7"
      ],
      "layout": "IPY_MODEL_769c790efbd44ad0878b33412aaca217"
     }
    },
    "c73bf20728b54837a6b0d9b73a671967": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cf7a5e2d83dd454d822cecafbb790ef7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "d694867018dd44bf9dda36f2bc50b2ff": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "da4a3cc21e4c4399bd66dab213beec9d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2b8d377685af4943a115acf3148f1c86",
      "max": 856712,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_a0e618e875dc4f5ca37e1a36faa4ed85",
      "value": 856712
     }
    },
    "ed64c3de037e431ca18e453da0b89a57": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b35b1a46fc144c31a0544a9b640dc1a6",
      "placeholder": "​",
      "style": "IPY_MODEL_a178dc61af344b19b966f2a51d873082",
      "value": "vocab.json: 100%"
     }
    },
    "fa556c4c30954072ac24226474e6a329": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "fb206ec77cfd4ae990e4eea0d6a1dcfb": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_92e96c2282b74512838dd3f76e129102",
      "max": 637,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_54edebef257e4f9e973aa76ce5bd69f7",
      "value": 637
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}