visualization_code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "def save_data(data, type):\n",
    "    # save your preprocessed data\n",
    "    with open(os.path.join(\"data/\", type + \".json\"), \"w\") as f:\n",
    "        json.dump(data, f, indent= 4)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open(\"data/train.json\", \"r\", encoding=\"utf-8\") as f:\n",
    "    train_data = json.load(f)\n",
    "f.close()\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "train_data, valid_data  = train_test_split(train_data[\"data\"], test_size= 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = {\"data\": train_data}\n",
    "valid_data = {\"data\": valid_data}\n",
    "\n",
    "save_data(train_data, \"new_train\")\n",
    "save_data(valid_data, \"valid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data & Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset visquad (C:\\Users\\phkha\\.cache\\huggingface\\datasets\\visquad\\plain_text\\1.0.0\\33e7735eec1e054b40673181eb3e6cbaf495119662fa6e6a993f5301d477abc1)\n",
      "100%|██████████| 2/2 [00:00<00:00, 222.19it/s]\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "raw_datasets = load_dataset(\"visquad.py\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetDict({\n",
      "    train: Dataset({\n",
      "        features: ['id', 'title', 'context', 'question', 'answers'],\n",
      "        num_rows: 26418\n",
      "    })\n",
      "    validation: Dataset({\n",
      "        features: ['id', 'title', 'context', 'question', 'answers'],\n",
      "        num_rows: 2039\n",
      "    })\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "print(raw_datasets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context:  Động vật lưỡng cư (danh pháp khoa học: Amphibia) là một lớp động vật có xương sống máu lạnh. Tất cả các loài lưỡng cư hiện đại đều là phân nhánh Lissamphibia của nhóm lớn Amphibia này. Động vật lưỡng cư phải trải qua quá trình biến thái từ ấu trùng sống dưới nước tới dạng trưởng thành có phổi thở không khí, mặc dù vài loài đã phát triển qua nhiều giai đoạn khác nhau để bảo vệ hoặc bỏ qua giai đoạn ấu trùng ở trong nước dễ gặp nguy hiểm. Da được dùng như cơ quan hô hấp phụ, một số loài kỳ giông và ếch thiếu phổi phụ thuộc hoàn toàn vào da. Động vật lưỡng cư có hình dáng giống bò sát, nhưng bò sát, cùng với chim và động vật có vú, là các loài động vật có màng ối và không cần có nước để sinh sản. Trong những thập kỷ gần đây, đã có sự suy giảm số lượng của nhiều loài lưỡng cư trên toàn cầu.\n",
      "Question:  Tên khoa học của động vật lưỡng cư là gì?\n",
      "Answer:  {'text': ['Amphibia'], 'answer_start': [39]}\n"
     ]
    }
   ],
   "source": [
    "print(\"Context: \", raw_datasets[\"train\"][0][\"context\"])\n",
    "print(\"Question: \", raw_datasets[\"train\"][0][\"question\"])\n",
    "print(\"Answer: \", raw_datasets[\"train\"][0][\"answers\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "During training, there is only **ONE** possible answer. We have to filter datasets which has only **ONE** possible answer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\phkha\\.cache\\huggingface\\datasets\\visquad\\plain_text\\1.0.0\\33e7735eec1e054b40673181eb3e6cbaf495119662fa6e6a993f5301d477abc1\\cache-37d2960dc868c755.arrow\n"
     ]
    }
   ],
   "source": [
    "raw_datasets[\"train\"] = raw_datasets[\"train\"].filter(lambda x: len(x[\"answers\"][\"text\"]) == 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\phkha\\.cache\\huggingface\\datasets\\visquad\\plain_text\\1.0.0\\33e7735eec1e054b40673181eb3e6cbaf495119662fa6e6a993f5301d477abc1\\cache-3c0aadcd14a87cac.arrow\n"
     ]
    }
   ],
   "source": [
    "raw_datasets[\"validation\"] = raw_datasets[\"validation\"].filter(lambda x: len(x[\"answers\"][\"text\"]) == 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'title', 'context', 'question', 'answers'],\n",
       "        num_rows: 17825\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['id', 'title', 'context', 'question', 'answers'],\n",
       "        num_rows: 1415\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Text You Want:  20%|██        | 100/500 [00:10<00:43,  9.19it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from time import sleep\n",
    " \n",
    " \n",
    "for i in tqdm(range(0, 100), total = 500,\n",
    "              desc =\"Text You Want\"):\n",
    "    sleep(.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "git clone --single-branch --branch fast_tokenizers_BARTpho_PhoBERT_BERTweet https://github.com/datquocnguyen/transformers.git\n",
    "\n",
    "cd transformers\n",
    "\n",
    "pip3 install -e ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import BartphoTokenizerFast\n",
    "\n",
    "tokenizer = BartphoTokenizerFast.from_pretrained(\"vinai/bartpho-syllable\")\n",
    "tokenizer.is_fast"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "or you can use:\n",
    "```(python)\n",
    "import torch\n",
    "from transformers import PreTrainedTokenizerFast\n",
    "\n",
    "tokenizer = PreTrainedTokenizerFast.from_pretrained(\"vinai/bartpho-syllable\")\n",
    "tokenizer.is_fast\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<s> Tên khoa học của động vật lưỡng cư là gì?</s></s> Động vật lưỡng cư (danh pháp khoa học: Amphibia) là một lớp động vật có xương sống máu lạnh. Tất cả các loài lưỡng cư hiện đại đều là phân nhánh Lissamphibia của nhóm lớn Amphibia này. Động vật lưỡng cư phải trải qua quá trình biến thái từ ấu trùng sống dưới nước tới dạng trưởng thành có phổi thở không khí, mặc dù vài loài đã phát triển qua nhiều giai đoạn khác nhau để bảo vệ hoặc bỏ qua giai đoạn ấu trùng ở trong nước dễ gặp nguy hiểm. Da được dùng như cơ quan hô hấp phụ, một số loài kỳ giông và ếch thiếu phổi phụ thuộc hoàn toàn vào da. Động vật lưỡng cư có hình dáng giống bò sát, nhưng bò sát, cùng với chim và động vật có vú, là các loài động vật có màng ối và không cần có nước để sinh sản. Trong những thập kỷ gần đây, đã có sự suy giảm số lượng của nhiều loài lưỡng cư trên toàn cầu.</s>'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "context = raw_datasets[\"train\"][0][\"context\"]\n",
    "question = raw_datasets[\"train\"][0][\"question\"]\n",
    "\n",
    "inputs = tokenizer(question, context)\n",
    "tokenizer.decode(inputs[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<s> Tên khoa học của động vật lưỡng cư là gì?</s></s> Động vật lưỡng cư (danh pháp khoa học: Amphibia) là một lớp động vật có xương sống máu lạnh. Tất cả các loài lưỡng cư hiện đại đều là phân nhánh Lissamphibia của nhóm lớn Amphibia này. Động vật lưỡng cư phải trải qua quá trình biến thái từ ấu trùng sống dưới nước tới dạng trưởng thành có phổi thở không khí,</s>\n",
      "\n",
      "<s> Tên khoa học của động vật lưỡng cư là gì?</s></s> hiện đại đều là phân nhánh Lissamphibia của nhóm lớn Amphibia này. Động vật lưỡng cư phải trải qua quá trình biến thái từ ấu trùng sống dưới nước tới dạng trưởng thành có phổi thở không khí, mặc dù vài loài đã phát triển qua nhiều giai đoạn khác nhau để bảo vệ hoặc bỏ qua giai đoạn ấu trùng ở trong nước dễ gặp nguy hiểm. Da</s>\n",
      "\n",
      "<s> Tên khoa học của động vật lưỡng cư là gì?</s></s>u trùng sống dưới nước tới dạng trưởng thành có phổi thở không khí, mặc dù vài loài đã phát triển qua nhiều giai đoạn khác nhau để bảo vệ hoặc bỏ qua giai đoạn ấu trùng ở trong nước dễ gặp nguy hiểm. Da được dùng như cơ quan hô hấp phụ, một số loài kỳ giông và ếch thiếu phổi phụ thuộc hoàn toàn vào da. Động vật lưỡng cư</s>\n",
      "\n",
      "<s> Tên khoa học của động vật lưỡng cư là gì?</s></s> qua giai đoạn ấu trùng ở trong nước dễ gặp nguy hiểm. Da được dùng như cơ quan hô hấp phụ, một số loài kỳ giông và ếch thiếu phổi phụ thuộc hoàn toàn vào da. Động vật lưỡng cư có hình dáng giống bò sát, nhưng bò sát, cùng với chim và động vật có vú, là các loài động vật có màng ối và không cần</s>\n",
      "\n",
      "<s> Tên khoa học của động vật lưỡng cư là gì?</s></s>ch thiếu phổi phụ thuộc hoàn toàn vào da. Động vật lưỡng cư có hình dáng giống bò sát, nhưng bò sát, cùng với chim và động vật có vú, là các loài động vật có màng ối và không cần có nước để sinh sản. Trong những thập kỷ gần đây, đã có sự suy giảm số lượng của nhiều loài lưỡng cư trên toàn cầu.</s>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "inputs = tokenizer(\n",
    "    question,\n",
    "    context,\n",
    "    max_length=100,\n",
    "    truncation=\"only_second\",\n",
    "    stride=50,\n",
    "    return_overflowing_tokens=True,\n",
    ")\n",
    "\n",
    "for ids in inputs[\"input_ids\"]:\n",
    "    print(tokenizer.decode(ids))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`max_length` to set the maximum length (here 100)\n",
    "\n",
    "`truncation=\"only_second\"` to truncate the context (which is in the second position) when the question with its context is too long\n",
    "\n",
    "`stride` to set the number of overlapping tokens between two successive chunks (here 50)\n",
    "\n",
    "`return_overflowing_tokens=True` to let the tokenizer know we want the overflowing tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is how `stride` works:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50\n"
     ]
    }
   ],
   "source": [
    "a = \" hiện đại đều là phân nhánh Lissamphibia của nhóm lớn Amphibia này. Động vật lưỡng cư phải trải qua quá trình biến thái từ ấu trùng sống dưới nước tới dạng trưởng thành có phổi thở không khí,\"\n",
    "c = tokenizer.tokenize(a)\n",
    "print(len(c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['input_ids', 'attention_mask', 'offset_mapping', 'overflow_to_sample_mapping'])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs = tokenizer(\n",
    "    question,\n",
    "    context,\n",
    "    max_length=100,\n",
    "    truncation=\"only_second\",\n",
    "    stride=50,\n",
    "    return_overflowing_tokens=True,\n",
    "    return_offsets_mapping=True,\n",
    ")\n",
    "inputs.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 0, 0, 0, 0]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[\"overflow_to_sample_mapping\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 3 examples gave 18 features.\n",
      "Here is where each comes from: [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2].\n"
     ]
    }
   ],
   "source": [
    "inputs = tokenizer(\n",
    "    raw_datasets[\"train\"][2:5][\"question\"],\n",
    "    raw_datasets[\"train\"][2:5][\"context\"],\n",
    "    max_length=100,\n",
    "    truncation=\"only_second\",\n",
    "    stride=50,\n",
    "    return_overflowing_tokens=True,\n",
    "    return_offsets_mapping=True,\n",
    ")\n",
    "\n",
    "print(f\"The 3 examples gave {len(inputs['input_ids'])} features.\")\n",
    "print(f\"Here is where each comes from: {inputs['overflow_to_sample_mapping']}.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'text': ['dưới nước'], 'answer_start': [254]},\n",
       " {'text': ['phổi'], 'answer_start': [289]},\n",
       " {'text': ['có màng ối và không cần có nước để sinh sản'],\n",
       "  'answer_start': [658]}]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets[\"train\"][2:5][\"answers\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [[0, 2104, 40, 78, 274, 7, 23, 178, 107, 1749, 171, 1694, 702, 168, 642, 9, 27, 995, 286, 2, 2, 2483, 405, 171, 1694, 702, 24, 3904, 140, 267, 465, 71, 32, 4016, 3306, 4474, 73, 9, 14, 830, 56, 405, 8, 2097, 274, 1115, 1376, 5, 1809, 110, 10, 745, 171, 1694, 702, 47, 257, 344, 9, 531, 2215, 1647, 345, 8625, 3306, 4474, 7, 639, 191, 4016, 3306, 4474, 21, 5, 2483, 405, 171, 1694, 702, 62, 1202, 109, 272, 122, 482, 1034, 38, 23, 178, 107, 1749, 274, 652, 55, 163, 1171, 305, 58, 8, 2], [0, 2104, 40, 78, 274, 7, 23, 178, 107, 1749, 171, 1694, 702, 168, 642, 9, 27, 995, 286, 2, 2, 110, 10, 745, 171, 1694, 702, 47, 257, 344, 9, 531, 2215, 1647, 345, 8625, 3306, 4474, 7, 639, 191, 4016, 3306, 4474, 21, 5, 2483, 405, 171, 1694, 702, 62, 1202, 109, 272, 122, 482, 1034, 38, 23, 178, 107, 1749, 274, 652, 55, 163, 1171, 305, 58, 8, 1140, 40, 2073, 17, 527, 4, 1013, 577, 832, 745, 16, 100, 180, 109, 45, 1054, 634, 160, 383, 22, 195, 421, 367, 433, 109, 1054, 634, 23, 2], [0, 2104, 40, 78, 274, 7, 23, 178, 107, 1749, 171, 1694, 702, 168, 642, 9, 27, 995, 286, 2, 2, 1694, 702, 62, 1202, 109, 272, 122, 482, 1034, 38, 23, 178, 107, 1749, 274, 652, 55, 163, 1171, 305, 58, 8, 1140, 40, 2073, 17, 527, 4, 1013, 577, 832, 745, 16, 100, 180, 109, 45, 1054, 634, 160, 383, 22, 195, 421, 367, 433, 109, 1054, 634, 23, 178, 107, 1749, 27, 12, 55, 746, 515, 807, 861, 5, 2378, 11, 375, 49, 82, 65, 1788, 1334, 445, 4, 14, 52, 745, 408, 1252, 657, 6, 2], [0, 2104, 40, 78, 274, 7, 23, 178, 107, 1749, 171, 1694, 702, 168, 642, 9, 27, 995, 286, 2, 2, 1013, 577, 832, 745, 16, 100, 180, 109, 45, 1054, 634, 160, 383, 22, 195, 421, 367, 433, 109, 1054, 634, 23, 178, 107, 1749, 27, 12, 55, 746, 515, 807, 861, 5, 2378, 11, 375, 49, 82, 65, 1788, 1334, 445, 4, 14, 52, 745, 408, 1252, 657, 6, 23, 944, 505, 776, 1140, 40, 445, 380, 354, 194, 37, 747, 5, 2483, 405, 171, 1694, 702, 8, 114, 1594, 864, 1791, 347, 4, 91, 1791, 347, 2], [0, 2104, 40, 78, 274, 7, 23, 178, 107, 1749, 171, 1694, 702, 168, 642, 9, 27, 995, 286, 2, 2, 746, 515, 807, 861, 5, 2378, 11, 375, 49, 82, 65, 1788, 1334, 445, 4, 14, 52, 745, 408, 1252, 657, 6, 23, 944, 505, 776, 1140, 40, 445, 380, 354, 194, 37, 747, 5, 2483, 405, 171, 1694, 702, 8, 114, 1594, 864, 1791, 347, 4, 91, 1791, 347, 4, 120, 15, 392, 35, 6, 56, 405, 8, 258, 1827, 4, 9, 10, 745, 56, 405, 8, 2930, 23, 718, 6, 17, 236, 8, 55, 22, 105, 2], [0, 2104, 40, 78, 274, 7, 23, 178, 107, 1749, 171, 1694, 702, 168, 642, 9, 27, 995, 286, 2, 2, 445, 380, 354, 194, 37, 747, 5, 2483, 405, 171, 1694, 702, 8, 114, 1594, 864, 1791, 347, 4, 91, 1791, 347, 4, 120, 15, 392, 35, 6, 56, 405, 8, 258, 1827, 4, 9, 10, 745, 56, 405, 8, 2930, 23, 718, 6, 17, 236, 8, 55, 22, 105, 118, 5, 222, 20, 1698, 774, 309, 155, 4, 16, 8, 54, 1219, 389, 52, 138, 7, 45, 745, 171, 1694, 702, 42, 194, 164, 5, 2], [0, 1221, 65, 1788, 1334, 74, 7, 171, 1694, 702, 9, 350, 286, 2, 2, 2483, 405, 171, 1694, 702, 24, 3904, 140, 267, 465, 71, 32, 4016, 3306, 4474, 73, 9, 14, 830, 56, 405, 8, 2097, 274, 1115, 1376, 5, 1809, 110, 10, 745, 171, 1694, 702, 47, 257, 344, 9, 531, 2215, 1647, 345, 8625, 3306, 4474, 7, 639, 191, 4016, 3306, 4474, 21, 5, 2483, 405, 171, 1694, 702, 62, 1202, 109, 272, 122, 482, 1034, 38, 23, 178, 107, 1749, 274, 652, 55, 163, 1171, 305, 58, 8, 1140, 40, 2073, 17, 527, 4, 2], [0, 1221, 65, 1788, 1334, 74, 7, 171, 1694, 702, 9, 350, 286, 2, 2, 47, 257, 344, 9, 531, 2215, 1647, 345, 8625, 3306, 4474, 7, 639, 191, 4016, 3306, 4474, 21, 5, 2483, 405, 171, 1694, 702, 62, 1202, 109, 272, 122, 482, 1034, 38, 23, 178, 107, 1749, 274, 652, 55, 163, 1171, 305, 58, 8, 1140, 40, 2073, 17, 527, 4, 1013, 577, 832, 745, 16, 100, 180, 109, 45, 1054, 634, 160, 383, 22, 195, 421, 367, 433, 109, 1054, 634, 23, 178, 107, 1749, 27, 12, 55, 746, 515, 807, 861, 5, 2378, 2], [0, 1221, 65, 1788, 1334, 74, 7, 171, 1694, 702, 9, 350, 286, 2, 2, 107, 1749, 274, 652, 55, 163, 1171, 305, 58, 8, 1140, 40, 2073, 17, 527, 4, 1013, 577, 832, 745, 16, 100, 180, 109, 45, 1054, 634, 160, 383, 22, 195, 421, 367, 433, 109, 1054, 634, 23, 178, 107, 1749, 27, 12, 55, 746, 515, 807, 861, 5, 2378, 11, 375, 49, 82, 65, 1788, 1334, 445, 4, 14, 52, 745, 408, 1252, 657, 6, 23, 944, 505, 776, 1140, 40, 445, 380, 354, 194, 37, 747, 5, 2483, 405, 171, 1694, 702, 2], [0, 1221, 65, 1788, 1334, 74, 7, 171, 1694, 702, 9, 350, 286, 2, 2, 109, 1054, 634, 23, 178, 107, 1749, 27, 12, 55, 746, 515, 807, 861, 5, 2378, 11, 375, 49, 82, 65, 1788, 1334, 445, 4, 14, 52, 745, 408, 1252, 657, 6, 23, 944, 505, 776, 1140, 40, 445, 380, 354, 194, 37, 747, 5, 2483, 405, 171, 1694, 702, 8, 114, 1594, 864, 1791, 347, 4, 91, 1791, 347, 4, 120, 15, 392, 35, 6, 56, 405, 8, 258, 1827, 4, 9, 10, 745, 56, 405, 8, 2930, 23, 718, 6, 17, 236, 2], [0, 1221, 65, 1788, 1334, 74, 7, 171, 1694, 702, 9, 350, 286, 2, 2, 505, 776, 1140, 40, 445, 380, 354, 194, 37, 747, 5, 2483, 405, 171, 1694, 702, 8, 114, 1594, 864, 1791, 347, 4, 91, 1791, 347, 4, 120, 15, 392, 35, 6, 56, 405, 8, 258, 1827, 4, 9, 10, 745, 56, 405, 8, 2930, 23, 718, 6, 17, 236, 8, 55, 22, 105, 118, 5, 222, 20, 1698, 774, 309, 155, 4, 16, 8, 54, 1219, 389, 52, 138, 7, 45, 745, 171, 1694, 702, 42, 194, 164, 5, 2], [0, 2384, 449, 36811, 191, 79, 29, 105, 118, 7, 1791, 347, 568, 15, 171, 1694, 702, 9, 350, 286, 2, 2, 2483, 405, 171, 1694, 702, 24, 3904, 140, 267, 465, 71, 32, 4016, 3306, 4474, 73, 9, 14, 830, 56, 405, 8, 2097, 274, 1115, 1376, 5, 1809, 110, 10, 745, 171, 1694, 702, 47, 257, 344, 9, 531, 2215, 1647, 345, 8625, 3306, 4474, 7, 639, 191, 4016, 3306, 4474, 21, 5, 2483, 405, 171, 1694, 702, 62, 1202, 109, 272, 122, 482, 1034, 38, 23, 178, 107, 1749, 274, 652, 55, 163, 1171, 305, 58, 2], [0, 2384, 449, 36811, 191, 79, 29, 105, 118, 7, 1791, 347, 568, 15, 171, 1694, 702, 9, 350, 286, 2, 2, 1809, 110, 10, 745, 171, 1694, 702, 47, 257, 344, 9, 531, 2215, 1647, 345, 8625, 3306, 4474, 7, 639, 191, 4016, 3306, 4474, 21, 5, 2483, 405, 171, 1694, 702, 62, 1202, 109, 272, 122, 482, 1034, 38, 23, 178, 107, 1749, 274, 652, 55, 163, 1171, 305, 58, 8, 1140, 40, 2073, 17, 527, 4, 1013, 577, 832, 745, 16, 100, 180, 109, 45, 1054, 634, 160, 383, 22, 195, 421, 367, 433, 109, 1054, 2], [0, 2384, 449, 36811, 191, 79, 29, 105, 118, 7, 1791, 347, 568, 15, 171, 1694, 702, 9, 350, 286, 2, 2, 405, 171, 1694, 702, 62, 1202, 109, 272, 122, 482, 1034, 38, 23, 178, 107, 1749, 274, 652, 55, 163, 1171, 305, 58, 8, 1140, 40, 2073, 17, 527, 4, 1013, 577, 832, 745, 16, 100, 180, 109, 45, 1054, 634, 160, 383, 22, 195, 421, 367, 433, 109, 1054, 634, 23, 178, 107, 1749, 27, 12, 55, 746, 515, 807, 861, 5, 2378, 11, 375, 49, 82, 65, 1788, 1334, 445, 4, 14, 52, 745, 408, 2], [0, 2384, 449, 36811, 191, 79, 29, 105, 118, 7, 1791, 347, 568, 15, 171, 1694, 702, 9, 350, 286, 2, 2, 17, 527, 4, 1013, 577, 832, 745, 16, 100, 180, 109, 45, 1054, 634, 160, 383, 22, 195, 421, 367, 433, 109, 1054, 634, 23, 178, 107, 1749, 27, 12, 55, 746, 515, 807, 861, 5, 2378, 11, 375, 49, 82, 65, 1788, 1334, 445, 4, 14, 52, 745, 408, 1252, 657, 6, 23, 944, 505, 776, 1140, 40, 445, 380, 354, 194, 37, 747, 5, 2483, 405, 171, 1694, 702, 8, 114, 1594, 864, 1791, 347, 2], [0, 2384, 449, 36811, 191, 79, 29, 105, 118, 7, 1791, 347, 568, 15, 171, 1694, 702, 9, 350, 286, 2, 2, 1749, 27, 12, 55, 746, 515, 807, 861, 5, 2378, 11, 375, 49, 82, 65, 1788, 1334, 445, 4, 14, 52, 745, 408, 1252, 657, 6, 23, 944, 505, 776, 1140, 40, 445, 380, 354, 194, 37, 747, 5, 2483, 405, 171, 1694, 702, 8, 114, 1594, 864, 1791, 347, 4, 91, 1791, 347, 4, 120, 15, 392, 35, 6, 56, 405, 8, 258, 1827, 4, 9, 10, 745, 56, 405, 8, 2930, 23, 718, 6, 17, 2], [0, 2384, 449, 36811, 191, 79, 29, 105, 118, 7, 1791, 347, 568, 15, 171, 1694, 702, 9, 350, 286, 2, 2, 944, 505, 776, 1140, 40, 445, 380, 354, 194, 37, 747, 5, 2483, 405, 171, 1694, 702, 8, 114, 1594, 864, 1791, 347, 4, 91, 1791, 347, 4, 120, 15, 392, 35, 6, 56, 405, 8, 258, 1827, 4, 9, 10, 745, 56, 405, 8, 2930, 23, 718, 6, 17, 236, 8, 55, 22, 105, 118, 5, 222, 20, 1698, 774, 309, 155, 4, 16, 8, 54, 1219, 389, 52, 138, 7, 45, 745, 171, 1694, 702, 2], [0, 2384, 449, 36811, 191, 79, 29, 105, 118, 7, 1791, 347, 568, 15, 171, 1694, 702, 9, 350, 286, 2, 2, 4, 120, 15, 392, 35, 6, 56, 405, 8, 258, 1827, 4, 9, 10, 745, 56, 405, 8, 2930, 23, 718, 6, 17, 236, 8, 55, 22, 105, 118, 5, 222, 20, 1698, 774, 309, 155, 4, 16, 8, 54, 1219, 389, 52, 138, 7, 45, 745, 171, 1694, 702, 42, 194, 164, 5, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'offset_mapping': [[(0, 0), (0, 2), (2, 3), (3, 10), (10, 15), (15, 19), (19, 20), (20, 21), (21, 22), (22, 28), (28, 30), (30, 34), (34, 37), (37, 41), (41, 45), (45, 48), (48, 50), (50, 54), (54, 55), (0, 0), (0, 0), (0, 4), (4, 8), (8, 10), (10, 14), (14, 17), (17, 19), (19, 22), (22, 23), (23, 28), (28, 33), (33, 37), (37, 38), (38, 41), (41, 44), (44, 47), (47, 48), (48, 51), (51, 55), (55, 59), (59, 64), (64, 68), (68, 71), (71, 77), (77, 82), (82, 86), (86, 91), (91, 92), (92, 96), (96, 99), (99, 103), (103, 108), (108, 110), (110, 114), (114, 117), (117, 122), (122, 126), (126, 130), (130, 133), (133, 138), (138, 144), (144, 147), (147, 148), (148, 151), (151, 154), (154, 157), (157, 161), (161, 166), (166, 170), (170, 173), (173, 176), (176, 179), (179, 183), (183, 184), (184, 189), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (0, 0)], [(0, 0), (0, 2), (2, 3), (3, 10), (10, 15), (15, 19), (19, 20), (20, 21), (21, 22), (22, 28), (28, 30), (30, 34), (34, 37), (37, 41), (41, 45), (45, 48), (48, 50), (50, 54), (54, 55), (0, 0), (0, 0), (96, 99), (99, 103), (103, 108), (108, 110), (110, 114), (114, 117), (117, 122), (122, 126), (126, 130), (130, 133), (133, 138), (138, 144), (144, 147), (147, 148), (148, 151), (151, 154), (154, 157), (157, 161), (161, 166), (166, 170), (170, 173), (173, 176), (176, 179), (179, 183), (183, 184), (184, 189), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (0, 0)], [(0, 0), (0, 2), (2, 3), (3, 10), (10, 15), (15, 19), (19, 20), (20, 21), (21, 22), (22, 28), (28, 30), (30, 34), (34, 37), (37, 41), (41, 45), (45, 48), (48, 50), (50, 54), (54, 55), (0, 0), (0, 0), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (0, 0)], [(0, 0), (0, 2), (2, 3), (3, 10), (10, 15), (15, 19), (19, 20), (20, 21), (21, 22), (22, 28), (28, 30), (30, 34), (34, 37), (37, 41), (41, 45), (45, 48), (48, 50), (50, 54), (54, 55), (0, 0), (0, 0), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (501, 502), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (0, 0)], [(0, 0), (0, 2), (2, 3), (3, 10), (10, 15), (15, 19), (19, 20), (20, 21), (21, 22), (22, 28), (28, 30), (30, 34), (34, 37), (37, 41), (41, 45), (45, 48), (48, 50), (50, 54), (54, 55), (0, 0), (0, 0), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (501, 502), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (677, 681), (681, 684), (684, 689), (689, 692), (692, 697), (0, 0)], [(0, 0), (0, 2), (2, 3), (3, 10), (10, 15), (15, 19), (19, 20), (20, 21), (21, 22), (22, 28), (28, 30), (30, 34), (34, 37), (37, 41), (41, 45), (45, 48), (48, 50), (50, 54), (54, 55), (0, 0), (0, 0), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (677, 681), (681, 684), (684, 689), (689, 692), (692, 697), (697, 701), (701, 702), (702, 708), (708, 714), (714, 719), (719, 722), (722, 726), (726, 730), (730, 731), (731, 734), (734, 737), (737, 740), (740, 744), (744, 749), (749, 752), (752, 758), (758, 762), (762, 768), (768, 773), (773, 775), (775, 779), (779, 782), (782, 787), (787, 792), (792, 796), (796, 797), (0, 0)], [(0, 0), (0, 2), (2, 7), (7, 10), (10, 14), (14, 20), (20, 24), (24, 26), (26, 30), (30, 33), (33, 36), (36, 39), (39, 40), (0, 0), (0, 0), (0, 4), (4, 8), (8, 10), (10, 14), (14, 17), (17, 19), (19, 22), (22, 23), (23, 28), (28, 33), (33, 37), (37, 38), (38, 41), (41, 44), (44, 47), (47, 48), (48, 51), (51, 55), (55, 59), (59, 64), (64, 68), (68, 71), (71, 77), (77, 82), (82, 86), (86, 91), (91, 92), (92, 96), (96, 99), (99, 103), (103, 108), (108, 110), (110, 114), (114, 117), (117, 122), (122, 126), (126, 130), (130, 133), (133, 138), (138, 144), (144, 147), (147, 148), (148, 151), (151, 154), (154, 157), (157, 161), (161, 166), (166, 170), (170, 173), (173, 176), (176, 179), (179, 183), (183, 184), (184, 189), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (0, 0)], [(0, 0), (0, 2), (2, 7), (7, 10), (10, 14), (14, 20), (20, 24), (24, 26), (26, 30), (30, 33), (33, 36), (36, 39), (39, 40), (0, 0), (0, 0), (117, 122), (122, 126), (126, 130), (130, 133), (133, 138), (138, 144), (144, 147), (147, 148), (148, 151), (151, 154), (154, 157), (157, 161), (161, 166), (166, 170), (170, 173), (173, 176), (176, 179), (179, 183), (183, 184), (184, 189), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (0, 0)], [(0, 0), (0, 2), (2, 7), (7, 10), (10, 14), (14, 20), (20, 24), (24, 26), (26, 30), (30, 33), (33, 36), (36, 39), (39, 40), (0, 0), (0, 0), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (501, 502), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (0, 0)], [(0, 0), (0, 2), (2, 7), (7, 10), (10, 14), (14, 20), (20, 24), (24, 26), (26, 30), (30, 33), (33, 36), (36, 39), (39, 40), (0, 0), (0, 0), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (501, 502), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (677, 681), (0, 0)], [(0, 0), (0, 2), (2, 7), (7, 10), (10, 14), (14, 20), (20, 24), (24, 26), (26, 30), (30, 33), (33, 36), (36, 39), (39, 40), (0, 0), (0, 0), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (677, 681), (681, 684), (684, 689), (689, 692), (692, 697), (697, 701), (701, 702), (702, 708), (708, 714), (714, 719), (719, 722), (722, 726), (726, 730), (730, 731), (731, 734), (734, 737), (737, 740), (740, 744), (744, 749), (749, 752), (752, 758), (758, 762), (762, 768), (768, 773), (773, 775), (775, 779), (779, 782), (782, 787), (787, 792), (792, 796), (796, 797), (0, 0)], [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (0, 4), (4, 8), (8, 10), (10, 14), (14, 17), (17, 19), (19, 22), (22, 23), (23, 28), (28, 33), (33, 37), (37, 38), (38, 41), (41, 44), (44, 47), (47, 48), (48, 51), (51, 55), (55, 59), (59, 64), (64, 68), (68, 71), (71, 77), (77, 82), (82, 86), (86, 91), (91, 92), (92, 96), (96, 99), (99, 103), (103, 108), (108, 110), (110, 114), (114, 117), (117, 122), (122, 126), (126, 130), (130, 133), (133, 138), (138, 144), (144, 147), (147, 148), (148, 151), (151, 154), (154, 157), (157, 161), (161, 166), (166, 170), (170, 173), (173, 176), (176, 179), (179, 183), (183, 184), (184, 189), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (0, 0)], [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (92, 96), (96, 99), (99, 103), (103, 108), (108, 110), (110, 114), (114, 117), (117, 122), (122, 126), (126, 130), (130, 133), (133, 138), (138, 144), (144, 147), (147, 148), (148, 151), (151, 154), (154, 157), (157, 161), (161, 166), (166, 170), (170, 173), (173, 176), (176, 179), (179, 183), (183, 184), (184, 189), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (0, 0)], [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (0, 0)], [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (501, 502), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (0, 0)], [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (501, 502), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (0, 0)], [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (677, 681), (681, 684), (684, 689), (689, 692), (692, 697), (697, 701), (701, 702), (702, 708), (708, 714), (714, 719), (719, 722), (722, 726), (726, 730), (730, 731), (731, 734), (734, 737), (737, 740), (740, 744), (744, 749), (749, 752), (752, 758), (758, 762), (762, 768), (768, 773), (773, 775), (775, 779), (779, 782), (0, 0)], [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (677, 681), (681, 684), (684, 689), (689, 692), (692, 697), (697, 701), (701, 702), (702, 708), (708, 714), (714, 719), (719, 722), (722, 726), (726, 730), (730, 731), (731, 734), (734, 737), (737, 740), (740, 744), (744, 749), (749, 752), (752, 758), (758, 762), (762, 768), (768, 773), (773, 775), (775, 779), (779, 782), (782, 787), (787, 792), (792, 796), (796, 797), (0, 0)]], 'overflow_to_sample_mapping': [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2]}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 [(0, 0), (0, 2), (2, 3), (3, 10), (10, 15), (15, 19), (19, 20), (20, 21), (21, 22), (22, 28), (28, 30), (30, 34), (34, 37), (37, 41), (41, 45), (45, 48), (48, 50), (50, 54), (54, 55), (0, 0), (0, 0), (0, 4), (4, 8), (8, 10), (10, 14), (14, 17), (17, 19), (19, 22), (22, 23), (23, 28), (28, 33), (33, 37), (37, 38), (38, 41), (41, 44), (44, 47), (47, 48), (48, 51), (51, 55), (55, 59), (59, 64), (64, 68), (68, 71), (71, 77), (77, 82), (82, 86), (86, 91), (91, 92), (92, 96), (96, 99), (99, 103), (103, 108), (108, 110), (110, 114), (114, 117), (117, 122), (122, 126), (126, 130), (130, 133), (133, 138), (138, 144), (144, 147), (147, 148), (148, 151), (151, 154), (154, 157), (157, 161), (161, 166), (166, 170), (170, 173), (173, 176), (176, 179), (179, 183), (183, 184), (184, 189), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (0, 0)]\n",
      "1 [(0, 0), (0, 2), (2, 3), (3, 10), (10, 15), (15, 19), (19, 20), (20, 21), (21, 22), (22, 28), (28, 30), (30, 34), (34, 37), (37, 41), (41, 45), (45, 48), (48, 50), (50, 54), (54, 55), (0, 0), (0, 0), (96, 99), (99, 103), (103, 108), (108, 110), (110, 114), (114, 117), (117, 122), (122, 126), (126, 130), (130, 133), (133, 138), (138, 144), (144, 147), (147, 148), (148, 151), (151, 154), (154, 157), (157, 161), (161, 166), (166, 170), (170, 173), (173, 176), (176, 179), (179, 183), (183, 184), (184, 189), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (0, 0)]\n",
      "2 [(0, 0), (0, 2), (2, 3), (3, 10), (10, 15), (15, 19), (19, 20), (20, 21), (21, 22), (22, 28), (28, 30), (30, 34), (34, 37), (37, 41), (41, 45), (45, 48), (48, 50), (50, 54), (54, 55), (0, 0), (0, 0), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (0, 0)]\n",
      "3 [(0, 0), (0, 2), (2, 3), (3, 10), (10, 15), (15, 19), (19, 20), (20, 21), (21, 22), (22, 28), (28, 30), (30, 34), (34, 37), (37, 41), (41, 45), (45, 48), (48, 50), (50, 54), (54, 55), (0, 0), (0, 0), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (501, 502), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (0, 0)]\n",
      "4 [(0, 0), (0, 2), (2, 3), (3, 10), (10, 15), (15, 19), (19, 20), (20, 21), (21, 22), (22, 28), (28, 30), (30, 34), (34, 37), (37, 41), (41, 45), (45, 48), (48, 50), (50, 54), (54, 55), (0, 0), (0, 0), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (501, 502), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (677, 681), (681, 684), (684, 689), (689, 692), (692, 697), (0, 0)]\n",
      "5 [(0, 0), (0, 2), (2, 3), (3, 10), (10, 15), (15, 19), (19, 20), (20, 21), (21, 22), (22, 28), (28, 30), (30, 34), (34, 37), (37, 41), (41, 45), (45, 48), (48, 50), (50, 54), (54, 55), (0, 0), (0, 0), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (677, 681), (681, 684), (684, 689), (689, 692), (692, 697), (697, 701), (701, 702), (702, 708), (708, 714), (714, 719), (719, 722), (722, 726), (726, 730), (730, 731), (731, 734), (734, 737), (737, 740), (740, 744), (744, 749), (749, 752), (752, 758), (758, 762), (762, 768), (768, 773), (773, 775), (775, 779), (779, 782), (782, 787), (787, 792), (792, 796), (796, 797), (0, 0)]\n",
      "6 [(0, 0), (0, 2), (2, 7), (7, 10), (10, 14), (14, 20), (20, 24), (24, 26), (26, 30), (30, 33), (33, 36), (36, 39), (39, 40), (0, 0), (0, 0), (0, 4), (4, 8), (8, 10), (10, 14), (14, 17), (17, 19), (19, 22), (22, 23), (23, 28), (28, 33), (33, 37), (37, 38), (38, 41), (41, 44), (44, 47), (47, 48), (48, 51), (51, 55), (55, 59), (59, 64), (64, 68), (68, 71), (71, 77), (77, 82), (82, 86), (86, 91), (91, 92), (92, 96), (96, 99), (99, 103), (103, 108), (108, 110), (110, 114), (114, 117), (117, 122), (122, 126), (126, 130), (130, 133), (133, 138), (138, 144), (144, 147), (147, 148), (148, 151), (151, 154), (154, 157), (157, 161), (161, 166), (166, 170), (170, 173), (173, 176), (176, 179), (179, 183), (183, 184), (184, 189), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (0, 0)]\n",
      "7 [(0, 0), (0, 2), (2, 7), (7, 10), (10, 14), (14, 20), (20, 24), (24, 26), (26, 30), (30, 33), (33, 36), (36, 39), (39, 40), (0, 0), (0, 0), (117, 122), (122, 126), (126, 130), (130, 133), (133, 138), (138, 144), (144, 147), (147, 148), (148, 151), (151, 154), (154, 157), (157, 161), (161, 166), (166, 170), (170, 173), (173, 176), (176, 179), (179, 183), (183, 184), (184, 189), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (0, 0)]\n",
      "8 [(0, 0), (0, 2), (2, 7), (7, 10), (10, 14), (14, 20), (20, 24), (24, 26), (26, 30), (30, 33), (33, 36), (36, 39), (39, 40), (0, 0), (0, 0), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (501, 502), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (0, 0)]\n",
      "9 [(0, 0), (0, 2), (2, 7), (7, 10), (10, 14), (14, 20), (20, 24), (24, 26), (26, 30), (30, 33), (33, 36), (36, 39), (39, 40), (0, 0), (0, 0), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (501, 502), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (677, 681), (0, 0)]\n",
      "10 [(0, 0), (0, 2), (2, 7), (7, 10), (10, 14), (14, 20), (20, 24), (24, 26), (26, 30), (30, 33), (33, 36), (36, 39), (39, 40), (0, 0), (0, 0), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (677, 681), (681, 684), (684, 689), (689, 692), (692, 697), (697, 701), (701, 702), (702, 708), (708, 714), (714, 719), (719, 722), (722, 726), (726, 730), (730, 731), (731, 734), (734, 737), (737, 740), (740, 744), (744, 749), (749, 752), (752, 758), (758, 762), (762, 768), (768, 773), (773, 775), (775, 779), (779, 782), (782, 787), (787, 792), (792, 796), (796, 797), (0, 0)]\n",
      "11 [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (0, 4), (4, 8), (8, 10), (10, 14), (14, 17), (17, 19), (19, 22), (22, 23), (23, 28), (28, 33), (33, 37), (37, 38), (38, 41), (41, 44), (44, 47), (47, 48), (48, 51), (51, 55), (55, 59), (59, 64), (64, 68), (68, 71), (71, 77), (77, 82), (82, 86), (86, 91), (91, 92), (92, 96), (96, 99), (99, 103), (103, 108), (108, 110), (110, 114), (114, 117), (117, 122), (122, 126), (126, 130), (130, 133), (133, 138), (138, 144), (144, 147), (147, 148), (148, 151), (151, 154), (154, 157), (157, 161), (161, 166), (166, 170), (170, 173), (173, 176), (176, 179), (179, 183), (183, 184), (184, 189), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (0, 0)]\n",
      "12 [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (92, 96), (96, 99), (99, 103), (103, 108), (108, 110), (110, 114), (114, 117), (117, 122), (122, 126), (126, 130), (130, 133), (133, 138), (138, 144), (144, 147), (147, 148), (148, 151), (151, 154), (154, 157), (157, 161), (161, 166), (166, 170), (170, 173), (173, 176), (176, 179), (179, 183), (183, 184), (184, 189), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (0, 0)]\n",
      "13 [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (189, 193), (193, 195), (195, 199), (199, 202), (202, 207), (207, 212), (212, 216), (216, 220), (220, 226), (226, 231), (231, 236), (236, 239), (239, 240), (240, 241), (241, 242), (242, 248), (248, 253), (253, 258), (258, 263), (263, 267), (267, 272), (272, 279), (279, 285), (285, 288), (288, 292), (292, 293), (293, 297), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (0, 0)]\n",
      "14 [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (297, 303), (303, 307), (307, 308), (308, 312), (312, 315), (315, 319), (319, 324), (324, 327), (327, 332), (332, 338), (338, 342), (342, 348), (348, 353), (353, 358), (358, 363), (363, 368), (368, 371), (371, 375), (375, 378), (378, 383), (383, 386), (386, 390), (390, 395), (395, 400), (400, 401), (401, 402), (402, 403), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (501, 502), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (0, 0)]\n",
      "15 [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (403, 409), (409, 411), (411, 417), (417, 422), (422, 425), (425, 429), (429, 434), (434, 439), (439, 440), (440, 443), (443, 448), (448, 453), (453, 457), (457, 460), (460, 465), (465, 468), (468, 472), (472, 476), (476, 477), (477, 481), (481, 484), (484, 489), (489, 492), (492, 495), (495, 498), (498, 501), (501, 502), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (0, 0)]\n",
      "16 [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (502, 503), (503, 505), (505, 511), (511, 515), (515, 516), (516, 520), (520, 526), (526, 531), (531, 536), (536, 540), (540, 543), (543, 544), (544, 549), (549, 553), (553, 555), (555, 559), (559, 562), (562, 565), (565, 570), (570, 575), (575, 581), (581, 584), (584, 588), (588, 589), (589, 595), (595, 598), (598, 602), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (677, 681), (681, 684), (684, 689), (689, 692), (692, 697), (697, 701), (701, 702), (702, 708), (708, 714), (714, 719), (719, 722), (722, 726), (726, 730), (730, 731), (731, 734), (734, 737), (737, 740), (740, 744), (744, 749), (749, 752), (752, 758), (758, 762), (762, 768), (768, 773), (773, 775), (775, 779), (779, 782), (0, 0)]\n",
      "17 [(0, 0), (0, 4), (4, 9), (9, 13), (13, 17), (17, 22), (22, 25), (25, 30), (30, 34), (34, 38), (38, 41), (41, 45), (45, 48), (48, 52), (52, 54), (54, 58), (58, 61), (61, 64), (64, 67), (67, 68), (0, 0), (0, 0), (602, 603), (603, 608), (608, 612), (612, 616), (616, 617), (617, 620), (620, 625), (625, 629), (629, 632), (632, 634), (634, 635), (635, 636), (636, 639), (639, 643), (643, 648), (648, 653), (653, 657), (657, 660), (660, 665), (665, 666), (666, 668), (668, 671), (671, 677), (677, 681), (681, 684), (684, 689), (689, 692), (692, 697), (697, 701), (701, 702), (702, 708), (708, 714), (714, 719), (719, 722), (722, 726), (726, 730), (730, 731), (731, 734), (734, 737), (737, 740), (740, 744), (744, 749), (749, 752), (752, 758), (758, 762), (762, 768), (768, 773), (773, 775), (775, 779), (779, 782), (782, 787), (787, 792), (792, 796), (796, 797), (0, 0)]\n"
     ]
    }
   ],
   "source": [
    "for i, offset in enumerate(inputs[\"offset_mapping\"]):\n",
    "    print(i, offset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`sequence_ids` similar to Next Sentence Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " None,\n",
       " None,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " None]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs.sequence_ids(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[\"overflow_to_sample_mapping\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([92, 64, 36, 0, 0, 0, 93, 59, 25, 0, 0, 0, 0, 0, 0, 0, 66, 39],\n",
       " [93, 65, 37, 0, 0, 0, 94, 60, 26, 0, 0, 0, 0, 0, 0, 0, 77, 50])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answers = raw_datasets[\"train\"][2:5][\"answers\"]\n",
    "start_positions = []\n",
    "end_positions = []\n",
    "\n",
    "for i, offset in enumerate(inputs[\"offset_mapping\"]):\n",
    "    sample_idx = inputs[\"overflow_to_sample_mapping\"][i]\n",
    "    answer = answers[sample_idx]\n",
    "    start_char = answer[\"answer_start\"][0]\n",
    "    end_char = answer[\"answer_start\"][0] + len(answer[\"text\"][0])\n",
    "    sequence_ids = inputs.sequence_ids(i)\n",
    "\n",
    "    # Find the start and end of the context\n",
    "    idx = 0\n",
    "    while sequence_ids[idx] != 1:\n",
    "        idx += 1\n",
    "    context_start = idx\n",
    "    while sequence_ids[idx] == 1:\n",
    "        idx += 1\n",
    "    context_end = idx - 1\n",
    "\n",
    "    # If the answer is not fully inside the context, label is (0, 0)\n",
    "    if offset[context_start][0] > start_char or offset[context_end][1] < end_char:\n",
    "        start_positions.append(0)\n",
    "        end_positions.append(0)\n",
    "    else:\n",
    "        # Otherwise it's the start and end token positions\n",
    "        idx = context_start\n",
    "        while idx <= context_end and offset[idx][0] <= start_char:\n",
    "            idx += 1\n",
    "        start_positions.append(idx - 1)\n",
    "\n",
    "        idx = context_end\n",
    "        while idx >= context_start and offset[idx][1] >= end_char:\n",
    "            idx -= 1\n",
    "        end_positions.append(idx + 1)\n",
    "\n",
    "start_positions, end_positions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Theoretical answer: dưới nước, labels give: dưới nước\n"
     ]
    }
   ],
   "source": [
    "idx = 0\n",
    "sample_idx = inputs[\"overflow_to_sample_mapping\"][idx]\n",
    "answer = answers[sample_idx][\"text\"][0]\n",
    "\n",
    "start = start_positions[idx]\n",
    "end = end_positions[idx]\n",
    "labeled_answer = tokenizer.decode(inputs[\"input_ids\"][idx][start : end + 1])\n",
    "\n",
    "print(f\"Theoretical answer: {answer}, labels give: {labeled_answer}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let’s check index 4, where we set the labels to (0, 0), which means the answer is not in the context chunk of that feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Theoretical answer: dưới nước, decoded example: <s> Môi trường sống của ấu trùng lưỡng cư chủ yếu là ở đâu?</s></s> dễ gặp nguy hiểm. Da được dùng như cơ quan hô hấp phụ, một số loài kỳ giông và ếch thiếu phổi phụ thuộc hoàn toàn vào da. Động vật lưỡng cư có hình dáng giống bò sát, nhưng bò sát, cùng với chim và động vật có vú, là các loài động vật có màng ối và không cần có nước để sinh</s>\n"
     ]
    }
   ],
   "source": [
    "idx = 4\n",
    "sample_idx = inputs[\"overflow_to_sample_mapping\"][idx]\n",
    "answer = answers[sample_idx][\"text\"][0]\n",
    "\n",
    "decoded_example = tokenizer.decode(inputs[\"input_ids\"][idx])\n",
    "print(f\"Theoretical answer: {answer}, decoded example: {decoded_example}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Indeed, we don’t see the answer inside the context."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_length = 1024\n",
    "stride = 128\n",
    "\n",
    "def preprocess_training_dataset(examples):\n",
    "    questions = [q.strip() for q in examples[\"question\"]]\n",
    "    inputs = tokenizer(\n",
    "        questions,\n",
    "        examples[\"context\"],\n",
    "         max_length=max_length,\n",
    "        truncation=\"only_second\",\n",
    "        stride=stride,\n",
    "        return_overflowing_tokens=True,\n",
    "        return_offsets_mapping=True,\n",
    "        padding=\"max_length\",\n",
    "    )\n",
    "\n",
    "    offset_mapping = inputs.pop(\"offset_mapping\")\n",
    "    sample_map = inputs.pop(\"overflow_to_sample_mapping\")\n",
    "    answers = examples[\"answers\"]\n",
    "    start_positions = []\n",
    "    end_positions = []\n",
    "\n",
    "    for i, offset in enumerate(offset_mapping):\n",
    "        sample_idx = sample_map[i]\n",
    "        answer = answers[sample_idx]\n",
    "        start_char = answer[\"answer_start\"][0]\n",
    "        end_char = answer[\"answer_start\"][0] + len(answer[\"text\"][0])\n",
    "        sequence_ids = inputs.sequence_ids(i)\n",
    "\n",
    "        # Find the start and end of the context\n",
    "        idx = 0\n",
    "        while sequence_ids[idx] != 1:\n",
    "            idx += 1\n",
    "        context_start = idx\n",
    "        while sequence_ids[idx] == 1:\n",
    "            idx += 1\n",
    "        context_end = idx - 1\n",
    "\n",
    "        # If the answer is not fully inside the context, label is (0, 0)\n",
    "        if offset[context_start][0] > start_char or offset[context_end][1] < end_char:\n",
    "            start_positions.append(0)\n",
    "            end_positions.append(0)\n",
    "        else:\n",
    "            # Otherwise it's the start and end token positions\n",
    "            idx = context_start\n",
    "            while idx <= context_end and offset[idx][0] <= start_char:\n",
    "                idx += 1\n",
    "            start_positions.append(idx - 1)\n",
    "\n",
    "            idx = context_end\n",
    "            while idx >= context_start and offset[idx][1] >= end_char:\n",
    "                idx -= 1\n",
    "            end_positions.append(idx + 1)\n",
    "\n",
    "    inputs[\"start_positions\"] = start_positions\n",
    "    inputs[\"end_positions\"] = end_positions\n",
    "    return inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 18/18 [01:00<00:00,  3.33s/ba]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(17825, 17825)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset = raw_datasets[\"train\"].map(\n",
    "    preprocess_training_dataset,\n",
    "    batched=True,\n",
    "    remove_columns=raw_datasets[\"train\"].column_names,\n",
    ")\n",
    "len(raw_datasets[\"train\"]), len(train_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs.pop(\"overflow_to_sample_mapping\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A tiny bit of cleanup of the **offset mappings**. They will contain offsets for the *question* and the *context*, but once we’re in the post-processing stage we won’t have any way to know which part of the input IDs corresponded to the *context* and which part was the *question*. So, we’ll set the offsets corresponding to the *question* to `None`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_validation_dataset(examples):\n",
    "    questions = [q.strip() for q in examples[\"question\"]]\n",
    "    inputs = tokenizer(\n",
    "        questions,\n",
    "        examples[\"context\"],\n",
    "        max_length=max_length,\n",
    "        truncation=\"only_second\",\n",
    "        stride=stride,\n",
    "        return_overflowing_tokens=True,\n",
    "        return_offsets_mapping=True,\n",
    "        padding=\"max_length\",\n",
    "    )\n",
    "\n",
    "    sample_map = inputs.pop(\"overflow_to_sample_mapping\")\n",
    "    example_ids = []\n",
    "\n",
    "    for i in range(len(inputs[\"input_ids\"])):\n",
    "        sample_idx = sample_map[i]\n",
    "        example_ids.append(examples[\"id\"][sample_idx])\n",
    "\n",
    "        sequence_ids = inputs.sequence_ids(i)\n",
    "        offset = inputs[\"offset_mapping\"][i]\n",
    "        inputs[\"offset_mapping\"][i] = [\n",
    "            o if sequence_ids[k] == 1 else None for k, o in enumerate(offset)\n",
    "        ]\n",
    "\n",
    "    inputs[\"example_id\"] = example_ids\n",
    "    return inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:22<00:00, 11.03s/ba]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1415, 1427)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_dataset = raw_datasets[\"validation\"].map(\n",
    "    preprocess_validation_dataset,\n",
    "    batched=True,\n",
    "    remove_columns=raw_datasets[\"validation\"].column_names,\n",
    ")\n",
    "len(raw_datasets[\"validation\"]), len(validation_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this case we’ve only added a few samples, so it appears the contexts in the validation dataset are a bit shorter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:01<00:00,  1.52s/ba]\n"
     ]
    }
   ],
   "source": [
    "small_eval_set = raw_datasets[\"validation\"].select(range(100))\n",
    "trained_checkpoint = \"distilbert-base-cased-distilled-squad\"\n",
    "\n",
    "eval_set = small_eval_set.map(\n",
    "    preprocess_validation_dataset,\n",
    "    batched=True,\n",
    "    remove_columns=raw_datasets[\"validation\"].column_names,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of MBartForQuestionAnswering were not initialized from the model checkpoint at vinai/bartpho-syllable and are newly initialized: ['qa_outputs.bias', 'qa_outputs.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForQuestionAnswering\n",
    "\n",
    "eval_set_for_model = eval_set.remove_columns([\"example_id\", \"offset_mapping\"])\n",
    "eval_set_for_model.set_format(\"torch\")\n",
    "\n",
    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "batch = {k: eval_set_for_model[k].to(device) for k in eval_set_for_model.column_names}\n",
    "trained_model = AutoModelForQuestionAnswering.from_pretrained(\"vinai/bartpho-syllable\").to(\n",
    "    device\n",
    ")\n",
    "\n",
    "with torch.no_grad():\n",
    "    outputs = trained_model(**batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_logits = outputs.start_logits.cpu().numpy()\n",
    "end_logits = outputs.end_logits.cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "example_to_features = collections.defaultdict(list)\n",
    "for idx, feature in enumerate(eval_set):\n",
    "    example_to_features[feature[\"example_id\"]].append(idx)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('vietai')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "29738e9c914f33cb0cb33e381a709df54f8597f8cbd29879658f9da23aacfbcc"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}