gpt_with_byte_pair_encoding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6fa94449",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f947b2db170>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from __future__ import annotations\n",
    "import typing\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import os\n",
    "import copy\n",
    "import pickle\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "\n",
    "torch.manual_seed(3654)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b20532c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cuda'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0e1c6932",
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_trainable_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd8e1971",
   "metadata": {},
   "source": [
    "### Get text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6e97ce57",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38739496\n"
     ]
    }
   ],
   "source": [
    "with open(\"data/text_corpus.txt\", encoding=\"utf-8\") as f:\n",
    "    text = f.read()\n",
    "\n",
    "print(len(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b6ff1d38",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "150\n",
      "['\\n', ' ', '!', '\"', '$', '%', '&', \"'\", '+', ',', '-', '.', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', '>', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', ']', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '´', 'É', 'Ö', 'Ü', 'á', 'ã', 'å', 'ç', 'è', 'é', 'ê', 'í', 'ó', 'ô', 'ö', 'ø', 'ù', 'ü', 'ć', 'ł', 'ő', 'Б', 'В', 'Г', 'Д', 'К', 'Н', 'П', 'Р', 'С', 'Т', 'У', 'Я', 'а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'х', 'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я', 'ё', '–', '—', '…', '가', '들', '어']\n"
     ]
    }
   ],
   "source": [
    "char_vocab = sorted(set(text))\n",
    "char_vocab_size = len(char_vocab)\n",
    "print(char_vocab_size)\n",
    "print(char_vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09f90281",
   "metadata": {},
   "source": [
    "### Map characters to integers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f56c27f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create encoder and decoder dicts\n",
    "\n",
    "char_int_mapping = dict()\n",
    "\n",
    "for i, c in enumerate(sorted(set(text))):\n",
    "    char_int_mapping[c] = i"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49113d03",
   "metadata": {},
   "source": [
    "### Byte Pair Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ffd8293a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the encoder dict\n",
    "int_char_mapping = dict()\n",
    "\n",
    "for key, item in char_int_mapping.items():\n",
    "    int_char_mapping[item] = key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "55bfcc9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/text_corpus.txt\", encoding=\"utf-8\") as f:\n",
    "    block_size = 10_000\n",
    "    text = f.read()\n",
    "    offset = np.random.randint(0, len(text) - block_size)\n",
    "    text = text[offset:offset+block_size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c0bb1d60",
   "metadata": {},
   "outputs": [],
   "source": [
    "def bpe_tokenize_text(text):\n",
    "    tokenized_text = []\n",
    "    max_pair_len = max([len(k) for k in char_int_mapping.keys()])\n",
    "    \n",
    "    text_cpy = copy.copy(text)\n",
    "    \n",
    "    while text_cpy:\n",
    "        # try to tokenize the different byte combinations from longest to shortest\n",
    "        for i in range(5, 0, -1):\n",
    "            byte = text_cpy[:i]\n",
    "            byte_len = len(byte)\n",
    "            #print(byte_len)\n",
    "            \n",
    "            if byte in char_int_mapping.keys():\n",
    "                #print(\"match:\", byte, \"with len\", byte_len)\n",
    "                \n",
    "                # append the tokenized byte to the list\n",
    "                tokenized_text.append(\n",
    "                    char_int_mapping[byte]\n",
    "                )\n",
    "                \n",
    "                # remove encoded byte from the text\n",
    "                text_cpy = text_cpy[byte_len:]\n",
    "                #print(text_cpy)\n",
    "                #print(tokenized_text)\n",
    "                \n",
    "                break\n",
    "    return tokenized_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1d975a14",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Initialize the Mapping with the N most common Byte Pairs \"\"\"\n",
    "\n",
    "def add_byte_pair_to_mapping():\n",
    "    \n",
    "    # get new text sample block\n",
    "    with open(\"data/text_corpus.txt\", encoding=\"utf-8\") as f:\n",
    "        block_size = 500\n",
    "        text = f.read()\n",
    "        offset = np.random.randint(0, len(text) - block_size)\n",
    "        text = text[offset:offset+block_size]\n",
    "    \n",
    "    # tokenize the text sample\n",
    "    tokenized_text = bpe_tokenize_text(text)\n",
    "\n",
    "    max_occurences = 0\n",
    "    max_pair = None\n",
    "\n",
    "    for byte1 in list(char_int_mapping.keys())[::-1]:\n",
    "        for byte2 in list(char_int_mapping.keys())[::-1]:\n",
    "            \n",
    "            if byte1+byte2 in char_int_mapping.keys():\n",
    "                break\n",
    "              \n",
    "            # encode the bytes\n",
    "            sublist = bpe_tokenize_text(byte1+byte2)\n",
    "            \n",
    "            # get the occurence count of the byte pair (sublist of tokens)\n",
    "            occurence_count = 0\n",
    "            for i in range(len(tokenized_text)):\n",
    "                if tokenized_text[i:i+len(sublist)] == sublist:\n",
    "                    occurence_count += 1\n",
    "            \n",
    "            if occurence_count > max_occurences:\n",
    "                max_occurences = occurence_count\n",
    "                max_pair = byte1 + byte2\n",
    "    \n",
    "    # add the max pair to the mapping as a byte pair\n",
    "    if max_pair is not None:\n",
    "        char_int_mapping[max_pair] = max(char_int_mapping.values()) + 1\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cf69762f",
   "metadata": {},
   "outputs": [],
   "source": [
    "create_bpe = False\n",
    "\n",
    "if create_bpe:\n",
    "    \n",
    "    max_mapping_size = 1_000\n",
    "    for i in range(max_mapping_size - len(char_int_mapping)):\n",
    "        added_pair = add_byte_pair_to_mapping()\n",
    "        if added_pair:\n",
    "            print(i, list(char_int_mapping.keys())[-1])\n",
    "        else:\n",
    "            print(\"---\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ee82473a",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_bpe_dict = False\n",
    "\n",
    "if save_bpe_dict:\n",
    "    with open('bpe_mapping.pkl', 'wb') as f:\n",
    "        pickle.dump(char_int_mapping, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e25d2540",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n"
     ]
    }
   ],
   "source": [
    "load_bpe_dict = True\n",
    "\n",
    "if load_bpe_dict:\n",
    "    with open('bpe_mapping.pkl', 'rb') as f:\n",
    "        char_int_mapping = pickle.load(f)\n",
    "        print(len(char_int_mapping))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5736983c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "534\n"
     ]
    }
   ],
   "source": [
    "\"\"\" Example: Encode the text using a Byte Pair Encoding (BPE) \"\"\"\n",
    "\n",
    "with open(\"data/text_corpus.txt\", encoding=\"utf-8\") as f:\n",
    "    block_size = 1_000\n",
    "    text = f.read()\n",
    "    offset = np.random.randint(0, len(text) - block_size)\n",
    "    text = text[offset:offset+block_size]\n",
    "\n",
    "tokenized_text = bpe_tokenize_text(text)\n",
    "# print(tokenized_text)\n",
    "print(len(tokenized_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "88f3bf27",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the encoder dict (reverse mapping)\n",
    "\n",
    "int_char_mapping = dict()\n",
    "\n",
    "for key, item in char_int_mapping.items():\n",
    "    int_char_mapping[item] = key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b6be7f23",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n"
     ]
    }
   ],
   "source": [
    "vocab_size = len(char_int_mapping)\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "278b0040",
   "metadata": {},
   "source": [
    "### Encode the text dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "dc127ac4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode(string: str) -> List[int]:\n",
    "    tokenized_text = bpe_tokenize_text(string)\n",
    "    result = torch.tensor(tokenized_text, dtype=torch.long, device=device)\n",
    "    return result\n",
    "\n",
    "def decode(data:torch.tensor) -> str:\n",
    "    string = [int_char_mapping[num] for num in data.tolist()]\n",
    "    return \"\".join(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4f3d70ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36.802521199999994"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2M text len -> 114s (~2min) load time\n",
    "\n",
    "114 / 60 * 38_739_496 / 2_000_000 # time in mins for encoding entire text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd657387",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 0-5M done. (769s, 2708019 tokens) -> bpe_dataset1.pkl\n",
    "# 5-10M TODO (837s, 2696978 tokens)-> bpe_dataset2.pkl\n",
    "# 10-15M TODO (595s, 2709264 tokens) -> bpe_dataset3\n",
    "# 15-20M TODO (781s, 2699014 tokens) => bpe_dataset4\n",
    "# 20-25M TODO (764s, 2702280 tokens) => bpe_dataset5\n",
    "# 25-30M TODO (797s, 2711570 tokens) => bpe_dataset6\n",
    "# 30-36RestM TODO (s, tokens) => bpe_dataset7,8,9,10,11,12\n",
    "#\n",
    "# COMBINE FILES TODO."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "1d7650c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Encode the text using a Byte Pair Encoding (BPE) \"\"\"\n",
    "\n",
    "encode_partial_dataset = False\n",
    "\n",
    "if encode_partial_dataset:\n",
    "    \n",
    "    start_time = time.time()\n",
    "    with open(\"data/text_corpus.txt\", encoding=\"utf-8\") as f:\n",
    "        text = f.read()[37_000_000:]\n",
    "    \n",
    "    train_data = encode(text)\n",
    "    print(train_data.shape)\n",
    "    \n",
    "    time.time() - start_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "134b88dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_encoded_dataset = False\n",
    "\n",
    "if save_encoded_dataset:\n",
    "    with open('bpe_dataset_complete.pkl', 'wb') as f:\n",
    "        pickle.dump(train_data, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "0372e30a",
   "metadata": {},
   "outputs": [],
   "source": [
    "load_encoded_dataset = False\n",
    "\n",
    "if load_encoded_dataset:\n",
    "    with open('byte_pair_encoded_dataset.pkl', 'rb') as f:\n",
    "        train_data = pickle.load(f)\n",
    "        print(train_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "d2ae70b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bpe_dataset1.pkl\n",
      "bpe_dataset2.pkl\n",
      "bpe_dataset3.pkl\n",
      "bpe_dataset4.pkl\n",
      "bpe_dataset5.pkl\n",
      "bpe_dataset6.pkl\n",
      "bpe_dataset7.pkl\n",
      "bpe_dataset8.pkl\n",
      "bpe_dataset9.pkl\n",
      "bpe_dataset10.pkl\n",
      "bpe_dataset11.pkl\n",
      "bpe_dataset12.pkl\n"
     ]
    }
   ],
   "source": [
    "\"\"\" Concat all encoded dataset parts \"\"\"\n",
    "train_data = torch.tensor([], dtype=torch.long, device=device)\n",
    "\n",
    "for i in range(1,13):\n",
    "    print(f\"bpe_dataset{i}.pkl\")\n",
    "    with open(f\"bpe_dataset{i}.pkl\", 'rb') as f:\n",
    "        partial_train_data = pickle.load(f)\n",
    "        train_data = torch.cat((train_data, partial_train_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "222f0765",
   "metadata": {},
   "source": [
    "### Train/Test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "8ffb6c69",
   "metadata": {},
   "outputs": [],
   "source": [
    "# N = int(0.9*len(data))\n",
    "# train_data = data[:N]\n",
    "# test_data = data[N:]\n",
    "# \n",
    "# print(len(train_data), len(test_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b72f353e",
   "metadata": {},
   "source": [
    "### Create minibatches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "032acc34",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_batch(split: torch.tensor):\n",
    "    offsets = np.random.randint(0, len(split) - block_size, size=batch_size)\n",
    "    x = torch.stack([split[i:i+block_size] for i in offsets]).to(device)\n",
    "    y = torch.stack([split[i+1:i+block_size+1] for i in offsets]).to(device)\n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "134c4b72",
   "metadata": {},
   "source": [
    "### Build the Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "6ffa661d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyperparameters\n",
    "training_steps = 10_000\n",
    "n_layers = 24\n",
    "embed_dims = 512 # is equivalent to d_model\n",
    "block_size = 256\n",
    "batch_size = 50\n",
    "n_heads = 16\n",
    "dropout = 0.6\n",
    "head_size = embed_dims // n_heads\n",
    "\n",
    "assert head_size * n_heads == embed_dims"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "c0ca7e1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SelfAttentionHead(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.proj_q = nn.Linear(embed_dims, head_size, bias=False)\n",
    "        self.proj_k = nn.Linear(embed_dims, head_size, bias=False)\n",
    "        self.proj_v = nn.Linear(embed_dims, head_size, bias=False)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \"\"\" \n",
    "        Applies masked scaled dot-product attention\n",
    "        between vectors of queries Q, keys K and values V. \n",
    "        \"\"\"\n",
    "        B,T,C = x.shape\n",
    "        \n",
    "        Q = self.proj_q(x)\n",
    "        K = self.proj_k(x)\n",
    "        V = self.proj_v(x)\n",
    "\n",
    "        W = (Q @ K.transpose(-1,-2)) # (B, T, C) @ (B, C, T) ==> (B,T,T)\n",
    "        W /= torch.sqrt(torch.tensor(head_size))\n",
    "        \n",
    "        # mask out forbidden connections\n",
    "        tril = torch.tril(torch.ones((block_size, block_size), device=device))\n",
    "        W = W.masked_fill(tril[:T, :T]==0, float(\"-inf\")) # make smaller so it fits if context < block_size\n",
    "        W = F.softmax(W, dim=1)\n",
    "        W = self.dropout(W)\n",
    "        \n",
    "        out = W @ V\n",
    "        return out # (B,T,C=head_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "9f44fb82",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadAttention(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "        self.heads = nn.ModuleList([SelfAttentionHead() for i in range(n_heads)])\n",
    "        self.proj = nn.Linear(embed_dims, embed_dims, bias=False) # embed_dims = n_heads * head_size\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \n",
    "        out = torch.cat([attn_head(x) for attn_head in self.heads], dim=-1)\n",
    "        out = self.dropout(self.proj(out))\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "74f2907c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Block(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.attn = MultiHeadAttention()\n",
    "        self.ln1 = nn.LayerNorm(embed_dims)\n",
    "        self.ln2 = nn.LayerNorm(embed_dims)\n",
    "        \n",
    "        self.mlp = nn.Sequential(\n",
    "            nn.Linear(embed_dims, 4*embed_dims), # following attention-is-all-you-need paper for num hidden units\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(4*embed_dims, embed_dims),\n",
    "            nn.Dropout(dropout),\n",
    "        )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \n",
    "        # Applies layernorm before self-attention.\n",
    "        # In the attention-is-all-you-need paper they apply it afterwards, \n",
    "        # but apparently pre-ln performs better. pre-ln paper: https://arxiv.org/pdf/2002.04745.pdf\n",
    "        \n",
    "        x = x + self.attn(self.ln1(x)) # (B,embed_dims)\n",
    "        x = x + self.mlp(self.ln2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "fddc8df6",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TransformerLanguageModel(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.token_embedding_table = nn.Embedding(vocab_size, embed_dims)\n",
    "        \n",
    "        # positional encoding\n",
    "        self.pos_embedding_table = nn.Embedding(block_size, embed_dims)\n",
    "        \n",
    "        # transformer layers\n",
    "        self.blocks = nn.Sequential(*[Block() for i in range(n_layers)])\n",
    "        \n",
    "        # output layers\n",
    "        self.lm_head = nn.Linear(embed_dims, vocab_size)\n",
    "        \n",
    "    \n",
    "    def forward(self, context, targets=None):\n",
    "        \n",
    "        B, T = context.shape\n",
    "        \n",
    "        # get the embedding vectors word-to-vec style\n",
    "        token_emb = self.token_embedding_table(context) # (Batch, Time, Channels) ==> [4, 8, 62]\n",
    "        \n",
    "        # add the positional embedding'\n",
    "        pos_emb = self.pos_embedding_table(torch.arange(T, device=device)) # (T,C)\n",
    "        \n",
    "        x = token_emb + pos_emb\n",
    "\n",
    "        # transformer forward pass\n",
    "        x = self.blocks(x)\n",
    "\n",
    "        # output layers\n",
    "        logits = self.lm_head(x)        \n",
    "        \n",
    "        if targets is None:\n",
    "            loss = None\n",
    "        else:\n",
    "            B, T, C = logits.shape\n",
    "            logits = logits.view(B*T,C) # [32,64]\n",
    "            targets = targets.view(B*T) # [32]\n",
    "            loss = F.cross_entropy(logits, targets)\n",
    "        \n",
    "        return logits, loss\n",
    "\n",
    "    def generate(self, context, max_new_tokens, topk=3):\n",
    "        if topk <= 1:\n",
    "            raise Exception(\"topk should be >= 2.\")\n",
    "        B, T = context.shape\n",
    "        \n",
    "        # context: (Batch, Time) ==> [4, 8]\n",
    "        # -> extend context in Time dimension for max_new_tokens\n",
    "        \n",
    "        for _ in range(max_new_tokens):\n",
    "            \n",
    "            # get prediction\n",
    "            # logits, loss = self(xb, yb)\n",
    "            logits, loss = self(context[:,-block_size:])\n",
    "            \n",
    "            # get logits for the last character (for the next token)\n",
    "            logits = logits.view(B,-1,vocab_size) # (B,T,C)\n",
    "            logits = logits[:, -1, :] # (B,C) for only the last character\n",
    "            \n",
    "            if np.random.rand() < 0.8:\n",
    "            \n",
    "                # top-k sampling\n",
    "                topk_logits, topk_vals = torch.topk(logits, topk)\n",
    "                topk_vals = topk_vals.squeeze()\n",
    "                \n",
    "                probs = F.softmax(topk_logits, dim=-1)\n",
    "                idx = torch.multinomial(probs, num_samples=1).item()\n",
    "                next_token = topk_vals[idx]\n",
    "                next_token = next_token.view(1,1)\n",
    "            \n",
    "            else:\n",
    "                \n",
    "                # sample from entire distribution\n",
    "                probs = F.softmax(logits, dim=-1)\n",
    "                next_token = torch.multinomial(probs, num_samples=1)\n",
    "            \n",
    "            \n",
    "            # append next token to the sequence\n",
    "            context = torch.cat((context, next_token), dim=1) # (B,T+1)\n",
    "        \n",
    "        return context\n",
    "    \n",
    "    def generate_to_text(self, context, max_new_tokens, topk=3):\n",
    "        context = self.generate(context, max_new_tokens, topk)\n",
    "        return decode(context[0])\n",
    "\n",
    "# model = TransformerLanguageModel().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "efd3ffc0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = TransformerLanguageModel().to(device)\n",
    "checkpoint = torch.load('weights/0.060485.weights')\n",
    "model.load_state_dict(checkpoint[\"model_state_dict\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "874b1227",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█████████████▋                                                                      | 1626/10000 [27:26<2:21:18,  1.01s/it]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [175], line 17\u001b[0m\n\u001b[1;32m     14\u001b[0m logits, loss \u001b[38;5;241m=\u001b[39m model(xb, yb)\n\u001b[1;32m     16\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad(set_to_none\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m---> 17\u001b[0m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     18\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m     20\u001b[0m losses\u001b[38;5;241m.\u001b[39mappend(loss\u001b[38;5;241m.\u001b[39mitem())\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/torch/_tensor.py:396\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    387\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    388\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m    389\u001b[0m         Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[1;32m    390\u001b[0m         (\u001b[38;5;28mself\u001b[39m,),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    394\u001b[0m         create_graph\u001b[38;5;241m=\u001b[39mcreate_graph,\n\u001b[1;32m    395\u001b[0m         inputs\u001b[38;5;241m=\u001b[39minputs)\n\u001b[0;32m--> 396\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/torch/autograd/__init__.py:173\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    168\u001b[0m     retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[1;32m    170\u001b[0m \u001b[38;5;66;03m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m    171\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m    172\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 173\u001b[0m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m    174\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    175\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "train = True\n",
    "\n",
    "if train:\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=4e-3)\n",
    "    losses = []\n",
    "    \n",
    "    for step in tqdm(range(training_steps)):\n",
    "        \n",
    "        # get a batch\n",
    "        xb, yb = get_batch(train_data)\n",
    "        \n",
    "        # predict and get loss\n",
    "        logits, loss = model(xb, yb)\n",
    "        \n",
    "        optimizer.zero_grad(set_to_none=True)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        losses.append(loss.item())\n",
    "    \n",
    "        if step % 500 == 0 or step == training_steps-1:\n",
    "            if not os.path.exists(\"weights/small_dataset\"):\n",
    "                os.mkdir(\"weights/small_dataset\")\n",
    "            torch.save({\n",
    "                    'epoch': step,\n",
    "                    'model_state_dict': model.state_dict(),\n",
    "                    'loss': loss,\n",
    "                    }, f\"weights/{np.mean(losses[-50:]):5f}.weights\")\n",
    "    \n",
    "    plt.plot(losses)\n",
    "    \n",
    "    # check the final performance\n",
    "    print(np.mean(losses[-50:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "914f1efa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAGxCAYAAAC0mWZZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABKBElEQVR4nO3de1xU1d4/8M/cGWAYBeSmSFjiDa+Z105qpUVqmvVUWt56flaaltlN6zmpnZLqlMdOpWYX044ePedkZlYqpmgdL6llmpZZoZKCKArDdWBmvr8/YLaMoIICe2B/3q/XfjGz9pqZtQac+bj2WnvrRERAREREVM/0ajeAiIiItIkhhIiIiFTBEEJERESqYAghIiIiVTCEEBERkSoYQoiIiEgVDCFERESkCoYQIiIiUgVDCBEREamCIYSuyL59+zB+/HjEx8cjICAAwcHB6NatG1599VWcOXNG7ebVizlz5mD16tVqN+OKXHXVVRg3bpzazfB7Op0Os2bNUrsZRI2GUe0GUMP17rvvYtKkSWjTpg2eeuoptG/fHqWlpdi9ezcWLlyI7du345NPPlG7mXVuzpw5uOuuuzB8+HC1m0J1bPv27WjRooXazSBqNBhC6LJs374dEydOxMCBA7F69WpYLBZl38CBA/HEE09g3bp1F32OoqIiWK3Wum6qXykqKkJAQAB0Op3aTaHL0KtXL7Wb4BcKCwsRGBiodjOoEeDhGLosc+bMgU6nw6JFi3wCiJfZbMbtt9+u3L/qqqswZMgQrFq1Cl27dkVAQABmz54NAPjxxx8xbNgwNG3aFAEBAejSpQuWLFni83wejwcvvvgi2rRpA6vViiZNmqBTp0544403lDqnTp3Cgw8+iNjYWFgsFjRr1gx9+/bFxo0bfZ5r48aNuOmmmxASEoLAwED07dsXX331lU+dWbNmQafT4cCBAxg5ciTsdjsiIyPxwAMPIDc3V6mn0+lQUFCAJUuWQKfTQafToX///gCADz/8EDqdDhs2bMADDzyAZs2aITAwEE6nEx6PB6+++iratm0Li8WCiIgIjBkzBn/88YdPO/r374/ExER8/fXX6NWrF6xWK5o3b44///nPcLvdAAARQevWrXHLLbdU+j3k5+fDbrfjkUceueDv8kKOHTuG+++/HxEREbBYLGjXrh1ef/11eDwen3oLFixA586dERwcDJvNhrZt2+LZZ59V9hcWFuLJJ59UDtmFhoaie/fu+Oc//3nR1/e+f5s2bcKECRMQFhaGkJAQjBkzBgUFBcjMzMTdd9+NJk2aIDo6Gk8++SRKS0t9nuPMmTOYNGkSmjdvDrPZjFatWuG5556D0+lU6nTt2hV/+tOfKr2+2+1G8+bNMWLECKXs/MMx3jZu3rwZEydORHh4OMLCwjBixAicOHHC5/mcTieeeOIJREVFITAwEDfccAP27NlT7UNhs2fPRs+ePREaGoqQkBB069YN77//Pipeg3T48OGIi4ur9DsCgJ49e6Jbt27KfRHB/Pnz0aVLF1itVjRt2hR33XUXfv/9d5/Hef8Gt27dij59+iAwMBAPPPAAAGDlypUYNGgQoqOjYbVa0a5dO0yfPh0FBQWVXv/dd99FQkICLBYL2rdvj+XLl2PcuHG46qqrfOqVlJTgxRdfVP5tNGvWDOPHj8epU6cu+R5RAyRENeRyuSQwMFB69uxZ7cfExcVJdHS0tGrVSj744APZvHmzfPvtt/Lzzz+LzWaTq6++WpYuXSqff/65jBw5UgDIK6+8ojw+OTlZDAaDzJw5U7766itZt26dzJs3T2bNmqXUueWWW6RZs2ayaNEiSU1NldWrV8vzzz8vK1asUOp89NFHotPpZPjw4bJq1Sr57LPPZMiQIWIwGGTjxo1KvZkzZwoAadOmjTz//POSkpIic+fOFYvFIuPHj1fqbd++XaxWq9x2222yfft22b59uxw4cEBERBYvXiwApHnz5vLggw/Kl19+Kf/5z3/E5XLJgw8+KABk8uTJsm7dOlm4cKE0a9ZMYmNj5dSpU8rz9+vXT8LCwiQmJkb+/ve/y/r16+XRRx8VAPLII48o9d544w3R6XTyyy+/+Lzvb7/9tgBQ2nSx38/YsWOV+1lZWdK8eXNp1qyZLFy4UNatWyeTJ08WADJx4kSl3j//+U8BIFOmTJENGzbIxo0bZeHChfLoo48qdR566CEJDAyUuXPnyubNm2Xt2rXy8ssvy5tvvnnRNnnfv/j4eHniiSdkw4YN8sorr4jBYJCRI0dKt27d5MUXX5SUlBR55plnBIC8/vrryuOLioqkU6dOEhQUJK+99pps2LBB/vznP4vRaJTbbrvN570DUOm9++KLLwSArFmzRikDIDNnzqzUxlatWsmUKVNk/fr18t5770nTpk1lwIABPs83cuRI0ev1Mn36dNmwYYPMmzdPYmNjxW63+7z3FzJu3Dh5//33JSUlRVJSUuQvf/mLWK1WmT17tlLn008/FQCSkpLi89iffvpJAMjf//53pWzChAliMpnkiSeekHXr1sny5culbdu2EhkZKZmZmUq9fv36SWhoqMTGxsqbb74pmzdvli1btoiIyF/+8hf529/+Jp9//rmkpqbKwoULJT4+vlLf33nnHQEgd955p6xdu1aWLVsmCQkJEhcXJ3FxcUo9t9stt956qwQFBcns2bMlJSVF3nvvPWnevLm0b99eCgsLL/k+UcPCEEI1lpmZKQDk3nvvrfZj4uLixGAwyKFDh3zK7733XrFYLHLs2DGf8qSkJAkMDJScnBwRERkyZIh06dLloq8RHBwsU6dOveD+goICCQ0NlaFDh/qUu91u6dy5s/To0UMp84aQV1991afupEmTJCAgQDwej1IWFBRU5ZeI9wtqzJgxPuXeL4RJkyb5lO/cuVMAyLPPPquU9evXTwDIp59+6lN3woQJotfr5ejRoyIi4nA4xGazyWOPPeZTr3379pW+EKpyfgiZPn26AJCdO3f61Js4caLodDrl9zh58mRp0qTJRZ87MTFRhg8ffsk2nM/7/k2ZMsWnfPjw4QJA5s6d61PepUsX6datm3J/4cKFAkD+9a9/+dR75ZVXBIBs2LBBREROnz4tZrPZ530XEbn77rslMjJSSktLlbILhZDzf5evvvqqAJCMjAwRETlw4IAAkGeeecannjfEVSeEVOR2u6W0tFReeOEFCQsLU/4eS0tLJTIyUkaNGuVT/+mnnxaz2SynT58WkbLwfH5oExFJT08Xq9UqTz/9tFLm/Rv86quvLtomj8cjpaWlsmXLFgEgP/zwg9LWqKioSv9pOXr0qJhMJp8Q4n0/Pv74Y5+6u3btEgAyf/78arw71JDwcAzVm06dOiEhIcGnbNOmTbjpppsQGxvrUz5u3DgUFhZi+/btAIAePXrghx9+wKRJk7B+/Xo4HI5Kz9+jRw98+OGHePHFF7Fjx45KQ/Pbtm3DmTNnMHbsWLhcLmXzeDy49dZbsWvXrkrDyBUPKXn7UFxcjKysrGr3+8477/S5v3nzZqWP57e/Xbt2lQ4N2Wy2Su0YNWoUPB4Ptm7dqtQZP348PvzwQ6UPmzZtwsGDBzF58uRqt9Vr06ZNaN++PXr06OFTPm7cOIgINm3apLQ5JycHI0eOxKefforTp09Xeq4ePXrgyy+/xPTp05GamoqioqIatWXIkCE+99u1awcAGDx4cKXyo0eP+vQhKCgId911V6U+AFDe57CwMAwdOhRLlixRDmOcPXsWn376KcaMGQOj8dJT56r6OwGgtGfLli0AgLvvvtun3l133VWt5/f25+abb4bdbofBYIDJZMLzzz+P7Oxs5e/RaDTi/vvvx6pVq5TDhm63Gx999BGGDRuGsLAwAMDatWuh0+lw//33+/xbiIqKQufOnZGamurz2k2bNsWNN95YqU2///47Ro0ahaioKKVN/fr1AwD89NNPAIBDhw4ph84qatmyJfr27etTtnbtWjRp0gRDhw71aVeXLl0QFRVVqV3U8DGEUI2Fh4cjMDAQaWlpNXpcdHR0pbLs7Owqy2NiYpT9ADBjxgy89tpr2LFjB5KSkhAWFoabbroJu3fvVh6zcuVKjB07Fu+99x569+6N0NBQjBkzBpmZmQCAkydPAij74DeZTD7bK6+8AhGptKzY+6Ht5Z3/UpMv0vP75+3Thfrt3e8VGRlZqV5UVJTPcwHAlClTkJeXh2XLlgEA3nrrLbRo0QLDhg2rdlsrtrE6v5fRo0fjgw8+wNGjR3HnnXciIiICPXv2REpKivKYv//973jmmWewevVqDBgwAKGhoRg+fDgOHz5crbaEhob63DebzRcsLy4u9ulDVFRUpUnAERERMBqNPu/dAw88gOPHjyvt/uc//wmn01ntZcuX+jvxvtb5v0uj0VjpsVX59ttvMWjQIABlcyv++9//YteuXXjuued8Xsfbl+LiYqxYsQIAsH79emRkZGD8+PFKnZMnT0JEEBkZWenfwo4dOyqFyar+FvLz8/GnP/0JO3fuxIsvvojU1FTs2rULq1atqlbfqyo7efIkcnJyYDabK7UrMzOzypBLDRtXx1CNGQwG3HTTTfjyyy/xxx9/VHvJYlUrQsLCwpCRkVGp3DupLzw8HEDZh/W0adMwbdo05OTkYOPGjXj22Wdxyy23ID09HYGBgQgPD8e8efMwb948HDt2DGvWrMH06dORlZWFdevWKc/15ptvXnCVQ1UflFfq/H57v3QyMjIqvXcnTpxQ2unlDU8VeYNVxS+wa665BklJSXj77beRlJSENWvWYPbs2TAYDDVuc3V/LwAwfvx4jB8/HgUFBdi6dStmzpyJIUOG4JdffkFcXByCgoIwe/ZszJ49GydPnlRGRYYOHYqff/65xm2rSR927twJEfH5HWRlZcHlcvn04ZZbbkFMTAwWL16MW265BYsXL0bPnj3Rvn37WmsLUPa7bN68uVLucrkqhc6qrFixAiaTCWvXrkVAQIBSXtX5abwjWIsXL8ZDDz2ExYsXIyYmRgkxQNnvT6fT4euvv65yYvn5ZVX92920aRNOnDiB1NRUZfQDAHJycnzqVez7+bx/xxXbFRYWdsGVdTabrcpyarg4EkKXZcaMGRARTJgwASUlJZX2l5aW4rPPPrvk89x0003Kh1lFS5cuRWBgYJVhoUmTJrjrrrvwyCOP4MyZMzhy5EilOi1btsTkyZMxcOBAfPfddwCAvn37okmTJjh48CC6d+9e5eb9X3ZNWCyWGo2MeIe1//GPf/iU79q1Cz/99BNuuukmn/K8vDysWbPGp2z58uXQ6/W44YYbfMofe+wx7Nu3D2PHjoXBYMCECRNq0hXFTTfdhIMHDyrvndfSpUuh0+kwYMCASo8JCgpCUlISnnvuOZSUlODAgQOV6kRGRmLcuHEYOXIkDh06hMLCwstqX3X7kJ+fX+mLeunSpcp+L4PBgNGjR2P16tX4+uuvsXv3bmUFSG3w/p5WrlzpU/6f//wHLpfrko/X6XQwGo0+gbKoqAgfffRRlfXHjx+PnTt34ptvvsFnn32m/D14DRkyBCKC48ePV/nvoGPHjtVqE1A5sLzzzjs+99u0aYOoqCj861//8ik/duwYtm3b5lM2ZMgQZGdnw+12V9muNm3aXLJd1LBwJIQuS+/evbFgwQJMmjQJ1157LSZOnIgOHTqgtLQU33//PRYtWoTExEQMHTr0os8zc+ZMrF27FgMGDMDzzz+P0NBQLFu2DJ9//jleffVV2O12AMDQoUORmJiI7t27o1mzZjh69CjmzZuHuLg4tG7dGrm5uRgwYABGjRqFtm3bwmazYdeuXVi3bp2yxDI4OBhvvvkmxo4dizNnzuCuu+5CREQETp06hR9++AGnTp3CggULavxedOzYEampqfjss88QHR0Nm8120Q/LNm3a4MEHH8Sbb74JvV6PpKQkHDlyBH/+858RGxuLxx9/3Kd+WFgYJk6ciGPHjiEhIQFffPEF3n33XUycOBEtW7b0qTtw4EC0b98emzdvVpbXXo7HH38cS5cuxeDBg/HCCy8gLi4On3/+OebPn4+JEycqc3smTJgAq9WKvn37Ijo6GpmZmUhOTobdbsd1110HoGxp6JAhQ9CpUyc0bdoUP/30Ez766CP07t27Ts81MWbMGLz99tsYO3Ysjhw5go4dO+Kbb77BnDlzcNttt+Hmm2/2qf/AAw/glVdewahRo2C1WnHPPffUWls6dOiAkSNH4vXXX4fBYMCNN96IAwcO4PXXX4fdbodef/H/Dw4ePBhz587FqFGj8OCDDyI7OxuvvfZalaMYADBy5EhMmzYNI0eOrPKwUt++ffHggw9i/Pjx2L17N2644QYEBQUhIyMD33zzDTp27IiJEydetE19+vRB06ZN8fDDD2PmzJkwmUxYtmwZfvjhB596er0es2fPxkMPPYS77roLDzzwAHJycjB79mxER0f79P3ee+/FsmXLcNttt+Gxxx5Djx49YDKZ8Mcff2Dz5s0YNmwY7rjjjou2ixoYNWfFUsO3d+9eGTt2rLRs2VLMZrMEBQVJ165d5fnnn5esrCylXlxcnAwePLjK59i/f78MHTpU7Ha7mM1m6dy5syxevNinzuuvvy59+vSR8PBwMZvN0rJlS/nf//1fOXLkiIiIFBcXy8MPPyydOnWSkJAQsVqt0qZNG5k5c6YUFBT4PNeWLVtk8ODBEhoaKiaTSZo3by6DBw+Wf//730od7+qYistlRc6thkhLS/N5D/r27SuBgYECQPr16+dTd9euXZX67Ha75ZVXXpGEhAQxmUwSHh4u999/v6Snp/vU69evn3To0EFSU1Ole/fuYrFYJDo6Wp599lmfVRsVzZo1SwDIjh07qtxflfNXx4iUrV4YNWqUhIWFiclkkjZt2shf//pXcbvdSp0lS5bIgAEDJDIyUsxms8TExMjdd98t+/btU+pMnz5dunfvLk2bNhWLxSKtWrWSxx9/XFmpcSEXev8u9LsZO3asBAUF+ZRlZ2fLww8/LNHR0WI0GiUuLk5mzJghxcXFVb5mnz59BIDcd999Ve7HBVbHnN/GzZs3CwDZvHmzUlZcXCzTpk2TiIgICQgIkF69esn27dvFbrfL448/ftH3QkTkgw8+kDZt2ijvYXJysrz//vuV/h69Ro0aJQCkb9++F33Onj17SlBQkFitVrn66qtlzJgxsnv3bqWO92+wKtu2bZPevXtLYGCgNGvWTP7f//t/8t133wmASv+GFy1aJNdcc42YzWZJSEiQDz74QIYNGyZdu3b1qVdaWiqvvfaadO7cWQICAiQ4OFjatm0rDz30kBw+fPiS7xM1LDqRCme6ISK/0r9/f5w+fRo//vhjtR/TvXt36HQ67Nq1qw5bRrVh27Zt6Nu3L5YtW4ZRo0ap3Zx6lZOTg4SEBAwfPhyLFi1SuzmkEh6OIWoEHA4HfvzxR6xduxZ79uzRxDV7GpqUlBRs374d1157LaxWK3744Qe8/PLLaN26tc9ZWRujzMxMvPTSSxgwYADCwsJw9OhR/O1vf0NeXh4ee+wxtZtHKmIIIWoEvvvuO+UDfubMmbyYnh8KCQnBhg0bMG/ePOTl5SE8PBxJSUlITk72WfHSGFksFhw5cgSTJk3CmTNnlEnnCxcuRIcOHdRuHqmIh2OIiIhIFVyiS0RERKpgCCEiIiJVMIQQERGRKvxuYqrH48GJEydgs9mqPFUwERER+R8RQV5eHmJiYi55Ar6KD6q2+fPnS8eOHcVms4nNZpNevXrJF198oewfO3asAPDZzr9886Wkp6dXeg5u3Lhx48aNW8PYzj/p4sXUaCSkRYsWePnll3HNNdcAAJYsWYJhw4bh+++/V5ZZ3XrrrVi8eLHymJpei8N7gaL09HSEhITU6LFERESkDofDgdjY2BpdaLBGIeT864C89NJLWLBgAXbs2KGEEIvFolxmvDqcTiecTqdyPy8vD0DZmnqGECIiooalJlMpLntiqtvtxooVK1BQUIDevXsr5ampqYiIiEBCQgImTJiArKysiz6P92JX3i02NvZym0REREQNSI1PVrZ//3707t0bxcXFCA4OxvLly3HbbbcBKLtMdXBwMOLi4pCWloY///nPcLlc2LNnzwWv9nj+SIh3OCc3N5cjIURERA2Ew+GA3W6v0fd3jUNISUkJjh07hpycHHz88cd47733sGXLFrRv375S3YyMDMTFxWHFihXVvjbC5XSCiIiI1HU53981XqJrNpuViandu3fHrl278MYbb+Cdd96pVDc6OhpxcXE4fPhwTV+GiIiIGrkrPlmZiPgcTqkoOzsb6enpiI6OvtKXISIiokamRiMhzz77LJKSkhAbG4u8vDysWLECqampWLduHfLz8zFr1izceeediI6OxpEjR/Dss88iPDwcd9xxR121n4iIiBqoGoWQkydPYvTo0cjIyIDdbkenTp2wbt06DBw4EEVFRdi/fz+WLl2KnJwcREdHY8CAAVi5cmWN1gwTERGRNtR4Ympd48RUIiKihudyvr95ATsiIiJSBUMIERERqYIhhIiIiFTBEEJERESq0FQIOZSZh3e3/g6ny612U4iIiDSvxmdMbchumbcVABBkMWJUz5Yqt4aIiEjbNDMSUur2KLczc4tUbAkREREBGgohZwpKlNuxoYEqtoSIiIgADYWQyJAA3Ng2AgDgX6dnIyIi0ibNhBAAMOh1AACXhymEiIhIbdoKIbqyEOLmUAgREZHqtBVCDGUhxMORECIiItVpK4ToeDiGiIjIX2grhOg5EkJEROQvNBlCOBJCRESkPm2FkPLDMR5OTCUiIlKdtkJI+cRUN0dCiIiIVKetEMKJqURERH5DWyGEE1OJiIj8hiZDCEdCiIiI1KfJEMKJqUREROrTZAjhxFQiIiL1aSuE6BhCiIiI/IW2QghHQoiIiPyGJkMIJ6YSERGpT5MhhEt0iYiI1KfJEOLm6hgiIiLVaSuEcGIqERGR39BWCOHEVCIiIr/BEEJERESq0FQI0TOEEBER+Q1NhRAjJ6YSERH5DU2FEE5MJSIi8h/aCiE8HENEROQ3GEKIiIhIFZoKIZyYSkRE5D80FUKMDCFERER+Q1MhRK/j6hgiIiJ/oakQwpEQIiIi/1GjELJgwQJ06tQJISEhCAkJQe/evfHll18q+0UEs2bNQkxMDKxWK/r3748DBw7UeqMvFyemEhER+Y8ahZAWLVrg5Zdfxu7du7F7927ceOONGDZsmBI0Xn31VcydOxdvvfUWdu3ahaioKAwcOBB5eXl10via4sRUIiIi/1GjEDJ06FDcdtttSEhIQEJCAl566SUEBwdjx44dEBHMmzcPzz33HEaMGIHExEQsWbIEhYWFWL58eV21v0Z4OIaIiMh/XPacELfbjRUrVqCgoAC9e/dGWloaMjMzMWjQIKWOxWJBv379sG3btgs+j9PphMPh8NnqCiemEhER+Y8ah5D9+/cjODgYFosFDz/8MD755BO0b98emZmZAIDIyEif+pGRkcq+qiQnJ8NutytbbGxsTZtUbUZDWQjxcCSEiIhIdTUOIW3atMHevXuxY8cOTJw4EWPHjsXBgweV/bry0QYvEalUVtGMGTOQm5urbOnp6TVtUrV5R0JcDCFERESqM9b0AWazGddccw0AoHv37ti1axfeeOMNPPPMMwCAzMxMREdHK/WzsrIqjY5UZLFYYLFYatqMy8LVMURERP7jis8TIiJwOp2Ij49HVFQUUlJSlH0lJSXYsmUL+vTpc6UvUys4MZWIiMh/1Ggk5Nlnn0VSUhJiY2ORl5eHFStWIDU1FevWrYNOp8PUqVMxZ84ctG7dGq1bt8acOXMQGBiIUaNG1VX7a4QTU4mIiPxHjULIyZMnMXr0aGRkZMBut6NTp05Yt24dBg4cCAB4+umnUVRUhEmTJuHs2bPo2bMnNmzYAJvNVieNrylOTCUiIvIfOhH/GhZwOByw2+3Izc1FSEhIrT73r1n5uHnuFtitJvwwc9ClH0BERETVcjnf35q6dox3YipHQoiIiNSnqRDinZjKJbpERETq01QIUa4d419HoIiIiDRJUyHEyMMxREREfkNTIYRnTCUiIvIfmgoh3pEQgKMhREREatNUCNFXCCEcDSEiIlKXpkKIoeJICCenEhERqUpTIaTi4RheP4aIiEhdmgoh3ompAA/HEBERqU1TIYQTU4mIiPyHpkIIJ6YSERH5D02FEKDC9WM4MZWIiEhVmg0hnJhKRESkLu2FEB1DCBERkT/QXAgxciSEiIjIL2guhHgnp3JiKhERkbo0F0I4MZWIiMg/aDaEuNwMIURERGrSXgjRcSSEiIjIH2gvhHBiKhERkV/QbAjhxFQiIiJ1aTaE8HAMERGRujQbQjgxlYiISF3aCyGcmEpEROQXtBdCODGViIjILzCEEBERkSo0F0L0DCFERER+QXMhxMglukRERH5BcyGEE1OJiIj8g/ZCCA/HEBER+QWGECIiIlKF5kIIJ6YSERH5B82FECNDCBERkV/QXAjRl09MdXNiKhERkao0F0I4EkJEROQfNBdCODGViIjIPzCEEBERkSoYQoiIiEgVNQohycnJuO6662Cz2RAREYHhw4fj0KFDPnXGjRsHnU7ns/Xq1atWG30lODGViIjIP9QohGzZsgWPPPIIduzYgZSUFLhcLgwaNAgFBQU+9W699VZkZGQo2xdffFGrjb4SnJhKRETkH4w1qbxu3Tqf+4sXL0ZERAT27NmDG264QSm3WCyIioqqnRbWMp6sjIiIyD9c0ZyQ3NxcAEBoaKhPeWpqKiIiIpCQkIAJEyYgKyvrgs/hdDrhcDh8trrEkRAiIiL/cNkhREQwbdo0XH/99UhMTFTKk5KSsGzZMmzatAmvv/46du3ahRtvvBFOp7PK50lOTobdble22NjYy21StXBiKhERkX+o0eGYiiZPnox9+/bhm2++8Sm/5557lNuJiYno3r074uLi8Pnnn2PEiBGVnmfGjBmYNm2act/hcNRpEOHEVCIiIv9wWSFkypQpWLNmDbZu3YoWLVpctG50dDTi4uJw+PDhKvdbLBZYLJbLacZlMRrKQoiHIyFERESqqlEIERFMmTIFn3zyCVJTUxEfH3/Jx2RnZyM9PR3R0dGX3cja5B0JcTGEEBERqapGc0IeeeQR/OMf/8Dy5cths9mQmZmJzMxMFBUVAQDy8/Px5JNPYvv27Thy5AhSU1MxdOhQhIeH44477qiTDtQUJ6YSERH5hxqNhCxYsAAA0L9/f5/yxYsXY9y4cTAYDNi/fz+WLl2KnJwcREdHY8CAAVi5ciVsNlutNfpKcIkuERGRf6jx4ZiLsVqtWL9+/RU1qK4ZODGViIjIL2ju2jGcmEpEROQfNBdCODGViIjIP2guhHgnpnIkhIiISF2aCyHeiakcCSEiIlKX5kJI+ZQQTkwlIiJSmfZCiKGsy243QwgREZGatBdCuESXiIjIL2guhHBiKhERkX/QXAjhxFQiIiL/oLkQUj4lBB4ejiEiIlKVBkNIWZddnJhKRESkKu2FEE5MJSIi8gvaCyGcmEpEROQXNBtCODGViIhIXRoMIWU/OTGViIhIXRoMIZyYSkRE5A+0F0LKJ6ZyJISIiEhd2gsh5XNC3JwTQkREpCqGECIiIlKFdkMID8cQERGpSrMhhBNTiYiI1KW9EMKJqURERH5BeyGEc0KIiIj8AkMIERERqUK7IYSHY4iIiFSl3RDCialERESq0l4I0XEkhIiIyB9oL4QYOCeEiIjIH2gvhOgYQoiIiPyB9kIIJ6YSERH5Bc2GEBHAw9EQIiIi1WgvhJQfjgE4GkJERKQm7YUQQ4UQwpEQIiIi1WgvhOgYQoiIiPyB9kKI/lwIcTGEEBERqUZzIcSo50gIERGRP9BcCNHrdcpoSKnbo3JriIiItEtzIQQ4NxrCEEJERKQeTYYQk6Gs2y5exI6IiEg1Gg0hZSMhLg9HQoiIiNRSoxCSnJyM6667DjabDRERERg+fDgOHTrkU0dEMGvWLMTExMBqtaJ///44cOBArTb6ShnLR0JKXBwJISIiUkuNQsiWLVvwyCOPYMeOHUhJSYHL5cKgQYNQUFCg1Hn11Vcxd+5cvPXWW9i1axeioqIwcOBA5OXl1XrjL5dJz5EQIiIitRlrUnndunU+9xcvXoyIiAjs2bMHN9xwA0QE8+bNw3PPPYcRI0YAAJYsWYLIyEgsX74cDz30UO21/Ap4R0JKOSeEiIhINVc0JyQ3NxcAEBoaCgBIS0tDZmYmBg0apNSxWCzo168ftm3bVuVzOJ1OOBwOn62ueeeEcHUMERGRei47hIgIpk2bhuuvvx6JiYkAgMzMTABAZGSkT93IyEhl3/mSk5Nht9uVLTY29nKbVG1cHUNERKS+yw4hkydPxr59+/DPf/6z0j5dheuzAGWB5fwyrxkzZiA3N1fZ0tPTL7dJ1Wb0joRwTggREZFqajQnxGvKlClYs2YNtm7dihYtWijlUVFRAMpGRKKjo5XyrKysSqMjXhaLBRaL5XKacdmMeo6EEBERqa1GIyEigsmTJ2PVqlXYtGkT4uPjffbHx8cjKioKKSkpSllJSQm2bNmCPn361E6La4FZmZjKkRAiIiK11Ggk5JFHHsHy5cvx6aefwmazKfM87HY7rFYrdDodpk6dijlz5qB169Zo3bo15syZg8DAQIwaNapOOnA5jJyYSkREpLoahZAFCxYAAPr37+9TvnjxYowbNw4A8PTTT6OoqAiTJk3C2bNn0bNnT2zYsAE2m61WGlwbjJyYSkREpLoahRCRS39p63Q6zJo1C7NmzbrcNtU5M0/bTkREpDpNXjvGOzG1hCMhREREqtFmCPGOhHBOCBERkWo0GUJ4sjIiIiL1aTSElI2ElHAkhIiISDWaDCFcHUNERKQ+TYYQk56rY4iIiNSmyRBiVM6YypEQIiIitWgyhJh42nYiIiLVaTSEcIkuERGR2jQZQrwnKyv18HAMERGRWrQZQjgSQkREpDpNhhAzJ6YSERGpTpMhxDsSwompRERE6tFoCOHJyoiIiNSmyRBi5kgIERGR6jQZQrg6hoiISH3aDCFcHUNERKQ6TYYQE+eEEBERqU7TIaSEIyFERESq0WQIUQ7H8Cq6REREqtFkCDHpeTiGiIhIbZoMITxZGRERkfo0GUJMPG07ERGR6jQaQrhEl4iISG2aDCE8WRkREZH6NBlCzEbOCSEiIlKbJkOIkatjiIiIVKfNEMLVMURERKrTZAhRTtvOOSFERESq0XQIcXsEHgYRIiIiVWgyhHgPxwBAKU/dTkREpApNhhDvadsBTk4lIiJSiyZDSMWREIYQIiIidWgzhOjPhZASrpAhIiJShSZDiE6nO3fqds4JISIiUoUmQwjAE5YRERGpTbMhxDsSwsMxRERE6tBwCOFICBERkZo0G0J46nYiIiJ11TiEbN26FUOHDkVMTAx0Oh1Wr17ts3/cuHHQ6XQ+W69evWqrvbXGOxLCwzFERETqqHEIKSgoQOfOnfHWW29dsM6tt96KjIwMZfviiy+uqJF1IcBkAACUuBhCiIiI1GCs6QOSkpKQlJR00ToWiwVRUVGX3aj6YDGW5a/iUrfKLSEiItKmOpkTkpqaioiICCQkJGDChAnIysq6YF2n0wmHw+Gz1QdvCHFyJISIiEgVtR5CkpKSsGzZMmzatAmvv/46du3ahRtvvBFOp7PK+snJybDb7coWGxtb202qkvdwDEdCiIiI1FHjwzGXcs899yi3ExMT0b17d8TFxeHzzz/HiBEjKtWfMWMGpk2bptx3OBz1EkQ4EkJERKSuWg8h54uOjkZcXBwOHz5c5X6LxQKLxVLXzaj8usaykRCGECIiInXU+XlCsrOzkZ6ejujo6Lp+qRoJMJWPhPBwDBERkSpqPBKSn5+PX3/9VbmflpaGvXv3IjQ0FKGhoZg1axbuvPNOREdH48iRI3j22WcRHh6OO+64o1YbfqU4EkJERKSuGoeQ3bt3Y8CAAcp973yOsWPHYsGCBdi/fz+WLl2KnJwcREdHY8CAAVi5ciVsNlvttboWcCSEiIhIXTUOIf3794fIha+3sn79+itqUH2xeFfHcCSEiIhIFZq9doyyOoYjIURERKrQbAg5d54QjoQQERGpQbMh5Nx5QjgSQkREpAaGEM4JISIiUoV2QwhP205ERKQq7YYQjoQQERGpSsMhhCMhREREatJsCAk0l4WQIq6OISIiUoXmQ0hhiUvllhAREWmTZkOIVQkhPBxDRESkBs2GkCBz2RnrixhCiIiIVKHZEOI9HFNQ4rrotXCIiIiobmg2hHgPx4hwmS4REZEaNBtCAs3nLiDMeSFERET1T7MhxKDXKScsK3ByhQwREVF902wIASqeK4QjIURERPVN4yGk7JAMD8cQERHVP42HkPJzhfBwDBERUb1jCAFHQoiIiNSg6RCinDWVc0KIiIjqnaZDyLmzpvJwDBERUX3TdAjxjoQUODkSQkREVN80HUK4RJeIiEg9Gg8h3iW6PBxDRERU3zQeQng4hoiISC0MIQCKuESXiIio3mk6hARZyg7H5PNwDBERUb1jCAEvYEdERKQGTYcQm3ckpJghhIiIqL5pOoQEB5SHEI6EEBER1Ttth5DykZA8joQQERHVO02HEBtHQoiIiFSj6RASbDEBKAshIqJya4iIiLRF2yGkfCTE7RE4XR6VW0NERKQtmg4hgSYDdLqy25wXQkREVL80HUL0eh2CzZwXQkREpAZNhxCgwjJdjoQQERHVK4YQ7zJdZ6nKLSEiItIWhhCOhBAREamCIcTCOSFERERqqHEI2bp1K4YOHYqYmBjodDqsXr3aZ7+IYNasWYiJiYHVakX//v1x4MCB2mpvreMJy4iIiNRR4xBSUFCAzp0746233qpy/6uvvoq5c+firbfewq5duxAVFYWBAwciLy/vihtbF3jqdiIiInUYa/qApKQkJCUlVblPRDBv3jw899xzGDFiBABgyZIliIyMxPLly/HQQw9dWWvrQMWzphIREVH9qdU5IWlpacjMzMSgQYOUMovFgn79+mHbtm1VPsbpdMLhcPhs9YkTU4mIiNRRqyEkMzMTABAZGelTHhkZqew7X3JyMux2u7LFxsbWZpMuycaJqURERKqok9UxOu+50MuJSKUyrxkzZiA3N1fZ0tPT66JJF+QdCeGcECIiovpV4zkhFxMVFQWgbEQkOjpaKc/Kyqo0OuJlsVhgsVhqsxk1cm6JLk9WRkREVJ9qdSQkPj4eUVFRSElJUcpKSkqwZcsW9OnTpzZfqtaEWMsmpuYUMoQQERHVpxqPhOTn5+PXX39V7qelpWHv3r0IDQ1Fy5YtMXXqVMyZMwetW7dG69atMWfOHAQGBmLUqFG12vDaEhZkBgCcKShRuSVERETaUuMQsnv3bgwYMEC5P23aNADA2LFj8eGHH+Lpp59GUVERJk2ahLNnz6Jnz57YsGEDbDZb7bW6FoUFnwshF5u7QkRERLVLJyKidiMqcjgcsNvtyM3NRUhISJ2/ntPlRpv/WwcA+OH5QbAHmur8NYmIiBqby/n+1vy1YyxGg3Lq9tMFTpVbQ0REpB2aDyEA54UQERGpgSEEQFhw2RLh7HyOhBAREdUXhhCcGwk5nc+RECIiovrCEIKKIyEMIURERPWFIQQV54TwcAwREVF9YQgBEF5+rpCTDoYQIiKi+sIQAiCmiRUAkJFbpHJLiIiItIMhBEDzpmUh5HhOscotISIi0g6GEADNy0dCTuc7UVzqVrk1RERE2sAQAsBuNSHQbAAAZORyNISIiKg+MIQA0Ol0yryQEzmcF0JERFQfGELKeQ/JHD/LEEJERFQfGELKeUdC/jhbqHJLiIiItIEhpNzVzYIAAL+dKlC5JURERNrAEFLu6ohgAMCvWfkqt4SIiEgbGELKXdOsLISknS6Ay+1RuTVERESNH0NIueZNrLCaDChxe3DsDOeFEBER1TWGkHJ6vQ6tyueF8JAMERFR3WMIqeAa77yQUwwhREREdY0hpALvvBCOhBAREdU9hpAKvCMhvzGEEBER1TmGkAqUEHKqACKicmuIiIgaN4aQCuLCgmDQ65DvdCHTwQvZERER1SWGkArMRj3iwgIBcF4IERFRXWMIOQ8npxIREdUPhpDztI2yAQB+ynCo3BIiIqLGjSHkPB2a2wEAPx5nCCEiIqpLDCHnSSwPIb+czENxqVvl1hARETVeDCHnibEHIDTIDJdH8MvJPLWbQ0RE1GgxhJxHp9MpoyG7j5xVuTVERESNF0NIFfpeHQYA+PrwKZVbQkRE1HgxhFThhoRmAIAdv5+B08V5IURERHWBIaQKbaNsiLBZUFTqxrZfs9VuDhERUaPEEFIFnU6HpMQoAMCne4+r3BoiIqLGiSHkAoZ1bQ4A2HDwJPKKS1VuDRERUePDEHIBXWOb4JqIYBSWuLFyV7razSEiImp0GEIuQKfTYcKf4gEAH3yThlK3R+UWERERNS4MIRcxrEtzhAdbcCK3GMt3HlO7OURERI1KrYeQWbNmQafT+WxRUVG1/TL1IsBkwNSbWwMA/rbxF+QUlqjcIiIiosajTkZCOnTogIyMDGXbv39/XbxMvbj3uli0jbIhp7AUf0v5Re3mEBERNRp1EkKMRiOioqKUrVmzZnXxMvXCaNDj+SHtAQBLdxzFtt9Oq9wiIiKixqFOQsjhw4cRExOD+Ph43Hvvvfj9998vWNfpdMLhcPhs/qbPNeG4p3ssRIDHV+5Fdr5T7SYRERE1eLUeQnr27ImlS5di/fr1ePfdd5GZmYk+ffogO7vqM48mJyfDbrcrW2xsbG03qVbMvL09WjULwkmHEw9+tAfFpTydOxER0ZXQiYjU5QsUFBTg6quvxtNPP41p06ZV2u90OuF0nhtZcDgciI2NRW5uLkJCQuqyaTX2a1Y+Rsz/LxzFLiQlRuHtUd2g1+vUbhYREZHqHA4H7HZ7jb6/63yJblBQEDp27IjDhw9Xud9isSAkJMRn81fXRATj3THdYTbo8eWPmZjzxU+o4wxHRETUaNV5CHE6nfjpp58QHR1d1y9VL3q2CsNf/6cTAOC9b9Lwf6t/5InMiIiILkOth5Ann3wSW7ZsQVpaGnbu3Im77roLDocDY8eOre2XUs2wLs0xa2h76HTAsp3HMG7xtzjpKFa7WURERA1KrYeQP/74AyNHjkSbNm0wYsQImM1m7NixA3FxcbX9Uqoa1zcei0Z3R6DZgP/+mo2b527B8p3H4Pbw8AwREVF11PnE1Jq6nIktavrlZB6e+s8+/JCeA6Bs3sjjNycgKTGKk1aJiEgzLuf7myGkFrg9gsX/TcObm35FblEpAKBVeBBG9WyJO7u1QNMgs8otJCIiqlsMISpzFJfig2/S8P7XachzugAARr0Ova8Ow6AOUehzdRhahQdBp+MICRERNS4MIX4i3+nCmr0nsGznURw44XsG2PBgM3rEh6JrbFO0ibKhTZQNETYLgwkRETVoDCF+KO10AdYfyMTmn7PwfXoOSlyVl/ParSYkRAYjIdKG+PAgxIUFoWVoIFo0tSLIYlSh1URERDXDEOLnnC439v2Ri2/TzuDH47k4dDIPR04X4GILakICjIhpYkW0PQDRTayIDglAlD0AYcFmNA00IyzIgtBgM4LMhgY/muJ0uZFf7ILT5YHT5YGIwGzUo3kTa4PvGxFRY8cQ0gAVl7rx+6kC/HIyD4dO5uFYdiGOnSnbvJNcq8Ns1CM00Ay71QS71YSQ8p/nNiPsgb5lIVYTQgJMCDAZ6rCHlbncHvycmYftv2Xj2yNn8NupfJzIKUJxadUnfYsPD8L9veLwP91bICTAVK9tJSKi6mEIaWTyikuRmVuME7nFyMgpUn6ezHPiTIETZ/JLkF1QAmcVh3hqwmzUIyTABFuAEcGW8i3AiCCzAUHl94PKt2CLAVazEYEmAwLNBljNBgSajRVuGxBgNFRanlxc6sbafRn4z5507E3PuWDg8LbHYtBDpwMKS9xwlQ8VBZj06N2qbJLvze0i0cxmuaJ+ExFR7WEI0ajCEhey80twtrAEuUWllTZHUSkcRa7K5cWlqKvfvtVUFkqsJgMCTHpkOZzKiiEAsAUY0a1lU1x/TTjaRYcgNtSKpkFmBJuNPgGmwOnCJ98fx5JtR3A4K9/nNdpG2fCn1uHo1rIp2kWHoGVoIM/NQkSkEoYQqhGPR1BQ4oKj2AVHUSnynS7kF7uQV/6zwOlCvrPsZ0GJC/lONwqcLhSWuFBU6kFRiQuFJW4UlbjLfpa6L/p6LZpaMapnSwxqH4VW4UE1Cgwigp8z87Dp5yysP5CJfX/kVqoTaDYgIdKGdtEhaBdtQ9uoELSJssFu5SEcIqK6xhBCqvJ4BMUut08wKSxxobjUgyCLAR1i7DDU0khFdr4T237LxrbfTuPACQcOZeZd8LBU8yZWJZR0amFHz/gw2AMZTIiIahNDCGmWy+3BkexC/JzpwE8ZDvyckYefM/NwPKeoUl2dDmgfHYJercLQq1UYelwVylBCRHSFGEKIzpNbWIqfMx34OTMPB084sPvoGfx2qsCnDkMJEdGVYwghqoasvGLs/P0MdvyejR2/ZzOUEBHVAoYQostwqVCi1wFdYptgQJsIDGgbgfbRIVyFQ0R0HoYQolpwqVDSzGZB/4RmuLFtBG5IaMZT6xMRgSGEqE5k5BYh9dApbP45C//99TQKSs4tRTYb9bihdbhyArXQILOKLSUiUg9DCFEdK3F5sPvIGWz6OQspP53E0exCZZ9eB1x3VShu6RCFQR0i0aJpoIotJSKqXwwhRPVIRHDoZB42HDiJ9QcyceCEw2d/k0ATmlhNCAu2ICEyGK3Cg9E22oYusU1g4zVwiKiRYQghUlH6mUJsOFgWSHYfOXPBqyPrdEC7qBAkJUZhSOcYxIcH1W9DiYjqAEMIkZ/ILSpFlqMYOUWlOJFThN+y8vHbqQLsP56LY2cKfeomNg/BkE4xGNwxGrGhPIRDRA0TQwhRA5CVV4zUQ6ewdl8G/vvrabgrDJl0iW2CoZ1jcM91sQjmqhsiakAYQogamOx8J9YdyMTaHzKwMy1bOYQTHmzB1Jtb497rYmE06NVtJBFRNTCEEDVgWXnF+HJ/Jhb/Nw1HylfdXN0sCE/d0haD2kfyBGlE5NcYQogagVK3B8t3HsMbXx3GmYISAECr8CA8cH08hndtzsM0ROSXGEKIGhFHcSkWbfkdS7YfQV6xCwBgMujQIz4U/RMi0PvqMLSLDoGBIyRE5AcYQogaoXynC//enY6Pth/F76d9TyEfEmBEj/gw9GoViq4tmyIhMpjnICEiVTCEEDVyv5/Kx6afs7Dtt2x8m3YG+U5XpTrNm1hxdUQwWoUHoXVkMNpE2tA60ga7leGEiOoOQwiRhrjcHhw44VAutPdTRh4yHcUXrB9tD0D76BC0jwlRfsY2DeSEVyKqFQwhRBqXW1iKX7Lyyk+Olo9fTubj8Mk8nMitOpwEW4xoF21D++gQtCsPJgmRNgSYDPXcciJq6BhCiKhKjuJSHMrMw8ETjrItw4FDJ/NQ4vJUqqvXAS1DAxEfHoT48GDENwtCXGggYppY0byJFVYzAwoRVcYQQkTVVur24PdTBTiYkYufMvKUcOJdFnwhoUFmNG9iRUyTAETYAmALMCI4wAhbgAkhAUaEBJgQGmRGWLAZ4cEWjqoQaQRDCBFdERHBqXwnfssqQNrpAqSdzsfvpwqQfrYQx88WoaDEXePntFmMZaMoTcuCS0wTK2KbBqJlaCBiQwPRNNAEnY7zUogaOoYQIqozIgJHsQvHzxbhRE4RjucUIbugBHnFpcgrdik/HcWlOJNfgtP5JShxVz7cc75gi1EZWfGGleZNrIgMCUBIgAkh1rJRFpvFyEm0RH7scr6/eepFIqoWnU4Hu9UEu9WE9jGX/oAREeQ5XchyFON4TrFPeEk/U4hjZwqRledEvtOFQyfzcOhk3iVevyywhASYYCs/7BNirXDfavK5fa6OCYFmAwx6HYx6XflPPSxGPUMNkcoYQoioTuh0urIQEGDCNRG2KusUl7rxx9lCHM8pLgsoFYLKqTwnHMUuOIpKUeL2QATlIy6Vz41yucxGPWyWsjktIQEmNLNZEFG+NbNZ0MwWgIgQC5oFl93n/Bai2sUQQkSqCTAZcE2E7YIhxau41K0c6skrDybe+xVve/edX6+w1A23p/KR5xKXB9muEmRfYjKuV0iAEfZAE6wmA6wmAwJMBgSaDbCay257y33um6v+GVCxzGTgyAxpEkMIEfm9gPIv7WY2y2U/h4jAI4DL44HLLXC6PCgscaHA6Ua+sxRnC0pxKt+JLIcTp/KLkeVwIivPiVPlW4nbUzYyU4sjMeezGPXlfdUrIcZiMiCgQrnFWBZYLBVvGw3l9y+0Xw+TUQ+TXg+jQQeTQQezocLzlT+WE4SpvjGEEJEm6HQ6GHSAQW+AxQgEWcqWG1eHiMBR5EJWXjEcxS4Ul7pRVOJGUWnZVum+cttTXu4q/+mpVLfiuVqcLg+cLg9yi+rqXbg4bwiynBd6yoKLHibl57nbRr0eeh2g1+mg06F800GHstt65Xb5fuigr1ivvKys7rnbVT2H1WxA00Bz+UiTHgHGsrk+Op33Oct+GvQ66HVlm0Gvg0Hv/f3ryuuX1THoyh6r3NbDt45ym+GsrjCEEBFdgk6ngz3QBHtg7V9/x+0RFJe6UVhSFmacLjeKy8OK8rNCWVlQccNZ6jl32+Upv+9WgoyztMJtlxul7rIRoFK3wOUpq1/scqPi+khvfarsXGip4nZ5UKk65FQORXp9eVjSVbh9fp3zA1V5PQAQAQRl4fhymQx6/O2eLrXy3lwJhhAiIhUZ9DoEWYwIstT/x7FIWShxnhdyvGHIG3RK3R4lvJS4PHB5RCkrdXvgESn7YvT+BHzLUPbF6alw+1x52WGyssede4ynwv2y6TyiTFSuGNDcHvF5PbdHlMe7ReDxCDwicHvKys7dLq9TxVyhqrg9Ajf86owWV8RsbOQhZP78+fjrX/+KjIwMdOjQAfPmzcOf/vSnuno5IiKqIZ1OB7NRV7ZKKEDt1qhHCSoi8HhQ4fa5oFIWWsoDTHXqVAg5Sp1Koaiqx57fHt/w5D005D1UdakDRReKTXo/OcRUJyFk5cqVmDp1KubPn4++ffvinXfeQVJSEg4ePIiWLVvWxUsSERFdFr1eBz10PDSggjo5Y2rPnj3RrVs3LFiwQClr164dhg8fjuTk5Is+lmdMJSIiangu5/tbX9uNKCkpwZ49ezBo0CCf8kGDBmHbtm2V6judTjgcDp+NiIiIGr9aDyGnT5+G2+1GZGSkT3lkZCQyMzMr1U9OTobdble22NjY2m4SERER+aFaDyFe56+rFpEq11rPmDEDubm5ypaenl5XTSIiIiI/UuvzcMLDw2EwGCqNemRlZVUaHQEAi8UCi+Xyz4JIREREDVOtj4SYzWZce+21SElJ8SlPSUlBnz59avvliIiIqIGqkxVJ06ZNw+jRo9G9e3f07t0bixYtwrFjx/Dwww/XxcsRERFRA1QnIeSee+5BdnY2XnjhBWRkZCAxMRFffPEF4uLi6uLliIiIqAGqk/OEXAmeJ4SIiKjh8YvzhBARERFVB0MIERERqYIhhIiIiFTBEEJERESq8LuLBnrnyfIaMkRERA2H93u7Jutd/C6E5OXlAQCvIUNERNQA5eXlwW63V6uu3y3R9Xg8OHHiBGw2W5XXmrkSDocDsbGxSE9Pb7TLf9nHxkML/WQfGwct9BHQRj+vpI8igry8PMTExECvr95sD78bCdHr9WjRokWdvkZISEij/QPyYh8bDy30k31sHLTQR0Ab/bzcPlZ3BMSLE1OJiIhIFQwhREREpApNhRCLxYKZM2fCYrGo3ZQ6wz42HlroJ/vYOGihj4A2+lnfffS7ialERESkDZoaCSEiIiL/wRBCREREqmAIISIiIlUwhBAREZEqGEKIiIhIFZoJIfPnz0d8fDwCAgJw7bXX4uuvv1a7SdWWnJyM6667DjabDRERERg+fDgOHTrkU0dEMGvWLMTExMBqtaJ///44cOCATx2n04kpU6YgPDwcQUFBuP322/HHH3/UZ1eqLTk5GTqdDlOnTlXKGkMfjx8/jvvvvx9hYWEIDAxEly5dsGfPHmV/Y+ijy+XC//3f/yE+Ph5WqxWtWrXCCy+8AI/Ho9RpaP3cunUrhg4dipiYGOh0Oqxevdpnf2315+zZsxg9ejTsdjvsdjtGjx6NnJycOu5dmYv1sbS0FM888ww6duyIoKAgxMTEYMyYMThx4oTPczTkPp7voYcegk6nw7x583zK/b2PQPX6+dNPP+H222+H3W6HzWZDr169cOzYMWV/vfVTNGDFihViMpnk3XfflYMHD8pjjz0mQUFBcvToUbWbVi233HKLLF68WH788UfZu3evDB48WFq2bCn5+flKnZdffllsNpt8/PHHsn//frnnnnskOjpaHA6HUufhhx+W5s2bS0pKinz33XcyYMAA6dy5s7hcLjW6dUHffvutXHXVVdKpUyd57LHHlPKG3sczZ85IXFycjBs3Tnbu3ClpaWmyceNG+fXXX5U6Db2PIiIvvviihIWFydq1ayUtLU3+/e9/S3BwsMybN0+p09D6+cUXX8hzzz0nH3/8sQCQTz75xGd/bfXn1ltvlcTERNm2bZts27ZNEhMTZciQIar3MScnR26++WZZuXKl/Pzzz7J9+3bp2bOnXHvttT7P0ZD7WNEnn3winTt3lpiYGPnb3/7ms8/f+yhy6X7++uuvEhoaKk899ZR899138ttvv8natWvl5MmTSp366qcmQkiPHj3k4Ycf9ilr27atTJ8+XaUWXZmsrCwBIFu2bBEREY/HI1FRUfLyyy8rdYqLi8Vut8vChQtFpOxDxGQyyYoVK5Q6x48fF71eL+vWravfDlxEXl6etG7dWlJSUqRfv35KCGkMfXzmmWfk+uuvv+D+xtBHEZHBgwfLAw884FM2YsQIuf/++0Wk4ffz/A/12urPwYMHBYDs2LFDqbN9+3YBID///HMd98rXxb6gvb799lsBoPxnrrH08Y8//pDmzZvLjz/+KHFxcT4hpKH1UaTqft5zzz3Kv8eq1Gc/G/3hmJKSEuzZsweDBg3yKR80aBC2bdumUquuTG5uLgAgNDQUAJCWlobMzEyfPlosFvTr10/p4549e1BaWupTJyYmBomJiX71PjzyyCMYPHgwbr75Zp/yxtDHNWvWoHv37vif//kfREREoGvXrnj33XeV/Y2hjwBw/fXX46uvvsIvv/wCAPjhhx/wzTff4LbbbgPQePrpVVv92b59O+x2O3r27KnU6dWrF+x2u9/1GSj7HNLpdGjSpAmAxtFHj8eD0aNH46mnnkKHDh0q7W8sffz888+RkJCAW265BREREejZs6fPIZv67GejDyGnT5+G2+1GZGSkT3lkZCQyMzNVatXlExFMmzYN119/PRITEwFA6cfF+piZmQmz2YymTZtesI7aVqxYge+++w7JycmV9jWGPv7+++9YsGABWrdujfXr1+Phhx/Go48+iqVLlwJoHH0EgGeeeQYjR45E27ZtYTKZ0LVrV0ydOhUjR44E0Hj66VVb/cnMzERERESl54+IiPC7PhcXF2P69OkYNWqUcqXVxtDHV155BUajEY8++miV+xtDH7OyspCfn4+XX34Zt956KzZs2IA77rgDI0aMwJYtWwDUbz+NV9CXBkWn0/ncF5FKZQ3B5MmTsW/fPnzzzTeV9l1OH/3lfUhPT8djjz2GDRs2ICAg4IL1GnIfPR4Punfvjjlz5gAAunbtigMHDmDBggUYM2aMUq8h9xEAVq5ciX/84x9Yvnw5OnTogL1792Lq1KmIiYnB2LFjlXoNvZ/nq43+VFXf3/pcWlqKe++9Fx6PB/Pnz79k/YbSxz179uCNN97Ad999V+O2NJQ+AlAmiA8bNgyPP/44AKBLly7Ytm0bFi5ciH79+l3wsXXRz0Y/EhIeHg6DwVApmWVlZVX6n4u/mzJlCtasWYPNmzejRYsWSnlUVBQAXLSPUVFRKCkpwdmzZy9YR0179uxBVlYWrr32WhiNRhiNRmzZsgV///vfYTQalTY25D5GR0ejffv2PmXt2rVTZqQ3ht8jADz11FOYPn067r33XnTs2BGjR4/G448/roxwNZZ+etVWf6KionDy5MlKz3/q1Cm/6XNpaSnuvvtupKWlISUlRRkFARp+H7/++mtkZWWhZcuWymfQ0aNH8cQTT+Cqq64C0PD7CJR9JxqNxkt+FtVXPxt9CDGbzbj22muRkpLiU56SkoI+ffqo1KqaERFMnjwZq1atwqZNmxAfH++zPz4+HlFRUT59LCkpwZYtW5Q+XnvttTCZTD51MjIy8OOPP/rF+3DTTTdh//792Lt3r7J1794d9913H/bu3YtWrVo1+D727du30tLqX375BXFxcQAax+8RAAoLC6HX+360GAwG5X9gjaWfXrXVn969eyM3NxfffvutUmfnzp3Izc31iz57A8jhw4exceNGhIWF+exv6H0cPXo09u3b5/MZFBMTg6eeegrr168H0PD7CJR9J1533XUX/Syq135WewprA+Zdovv+++/LwYMHZerUqRIUFCRHjhxRu2nVMnHiRLHb7ZKamioZGRnKVlhYqNR5+eWXxW63y6pVq2T//v0ycuTIKpcItmjRQjZu3Cjfffed3HjjjX61tPN8FVfHiDT8Pn777bdiNBrlpZdeksOHD8uyZcskMDBQ/vGPfyh1GnofRUTGjh0rzZs3V5borlq1SsLDw+Xpp59W6jS0fubl5cn3338v33//vQCQuXPnyvfff6+sDKmt/tx6663SqVMn2b59u2zfvl06duxYb0s7L9bH0tJSuf3226VFixayd+9en88hp9PZKPpYlfNXx4j4fx9FLt3PVatWiclkkkWLFsnhw4flzTffFIPBIF9//XW991MTIURE5O2335a4uDgxm83SrVs3ZXlrQwCgym3x4sVKHY/HIzNnzpSoqCixWCxyww03yP79+32ep6ioSCZPniyhoaFitVplyJAhcuzYsXruTfWdH0IaQx8/++wzSUxMFIvFIm3btpVFixb57G8MfXQ4HPLYY49Jy5YtJSAgQFq1aiXPPfecz5dVQ+vn5s2bq/w3OHbsWBGpvf5kZ2fLfffdJzabTWw2m9x3331y9uxZ1fuYlpZ2wc+hzZs3N4o+VqWqEOLvfRSpXj/ff/99ueaaayQgIEA6d+4sq1ev9nmO+uqnTkSk+uMmRERERLWj0c8JISIiIv/EEEJERESqYAghIiIiVTCEEBERkSoYQoiIiEgVDCFERESkCoYQIiIiUgVDCBEREamCIYSIiIhUwRBCREREqmAIISIiIlX8f/j0F2FhKEYZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if train:\n",
    "    \n",
    "    rolling_length = 50\n",
    "    \n",
    "    losses_mov_avg = (\n",
    "        np.convolve(\n",
    "            np.array(losses),\n",
    "            np.ones(rolling_length),\n",
    "            mode=\"valid\",\n",
    "        )\n",
    "        / rolling_length\n",
    "    )\n",
    "    \n",
    "    plt.title(\"Crossentropy loss moving average\")\n",
    "    plt.plot(losses_mov_avg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "cf065d0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prompt_model(model, prompt=None, max_new_tokens=200, topk=3):\n",
    "    model.eval();\n",
    "    \n",
    "    # pad the prompt so that it has at least the length of block_size\n",
    "    # and so that the model is guided into giving an answer\n",
    "    pre_padding = \"Ultimately, what we need to remember is that every person has a unique perspective and experiences that shape their opinions and actions. We must strive to approach each conversation with empathy and understanding, and be willing to listen and learn from one another. Only then can we truly progress and create a better world for ourselves and future generations.\\n Unfortunately we only have time for one final question. There is one thing that I have to ask you. \\n\"\n",
    "    post_padding = \"\\nThat's such an interesting question. I think that \"\n",
    "    \n",
    "    if prompt is None or prompt == \"\":\n",
    "        # give zero context\n",
    "        prompt = \"\"\n",
    "    \n",
    "    context = pre_padding + prompt + post_padding\n",
    "    \n",
    "    # convert prompt to a batched tensor\n",
    "    context_tensor = encode(context)\n",
    "    context_tensor = torch.unsqueeze(context_tensor, 0)\n",
    "    \n",
    "    output = model.generate_to_text(context_tensor, max_new_tokens, topk)\n",
    "    answer = output\n",
    "    return answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "61e6c6f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ultimately, what we need to remember is that every person has a unique perspective and experiences that shape their opinions and actions. We must strive to approach each conversation with empathy and understanding, and be willing to listen and learn from one another. Only then can we truly progress and create a better world for ourselves and future generations.\n",
      " Unfortunately we only have time for one final question. There is one thing that I have to ask you. \n",
      "What do you think about Elon?\n",
      "That's such an interesting question. I think that it's this this a sirasionart of fureareararareartionasionacatias it it with a saicacicactiople it's like this uppaseareampicicici\n"
     ]
    }
   ],
   "source": [
    "prompt = \"What do you think about Elon?\"\n",
    "answer = prompt_model(model, prompt, max_new_tokens=100, topk=2)\n",
    "print(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "c6082f37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([50, 256])"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_batch(train_data)[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "e7ca7c62",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Ultimately, what we need to remember is that every person has a unique perspective and experiences that shape their opinions and actions. We must strive to approach each conversation with empathy and understanding, and be willing to listen and learn from one another. Only then can we truly progress and create a better world for ourselves and future generations.\\n Unfortunately we only have time for one final question. There is one thing that I have to ask you. \\ns a sign of the times being good,\\n is that it's just entertaining\\n on some unpleasant mammalian level to watch,\\n to disagree with others.\\n And Twitter and Facebook are actually taking advantage\\n of that in a sense because it brings you back\\n to the platform and they're advertiser driven,\\n so they make a lot of money.\\n So you go back and you click.\\n Love doesn't sell quite as well in terms of advertisement.\\n It doesn't.\\n So you've started your career\\n at NASA Jet \\nThat's such an interesting question. I think that that's most intiditiont, you know, this that you have that eally setuntake that is a that it pasuntistis,\\n things that wen't have to exple that that you have that eally be all ach that that you seally have to tenenti\""
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt_model(model, decode(get_batch(train_data)[0][0]), max_new_tokens=100, topk=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30153611",
   "metadata": {},
   "source": [
    "### Parameter count and losses for different models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "86d86645",
   "metadata": {},
   "outputs": [],
   "source": [
    "# report the parameters and loss with 5k training steps:\n",
    "\n",
    "# --- tiny dataset (10k tokens) ---\n",
    "# bigram model: 3844 ==> loss to 2.5\n",
    "# with one multihead attention (5 heads): 53262 ==> loss to 0.35\n",
    "# with one multihead attention (5 heads) and residual connection: 53262 ==> loss to 0.30\n",
    "# with one full block: 134162 ==> loss to 0.24\n",
    "# with 5 blocks: 617762 ==> loss to 0.12\n",
    "\n",
    "# --- large dataset (38M tokens)--- \n",
    "# bigger model: 5 015 318 (5M) ==> loss to 0.0116\n",
    "# large model: 75 892 886 (75M) ==> loss to 0.0074 (4.5h training = 15k steps)\n",
    "# large model: 75 892 886 (75M) with longer training ==> loss to 0.0067 (20k steps with 3e-4 + 5k steps with 1e-4 lr)\n",
    "\n",
    "# --- byte-pair encoding\n",
    "# 3k steps: 0.06048505887389183"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a03abbd8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}