Lecture6_Recurrent_Neural_Network.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "[FULL CODE] Lecture 6_Recurrent Neural Network",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMGpld/29q36js+OrB3Q3jH",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "2b783b6ecb9343bcb0a68800026661ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3572fcc9e82c455b8ca5e84515f919a4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_809a7181f8b044e3aacd57d5a0847936",
              "IPY_MODEL_1c57408da40e475a9f30d3b27c6ceb38"
            ]
          }
        },
        "3572fcc9e82c455b8ca5e84515f919a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "809a7181f8b044e3aacd57d5a0847936": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_63ebdde702294f4d816f27a9c534d8fa",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 100,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e77f7ae79a5e41cc89e7c94fe0391c29"
          }
        },
        "1c57408da40e475a9f30d3b27c6ceb38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_879c228a7f23411f8b7ac4ce90a3e694",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100/100 [00:34&lt;00:00,  2.91it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a33530a4c2c846ab8dbabf96a3c37aad"
          }
        },
        "63ebdde702294f4d816f27a9c534d8fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e77f7ae79a5e41cc89e7c94fe0391c29": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "879c228a7f23411f8b7ac4ce90a3e694": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a33530a4c2c846ab8dbabf96a3c37aad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cotton-ahn/TUM_20WS_Intro2Deep/blob/main/Lecture6_Recurrent_Neural_Network.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rQO90yEK24X8"
      },
      "source": [
        "import numpy as np\r\n",
        "import torch\r\n",
        "import torch.nn as nn\r\n",
        "from torch.utils.data import Dataset, DataLoader\r\n",
        "\r\n",
        "from tqdm.notebook import tqdm"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xcdpjvBb2psF",
        "outputId": "661e4ef3-8672-45fd-d700-4dfb6c068764"
      },
      "source": [
        "!wget http://www.cs.cmu.edu/afs/cs/project/ai-repository/ai/areas/nlp/corpora/names/female.txt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-12-18 12:00:06--  http://www.cs.cmu.edu/afs/cs/project/ai-repository/ai/areas/nlp/corpora/names/female.txt\n",
            "Resolving www.cs.cmu.edu (www.cs.cmu.edu)... 128.2.42.95\n",
            "Connecting to www.cs.cmu.edu (www.cs.cmu.edu)|128.2.42.95|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 35751 (35K) [text/plain]\n",
            "Saving to: ‘female.txt’\n",
            "\n",
            "female.txt          100%[===================>]  34.91K   167KB/s    in 0.2s    \n",
            "\n",
            "2020-12-18 12:00:07 (167 KB/s) - ‘female.txt’ saved [35751/35751]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M494X00Js1ig"
      },
      "source": [
        "device = 'cuda'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "toeJz1ewgdmF",
        "outputId": "b7742b17-8f4b-483d-ea4f-ca63feebfa76"
      },
      "source": [
        "with open('female.txt', 'r') as f:\r\n",
        "    lines = f.readlines()\r\n",
        "names = []\r\n",
        "max_len = 0\r\n",
        "for l in lines[6:]:\r\n",
        "    curr_name = l[:-1].lower()\r\n",
        "    if curr_name.isalpha():\r\n",
        "        names.append(l[:-1].lower())\r\n",
        "        max_len = max(len(names[-1]), max_len)\r\n",
        "max_len += 1 # consider the 'EOS' (end of signal)\r\n",
        "print('Maximum Length : ' + str(max_len))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Maximum Length : 14\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AclIiMX9f4ta"
      },
      "source": [
        "class NameDataset(Dataset):\r\n",
        "    def __init__(self, names, max_len):\r\n",
        "        self.names = names\r\n",
        "        self.max_len = max_len\r\n",
        "        self.a_order = ord('a') \r\n",
        "        self.z_order = ord('z') \r\n",
        "        self.num_classes = 26 + 1 # a-z + include the end of signal\r\n",
        "\r\n",
        "    def __len__(self):\r\n",
        "        return len(names)\r\n",
        "\r\n",
        "    def __getitem__(self, idx):\r\n",
        "        padding_name = [self.num_classes-1 for _ in range(self.max_len)]\r\n",
        "        curr_name = [ord(n)-self.a_order for n in names[idx]]\r\n",
        "        padding_name[:len(curr_name)] = curr_name\r\n",
        "        \r\n",
        "        # Slide the input to make a output\r\n",
        "        sample = dict()\r\n",
        "        sample['input'] = torch.LongTensor(padding_name[:-1]) # h y e m i n  -1 -1 -1\r\n",
        "        sample['output'] = torch.LongTensor(padding_name[1:]) # y e m i n -1 -1 -1 -1\r\n",
        "        sample['length'] = len(names[idx])\r\n",
        "        sample['original'] = names[idx]\r\n",
        "\r\n",
        "        return sample"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6eM0TZVpiuCf"
      },
      "source": [
        "batch_size = 64\r\n",
        "dataset = NameDataset(names, max_len)\r\n",
        "dataloader = DataLoader(dataset, batch_size=64, shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6yQkTkHMk9m1",
        "outputId": "8085da9e-17af-4c1c-a75a-7e14b07f6a95"
      },
      "source": [
        "sample = next(iter(dataloader))\r\n",
        "print(sample['input'][0])\r\n",
        "print(sample['output'][0])\r\n",
        "print(sample['length'][0])\r\n",
        "print(sample['original'][0])\r\n",
        "print(sample['input'].shape, sample['output'].shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([11,  0, 20, 17,  4, 13, 26, 26, 26, 26, 26, 26, 26])\n",
            "tensor([ 0, 20, 17,  4, 13, 26, 26, 26, 26, 26, 26, 26, 26])\n",
            "tensor(6)\n",
            "lauren\n",
            "torch.Size([64, 13]) torch.Size([64, 13])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aw0rHh8hlJ4z",
        "outputId": "54dce497-9c07-4c20-93cc-a157922b0cf8"
      },
      "source": [
        "# This could be useful with variable lengths\r\n",
        "total_lengths = sample['length']\r\n",
        "sort_length, sort_idx = torch.sort(total_lengths, descending=True)\r\n",
        "sort_input = sample['input'][sort_idx]\r\n",
        "sort_output = sample['output'][sort_idx]\r\n",
        "print(sort_length)\r\n",
        "print(sort_input.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([10, 10, 10,  9,  9,  8,  8,  8,  8,  8,  8,  8,  8,  7,  7,  7,  7,  7,\n",
            "         7,  7,  7,  6,  6,  6,  6,  6,  6,  6,  6,  6,  6,  6,  6,  6,  6,  6,\n",
            "         6,  6,  6,  6,  6,  6,  6,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,\n",
            "         5,  5,  5,  4,  4,  4,  4,  4,  3,  3])\n",
            "torch.Size([64, 13])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j_i2kCPlnXO7"
      },
      "source": [
        "class RNNmodel(nn.Module):\r\n",
        "    def __init__(self, lstm_dim=256, num_classes=dataset.num_classes, max_len=max_len):\r\n",
        "        super(RNNmodel, self).__init__()\r\n",
        "        self.lstm_dim = lstm_dim\r\n",
        "        self.num_classes = num_classes\r\n",
        "        self.max_len = max_len\r\n",
        "        self.char_embedding = nn.Embedding(num_embeddings=num_classes, \r\n",
        "                                           embedding_dim=lstm_dim)\r\n",
        "        self.lstm = nn.LSTM(input_size=lstm_dim, \r\n",
        "                            hidden_size=lstm_dim,\r\n",
        "                            num_layers=1, \r\n",
        "                            batch_first=True,\r\n",
        "                            )\r\n",
        "        \r\n",
        "        self.out_linear = nn.Linear(lstm_dim, num_classes)\r\n",
        "\r\n",
        "    def forward(self, sort_input, sort_output, sort_length):\r\n",
        "        ## originally, recommended to use torch.nn.utils.rnn.pack_padded_sequence,when we have variable lengths\r\n",
        "        ## but in this case, I just neglected it because beginners can be more confused with this\r\n",
        "        lstm_input = self.char_embedding(sort_input)\r\n",
        "        lstm_out, (h, c) = self.lstm(lstm_input)\r\n",
        "        out = self.out_linear(lstm_out)\r\n",
        "\r\n",
        "        return out\r\n",
        "\r\n",
        "    def test(self, start_char):\r\n",
        "        generated_name = list()\r\n",
        "        generated_name.append(start_char)\r\n",
        "\r\n",
        "        start_order = torch.LongTensor([ord(start_char)]).to(device) - ord('a')\r\n",
        "        start_order = start_order.reshape(1, 1)\r\n",
        "        cnt = 0\r\n",
        "\r\n",
        "        while cnt <= self.max_len:\r\n",
        "            curr_embed = self.char_embedding(start_order)\r\n",
        "            if cnt == 0:\r\n",
        "                lstm_out, (h, c) = self.lstm(curr_embed)\r\n",
        "            else:\r\n",
        "                lstm_out, (h, c) = self.lstm(curr_embed, (h, c))\r\n",
        "            out = self.out_linear(lstm_out)\r\n",
        "\r\n",
        "            sample_next = torch.distributions.Categorical(logits = out[0, 0, :]).sample().item()\r\n",
        "            if sample_next == 26:\r\n",
        "                break\r\n",
        "            else:\r\n",
        "                generated_name.append(chr(ord('a')+sample_next))\r\n",
        "                sample_next = torch.LongTensor([sample_next]).to(device)\r\n",
        "                start_order = sample_next.reshape(1, 1)\r\n",
        "\r\n",
        "                cnt += 1\r\n",
        "\r\n",
        "        return ''.join(generated_name)\r\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XPuT8GV8pJLX"
      },
      "source": [
        "model = RNNmodel()\r\n",
        "model = model.to(device)\r\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Q-ep_yepepC"
      },
      "source": [
        "def train(model, optimizer, sample):\r\n",
        "    optimizer.zero_grad()\r\n",
        "    criteria = nn.CrossEntropyLoss()\r\n",
        "\r\n",
        "    total_lengths = sample['length']\r\n",
        "    sort_length, sort_idx = torch.sort(total_lengths, descending=True)\r\n",
        "\r\n",
        "    sort_input = sample['input'][sort_idx].to(device)\r\n",
        "    sort_output = sample['output'][sort_idx].to(device)\r\n",
        "    sort_length = sort_length.to(device)\r\n",
        "\r\n",
        "    pred = model(sort_input, sort_output, sort_length) # B T C\r\n",
        "    B, T, C = pred.shape\r\n",
        "    \r\n",
        "    curr_loss = criteria(pred.reshape(B*T, C), sort_output.reshape(B*T))\r\n",
        "\r\n",
        "    curr_loss.backward()\r\n",
        "    optimizer.step()\r\n",
        "\r\n",
        "    return curr_loss.item()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "2b783b6ecb9343bcb0a68800026661ba",
            "3572fcc9e82c455b8ca5e84515f919a4",
            "809a7181f8b044e3aacd57d5a0847936",
            "1c57408da40e475a9f30d3b27c6ceb38",
            "63ebdde702294f4d816f27a9c534d8fa",
            "e77f7ae79a5e41cc89e7c94fe0391c29",
            "879c228a7f23411f8b7ac4ce90a3e694",
            "a33530a4c2c846ab8dbabf96a3c37aad"
          ]
        },
        "id": "UaMHeot3wzm_",
        "outputId": "b787399f-e792-4d60-b8a9-da2b4c37f4f5"
      },
      "source": [
        "max_epoch = 100\r\n",
        "for epoch in tqdm(range(max_epoch)):\r\n",
        "    total_loss = 0.0\r\n",
        "    for sample in dataloader:\r\n",
        "        curr_loss = train(model, optimizer, sample)\r\n",
        "        total_loss += curr_loss / len(dataloader)\r\n",
        "\r\n",
        "    start_char = chr(np.random.randint(ord('a'), ord('z')))\r\n",
        "    print('[EPOCH {}] TRAIN LOSS: {}, SAMPLED NAME: {}'.format(epoch,\r\n",
        "                                                               total_loss, \r\n",
        "                                                               model.test(start_char)))\r\n",
        "\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2b783b6ecb9343bcb0a68800026661ba",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "[EPOCH 0] TRAIN LOSS: 1.8830124307901437, SAMPLED NAME: yzvvgtcwbdbapnfi\n",
            "[EPOCH 1] TRAIN LOSS: 1.2530618921304353, SAMPLED NAME: yfndnha\n",
            "[EPOCH 2] TRAIN LOSS: 1.1317325356679084, SAMPLED NAME: wiilit\n",
            "[EPOCH 3] TRAIN LOSS: 1.0657492180665333, SAMPLED NAME: xpeh\n",
            "[EPOCH 4] TRAIN LOSS: 1.021871271805885, SAMPLED NAME: ninse\n",
            "[EPOCH 5] TRAIN LOSS: 0.9894907497442683, SAMPLED NAME: qqesisolle\n",
            "[EPOCH 6] TRAIN LOSS: 0.9635974642557978, SAMPLED NAME: qobuyna\n",
            "[EPOCH 7] TRAIN LOSS: 0.9425856471061705, SAMPLED NAME: yna\n",
            "[EPOCH 8] TRAIN LOSS: 0.9253952258672469, SAMPLED NAME: kopina\n",
            "[EPOCH 9] TRAIN LOSS: 0.9099500721845873, SAMPLED NAME: brijalone\n",
            "[EPOCH 10] TRAIN LOSS: 0.8973082074752216, SAMPLED NAME: w\n",
            "[EPOCH 11] TRAIN LOSS: 0.8853509517816397, SAMPLED NAME: uktie\n",
            "[EPOCH 12] TRAIN LOSS: 0.8750358804678307, SAMPLED NAME: bgarra\n",
            "[EPOCH 13] TRAIN LOSS: 0.8649777708909453, SAMPLED NAME: ynemena\n",
            "[EPOCH 14] TRAIN LOSS: 0.8561465793695203, SAMPLED NAME: neruwsa\n",
            "[EPOCH 15] TRAIN LOSS: 0.847805723165854, SAMPLED NAME: fumarita\n",
            "[EPOCH 16] TRAIN LOSS: 0.840166321167579, SAMPLED NAME: pa\n",
            "[EPOCH 17] TRAIN LOSS: 0.832294823267521, SAMPLED NAME: vachryne\n",
            "[EPOCH 18] TRAIN LOSS: 0.8250684730517561, SAMPLED NAME: quzinen\n",
            "[EPOCH 19] TRAIN LOSS: 0.8181619605956937, SAMPLED NAME: mami\n",
            "[EPOCH 20] TRAIN LOSS: 0.8115072005834333, SAMPLED NAME: frda\n",
            "[EPOCH 21] TRAIN LOSS: 0.8051936114445711, SAMPLED NAME: il\n",
            "[EPOCH 22] TRAIN LOSS: 0.7995012685274465, SAMPLED NAME: margianne\n",
            "[EPOCH 23] TRAIN LOSS: 0.792792749710572, SAMPLED NAME: rillle\n",
            "[EPOCH 24] TRAIN LOSS: 0.7872253090907366, SAMPLED NAME: k\n",
            "[EPOCH 25] TRAIN LOSS: 0.7818051805863013, SAMPLED NAME: sussal\n",
            "[EPOCH 26] TRAIN LOSS: 0.7760848899682363, SAMPLED NAME: nina\n",
            "[EPOCH 27] TRAIN LOSS: 0.7711567168052379, SAMPLED NAME: lise\n",
            "[EPOCH 28] TRAIN LOSS: 0.7657666924672247, SAMPLED NAME: qucinde\n",
            "[EPOCH 29] TRAIN LOSS: 0.7610198175295804, SAMPLED NAME: jatola\n",
            "[EPOCH 30] TRAIN LOSS: 0.7564570246598663, SAMPLED NAME: arylee\n",
            "[EPOCH 31] TRAIN LOSS: 0.7518286086045778, SAMPLED NAME: byll\n",
            "[EPOCH 32] TRAIN LOSS: 0.7471545460896618, SAMPLED NAME: frosela\n",
            "[EPOCH 33] TRAIN LOSS: 0.7425589110606758, SAMPLED NAME: winga\n",
            "[EPOCH 34] TRAIN LOSS: 0.7384395202000934, SAMPLED NAME: orissa\n",
            "[EPOCH 35] TRAIN LOSS: 0.7342337308785857, SAMPLED NAME: harmia\n",
            "[EPOCH 36] TRAIN LOSS: 0.7296120463273463, SAMPLED NAME: coritta\n",
            "[EPOCH 37] TRAIN LOSS: 0.725595070765569, SAMPLED NAME: yangley\n",
            "[EPOCH 38] TRAIN LOSS: 0.7220079195805085, SAMPLED NAME: inola\n",
            "[EPOCH 39] TRAIN LOSS: 0.7177887459596, SAMPLED NAME: katherina\n",
            "[EPOCH 40] TRAIN LOSS: 0.7144224032377587, SAMPLED NAME: dealina\n",
            "[EPOCH 41] TRAIN LOSS: 0.7103537412790151, SAMPLED NAME: parla\n",
            "[EPOCH 42] TRAIN LOSS: 0.7071195634511802, SAMPLED NAME: unnie\n",
            "[EPOCH 43] TRAIN LOSS: 0.7033007611066868, SAMPLED NAME: nerdie\n",
            "[EPOCH 44] TRAIN LOSS: 0.699948883209473, SAMPLED NAME: fitobie\n",
            "[EPOCH 45] TRAIN LOSS: 0.6967069651836004, SAMPLED NAME: licins\n",
            "[EPOCH 46] TRAIN LOSS: 0.6933302000547067, SAMPLED NAME: fedie\n",
            "[EPOCH 47] TRAIN LOSS: 0.6900569529105453, SAMPLED NAME: xenni\n",
            "[EPOCH 48] TRAIN LOSS: 0.6867835300090988, SAMPLED NAME: madol\n",
            "[EPOCH 49] TRAIN LOSS: 0.6837741679106005, SAMPLED NAME: quette\n",
            "[EPOCH 50] TRAIN LOSS: 0.6809374781755299, SAMPLED NAME: ura\n",
            "[EPOCH 51] TRAIN LOSS: 0.6781047735458764, SAMPLED NAME: inora\n",
            "[EPOCH 52] TRAIN LOSS: 0.6745225794804403, SAMPLED NAME: miselle\n",
            "[EPOCH 53] TRAIN LOSS: 0.672203873976683, SAMPLED NAME: deliandra\n",
            "[EPOCH 54] TRAIN LOSS: 0.6689148850930045, SAMPLED NAME: uldika\n",
            "[EPOCH 55] TRAIN LOSS: 0.6666179857192897, SAMPLED NAME: margri\n",
            "[EPOCH 56] TRAIN LOSS: 0.6638147227275063, SAMPLED NAME: nikil\n",
            "[EPOCH 57] TRAIN LOSS: 0.6614588094063293, SAMPLED NAME: coretta\n",
            "[EPOCH 58] TRAIN LOSS: 0.658344934384028, SAMPLED NAME: debbe\n",
            "[EPOCH 59] TRAIN LOSS: 0.65584884927823, SAMPLED NAME: joyne\n",
            "[EPOCH 60] TRAIN LOSS: 0.6534307988790364, SAMPLED NAME: uregan\n",
            "[EPOCH 61] TRAIN LOSS: 0.6511718088235613, SAMPLED NAME: katharina\n",
            "[EPOCH 62] TRAIN LOSS: 0.6488180206372189, SAMPLED NAME: valena\n",
            "[EPOCH 63] TRAIN LOSS: 0.6461894260003017, SAMPLED NAME: joilee\n",
            "[EPOCH 64] TRAIN LOSS: 0.644411732753118, SAMPLED NAME: chressa\n",
            "[EPOCH 65] TRAIN LOSS: 0.6418317884970933, SAMPLED NAME: feani\n",
            "[EPOCH 66] TRAIN LOSS: 0.639355623569244, SAMPLED NAME: jerry\n",
            "[EPOCH 67] TRAIN LOSS: 0.6371335096848315, SAMPLED NAME: quenica\n",
            "[EPOCH 68] TRAIN LOSS: 0.635012151339115, SAMPLED NAME: fudileen\n",
            "[EPOCH 69] TRAIN LOSS: 0.6328473908778949, SAMPLED NAME: xanissie\n",
            "[EPOCH 70] TRAIN LOSS: 0.6305170341944083, SAMPLED NAME: kathy\n",
            "[EPOCH 71] TRAIN LOSS: 0.6289020249476802, SAMPLED NAME: dolly\n",
            "[EPOCH 72] TRAIN LOSS: 0.6266228671257313, SAMPLED NAME: queince\n",
            "[EPOCH 73] TRAIN LOSS: 0.6245588217026149, SAMPLED NAME: yonalie\n",
            "[EPOCH 74] TRAIN LOSS: 0.6230149941566665, SAMPLED NAME: yetri\n",
            "[EPOCH 75] TRAIN LOSS: 0.6209051104692314, SAMPLED NAME: wally\n",
            "[EPOCH 76] TRAIN LOSS: 0.6189646155406264, SAMPLED NAME: xiphani\n",
            "[EPOCH 77] TRAIN LOSS: 0.6173802789969325, SAMPLED NAME: yeotsia\n",
            "[EPOCH 78] TRAIN LOSS: 0.6150018595732177, SAMPLED NAME: illamie\n",
            "[EPOCH 79] TRAIN LOSS: 0.6132937692678891, SAMPLED NAME: kibby\n",
            "[EPOCH 80] TRAIN LOSS: 0.6117584827618723, SAMPLED NAME: margaele\n",
            "[EPOCH 81] TRAIN LOSS: 0.6098990249328123, SAMPLED NAME: leona\n",
            "[EPOCH 82] TRAIN LOSS: 0.608051819679065, SAMPLED NAME: ashalie\n",
            "[EPOCH 83] TRAIN LOSS: 0.6064682847414259, SAMPLED NAME: loni\n",
            "[EPOCH 84] TRAIN LOSS: 0.6043610068467947, SAMPLED NAME: willi\n",
            "[EPOCH 85] TRAIN LOSS: 0.6031536757946016, SAMPLED NAME: gwenna\n",
            "[EPOCH 86] TRAIN LOSS: 0.6017358738642474, SAMPLED NAME: jennine\n",
            "[EPOCH 87] TRAIN LOSS: 0.5998151042522527, SAMPLED NAME: garthie\n",
            "[EPOCH 88] TRAIN LOSS: 0.5983164012432101, SAMPLED NAME: bercie\n",
            "[EPOCH 89] TRAIN LOSS: 0.5964683462411929, SAMPLED NAME: glonda\n",
            "[EPOCH 90] TRAIN LOSS: 0.5950011702684257, SAMPLED NAME: wenny\n",
            "[EPOCH 91] TRAIN LOSS: 0.5933413834143911, SAMPLED NAME: othelia\n",
            "[EPOCH 92] TRAIN LOSS: 0.5920507785601492, SAMPLED NAME: nothe\n",
            "[EPOCH 93] TRAIN LOSS: 0.5905433770937799, SAMPLED NAME: ricki\n",
            "[EPOCH 94] TRAIN LOSS: 0.5892694760591555, SAMPLED NAME: yanna\n",
            "[EPOCH 95] TRAIN LOSS: 0.5878258454493989, SAMPLED NAME: ninet\n",
            "[EPOCH 96] TRAIN LOSS: 0.5861708269669459, SAMPLED NAME: fama\n",
            "[EPOCH 97] TRAIN LOSS: 0.5848483030612651, SAMPLED NAME: pelpi\n",
            "[EPOCH 98] TRAIN LOSS: 0.5836295798802987, SAMPLED NAME: ilyse\n",
            "[EPOCH 99] TRAIN LOSS: 0.5819314962778336, SAMPLED NAME: gralia\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4iXvoXk-w2Gd"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}