electra.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WpENGJ-_zihO",
        "outputId": "69a2d38c-0968-4cec-e73a-f84cde725952"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.35.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.20.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.11.17)\n",
            "Requirement already satisfied: torchsummary in /usr/local/lib/python3.10/dist-packages (1.5.1)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (2.16.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.13.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.23.5)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (10.0.1)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.7)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.1)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.15)\n",
            "Requirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.1)\n",
            "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.20.2)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->datasets) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2023.11.17)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3.post1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "!pip install torchsummary\n",
        "!pip install datasets"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "# Create a folder\n",
        "!mkdir -p \"/content/drive/My Drive/LLM-projects-models\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q--V0YUJn4lx",
        "outputId": "2a153070-836f-4c60-86ce-2eb81df779e8"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "FF8RjUGizihT"
      },
      "outputs": [],
      "source": [
        "from transformers import ElectraModel, ElectraConfig, ElectraForMaskedLM, ElectraTokenizer, PretrainedConfig, ElectraForTokenClassification\n",
        "import numpy as np\n",
        "from torchsummary import summary\n",
        "import torch\n",
        "from torch import nn\n",
        "from datasets import load_dataset\n",
        "from torch.utils.data import DataLoader\n",
        "import random\n",
        "from tqdm import tqdm\n",
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4rIXgnDpzihV"
      },
      "source": [
        "# Some tests, you can directly go to next part"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "61HNCwU5zihY",
        "outputId": "7bc456ad-034e-4cdc-9391-781628687eb9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': tensor([[ 101, 1996, 3007, 1997, 2605, 2003,  103, 1012,  102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1]])} {'input_ids': tensor([[ 101, 1996, 3007, 1997, 2605, 2003, 3000, 1012,  102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n",
            "tensor([[1012, 1996, 3007, 1997, 2605, 2003, 3000, 1012, 1012]])\n",
            ". the capital of france is paris..\n"
          ]
        }
      ],
      "source": [
        "config = ElectraConfig()\n",
        "tokenizer = ElectraTokenizer.from_pretrained('google/electra-small-discriminator')\n",
        "\n",
        "\n",
        "discriminator = ElectraModel.from_pretrained('google/electra-small-discriminator')\n",
        "inputs = tokenizer(\"hello my dog is cute\", return_tensors=\"pt\")\n",
        "#print(inputs)\n",
        "outputs = discriminator(**inputs, )\n",
        "last_hidden_states = outputs.last_hidden_state\n",
        "#print(last_hidden_states, last_hidden_states.shape)\n",
        "\n",
        "inputs = tokenizer(\"The capital of France is [MASK].\", return_tensors=\"pt\")\n",
        "labels = tokenizer(\"The capital of France is Paris.\", return_tensors=\"pt\")\n",
        "print(inputs, labels)\n",
        "model = ElectraForMaskedLM.from_pretrained('google/electra-small-generator')\n",
        "outputs = model(**inputs, labels=labels[\"input_ids\"])\n",
        "loss = outputs.loss\n",
        "logits = outputs.logits\n",
        "print(torch.argmax(logits, dim = -1))\n",
        "print(tokenizer.decode(torch.squeeze(torch.argmax(logits, dim = -1)).tolist()))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vg80oTcaziha",
        "outputId": "d1347c64-9b1c-4ab0-fe13-571511aac416"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "103\n"
          ]
        }
      ],
      "source": [
        "print(tokenizer.mask_token_id)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "47iVdOzhziha",
        "outputId": "096b5b19-ab27-4e9c-c88d-a45312ead776"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ElectraModel(\n",
            "  (embeddings): ElectraEmbeddings(\n",
            "    (word_embeddings): Embedding(30522, 128, padding_idx=0)\n",
            "    (position_embeddings): Embedding(512, 128)\n",
            "    (token_type_embeddings): Embedding(2, 128)\n",
            "    (LayerNorm): LayerNorm((128,), eps=1e-12, elementwise_affine=True)\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "  )\n",
            "  (embeddings_project): Linear(in_features=128, out_features=256, bias=True)\n",
            "  (encoder): ElectraEncoder(\n",
            "    (layer): ModuleList(\n",
            "      (0-11): 12 x ElectraLayer(\n",
            "        (attention): ElectraAttention(\n",
            "          (self): ElectraSelfAttention(\n",
            "            (query): Linear(in_features=256, out_features=256, bias=True)\n",
            "            (key): Linear(in_features=256, out_features=256, bias=True)\n",
            "            (value): Linear(in_features=256, out_features=256, bias=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "          (output): ElectraSelfOutput(\n",
            "            (dense): Linear(in_features=256, out_features=256, bias=True)\n",
            "            (LayerNorm): LayerNorm((256,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (intermediate): ElectraIntermediate(\n",
            "          (dense): Linear(in_features=256, out_features=1024, bias=True)\n",
            "          (intermediate_act_fn): GELUActivation()\n",
            "        )\n",
            "        (output): ElectraOutput(\n",
            "          (dense): Linear(in_features=1024, out_features=256, bias=True)\n",
            "          (LayerNorm): LayerNorm((256,), eps=1e-12, elementwise_affine=True)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            ")\n"
          ]
        }
      ],
      "source": [
        "print(discriminator)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "D9ragiRAzihc",
        "outputId": "b0d9aacc-87da-4c55-932c-749f976beb62"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'generator_config = {\\n  \"attention_probs_dropout_prob\": 0.1,\\n  \"embedding_size\": 128,\\n  \"hidden_act\": \"gelu\",\\n  \"hidden_dropout_prob\": 0.1,\\n  \"hidden_size\": 256,\\n  \"initializer_range\": 0.02,\\n  \"intermediate_size\": 1024,\\n  \"layer_norm_eps\": 1e-12,\\n  \"max_position_embeddings\": 512,\\n  \"model_type\": \"electra\",\\n  \"num_attention_heads\": 4,\\n  \"num_hidden_layers\": 12,\\n  \"pad_token_id\": 0,\\n  \"position_embedding_type\": \"absolute\",\\n  \"summary_activation\": \"gelu\",\\n  \"summary_last_dropout\": 0.1,\\n  \"summary_type\": \"first\",\\n  \"summary_use_proj\": True,\\n  \"transformers_version\": \"4.36.2\",\\n  \"type_vocab_size\": 2,\\n  \"use_cache\": True,\\n  \"vocab_size\": 30522\\n}\\n\\ndiscriminator_config = {\\n  \"attention_probs_dropout_prob\": 0.1,\\n  \"embedding_size\": 128,\\n  \"hidden_act\": \"gelu\",\\n  \"hidden_dropout_prob\": 0.1,\\n  \"hidden_size\": 256,\\n  \"initializer_range\": 0.02,\\n  \"intermediate_size\": 1024,\\n  \"layer_norm_eps\": 1e-12,\\n  \"max_position_embeddings\": 512,\\n  \"model_type\": \"electra\",\\n  \"num_attention_heads\": 4,\\n  \"num_hidden_layers\": 12,\\n  \"pad_token_id\": 0,\\n  \"position_embedding_type\": \"absolute\",\\n  \"summary_activation\": \"gelu\",\\n  \"summary_last_dropout\": 0.1,\\n  \"summary_type\": \"first\",\\n  \"summary_use_proj\": True,\\n  \"transformers_version\": \"4.36.2\",\\n  \"type_vocab_size\": 2,\\n  \"use_cache\": True,\\n  \"vocab_size\": 30522\\n}'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "\"\"\"generator_config = {\n",
        "  \"attention_probs_dropout_prob\": 0.1,\n",
        "  \"embedding_size\": 128,\n",
        "  \"hidden_act\": \"gelu\",\n",
        "  \"hidden_dropout_prob\": 0.1,\n",
        "  \"hidden_size\": 256,\n",
        "  \"initializer_range\": 0.02,\n",
        "  \"intermediate_size\": 1024,\n",
        "  \"layer_norm_eps\": 1e-12,\n",
        "  \"max_position_embeddings\": 512,\n",
        "  \"model_type\": \"electra\",\n",
        "  \"num_attention_heads\": 4,\n",
        "  \"num_hidden_layers\": 12,\n",
        "  \"pad_token_id\": 0,\n",
        "  \"position_embedding_type\": \"absolute\",\n",
        "  \"summary_activation\": \"gelu\",\n",
        "  \"summary_last_dropout\": 0.1,\n",
        "  \"summary_type\": \"first\",\n",
        "  \"summary_use_proj\": True,\n",
        "  \"transformers_version\": \"4.36.2\",\n",
        "  \"type_vocab_size\": 2,\n",
        "  \"use_cache\": True,\n",
        "  \"vocab_size\": 30522\n",
        "}\n",
        "\n",
        "discriminator_config = {\n",
        "  \"attention_probs_dropout_prob\": 0.1,\n",
        "  \"embedding_size\": 128,\n",
        "  \"hidden_act\": \"gelu\",\n",
        "  \"hidden_dropout_prob\": 0.1,\n",
        "  \"hidden_size\": 256,\n",
        "  \"initializer_range\": 0.02,\n",
        "  \"intermediate_size\": 1024,\n",
        "  \"layer_norm_eps\": 1e-12,\n",
        "  \"max_position_embeddings\": 512,\n",
        "  \"model_type\": \"electra\",\n",
        "  \"num_attention_heads\": 4,\n",
        "  \"num_hidden_layers\": 12,\n",
        "  \"pad_token_id\": 0,\n",
        "  \"position_embedding_type\": \"absolute\",\n",
        "  \"summary_activation\": \"gelu\",\n",
        "  \"summary_last_dropout\": 0.1,\n",
        "  \"summary_type\": \"first\",\n",
        "  \"summary_use_proj\": True,\n",
        "  \"transformers_version\": \"4.36.2\",\n",
        "  \"type_vocab_size\": 2,\n",
        "  \"use_cache\": True,\n",
        "  \"vocab_size\": 30522\n",
        "}\"\"\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HoSz3T3kzihd"
      },
      "source": [
        "# Load dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "26e6569062204c2d850d071f669271bc",
            "91bffffd6d1340e8b9808c43e0fba036",
            "c1d462aa5bab44f4b46a361b961620d9",
            "1f9cdb0023eb444380e57465cfba788d",
            "bcad8bc436e24b9984cc4f862e88a65b",
            "351f7fb8bccd4c9bb4b0deb3c72d1c20",
            "296489e9dd2d437384d74f732c7769da",
            "9ac1dabbe87a4e159e78426468979ffb",
            "fb84f92841f9492ea94750115934a48d",
            "9f7f6dd4e5d244feae338af7c0a12736",
            "e465d4e6a51447d58b32681115371d1c"
          ]
        },
        "id": "N9PGNOZmzihd",
        "outputId": "2a0af46d-0772-470f-83b4-6c038997094f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "50000\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/1000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "26e6569062204c2d850d071f669271bc"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "#Load IMDB dataset\n",
        "\n",
        "#Input = masked data\n",
        "#label = real data\n",
        "\n",
        "#input = prediction of generator\n",
        "#output = binary output\n",
        "\n",
        "#We will masked 20% of the input\n",
        "\n",
        "#103 is the encoding for [MASK]\n",
        "\n",
        "def masking(x , mask_token = 103):\n",
        "    k = int(len(x)*1/10)\n",
        "    ids = random.sample(range(1, len(x)-1), k)\n",
        "    for i in ids:\n",
        "        x[i] = mask_token\n",
        "    return x, ids\n",
        "\n",
        "\n",
        "tokenizer = ElectraTokenizer.from_pretrained('google/electra-small-generator')\n",
        "dataset = load_dataset(\"scikit-learn/imdb\", split=\"train\")\n",
        "\n",
        "def preprocessing_fn(x, tokenizer):\n",
        "    generator_labels = tokenizer.encode(\n",
        "        x[\"review\"],\n",
        "        add_special_tokens=False,\n",
        "        truncation=True,\n",
        "        max_length=128,\n",
        "        padding=False,\n",
        "        return_attention_mask=False,\n",
        "    )\n",
        "    x[\"input_ids\"], masked_ids = masking(generator_labels, 103)\n",
        "\n",
        "    attention_mask = torch.ones((len(x[\"input_ids\"])))\n",
        "    x[\"labels\"] = generator_labels\n",
        "    x[\"attention_mask\"] = attention_mask\n",
        "    x[\"masked_ids\"] = masked_ids\n",
        "    return x\n",
        "\n",
        "print(len(dataset))\n",
        "\n",
        "n_samples = 1000\n",
        "\n",
        "# We first shuffle the data !\n",
        "dataset = dataset.shuffle()\n",
        "\n",
        "# Select n_samples\n",
        "splitted_dataset = dataset.select(range(n_samples))\n",
        "\n",
        "# Tokenize the dataset\n",
        "splitted_dataset = splitted_dataset.map(\n",
        "    preprocessing_fn, fn_kwargs={\"tokenizer\": tokenizer}\n",
        ")\n",
        "\n",
        "\n",
        "# Remove useless columns\n",
        "splitted_dataset = splitted_dataset.select_columns([\"input_ids\", \"attention_mask\", \"labels\", \"masked_ids\"])\n",
        "\n",
        "# Split the train and validation\n",
        "splitted_dataset = splitted_dataset.train_test_split(test_size=0.2)\n",
        "\n",
        "train_set = splitted_dataset[\"train\"]\n",
        "valid_set = splitted_dataset[\"test\"]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AENOVjyBzihg",
        "outputId": "8c10806b-49ef-44a9-c5c1-edd7b7b15267"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dict_keys(['input_ids', 'attention_mask', 'labels', 'masked_ids'])\n",
            "128 128 128\n",
            "128 128 128\n",
            "torch.Size([32, 127]) torch.Size([32, 127]) torch.Size([32, 127])\n"
          ]
        }
      ],
      "source": [
        "class DataCollator:\n",
        "    def __init__(self, tokenizer):\n",
        "        self.tokenizer = tokenizer\n",
        "\n",
        "    def __call__(self, batch):\n",
        "        max_label_length = max(len(feature[\"input_ids\"]) for feature in batch)\n",
        "        max_length_ids = max(len(feature[\"masked_ids\"]) for feature in batch)\n",
        "        for feature in batch:\n",
        "            current_length = len(feature[\"input_ids\"])\n",
        "            current_length_mask = len(feature[\"masked_ids\"])\n",
        "            remainder = [tokenizer.pad_token_id] * (max_label_length - current_length)\n",
        "\n",
        "            feature[\"input_ids\"] = feature[\"input_ids\"] + remainder\n",
        "            feature[\"labels\"] = feature[\"input_ids\"][1:]\n",
        "            feature[\"labels\"] = [\n",
        "                feature[\"labels\"][i] if i < current_length - 1 else -100\n",
        "                for i in range(max_label_length - 1)\n",
        "            ]\n",
        "            feature[\"input_ids\"] = feature[\"input_ids\"][:-1]\n",
        "            feature[\"attention_mask\"] = [\n",
        "                1 if x < current_length else 0 for x in range(max_label_length - 1)\n",
        "            ]\n",
        "            feature[\"masked_ids\"] = [\n",
        "                feature[\"masked_ids\"][i] if i < current_length_mask - 1 else -1\n",
        "                for i in range(max_length_ids - 1)\n",
        "            ]\n",
        "        features = {\n",
        "            \"input_ids\": torch.tensor([f[\"input_ids\"] for f in batch]),\n",
        "            \"attention_mask\": torch.tensor([f[\"attention_mask\"] for f in batch]),\n",
        "            \"labels\": torch.tensor([f[\"labels\"] for f in batch]),\n",
        "            \"masked_ids\": torch.tensor([f[\"masked_ids\"] for f in batch]),\n",
        "        }\n",
        "        return features\n",
        "\n",
        "print(train_set[0].keys())\n",
        "print(len(train_set[0][\"input_ids\"]), len(train_set[0][\"attention_mask\"]), len(train_set[0][\"labels\"]))\n",
        "print(len(train_set[1][\"input_ids\"]), len(train_set[1][\"attention_mask\"]), len(train_set[1][\"labels\"]))\n",
        "\n",
        "data_collator = DataCollator(tokenizer)\n",
        "\n",
        "batch_size = 32\n",
        "\n",
        "train_dataloader = DataLoader(\n",
        "    train_set, batch_size=batch_size, collate_fn=data_collator\n",
        ")\n",
        "valid_dataloader = DataLoader(\n",
        "    valid_set, batch_size=batch_size, collate_fn=data_collator\n",
        ")\n",
        "n_valid = len(valid_set)\n",
        "n_train = len(train_set)\n",
        "\n",
        "batch = next(iter(train_dataloader))\n",
        "print(batch[\"input_ids\"].shape, batch[\"labels\"].shape, batch[\"attention_mask\"].shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lKKgXh3Pzihh"
      },
      "source": [
        "# Model instantiation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "7iRCytB0zihh"
      },
      "outputs": [],
      "source": [
        "class GeneratorHead(nn.Module):\n",
        "    \"\"\"Prediction module for the generator, made up of two dense layers.\"\"\"\n",
        "    def __init__(self, config):\n",
        "        super().__init__()\n",
        "\n",
        "        self.LayerNorm = nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps)\n",
        "        self.dense = nn.Linear(config.hidden_size, config.embedding_size)\n",
        "        self.dense2 = nn.Linear(config.embedding_size, config.vocab_size)\n",
        "\n",
        "    def forward(self, generator_hidden_states):\n",
        "        hidden_states = self.dense(generator_hidden_states)\n",
        "        hidden_states = torch.nn.GELU()(hidden_states)\n",
        "        hidden_states = self.LayerNorm(hidden_states)\n",
        "        hidden_states = self.dense2(hidden_states)\n",
        "        #hidden_states = torch.nn.Softmax(dim = -1)(hidden_states)\n",
        "\n",
        "        return hidden_states\n",
        "\n",
        "class Generator(nn.Module):\n",
        "    \"\"\"Complete generator module\"\"\"\n",
        "    def __init__(self, generator_body, generator_head):\n",
        "        super().__init__()\n",
        "        self.generator_body = generator_body\n",
        "        self.generator_head = generator_head\n",
        "\n",
        "    def forward(self, input):\n",
        "      output = self.generator_body(input_ids=input[\"input_ids\"], attention_mask=input[\"attention_mask\"]).last_hidden_state\n",
        "      output = self.generator_head(output)\n",
        "      return output\n",
        "\n",
        "class DiscriminatorHead(nn.Module):\n",
        "    \"\"\"Discriminator module for the generator, made up of two dense layers.\"\"\"\n",
        "    def __init__(self, config):\n",
        "        super().__init__()\n",
        "\n",
        "        self.LayerNorm = nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps)\n",
        "        self.dense = nn.Linear(config.hidden_size, config.embedding_size)\n",
        "        self.dense2 = nn.Linear(config.embedding_size, 2)\n",
        "\n",
        "    def forward(self, discriminator_hidden_states):\n",
        "        hidden_states = self.dense(discriminator_hidden_states)\n",
        "        hidden_states = torch.nn.GELU()(hidden_states)\n",
        "        hidden_states = self.LayerNorm(hidden_states)\n",
        "        hidden_states = self.dense2(hidden_states)\n",
        "\n",
        "        return hidden_states\n",
        "\n",
        "class Discriminator(nn.Module):\n",
        "    \"\"\"Complete Discriminator\"\"\"\n",
        "    def __init__(self, discriminator_body, discriminator_head):\n",
        "        super().__init__()\n",
        "        self.discriminator_body = discriminator_body\n",
        "        self.discriminator_head = discriminator_head\n",
        "\n",
        "    def forward(self, input):\n",
        "      output = self.discriminator_body(input_ids=input[\"input_ids\"], attention_mask=input[\"attention_mask\"]).last_hidden_state\n",
        "      output = self.discriminator_head(output)\n",
        "      return output\n",
        "\n",
        "class ELECTRALoss():\n",
        "  def __init__(self, loss_weights=(1.0, 50.0)):\n",
        "    self.loss_weights = loss_weights\n",
        "    self.gen_loss_fc = nn.CrossEntropyLoss(ignore_index = -100)\n",
        "    self.disc_loss_fc = nn.BCEWithLogitsLoss()\n",
        "\n",
        "  def __call__(self, generator_pred, discriminator_pred, generator_labels, masked_ids):\n",
        "    list_gen_pred = []\n",
        "    list_gen_labels = []\n",
        "    for i in range(masked_ids.shape[0]):\n",
        "      for j in range(masked_ids.shape[1]):\n",
        "        masked_index = masked_ids[i,j]\n",
        "        list_gen_pred.append(torch.unsqueeze(generator_pred[i,masked_index,:],dim =0))\n",
        "        list_gen_labels.append(torch.unsqueeze(generator_labels[i,masked_index], dim = 0))\n",
        "    list_gen_pred = torch.concat(list_gen_pred, axis = 0)\n",
        "    list_gen_labels = torch.concat(list_gen_labels, axis = 0)\n",
        "    gen_loss = self.gen_loss_fc(list_gen_pred, list_gen_labels)\n",
        "    gen_acc = torch.sum(torch.argmax(list_gen_pred, dim = -1) == list_gen_labels)/ len(list_gen_labels)*100\n",
        "\n",
        "    generated_tokens = torch.argmax(generator_pred, dim = -1)\n",
        "\n",
        "    discriminator_labels =torch.ones((discriminator_pred.shape[0], discriminator_pred.shape[1])).cuda()\n",
        "    for i in range(masked_ids.shape[0]):\n",
        "      for j in range(masked_ids.shape[1]):\n",
        "        masked_index = masked_ids[i,j]\n",
        "        if generated_tokens[i,masked_index] != generator_labels[i,masked_index]:\n",
        "          discriminator_labels[i,masked_index] = 0\n",
        "    discriminator_labels_one_hot =  nn.functional.one_hot(discriminator_labels.to(torch.int64).view(discriminator_labels.shape[0]*discriminator_labels.shape[1])).float()\n",
        "    discriminator_pred = discriminator_pred.view(discriminator_pred.shape[0]*discriminator_pred.shape[1],discriminator_pred.shape[2])\n",
        "    disc_loss = self.disc_loss_fc(discriminator_pred, discriminator_labels_one_hot)\n",
        "    disc_acc = torch.sum(torch.argmax(discriminator_pred.view(-1), dim = -1) == discriminator_labels)/ len(discriminator_labels)*100\n",
        "    return gen_loss * self.loss_weights[0] + disc_loss * self.loss_weights[1], gen_acc, disc_acc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "id": "tX93YQc_zihi"
      },
      "outputs": [],
      "source": [
        "generator_config = ElectraConfig(vocab_size = tokenizer.vocab_size, embedding_size= 64, hidden_size = 128)\n",
        "generator_body = ElectraModel(generator_config)\n",
        "generator_head = GeneratorHead(generator_config)\n",
        "generator = Generator(generator_body, generator_head)\n",
        "\n",
        "discriminator_config = ElectraConfig(vocab_size = tokenizer.vocab_size)\n",
        "discriminator_body = ElectraModel(discriminator_config)\n",
        "discriminator_head = DiscriminatorHead(discriminator_config)\n",
        "discriminator = Discriminator(discriminator_body,discriminator_head)\n",
        "\n",
        "loss = ELECTRALoss()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qihQkW3Yzihi"
      },
      "source": [
        "# Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "id": "iYutY6LBzihi"
      },
      "outputs": [],
      "source": [
        "def training(generator, discriminator, n_epochs, train_dataloader, val_dataloader, loss, tokenizer, lr=5e-5):\n",
        "\n",
        "    #trained_gen = ElectraForMaskedLM.from_pretrained('google/electra-small-generator').cuda()\n",
        "    optimizer_gen = torch.optim.AdamW(\n",
        "        generator.parameters(),\n",
        "        lr=lr,\n",
        "        eps=1e-08,\n",
        "    )\n",
        "    optimizer_disc = torch.optim.AdamW(\n",
        "        discriminator.parameters(),\n",
        "        lr=lr,\n",
        "        eps=1e-08,\n",
        "    )\n",
        "    list_train_loss = []\n",
        "    list_val_loss = []\n",
        "    best_val_loss = 10**50\n",
        "    list_train_gen_acc = []\n",
        "    list_train_disc_acc = []\n",
        "    list_val_gen_acc = []\n",
        "    list_val_disc_acc = []\n",
        "    generator.cuda()\n",
        "    discriminator.cuda()\n",
        "    for e in range(n_epochs):\n",
        "        # ========== Training ==========\n",
        "\n",
        "        # Set model to training mode\n",
        "        generator.train()\n",
        "        discriminator.train()\n",
        "\n",
        "        # Tracking variables\n",
        "        train_loss = 0\n",
        "        train_gen_acc = 0\n",
        "        train_disc_acc = 0\n",
        "        for batch in tqdm(train_dataloader):\n",
        "            batch = {k: v.cuda() for k, v in batch.items()}\n",
        "\n",
        "            input_ids, attention_mask, labels, masked_ids =(\n",
        "                batch[\"input_ids\"],\n",
        "                batch[\"attention_mask\"],\n",
        "                batch[\"labels\"],\n",
        "                batch[\"masked_ids\"]\n",
        "            )\n",
        "            batch.pop(\"labels\")\n",
        "            batch.pop(\"masked_ids\")\n",
        "            optimizer_gen.zero_grad()\n",
        "            optimizer_disc.zero_grad()\n",
        "            # Forward pass\n",
        "            generator_pred = generator(batch)\n",
        "            #generator_pred = trained_gen(**batch).logits\n",
        "            predicted_tokens = torch.argmax(generator_pred, dim = -1)\n",
        "            batch_discriminator = batch\n",
        "\n",
        "            input_ids_for_discriminator = labels\n",
        "            for i in range(len(batch)):\n",
        "              for id in masked_ids[i,:]:\n",
        "                if id != -1:\n",
        "                  input_ids_for_discriminator[i][id] = predicted_tokens[i][id]\n",
        "            input_ids_for_discriminator[input_ids_for_discriminator == -100] = tokenizer.pad_token_id\n",
        "            batch_discriminator[\"input_ids\"] = input_ids_for_discriminator\n",
        "\n",
        "            batch_discriminator = {k: v.cuda() for k, v in batch_discriminator.items()}\n",
        "            discriminator_pred = discriminator(batch_discriminator)\n",
        "\n",
        "            # Backward pass\n",
        "            loss_value, train_gen_acc_value, train_disc_acc_value = loss(generator_pred, discriminator_pred,labels, masked_ids)\n",
        "            loss_value.backward()\n",
        "            optimizer_gen.step()\n",
        "            optimizer_disc.step()\n",
        "            train_loss += loss_value.detach().cpu().item()\n",
        "            train_gen_acc += train_gen_acc_value\n",
        "            train_disc_acc += train_disc_acc_value\n",
        "            print(\"Batch training loss is\", loss_value)\n",
        "        list_train_loss.append(train_loss / len(train_dataloader))\n",
        "        list_train_gen_acc.append(train_gen_acc / len(train_dataloader))\n",
        "        list_train_disc_acc.append(train_disc_acc / len(train_dataloader))\n",
        "        # ========== Validation ==========\n",
        "        if e%1 == 0:\n",
        "          generator.eval()\n",
        "          discriminator.eval()\n",
        "          valid_loss = 0\n",
        "          val_gen_acc = 0\n",
        "          val_disc_acc = 0\n",
        "          for batch in tqdm(val_dataloader):\n",
        "              batch = {k: v.cuda() for k, v in batch.items()}\n",
        "\n",
        "              input_ids, attention_mask, labels, masked_ids =(\n",
        "                  batch[\"input_ids\"],\n",
        "                  batch[\"attention_mask\"],\n",
        "                  batch[\"labels\"],\n",
        "                  batch[\"masked_ids\"]\n",
        "              )\n",
        "              batch.pop(\"labels\")\n",
        "              batch.pop(\"masked_ids\")\n",
        "\n",
        "              # Forward pass\n",
        "              generator_pred = generator(batch)\n",
        "              predicted_tokens = torch.argmax(generator_pred, dim = -1)\n",
        "              batch_discriminator = batch\n",
        "\n",
        "              input_ids_for_discriminator = labels\n",
        "              for i in range(len(batch)):\n",
        "                for id in masked_ids[i,:]:\n",
        "                  if id != -1:\n",
        "                    input_ids_for_discriminator[i][id] = predicted_tokens[i][id]\n",
        "              input_ids_for_discriminator[input_ids_for_discriminator == -100] = tokenizer.pad_token_id\n",
        "              batch_discriminator[\"input_ids\"] = input_ids_for_discriminator\n",
        "\n",
        "              batch_discriminator = {k: v.cuda() for k, v in batch_discriminator.items()}\n",
        "              discriminator_pred = discriminator(batch_discriminator)\n",
        "\n",
        "              # Backward pass\n",
        "              loss_value, val_gen_acc_value, val_disc_acc_value = loss(generator_pred, discriminator_pred,labels, masked_ids)\n",
        "\n",
        "              valid_loss += loss_value.detach().cpu().item()\n",
        "              val_gen_acc += val_gen_acc_value\n",
        "              val_disc_acc += val_disc_acc_value\n",
        "        list_val_loss.append(valid_loss / len(val_dataloader))\n",
        "        list_val_gen_acc.append(val_gen_acc_value / len(val_dataloader))\n",
        "        list_val_disc_acc.append(val_disc_acc / len(val_dataloader))\n",
        "        if list_val_loss[-1] < best_val_loss:\n",
        "          best_val_loss = list_val_loss[-1]\n",
        "          generator_path = f\"/content/drive/My Drive/LLM-projects-models/generator_epoch{e}_lr{lr}\"\n",
        "          discriminator_path =  f\"/content/drive/My Drive/LLM-projects-models/discriminator_epoch{e}_lr{lr}\"\n",
        "          torch.save(generator.state_dict(), generator_path)\n",
        "          torch.save(discriminator.state_dict(), discriminator_path)\n",
        "\n",
        "\n",
        "        print(\n",
        "            e,\n",
        "            \"\\n\\t - Train loss: {:.4f}\".format(list_train_loss[-1]),\n",
        "            \"\\n\\t - Val loss: {:.4f}\".format(list_val_loss[-1]),\n",
        "            \"\\n\\t - Train gen acc: {:.4f}\".format(list_train_gen_acc[-1]),\n",
        "            \"\\n\\t - Train disc acc: {:.4f}\".format(list_train_disc_acc[-1]),\n",
        "            \"\\n\\t - Val gen acc: {:.4f}\".format(list_val_gen_acc[-1]),\n",
        "            \"\\n\\t - Val disc acc: {:.4f}\".format(list_val_disc_acc[-1]),\n",
        "        )\n",
        "    return list_train_loss, list_val_loss, list_train_gen_acc, list_train_disc_acc, list_val_gen_acc, list_val_disc_acc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "ygYX8GCqzihj",
        "outputId": "cd2538a6-2663-45f1-f966-7001d5a0e6e3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  4%|▍         | 1/25 [00:03<01:22,  3.44s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(42.5251, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  8%|▊         | 2/25 [00:06<01:16,  3.31s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(25.6422, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 12%|█▏        | 3/25 [00:09<01:12,  3.28s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(123.6281, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 16%|█▌        | 4/25 [00:13<01:08,  3.26s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(28.2587, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 20%|██        | 5/25 [00:16<01:04,  3.24s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(149.5104, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 24%|██▍       | 6/25 [00:19<01:01,  3.24s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(31.5858, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 28%|██▊       | 7/25 [00:22<00:58,  3.24s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(37.0925, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 32%|███▏      | 8/25 [00:26<00:55,  3.24s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(35.6216, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 36%|███▌      | 9/25 [00:29<00:51,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(31.1903, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 40%|████      | 10/25 [00:32<00:48,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(26.1196, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 44%|████▍     | 11/25 [00:35<00:45,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(20.8131, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 48%|████▊     | 12/25 [00:38<00:41,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(25.1059, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 52%|█████▏    | 13/25 [00:42<00:38,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(27.9345, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 56%|█████▌    | 14/25 [00:45<00:35,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(22.8789, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 60%|██████    | 15/25 [00:48<00:32,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(20.2255, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 64%|██████▍   | 16/25 [00:51<00:28,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(21.4555, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 68%|██████▊   | 17/25 [00:55<00:25,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(22.3020, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 72%|███████▏  | 18/25 [00:58<00:22,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(23.5743, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 76%|███████▌  | 19/25 [01:01<00:19,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(24.0664, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 80%|████████  | 20/25 [01:04<00:16,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(22.7715, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 84%|████████▍ | 21/25 [01:07<00:12,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(22.5859, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 88%|████████▊ | 22/25 [01:11<00:09,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(20.3885, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 92%|█████████▏| 23/25 [01:14<00:06,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.7525, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 96%|█████████▌| 24/25 [01:17<00:03,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(20.9310, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 25/25 [01:20<00:00,  3.23s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(21.5125, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 7/7 [00:00<00:00,  7.03it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0 \n",
            "\t - Train loss: 34.6989 \n",
            "\t - Val loss: 20.4537 \n",
            "\t - Train gen acc: 12.0000 \n",
            "\t - Train disc acc: 0.0000 \n",
            "\t - Val gen acc: 5.1948 \n",
            "\t - Val disc acc: 0.0000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  4%|▍         | 1/25 [00:03<01:17,  3.25s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.8805, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  8%|▊         | 2/25 [00:06<01:14,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0612, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 12%|█▏        | 3/25 [00:09<01:10,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.7545, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 16%|█▌        | 4/25 [00:12<01:07,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(20.1560, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 20%|██        | 5/25 [00:16<01:04,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(20.4644, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 24%|██▍       | 6/25 [00:19<01:01,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(21.2383, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 28%|██▊       | 7/25 [00:22<00:57,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.7409, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 32%|███▏      | 8/25 [00:25<00:54,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(20.1856, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 36%|███▌      | 9/25 [00:28<00:51,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.7089, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 40%|████      | 10/25 [00:32<00:48,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.6779, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 44%|████▍     | 11/25 [00:35<00:45,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.6365, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 48%|████▊     | 12/25 [00:38<00:41,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.8504, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 52%|█████▏    | 13/25 [00:41<00:38,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3805, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 56%|█████▌    | 14/25 [00:45<00:35,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3060, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 60%|██████    | 15/25 [00:48<00:32,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3683, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 64%|██████▍   | 16/25 [00:51<00:28,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.4697, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 68%|██████▊   | 17/25 [00:54<00:25,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1860, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 72%|███████▏  | 18/25 [00:57<00:22,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3401, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 76%|███████▌  | 19/25 [01:01<00:19,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.5649, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 80%|████████  | 20/25 [01:04<00:16,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1676, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 84%|████████▍ | 21/25 [01:07<00:12,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.9355, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 88%|████████▊ | 22/25 [01:10<00:09,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.4291, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 92%|█████████▏| 23/25 [01:14<00:06,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3024, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 96%|█████████▌| 24/25 [01:17<00:03,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.5564, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 25/25 [01:20<00:00,  3.22s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.9245, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 7/7 [00:00<00:00,  8.73it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1 \n",
            "\t - Train loss: 19.6114 \n",
            "\t - Val loss: 19.1809 \n",
            "\t - Train gen acc: 13.3295 \n",
            "\t - Train disc acc: 0.0000 \n",
            "\t - Val gen acc: 5.1948 \n",
            "\t - Val disc acc: 0.0000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  4%|▍         | 1/25 [00:03<01:17,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.8982, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  8%|▊         | 2/25 [00:06<01:14,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.9092, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 12%|█▏        | 3/25 [00:09<01:11,  3.24s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0770, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 16%|█▌        | 4/25 [00:12<01:07,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.4441, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 20%|██        | 5/25 [00:16<01:04,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0158, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 24%|██▍       | 6/25 [00:19<01:01,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3734, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 28%|██▊       | 7/25 [00:22<00:58,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.5436, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 32%|███▏      | 8/25 [00:25<00:54,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1703, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 36%|███▌      | 9/25 [00:28<00:51,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1292, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 40%|████      | 10/25 [00:32<00:48,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.9275, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 44%|████▍     | 11/25 [00:35<00:45,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.5513, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 48%|████▊     | 12/25 [00:38<00:41,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0601, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 52%|█████▏    | 13/25 [00:41<00:38,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.2129, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 56%|█████▌    | 14/25 [00:45<00:35,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3990, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 60%|██████    | 15/25 [00:48<00:32,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.2679, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 64%|██████▍   | 16/25 [00:51<00:28,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.4892, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 68%|██████▊   | 17/25 [00:54<00:25,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.9923, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 72%|███████▏  | 18/25 [00:57<00:22,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1769, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 76%|███████▌  | 19/25 [01:01<00:19,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3913, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 80%|████████  | 20/25 [01:04<00:16,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.8002, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 84%|████████▍ | 21/25 [01:07<00:12,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.5295, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 88%|████████▊ | 22/25 [01:10<00:09,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.7804, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 92%|█████████▏| 23/25 [01:14<00:06,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.7714, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 96%|█████████▌| 24/25 [01:17<00:03,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0982, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 25/25 [01:20<00:00,  3.22s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.5606, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 7/7 [00:00<00:00,  8.27it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2 \n",
            "\t - Train loss: 19.1028 \n",
            "\t - Val loss: 19.1319 \n",
            "\t - Train gen acc: 13.3295 \n",
            "\t - Train disc acc: 0.0000 \n",
            "\t - Val gen acc: 5.1948 \n",
            "\t - Val disc acc: 1684.3751\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  4%|▍         | 1/25 [00:03<01:16,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0533, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  8%|▊         | 2/25 [00:06<01:13,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1744, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 12%|█▏        | 3/25 [00:09<01:10,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.9882, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 16%|█▌        | 4/25 [00:12<01:07,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3847, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 20%|██        | 5/25 [00:16<01:04,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.7647, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 24%|██▍       | 6/25 [00:19<01:01,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.5236, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 28%|██▊       | 7/25 [00:22<00:57,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.5406, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 32%|███▏      | 8/25 [00:25<00:54,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.2597, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 36%|███▌      | 9/25 [00:28<00:51,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.2286, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 40%|████      | 10/25 [00:32<00:48,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0169, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 44%|████▍     | 11/25 [00:35<00:44,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.5606, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 48%|████▊     | 12/25 [00:38<00:41,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.9272, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 52%|█████▏    | 13/25 [00:41<00:38,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0291, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 56%|█████▌    | 14/25 [00:45<00:35,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.2791, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 60%|██████    | 15/25 [00:48<00:32,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0343, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 64%|██████▍   | 16/25 [00:51<00:28,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0518, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 68%|██████▊   | 17/25 [00:54<00:25,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.5833, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 72%|███████▏  | 18/25 [00:57<00:22,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.8093, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 76%|███████▌  | 19/25 [01:01<00:19,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0916, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 80%|████████  | 20/25 [01:04<00:16,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.6285, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 84%|████████▍ | 21/25 [01:07<00:12,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3718, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 88%|████████▊ | 22/25 [01:10<00:09,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.8421, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 92%|█████████▏| 23/25 [01:14<00:06,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.9032, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 96%|█████████▌| 24/25 [01:17<00:03,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3168, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 25/25 [01:20<00:00,  3.22s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.8312, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 7/7 [00:00<00:00,  8.56it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "3 \n",
            "\t - Train loss: 19.0078 \n",
            "\t - Val loss: 18.9445 \n",
            "\t - Train gen acc: 13.3295 \n",
            "\t - Train disc acc: 0.0000 \n",
            "\t - Val gen acc: 5.1948 \n",
            "\t - Val disc acc: 0.0000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  4%|▍         | 1/25 [00:03<01:17,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.5450, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  8%|▊         | 2/25 [00:06<01:14,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.2960, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 12%|█▏        | 3/25 [00:09<01:10,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.8023, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 16%|█▌        | 4/25 [00:12<01:07,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0524, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 20%|██        | 5/25 [00:16<01:04,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.6884, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 24%|██▍       | 6/25 [00:19<01:01,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3399, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 28%|██▊       | 7/25 [00:22<00:57,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.4361, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 32%|███▏      | 8/25 [00:25<00:54,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1674, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 36%|███▌      | 9/25 [00:28<00:51,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1550, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 40%|████      | 10/25 [00:32<00:48,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.9098, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 44%|████▍     | 11/25 [00:35<00:45,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.4057, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 48%|████▊     | 12/25 [00:38<00:41,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.7956, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 52%|█████▏    | 13/25 [00:41<00:38,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0099, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 56%|█████▌    | 14/25 [00:45<00:35,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1557, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 60%|██████    | 15/25 [00:48<00:32,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0074, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 64%|██████▍   | 16/25 [00:51<00:28,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.9759, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 68%|██████▊   | 17/25 [00:54<00:25,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.6087, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 72%|███████▏  | 18/25 [00:57<00:22,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.8317, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 76%|███████▌  | 19/25 [01:01<00:19,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1323, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 80%|████████  | 20/25 [01:04<00:16,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.7007, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 84%|████████▍ | 21/25 [01:07<00:12,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.4601, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 88%|████████▊ | 22/25 [01:10<00:09,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.8891, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 92%|█████████▏| 23/25 [01:13<00:06,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.9183, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 96%|█████████▌| 24/25 [01:17<00:03,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3043, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 25/25 [01:20<00:00,  3.22s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.7626, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 7/7 [00:00<00:00,  8.67it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "4 \n",
            "\t - Train loss: 18.9340 \n",
            "\t - Val loss: 18.9689 \n",
            "\t - Train gen acc: 13.3295 \n",
            "\t - Train disc acc: 0.0000 \n",
            "\t - Val gen acc: 5.1948 \n",
            "\t - Val disc acc: 0.0000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  4%|▍         | 1/25 [00:03<01:16,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.5307, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  8%|▊         | 2/25 [00:06<01:13,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1244, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 12%|█▏        | 3/25 [00:09<01:10,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.6713, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 16%|█▌        | 4/25 [00:12<01:07,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.9471, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 20%|██        | 5/25 [00:16<01:04,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.5652, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 24%|██▍       | 6/25 [00:19<01:01,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.2418, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 28%|██▊       | 7/25 [00:22<00:57,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.4393, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 32%|███▏      | 8/25 [00:25<00:54,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1477, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 36%|███▌      | 9/25 [00:28<00:51,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1010, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 40%|████      | 10/25 [00:32<00:48,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.9266, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 44%|████▍     | 11/25 [00:35<00:45,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.4262, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 48%|████▊     | 12/25 [00:38<00:41,  3.21s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.8260, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 52%|█████▏    | 13/25 [00:41<00:38,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0004, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 56%|█████▌    | 14/25 [00:45<00:35,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.1770, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 60%|██████    | 15/25 [00:48<00:32,  3.23s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0434, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 64%|██████▍   | 16/25 [00:51<00:28,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0562, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 68%|██████▊   | 17/25 [00:54<00:25,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.6443, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 72%|███████▏  | 18/25 [00:57<00:22,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.7938, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 76%|███████▌  | 19/25 [01:01<00:19,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.0702, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 80%|████████  | 20/25 [01:04<00:16,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(18.6381, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 84%|████████▍ | 21/25 [01:07<00:12,  3.22s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Batch training loss is tensor(19.3920, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r 84%|████████▍ | 21/25 [01:10<00:13,  3.38s/it]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-40-ab5d47956dd4>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlist_train_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist_val_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist_train_gen_acc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist_train_disc_acc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist_val_gen_acc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist_val_disc_acc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgenerator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdiscriminator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_dataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_dataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5e-2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-39-e543bec6b9a4>\u001b[0m in \u001b[0;36mtraining\u001b[0;34m(generator, discriminator, n_epochs, train_dataloader, val_dataloader, loss, tokenizer, lr)\u001b[0m\n\u001b[1;32m     64\u001b[0m             \u001b[0;31m# Backward pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m             \u001b[0mloss_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_gen_acc_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_disc_acc_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgenerator_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdiscriminator_pred\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmasked_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m             \u001b[0mloss_value\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m             \u001b[0moptimizer_gen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m             \u001b[0moptimizer_disc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    490\u001b[0m                 \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             )\n\u001b[0;32m--> 492\u001b[0;31m         torch.autograd.backward(\n\u001b[0m\u001b[1;32m    493\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    494\u001b[0m         )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    249\u001b[0m     \u001b[0;31m# some Python versions print out the first line of a multi-line function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m     \u001b[0;31m# calls in the traceback and some print out the last line\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 251\u001b[0;31m     Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n\u001b[0m\u001b[1;32m    252\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    253\u001b[0m         \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "list_train_loss, list_val_loss, list_train_gen_acc, list_train_disc_acc, list_val_gen_acc, list_val_disc_acc = training(generator, discriminator, 10, train_dataloader, valid_dataloader, loss, tokenizer, lr=5e-2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8UEFu9V9zihj"
      },
      "outputs": [],
      "source": [
        "plt.figure()\n",
        "x = [i for i in range(len(list_train_loss))]\n",
        "plt.plot(x, list_train_loss)\n",
        "plt.plot(x, list_val_loss)\n",
        "plt.show()\n",
        "\n",
        "x = [i for i in range(len(list_train_loss))]\n",
        "plt.plot(x, list_train_gen_acc, label=\"train gen acc\")\n",
        "plt.plot(x, list_train_disc_acc, label = \"train disc acc\")\n",
        "plt.plot(x, list_val_gen_acc, label =\"Valid gen acc\" )\n",
        "plt.plot(x, list_val_loss, label =\"Valid disc acc\" )\n",
        "plt.legend()\n",
        "plt.show()\n",
        "list_train_gen_acc, list_train_disc_acc, list_val_gen_acc, list_val_gen_acc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "TSPPo4Wo4oIO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "77fa8caf-d7a6-4eb4-bc32-29460ae5702e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[ 0.8779,  0.9025, -0.1308, -0.1728,  0.3057],\n",
            "        [ 0.5910,  1.8592, -1.4776,  1.7094, -0.2206],\n",
            "        [ 1.2816,  0.5546,  0.1915, -0.2074, -1.3647]], requires_grad=True) tensor([3, 0, 0]) tensor(1.6992, grad_fn=<NllLossBackward0>)\n",
            "tensor([[ 2.0994, -0.0871,  0.5965, -1.5593, -0.1185],\n",
            "        [-0.9349,  1.0487, -0.9134, -1.3678, -0.3626],\n",
            "        [ 0.6379, -1.1598,  0.7783, -1.7011,  1.0480]], requires_grad=True) tensor([[0.2759, 0.2942, 0.1516, 0.1326, 0.1457],\n",
            "        [0.1417, 0.0869, 0.0849, 0.4153, 0.2712],\n",
            "        [0.1582, 0.0491, 0.0480, 0.5533, 0.1914]]) tensor(2.3461, grad_fn=<DivBackward1>)\n"
          ]
        }
      ],
      "source": [
        "# Example of target with class indice\n",
        "loss = nn.CrossEntropyLoss(reduction = \"mean\")\n",
        "input = torch.randn(3, 5, requires_grad=True)\n",
        "target = torch.empty(3, dtype=torch.long).random_(5)\n",
        "output = loss(input, target)\n",
        "print(input, target,output)\n",
        "\n",
        "# Example of target with class probabilities\n",
        "input = torch.randn(3, 5, requires_grad=True)\n",
        "target = torch.randn(3, 5).softmax(dim=1)\n",
        "output = loss(input, target)\n",
        "print(input, target,output)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n"
      ],
      "metadata": {
        "id": "9nl7HoLrceCs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "XTpIXN4H8_tu"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "4rIXgnDpzihV"
      ],
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.7"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "26e6569062204c2d850d071f669271bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_91bffffd6d1340e8b9808c43e0fba036",
              "IPY_MODEL_c1d462aa5bab44f4b46a361b961620d9",
              "IPY_MODEL_1f9cdb0023eb444380e57465cfba788d"
            ],
            "layout": "IPY_MODEL_bcad8bc436e24b9984cc4f862e88a65b"
          }
        },
        "91bffffd6d1340e8b9808c43e0fba036": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_351f7fb8bccd4c9bb4b0deb3c72d1c20",
            "placeholder": "​",
            "style": "IPY_MODEL_296489e9dd2d437384d74f732c7769da",
            "value": "Map: 100%"
          }
        },
        "c1d462aa5bab44f4b46a361b961620d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ac1dabbe87a4e159e78426468979ffb",
            "max": 1000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fb84f92841f9492ea94750115934a48d",
            "value": 1000
          }
        },
        "1f9cdb0023eb444380e57465cfba788d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9f7f6dd4e5d244feae338af7c0a12736",
            "placeholder": "​",
            "style": "IPY_MODEL_e465d4e6a51447d58b32681115371d1c",
            "value": " 1000/1000 [00:07&lt;00:00, 163.88 examples/s]"
          }
        },
        "bcad8bc436e24b9984cc4f862e88a65b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "351f7fb8bccd4c9bb4b0deb3c72d1c20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "296489e9dd2d437384d74f732c7769da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9ac1dabbe87a4e159e78426468979ffb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fb84f92841f9492ea94750115934a48d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9f7f6dd4e5d244feae338af7c0a12736": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e465d4e6a51447d58b32681115371d1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}