Homework5_TNiven_34.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNFi0g/JbholPzf0tG9CsWR",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TimotheeeNiven/IntroML_TNiven/blob/main/Homework5_TNiven_34.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.utils.data as data\n",
        "import math\n",
        "import copy\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "import math\n",
        "import random"
      ],
      "metadata": {
        "id": "OvfhXzfWNaU1"
      },
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "metadata": {
        "id": "w0IveXmdNRmG"
      },
      "outputs": [],
      "source": [
        "# Dataset containing pairs of synonyms\n",
        "dataset = [\n",
        "    (\"I am cold\", \"J'ai froid\"),\n",
        "    (\"You are tired\", \"Tu es fatigué\"),\n",
        "    (\"He is hungry\", \"Il a faim\"),\n",
        "    (\"She is happy\", \"Elle est heureuse\"),\n",
        "    (\"We are friends\", \"Nous sommes amis\"),\n",
        "    (\"They are students\", \"Ils sont étudiants\"),\n",
        "    (\"The cat is sleeping\", \"Le chat dort\"),\n",
        "    (\"The sun is shining\", \"Le soleil brille\"),\n",
        "    (\"We love music\", \"Nous aimons la musique\"),\n",
        "    (\"She speaks French fluently\", \"Elle parle français couramment\"),\n",
        "    (\"He enjoys reading books\", \"Il aime lire des livres\"),\n",
        "    (\"They play soccer every weekend\", \"Ils jouent au football chaque week-end\"),\n",
        "    (\"The movie starts at 7 PM\", \"Le film commence à 19 heures\"),\n",
        "    (\"She wears a red dress\", \"Elle porte une robe rouge\"),\n",
        "    (\"We cook dinner together\", \"Nous cuisinons le dîner ensemble\"),\n",
        "    (\"He drives a blue car\", \"Il conduit une voiture bleue\"),\n",
        "    (\"They visit museums often\", \"Ils visitent souvent des musées\"),\n",
        "    (\"The restaurant serves delicious food\", \"Le restaurant sert une délicieuse cuisine\"),\n",
        "    (\"She studies mathematics at university\", \"Elle étudie les mathématiques à l'université\"),\n",
        "    (\"We watch movies on Fridays\", \"Nous regardons des films le vendredi\"),\n",
        "    (\"He listens to music while jogging\", \"Il écoute de la musique en faisant du jogging\"),\n",
        "    (\"They travel around the world\", \"Ils voyagent autour du monde\"),\n",
        "    (\"The book is on the table\", \"Le livre est sur la table\"),\n",
        "    (\"She dances gracefully\", \"Elle danse avec grâce\"),\n",
        "    (\"We celebrate birthdays with cake\", \"Nous célébrons les anniversaires avec un gâteau\"),\n",
        "    (\"He works hard every day\", \"Il travaille dur tous les jours\"),\n",
        "    (\"They speak different languages\", \"Ils parlent différentes langues\"),\n",
        "    (\"The flowers bloom in spring\", \"Les fleurs fleurissent au printemps\"),\n",
        "    (\"She writes poetry in her free time\", \"Elle écrit de la poésie pendant son temps libre\"),\n",
        "    (\"We learn something new every day\", \"Nous apprenons quelque chose de nouveau chaque jour\"),\n",
        "    (\"The dog barks loudly\", \"Le chien aboie bruyamment\"),\n",
        "    (\"He sings beautifully\", \"Il chante magnifiquement\"),\n",
        "    (\"They swim in the pool\", \"Ils nagent dans la piscine\"),\n",
        "    (\"The birds chirp in the morning\", \"Les oiseaux gazouillent le matin\"),\n",
        "    (\"She teaches English at school\", \"Elle enseigne l'anglais à l'école\"),\n",
        "    (\"We eat breakfast together\", \"Nous prenons le petit déjeuner ensemble\"),\n",
        "    (\"He paints landscapes\", \"Il peint des paysages\"),\n",
        "    (\"They laugh at the joke\", \"Ils rient de la blague\"),\n",
        "    (\"The clock ticks loudly\", \"L'horloge tic-tac bruyamment\"),\n",
        "    (\"She runs in the park\", \"Elle court dans le parc\"),\n",
        "    (\"We travel by train\", \"Nous voyageons en train\"),\n",
        "    (\"He writes a letter\", \"Il écrit une lettre\"),\n",
        "    (\"They read books at the library\", \"Ils lisent des livres à la bibliothèque\"),\n",
        "    (\"The baby cries\", \"Le bébé pleure\"),\n",
        "    (\"She studies hard for exams\", \"Elle étudie dur pour les examens\"),\n",
        "    (\"We plant flowers in the garden\", \"Nous plantons des fleurs dans le jardin\"),\n",
        "    (\"He fixes the car\", \"Il répare la voiture\"),\n",
        "    (\"They drink coffee in the morning\", \"Ils boivent du café le matin\"),\n",
        "    (\"The sun sets in the evening\", \"Le soleil se couche le soir\"),\n",
        "    (\"She dances at the party\", \"Elle danse à la fête\"),\n",
        "    (\"We play music at the concert\", \"Nous jouons de la musique au concert\"),\n",
        "    (\"He cooks dinner for his family\", \"Il cuisine le dîner pour sa famille\"),\n",
        "    (\"They study French grammar\", \"Ils étudient la grammaire française\"),\n",
        "    (\"The rain falls gently\", \"La pluie tombe doucement\"),\n",
        "    (\"She sings a song\", \"Elle chante une chanson\"),\n",
        "    (\"We watch a movie together\", \"Nous regardons un film ensemble\"),\n",
        "    (\"He sleeps deeply\", \"Il dort profondément\"),\n",
        "    (\"They travel to Paris\", \"Ils voyagent à Paris\"),\n",
        "    (\"The children play in the park\", \"Les enfants jouent dans le parc\"),\n",
        "    (\"She walks along the beach\", \"Elle se promène le long de la plage\"),\n",
        "    (\"We talk on the phone\", \"Nous parlons au téléphone\"),\n",
        "    (\"He waits for the bus\", \"Il attend le bus\"),\n",
        "    (\"They visit the Eiffel Tower\", \"Ils visitent la tour Eiffel\"),\n",
        "    (\"The stars twinkle at night\", \"Les étoiles scintillent la nuit\"),\n",
        "    (\"She dreams of flying\", \"Elle rêve de voler\"),\n",
        "    (\"We work in the office\", \"Nous travaillons au bureau\"),\n",
        "    (\"He studies history\", \"Il étudie l'histoire\"),\n",
        "    (\"They listen to the radio\", \"Ils écoutent la radio\"),\n",
        "    (\"The wind blows gently\", \"Le vent souffle doucement\"),\n",
        "    (\"She swims in the ocean\", \"Elle nage dans l'océan\"),\n",
        "    (\"We dance at the wedding\", \"Nous dansons au mariage\"),\n",
        "    (\"He climbs the mountain\", \"Il gravit la montagne\"),\n",
        "    (\"They hike in the forest\", \"Ils font de la randonnée dans la forêt\"),\n",
        "    (\"The cat meows loudly\", \"Le chat miaule bruyamment\"),\n",
        "    (\"She paints a picture\", \"Elle peint un tableau\"),\n",
        "    (\"We build a sandcastle\", \"Nous construisons un château de sable\"),\n",
        "    (\"He sings in the choir\", \"Il chante dans le chœur\")\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Special tokens for the start and end of sequences\n",
        "SOS_token = 0  # Start Of Sequence Token\n",
        "EOS_token = 1  # End Of Sequence Token\n",
        "\n",
        "# Preparing the word to index mapping and vice versa\n",
        "word_to_index = {\"SOS\": SOS_token, \"EOS\": EOS_token}\n",
        "for pair in dataset:\n",
        "    for word in pair[0].split() + pair[1].split():\n",
        "        if word not in word_to_index:\n",
        "            word_to_index[word] = len(word_to_index)\n",
        "\n",
        "index_to_word = {i: word for word, i in word_to_index.items()}\n",
        "\n",
        "def generate_padding_mask(self, sequence):\n",
        "    mask = (sequence == word_to_index[\"EOS\"]).to(sequence.dtype)\n",
        "    return mask\n",
        "\n",
        "def generate_subsequent_mask(self, sequence):\n",
        "    mask = (torch.triu(torch.ones(sequence.size(0), sequence.size(0)), diagonal=1) == 1).to(sequence.dtype)\n",
        "    mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n",
        "    return mask\n",
        "\n",
        "\n",
        "class TranslationDataset(Dataset):\n",
        "    \"\"\"Custom Dataset class for handling translation pairs.\"\"\"\n",
        "    def __init__(self, dataset, word_to_index):\n",
        "        self.dataset = dataset\n",
        "        self.word_to_index = word_to_index\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.dataset)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        input_sentence, target_sentence = self.dataset[idx]\n",
        "        input_indices = [self.word_to_index[word] for word in input_sentence.split()] + [EOS_token]\n",
        "        target_indices = [self.word_to_index[word] for word in target_sentence.split()] + [EOS_token]\n",
        "        return torch.tensor(input_indices, dtype=torch.long), torch.tensor(target_indices, dtype=torch.long)\n",
        "\n",
        "# Creating a DataLoader to batch and shuffle the dataset\n",
        "translation_dataset = TranslationDataset(dataset, word_to_index)\n",
        "dataloader = DataLoader(translation_dataset, batch_size=1, shuffle=True)\n",
        "\n",
        "class Transformer(nn.Module):\n",
        "    def __init__(self, input_vocab_size, target_vocab_size, hidden_size, num_layers=2, num_heads=8, dropout=0.1):\n",
        "        super(Transformer, self).__init__()\n",
        "        self.embedding_input = nn.Embedding(input_vocab_size, hidden_size)\n",
        "        self.embedding_target = nn.Embedding(target_vocab_size, hidden_size)\n",
        "        self.transformer = nn.Transformer(\n",
        "            d_model=hidden_size,\n",
        "            nhead=num_heads,\n",
        "            num_encoder_layers=num_layers,\n",
        "            num_decoder_layers=num_layers,\n",
        "            dim_feedforward=hidden_size * 4,\n",
        "            dropout=dropout\n",
        "        )\n",
        "        self.fc = nn.Linear(hidden_size, target_vocab_size)\n",
        "\n",
        "    def forward(self, input, target):\n",
        "        input_embedded = self.embedding_input(input)\n",
        "        target_embedded = self.embedding_target(target)\n",
        "\n",
        "        input_pad_mask = self.generate_padding_mask(input)\n",
        "        target_pad_mask = self.generate_padding_mask(target)\n",
        "        target_subsequent_mask = self.generate_subsequent_mask(target)\n",
        "\n",
        "        input_embedded = input_embedded.permute(1, 0, 2)\n",
        "        target_embedded = target_embedded.permute(1, 0, 2)\n",
        "\n",
        "        output = self.transformer(\n",
        "            input_embedded,\n",
        "            target_embedded,\n",
        "            src_key_padding_mask=input_pad_mask,\n",
        "            tgt_key_padding_mask=target_pad_mask,\n",
        "            memory_key_padding_mask=input_pad_mask,\n",
        "            tgt_mask=target_subsequent_mask\n",
        "        )\n",
        "\n",
        "        output = self.fc(output)\n",
        "        return output.permute(1, 0, 2)\n",
        "\n",
        "    def generate_padding_mask(self, sequence):\n",
        "        mask = (sequence == word_to_index[\"EOS\"])\n",
        "        return mask\n",
        "\n",
        "    def generate_subsequent_mask(self, sequence):\n",
        "        mask = (torch.triu(torch.ones(sequence.size(1), sequence.size(1))) == 1).transpose(0, 1)\n",
        "        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n",
        "        return mask\n",
        "\n",
        "# Assuming all words in the dataset + 'SOS' and 'EOS' tokens are included in word_to_index\n",
        "input_size = len(word_to_index)\n",
        "hidden_size = 64\n",
        "output_size = len(word_to_index)\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "model = Transformer(input_size, output_size, hidden_size).to(device)\n",
        "\n",
        "# Set the learning rate for optimization\n",
        "learning_rate = 0.0005\n",
        "\n",
        "# Define the loss function and optimizer\n",
        "criterion = nn.CrossEntropyLoss(ignore_index=word_to_index[\"EOS\"])\n",
        "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
        "\n",
        "# Set number of epochs for training\n",
        "n_epochs = 20\n",
        "\n",
        "# Training loop\n",
        "for epoch in range(n_epochs):\n",
        "    total_loss = 0\n",
        "    total_correct = 0\n",
        "    total_examples = 0\n",
        "\n",
        "    model.train()  # Set the model to training mode\n",
        "\n",
        "    for input_tensor, target_tensor in dataloader:\n",
        "        input_tensor = input_tensor.to(device)\n",
        "        target_tensor = target_tensor.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        output = model(input_tensor, target_tensor[:, :-1])  # Exclude EOS token from target\n",
        "        output_dim = output.shape[-1]\n",
        "\n",
        "        output = output.contiguous().view(-1, output_dim)\n",
        "        target_tensor = target_tensor[:, 1:].contiguous().view(-1)  # Exclude SOS token from target\n",
        "\n",
        "        loss = criterion(output, target_tensor)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        total_loss += loss.item()\n",
        "\n",
        "        _, predicted = torch.max(output, 1)\n",
        "        correct = (predicted == target_tensor).sum().item()\n",
        "        total_correct += correct\n",
        "        total_examples += target_tensor.size(0)  # Use target tensor size for total examples\n",
        "\n",
        "    avg_loss = total_loss / len(dataloader)\n",
        "    training_accuracy = total_correct / total_examples  # Calculate training accuracy\n",
        "\n",
        "    print(f\"Epoch [{epoch+1}/{n_epochs}], Loss: {avg_loss:.4f}, Training Accuracy: {training_accuracy:.4f}\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LEtG272Ak7vT",
        "outputId": "b43ef835-1ef8-44ce-caf8-d55c6f9208e9"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch [1/20], Loss: 5.9733, Training Accuracy: 0.0320\n",
            "Epoch [2/20], Loss: 5.1066, Training Accuracy: 0.0640\n",
            "Epoch [3/20], Loss: 4.5633, Training Accuracy: 0.1093\n",
            "Epoch [4/20], Loss: 4.1187, Training Accuracy: 0.1627\n",
            "Epoch [5/20], Loss: 3.7050, Training Accuracy: 0.2480\n",
            "Epoch [6/20], Loss: 3.3437, Training Accuracy: 0.2960\n",
            "Epoch [7/20], Loss: 2.9607, Training Accuracy: 0.3787\n",
            "Epoch [8/20], Loss: 2.6432, Training Accuracy: 0.4240\n",
            "Epoch [9/20], Loss: 2.3154, Training Accuracy: 0.5147\n",
            "Epoch [10/20], Loss: 2.0343, Training Accuracy: 0.5573\n",
            "Epoch [11/20], Loss: 1.7958, Training Accuracy: 0.5760\n",
            "Epoch [12/20], Loss: 1.5184, Training Accuracy: 0.6533\n",
            "Epoch [13/20], Loss: 1.2949, Training Accuracy: 0.6827\n",
            "Epoch [14/20], Loss: 1.1117, Training Accuracy: 0.7227\n",
            "Epoch [15/20], Loss: 0.9492, Training Accuracy: 0.7200\n",
            "Epoch [16/20], Loss: 0.8171, Training Accuracy: 0.7573\n",
            "Epoch [17/20], Loss: 0.6765, Training Accuracy: 0.7653\n",
            "Epoch [18/20], Loss: 0.5470, Training Accuracy: 0.7840\n",
            "Epoch [19/20], Loss: 0.4505, Training Accuracy: 0.7920\n",
            "Epoch [20/20], Loss: 0.3788, Training Accuracy: 0.7867\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_model(model, dataloader, criterion):\n",
        "    total_loss = 0\n",
        "    total_correct = 0\n",
        "    total_examples = 0\n",
        "\n",
        "    model.eval()  # Set the model to evaluation mode\n",
        "\n",
        "    with torch.no_grad():  # No gradient calculation during evaluation\n",
        "        for input_tensor, target_tensor in dataloader:\n",
        "            input_tensor = input_tensor.to(device)\n",
        "            target_tensor = target_tensor.to(device)\n",
        "\n",
        "            output = model(input_tensor, target_tensor[:, :-1])  # Exclude EOS token from target\n",
        "            output_dim = output.shape[-1]\n",
        "\n",
        "            output = output.contiguous().view(-1, output_dim)\n",
        "            target_tensor = target_tensor[:, 1:].contiguous().view(-1)  # Exclude SOS token from target\n",
        "\n",
        "            loss = criterion(output, target_tensor)\n",
        "\n",
        "            total_loss += loss.item()\n",
        "\n",
        "            _, predicted = torch.max(output, 1)\n",
        "            correct = (predicted == target_tensor).sum().item()\n",
        "            total_correct += correct\n",
        "            total_examples += target_tensor.size(0)  # Use target tensor size for total examples\n",
        "\n",
        "    avg_loss = total_loss / len(dataloader)\n",
        "    accuracy = total_correct / total_examples  # Calculate accuracy\n",
        "\n",
        "    return avg_loss, accuracy\n",
        "\n",
        "eval_loss, eval_accuracy = evaluate_model(model, dataloader, criterion)\n",
        "print(f\"Evaluation Loss: {eval_loss:.4f}, Evaluation Accuracy: {eval_accuracy:.4f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BXcMWyBgp19D",
        "outputId": "e496b66c-7af8-4e49-99a1-20d5c2548f70"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Evaluation Loss: 0.1589, Evaluation Accuracy: 0.7947\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Problem 4"
      ],
      "metadata": {
        "id": "MT4mUwzSbiKR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "reversed_dataset = [(french, english) for english, french in dataset]"
      ],
      "metadata": {
        "id": "_TfLoWuwbhtY"
      },
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 1. Reverse the order of sentences in your dataset\n",
        "reversed_dataset = [(target_sentence, input_sentence) for input_sentence, target_sentence in dataset]\n",
        "\n",
        "# 2. Modify the word-to-index mapping accordingly\n",
        "# No need to modify the SOS and EOS tokens\n",
        "reversed_word_to_index = {\"SOS\": SOS_token, \"EOS\": EOS_token}\n",
        "for pair in reversed_dataset:\n",
        "    for word in pair[0].split() + pair[1].split():\n",
        "        if word not in reversed_word_to_index:\n",
        "            reversed_word_to_index[word] = len(reversed_word_to_index)\n",
        "\n",
        "# 3. Adjust the TranslationDataset class to generate input and target tensors in the reversed order\n",
        "class ReversedTranslationDataset(Dataset):\n",
        "    def __init__(self, dataset, word_to_index):\n",
        "        self.dataset = dataset\n",
        "        self.word_to_index = word_to_index\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.dataset)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        target_sentence, input_sentence = self.dataset[idx]  # Reversed order\n",
        "        input_indices = [self.word_to_index[word] for word in input_sentence.split()] + [EOS_token]\n",
        "        target_indices = [self.word_to_index[word] for word in target_sentence.split()] + [EOS_token]\n",
        "        return torch.tensor(input_indices, dtype=torch.long), torch.tensor(target_indices, dtype=torch.long)\n",
        "\n",
        "# Creating a DataLoader to batch and shuffle the reversed dataset\n",
        "reversed_translation_dataset = ReversedTranslationDataset(reversed_dataset, reversed_word_to_index)\n",
        "reversed_dataloader = DataLoader(reversed_translation_dataset, batch_size=1, shuffle=True)\n",
        "\n",
        "# 4. Update the training loop to handle the reversed input-target pairs\n",
        "for epoch in range(n_epochs):\n",
        "    total_loss = 0\n",
        "    total_correct = 0\n",
        "    total_examples = 0\n",
        "\n",
        "    model.train()\n",
        "\n",
        "    for input_tensor, target_tensor in reversed_dataloader:  # Use reversed dataloader\n",
        "        input_tensor = input_tensor.to(device)\n",
        "        target_tensor = target_tensor.to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        output = model(input_tensor, target_tensor[:, :-1])  # Exclude EOS token from target\n",
        "        output_dim = output.shape[-1]\n",
        "\n",
        "        output = output.contiguous().view(-1, output_dim)\n",
        "        target_tensor = target_tensor[:, 1:].contiguous().view(-1)\n",
        "\n",
        "        loss = criterion(output, target_tensor)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        total_loss += loss.item()\n",
        "\n",
        "        _, predicted = torch.max(output, 1)\n",
        "        correct = (predicted == target_tensor).sum().item()\n",
        "        total_correct += correct\n",
        "        total_examples += target_tensor.size(0)\n",
        "\n",
        "    avg_loss = total_loss / len(reversed_dataloader)\n",
        "    training_accuracy = total_correct / total_examples\n",
        "\n",
        "    print(f\"Epoch [{epoch+1}/{n_epochs}], Loss: {avg_loss:.4f}, Training Accuracy: {training_accuracy:.4f}\")\n"
      ],
      "metadata": {
        "id": "WGp3VlJHbqh4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b93b069f-7595-4571-cc5c-7f9df32ea93d"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch [1/20], Loss: 6.0538, Training Accuracy: 0.0160\n",
            "Epoch [2/20], Loss: 4.7351, Training Accuracy: 0.0800\n",
            "Epoch [3/20], Loss: 3.8820, Training Accuracy: 0.1787\n",
            "Epoch [4/20], Loss: 3.2004, Training Accuracy: 0.2773\n",
            "Epoch [5/20], Loss: 2.6263, Training Accuracy: 0.3840\n",
            "Epoch [6/20], Loss: 2.2174, Training Accuracy: 0.4827\n",
            "Epoch [7/20], Loss: 1.8688, Training Accuracy: 0.5387\n",
            "Epoch [8/20], Loss: 1.5444, Training Accuracy: 0.6107\n",
            "Epoch [9/20], Loss: 1.2863, Training Accuracy: 0.6507\n",
            "Epoch [10/20], Loss: 1.0309, Training Accuracy: 0.7093\n",
            "Epoch [11/20], Loss: 0.8329, Training Accuracy: 0.7413\n",
            "Epoch [12/20], Loss: 0.6750, Training Accuracy: 0.7680\n",
            "Epoch [13/20], Loss: 0.5492, Training Accuracy: 0.7760\n",
            "Epoch [14/20], Loss: 0.4519, Training Accuracy: 0.7813\n",
            "Epoch [15/20], Loss: 0.3716, Training Accuracy: 0.7920\n",
            "Epoch [16/20], Loss: 0.3134, Training Accuracy: 0.7867\n",
            "Epoch [17/20], Loss: 0.2664, Training Accuracy: 0.7893\n",
            "Epoch [18/20], Loss: 0.2293, Training Accuracy: 0.7920\n",
            "Epoch [19/20], Loss: 0.1966, Training Accuracy: 0.7840\n",
            "Epoch [20/20], Loss: 0.1630, Training Accuracy: 0.7947\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_model(model, dataloader, criterion):\n",
        "    total_loss = 0\n",
        "    total_correct = 0\n",
        "    total_examples = 0\n",
        "\n",
        "    model.eval()  # Set the model to evaluation mode\n",
        "\n",
        "    with torch.no_grad():  # No gradient calculation during evaluation\n",
        "        for input_tensor, target_tensor in dataloader:\n",
        "            input_tensor = input_tensor.to(device)\n",
        "            target_tensor = target_tensor.to(device)\n",
        "\n",
        "            output = model(input_tensor, target_tensor[:, :-1])  # Exclude EOS token from target\n",
        "            output_dim = output.shape[-1]\n",
        "\n",
        "            output = output.contiguous().view(-1, output_dim)\n",
        "            target_tensor = target_tensor[:, 1:].contiguous().view(-1)  # Exclude SOS token from target\n",
        "\n",
        "            loss = criterion(output, target_tensor)\n",
        "\n",
        "            total_loss += loss.item()\n",
        "\n",
        "            _, predicted = torch.max(output, 1)\n",
        "            correct = (predicted == target_tensor).sum().item()\n",
        "            total_correct += correct\n",
        "            total_examples += target_tensor.size(0)  # Use target tensor size for total examples\n",
        "\n",
        "    avg_loss = total_loss / len(dataloader)\n",
        "    accuracy = total_correct / total_examples  # Calculate accuracy\n",
        "\n",
        "    return avg_loss, accuracy\n",
        "eval_loss, eval_accuracy = evaluate_model(model, reversed_dataloader, criterion)\n",
        "print(f\"Evaluation Loss: {eval_loss:.4f}, Evaluation Accuracy: {eval_accuracy:.4f}\")"
      ],
      "metadata": {
        "id": "2v9oAXmab6gs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "637e876b-7544-4c0f-c1b7-be2e9328b498"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Evaluation Loss: 0.0578, Evaluation Accuracy: 0.7947\n"
          ]
        }
      ]
    }
  ]
}