Homework5_TNiven_12.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TimotheeeNiven/IntroML_TNiven/blob/main/Homework5_TNiven_12.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zq8OJyc6Uc26"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n",
        "from torch.utils.data import Dataset, DataLoader, IterableDataset, TensorDataset\n",
        "import torch.utils.data as data\n",
        "import time\n",
        "import requests\n",
        "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
        "import torch.nn.functional as F\n",
        "from torch.optim.lr_scheduler import LambdaLR\n",
        "import random\n",
        "from transformers import BertTokenizer, BertForSequenceClassification\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wl5WkVpyMnkW",
        "outputId": "6fe31a16-401f-4b0c-c956-02a6aeca634f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Training and validating for sequence length 10\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/transformer.py:286: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance)\n",
            "  warnings.warn(f\"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}\")\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, Loss: 3.0526015758514404, Validation Loss: 2.9355547428131104, Validation Accuracy: 0.14255765080451965\n",
            "Epoch 10, Loss: 2.856675148010254, Validation Loss: 2.712320566177368, Validation Accuracy: 0.2159329205751419\n",
            "Epoch 15, Loss: 2.5810840129852295, Validation Loss: 2.5576460361480713, Validation Accuracy: 0.2515723407268524\n",
            "Epoch 20, Loss: 2.4728055000305176, Validation Loss: 2.5014548301696777, Validation Accuracy: 0.2725366950035095\n",
            "Epoch 25, Loss: 2.3905084133148193, Validation Loss: 2.4713757038116455, Validation Accuracy: 0.2704402506351471\n",
            "Epoch 30, Loss: 2.3448147773742676, Validation Loss: 2.441974639892578, Validation Accuracy: 0.26834380626678467\n",
            "Epoch 35, Loss: 2.3021578788757324, Validation Loss: 2.4251315593719482, Validation Accuracy: 0.276729553937912\n",
            "Epoch 40, Loss: 2.2780842781066895, Validation Loss: 2.409914255142212, Validation Accuracy: 0.2704402506351471\n",
            "Epoch 45, Loss: 2.250457763671875, Validation Loss: 2.3948702812194824, Validation Accuracy: 0.26834380626678467\n",
            "Epoch 50, Loss: 2.238884210586548, Validation Loss: 2.387345314025879, Validation Accuracy: 0.276729553937912\n",
            "Execution time for training: 270.5518856048584 seconds\n",
            "\n",
            "\n",
            "Training and validating for sequence length 20\n",
            "Epoch 5, Loss: 3.0259556770324707, Validation Loss: 3.0059196949005127, Validation Accuracy: 0.12842105329036713\n",
            "Epoch 10, Loss: 2.789217472076416, Validation Loss: 2.7607150077819824, Validation Accuracy: 0.2210526317358017\n",
            "Epoch 15, Loss: 2.569000005722046, Validation Loss: 2.640048027038574, Validation Accuracy: 0.2336842119693756\n",
            "Epoch 20, Loss: 2.459080934524536, Validation Loss: 2.584430456161499, Validation Accuracy: 0.22315789759159088\n",
            "Epoch 25, Loss: 2.3874447345733643, Validation Loss: 2.545837879180908, Validation Accuracy: 0.22315789759159088\n",
            "Epoch 30, Loss: 2.319239616394043, Validation Loss: 2.5153255462646484, Validation Accuracy: 0.2336842119693756\n",
            "Epoch 35, Loss: 2.2776100635528564, Validation Loss: 2.487255334854126, Validation Accuracy: 0.23157894611358643\n",
            "Epoch 40, Loss: 2.2535040378570557, Validation Loss: 2.4848263263702393, Validation Accuracy: 0.22736841440200806\n",
            "Epoch 45, Loss: 2.2270071506500244, Validation Loss: 2.495347499847412, Validation Accuracy: 0.22947368025779724\n",
            "Epoch 50, Loss: 2.2170634269714355, Validation Loss: 2.480761766433716, Validation Accuracy: 0.22736841440200806\n",
            "Execution time for training: 530.8911323547363 seconds\n",
            "\n",
            "\n",
            "Training and validating for sequence length 30\n",
            "Epoch 5, Loss: 3.017021656036377, Validation Loss: 3.038987636566162, Validation Accuracy: 0.1437632143497467\n",
            "Epoch 10, Loss: 2.7562129497528076, Validation Loss: 2.812053680419922, Validation Accuracy: 0.20930232107639313\n",
            "Epoch 15, Loss: 2.5399293899536133, Validation Loss: 2.6822352409362793, Validation Accuracy: 0.2431289702653885\n",
            "Epoch 20, Loss: 2.437251567840576, Validation Loss: 2.608985662460327, Validation Accuracy: 0.23255814611911774\n",
            "Epoch 25, Loss: 2.356602907180786, Validation Loss: 2.601389169692993, Validation Accuracy: 0.21987314522266388\n",
            "Epoch 30, Loss: 2.298011064529419, Validation Loss: 2.5583956241607666, Validation Accuracy: 0.23044396936893463\n",
            "Epoch 35, Loss: 2.2672488689422607, Validation Loss: 2.5284736156463623, Validation Accuracy: 0.2262156456708908\n",
            "Epoch 40, Loss: 2.2352144718170166, Validation Loss: 2.508700132369995, Validation Accuracy: 0.2241014838218689\n",
            "Epoch 45, Loss: 2.217587471008301, Validation Loss: 2.4988160133361816, Validation Accuracy: 0.2241014838218689\n",
            "Epoch 50, Loss: 2.194214344024658, Validation Loss: 2.4961514472961426, Validation Accuracy: 0.23044396936893463\n",
            "Execution time for training: 769.2569558620453 seconds\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Provided text sequence\n",
        "text = \"\"\"Next character prediction is a fundamental task in the field of natural language processing (NLP) that involves predicting the next character in a sequence of text based on the characters that precede it. This task is essential for various applications, including text auto-completion, spell checking, and even in the development of sophisticated AI models capable of generating human-like text.\n",
        "\n",
        "At its core, next character prediction relies on statistical models or deep learning algorithms to analyze a given sequence of text and predict which character is most likely to follow. These predictions are based on patterns and relationships learned from large datasets of text during the training phase of the model.\n",
        "\n",
        "One of the most popular approaches to next character prediction involves the use of Recurrent Neural Networks (RNNs), and more specifically, a variant called Long Short-Term Memory (LSTM) networks. RNNs are particularly well-suited for sequential data like text, as they can maintain information in 'memory' about previous characters to inform the prediction of the next character. LSTM networks enhance this capability by being able to remember long-term dependencies, making them even more effective for next character prediction tasks.\n",
        "\n",
        "Training a model for next character prediction involves feeding it large amounts of text data, allowing it to learn the probability of each character's appearance following a sequence of characters. During this training process, the model adjusts its parameters to minimize the difference between its predictions and the actual outcomes, thus improving its predictive accuracy over time.\n",
        "\n",
        "Once trained, the model can be used to predict the next character in a given piece of text by considering the sequence of characters that precede it. This can enhance user experience in text editing software, improve efficiency in coding environments with auto-completion features, and enable more natural interactions with AI-based chatbots and virtual assistants.\n",
        "\n",
        "In summary, next character prediction plays a crucial role in enhancing the capabilities of various NLP applications, making text-based interactions more efficient, accurate, and human-like. Through the use of advanced machine learning models like RNNs and LSTMs, next character prediction continues to evolve, opening new possibilities for the future of text-based technology.\"\"\"\n",
        "\n",
        "# Function to preprocess text into sequences and labels\n",
        "def create_sequences_and_labels(text, max_length):\n",
        "    sequences = []\n",
        "    labels = []\n",
        "    for i in range(len(text) - max_length):\n",
        "        sequences.append(text[i:i + max_length])\n",
        "        labels.append(text[i + max_length])\n",
        "    return sequences, labels\n",
        "\n",
        "# Function to create vocabulary and mappings\n",
        "def create_vocab_mappings(text):\n",
        "    chars = sorted(list(set(text)))\n",
        "    ix_to_char = {i: ch for i, ch in enumerate(chars)}\n",
        "    char_to_ix = {ch: i for i, ch in enumerate(chars)}\n",
        "    return chars, ix_to_char, char_to_ix\n",
        "\n",
        "# Function to convert sequences and labels to tensors\n",
        "def convert_to_tensors(sequences, labels, char_to_ix):\n",
        "    X = torch.tensor([[char_to_ix[ch] for ch in seq] for seq in sequences], dtype=torch.long)\n",
        "    y = torch.tensor([char_to_ix[label] for label in labels], dtype=torch.long)\n",
        "    return X, y\n",
        "\n",
        "# Prepare dataset for different sequence lengths\n",
        "max_lengths = [10, 20, 30]\n",
        "datasets = {}\n",
        "\n",
        "for max_length in max_lengths:\n",
        "    sequences, labels = create_sequences_and_labels(text, max_length)\n",
        "    chars, ix_to_char, char_to_ix = create_vocab_mappings(text)\n",
        "    X, y = convert_to_tensors(sequences, labels, char_to_ix)\n",
        "    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n",
        "    datasets[max_length] = (X_train, X_val, y_train, y_val, len(chars))\n",
        "\n",
        "# Define Transformer model\n",
        "class CharTransformer(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size, output_size, num_layers, nhead):\n",
        "        super(CharTransformer, self).__init__()\n",
        "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
        "        encoder_layers = nn.TransformerEncoderLayer(hidden_size, nhead)\n",
        "        self.transformer_encoder = nn.TransformerEncoder(encoder_layers, num_layers)\n",
        "        self.fc = nn.Linear(hidden_size, output_size)\n",
        "\n",
        "    def forward(self, x):\n",
        "        embedded = self.embedding(x)\n",
        "        transformer_output = self.transformer_encoder(embedded)\n",
        "        output = self.fc(transformer_output[:, -1, :])  # Get output of last Transformer block\n",
        "        return output\n",
        "\n",
        "# Hyperparameters\n",
        "hidden_size = 128\n",
        "num_layers = 4\n",
        "nhead = 2\n",
        "learning_rate = 0.001\n",
        "epochs = 50\n",
        "\n",
        "# Train and validate models for different sequence lengths\n",
        "for max_length, (X_train, X_val, y_train, y_val, vocab_size) in datasets.items():\n",
        "    print(f\"Training and validating for sequence length {max_length}\")\n",
        "    model = CharTransformer(vocab_size, hidden_size, vocab_size, num_layers, nhead)\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
        "\n",
        "    start_time = time.time()\n",
        "    for epoch in range(epochs):\n",
        "        model.train()\n",
        "        optimizer.zero_grad()\n",
        "        output = model(X_train)\n",
        "        loss = criterion(output, y_train)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # Validation\n",
        "        model.eval()\n",
        "        with torch.no_grad():\n",
        "            val_output = model(X_val)\n",
        "            val_loss = criterion(val_output, y_val)\n",
        "            _, predicted = torch.max(val_output, 1)\n",
        "            val_accuracy = (predicted == y_val).float().mean()\n",
        "\n",
        "        if (epoch+1) % 5 == 0:\n",
        "            print(f'Epoch {epoch+1}, Loss: {loss.item()}, Validation Loss: {val_loss.item()}, Validation Accuracy: {val_accuracy.item()}')\n",
        "\n",
        "    end_time = time.time()\n",
        "    execution_time = end_time - start_time\n",
        "    print(f\"Execution time for training: {execution_time} seconds\")\n",
        "    print(\"\\n\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JMMSp6xFf0gB"
      },
      "source": [
        "Problem 2"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Step 1: Download the dataset\n",
        "url = \"https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt\"\n",
        "response = requests.get(url)\n",
        "text = response.text  # This is the entire text data\n",
        "\n",
        "# Step 2: Prepare the dataset\n",
        "sequence_length = 20\n",
        "# Create a character mapping to integers\n",
        "chars = sorted(list(set(text)))\n",
        "char_to_int = {ch: i for i, ch in enumerate(chars)}\n",
        "int_to_char = {i: ch for i, ch in enumerate(chars)}\n",
        "\n",
        "# Encode the text into integers\n",
        "encoded_text = [char_to_int[ch] for ch in text]\n",
        "\n",
        "# Create sequences and targets\n",
        "sequences = []\n",
        "targets = []\n",
        "for i in range(0, len(encoded_text) - sequence_length):\n",
        "    seq = encoded_text[i:i+sequence_length]\n",
        "    target = encoded_text[i+sequence_length]\n",
        "    sequences.append(seq)\n",
        "    targets.append(target)\n",
        "\n",
        "# Convert lists to PyTorch tensors\n",
        "sequences = torch.tensor(sequences, dtype=torch.long)\n",
        "targets = torch.tensor(targets, dtype=torch.long)\n",
        "\n",
        "# Step 3: Create a dataset class\n",
        "class CharDataset(Dataset):\n",
        "    def __init__(self, sequences, targets):\n",
        "        self.sequences = sequences\n",
        "        self.targets = targets\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.sequences)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        return self.sequences[index], self.targets[index]\n",
        "\n",
        "# Instantiate the dataset\n",
        "dataset = CharDataset(sequences, targets)\n",
        "\n",
        "# Step 4: Create data loaders\n",
        "batch_size = 128\n",
        "train_size = int(len(dataset) * 0.8)\n",
        "test_size = len(dataset) - train_size\n",
        "train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])\n",
        "\n",
        "train_loader = DataLoader(train_dataset, shuffle=True, batch_size=batch_size)\n",
        "test_loader = DataLoader(test_dataset, shuffle=False, batch_size=batch_size)"
      ],
      "metadata": {
        "id": "S3LHnB5-j06G"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class CharModel(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size, output_size, model_type='Transformer', num_layers=2, num_heads=2, dim_feedforward=256, dropout=0.1):\n",
        "        super(CharModel, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
        "        if model_type == 'Transformer':\n",
        "            encoder_layer = nn.TransformerEncoderLayer(d_model=hidden_size, nhead=num_heads, dim_feedforward=dim_feedforward, dropout=dropout)\n",
        "            self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)\n",
        "        else:\n",
        "            raise ValueError(\"Invalid model type. Choose 'Transformer'.\")\n",
        "        self.fc = nn.Linear(hidden_size, output_size)\n",
        "\n",
        "    def forward(self, x):\n",
        "        embedded = self.embedding(x)\n",
        "        transformer_output = self.transformer_encoder(embedded)\n",
        "        output = self.fc(transformer_output[:, -1, :])\n",
        "        return output"
      ],
      "metadata": {
        "id": "3T-P-DM2oAp9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Train and evaluate function\n",
        "def train_evaluate(model_type, train_loader, val_loader, device):\n",
        "    model = CharModel(len(chars), hidden_size, len(chars), model_type).to(device)\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
        "\n",
        "    start_time = time.time()\n",
        "    for epoch in range(epochs):\n",
        "        model.train()\n",
        "        train_loss = 0.0\n",
        "        for inputs, targets in train_loader:\n",
        "            inputs, targets = inputs.to(device, non_blocking=True), targets.to(device, non_blocking=True)  # Move data to device\n",
        "            optimizer.zero_grad()\n",
        "            output = model(inputs)\n",
        "            loss = criterion(output, targets)\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "            train_loss += loss.item() * inputs.size(0)\n",
        "\n",
        "        epoch_train_loss = train_loss / len(train_loader.dataset)\n",
        "\n",
        "        # Validation\n",
        "        model.eval()\n",
        "        val_loss = 0.0\n",
        "        correct = 0\n",
        "        total = 0\n",
        "        with torch.no_grad():\n",
        "            for inputs, targets in val_loader:\n",
        "                inputs, targets = inputs.to(device, non_blocking=True), targets.to(device, non_blocking=True)  # Move data to device\n",
        "                val_output = model(inputs)\n",
        "                loss = criterion(val_output, targets)\n",
        "                val_loss += loss.item() * inputs.size(0)\n",
        "                _, predicted = torch.max(val_output, 1)\n",
        "                total += targets.size(0)\n",
        "                correct += (predicted == targets).sum().item()\n",
        "\n",
        "        epoch_val_loss = val_loss / len(val_loader.dataset)\n",
        "        epoch_val_accuracy = correct / total\n",
        "\n",
        "        if (epoch+1) % 1 == 0:\n",
        "            print(f'Epoch {epoch+1}, Train Loss: {epoch_train_loss}, Validation Loss: {epoch_val_loss}, Validation Accuracy: {epoch_val_accuracy}')\n",
        "\n",
        "    end_time = time.time()\n",
        "    execution_time = end_time - start_time\n",
        "\n",
        "    return epoch_train_loss, epoch_val_loss, epoch_val_accuracy, execution_time\n"
      ],
      "metadata": {
        "id": "mRMMlSZ_l-Dx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "maxlen_values = [20, 30, 50]\n",
        "# Define parameters\n",
        "hidden_size = 512\n",
        "num_layers = 2\n",
        "num_heads = 2\n",
        "dim_feedforward = 256\n",
        "dropout = 0.1\n",
        "learning_rate = 0.0001\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "epochs = 20\n",
        "# Train and evaluate models for different sequence lengths\n",
        "results = {}\n",
        "for maxlen in maxlen_values:\n",
        "    print(f\"\\nTraining models for sequence length: {maxlen}\")\n",
        "    results[maxlen] = {}\n",
        "    for model_type in ['Transformer']:\n",
        "        print(f\"\\nTraining {model_type} model...\")\n",
        "        loss, val_loss, val_accuracy, execution_time = train_evaluate(model_type, train_loader, test_loader, device)\n",
        "        results[maxlen][model_type] = {\n",
        "            'loss': loss,\n",
        "            'val_loss': val_loss,\n",
        "            'val_accuracy': val_accuracy,\n",
        "            'execution_time': execution_time\n",
        "        }\n",
        "\n",
        "# Print and compare results\n",
        "for maxlen, models_data in results.items():\n",
        "    print(f\"\\nResults for sequence length: {maxlen}\")\n",
        "    for model_type, data in models_data.items():\n",
        "        print(f\"\\n{model_type} Model:\")\n",
        "        print(f\"Training Loss: {data['loss']}\")\n",
        "        print(f\"Validation Loss: {data['val_loss']}\")\n",
        "        print(f\"Validation Accuracy: {data['val_accuracy']}\")\n",
        "        print(f\"Execution Time: {data['execution_time']} seconds\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rUXz-NXxoLMW",
        "outputId": "55f940e9-371e-46e2-dda7-5788e999be28"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Training models for sequence length: 20\n",
            "\n",
            "Training Transformer model...\n",
            "Epoch 1, Train Loss: 2.511823572262447, Validation Loss: 2.4801008787083543, Validation Accuracy: 0.27047405581082595\n",
            "Epoch 2, Train Loss: 2.4833527778428857, Validation Loss: 2.470777820613664, Validation Accuracy: 0.27020957077216184\n",
            "Epoch 3, Train Loss: 2.478055991993177, Validation Loss: 2.4696921855142455, Validation Accuracy: 0.2701512944077104\n",
            "Epoch 4, Train Loss: 2.4750124252402315, Validation Loss: 2.4670280606407604, Validation Accuracy: 0.2691157682393814\n",
            "Epoch 5, Train Loss: 2.472941410287177, Validation Loss: 2.466399253943329, Validation Accuracy: 0.2695012888042138\n",
            "Epoch 6, Train Loss: 2.4716557105294377, Validation Loss: 2.465298949567296, Validation Accuracy: 0.2681609324218312\n",
            "Epoch 7, Train Loss: 2.4707366181172024, Validation Loss: 2.4655798448111383, Validation Accuracy: 0.26311778549815085\n",
            "Epoch 8, Train Loss: 2.4699570494612284, Validation Loss: 2.4663895775701894, Validation Accuracy: 0.27028577832567524\n",
            "Epoch 9, Train Loss: 2.4693051503035326, Validation Loss: 2.4640024262004725, Validation Accuracy: 0.26982405020732936\n",
            "Epoch 10, Train Loss: 2.4685326474640776, Validation Loss: 2.4647947633799503, Validation Accuracy: 0.26989577496357725\n",
            "Epoch 11, Train Loss: 2.4679650999595513, Validation Loss: 2.4641304946979785, Validation Accuracy: 0.26929508013000114\n",
            "Epoch 12, Train Loss: 2.4676398682826144, Validation Loss: 2.461557034989797, Validation Accuracy: 0.269348873697187\n",
            "Epoch 13, Train Loss: 2.4671447872376073, Validation Loss: 2.4615053024877684, Validation Accuracy: 0.27032164070379916\n",
            "Epoch 14, Train Loss: 2.4668980842395234, Validation Loss: 2.4624555724273414, Validation Accuracy: 0.269012663902275\n",
            "Epoch 15, Train Loss: 2.4664853432889347, Validation Loss: 2.46199012842919, Validation Accuracy: 0.27033060629833017\n",
            "Epoch 16, Train Loss: 2.4662793305548, Validation Loss: 2.461388938510069, Validation Accuracy: 0.27055026336433935\n",
            "Epoch 17, Train Loss: 2.4656637185141195, Validation Loss: 2.462964135225239, Validation Accuracy: 0.2705771601479323\n",
            "Epoch 18, Train Loss: 2.4654406009557186, Validation Loss: 2.4606514614440393, Validation Accuracy: 0.27069819567410064\n",
            "Epoch 19, Train Loss: 2.4654421444570986, Validation Loss: 2.46114900151323, Validation Accuracy: 0.26894542194329263\n",
            "Epoch 20, Train Loss: 2.464954891251369, Validation Loss: 2.459702318759693, Validation Accuracy: 0.2703843998655161\n",
            "\n",
            "Training models for sequence length: 30\n",
            "\n",
            "Training Transformer model...\n",
            "Epoch 1, Train Loss: 2.513062586225245, Validation Loss: 2.4811034426977185, Validation Accuracy: 0.2640098621539841\n",
            "Epoch 2, Train Loss: 2.4845249872914352, Validation Loss: 2.473996302174918, Validation Accuracy: 0.2692412865628152\n",
            "Epoch 3, Train Loss: 2.478878096742564, Validation Loss: 2.4698727929697677, Validation Accuracy: 0.2695416339796033\n",
            "Epoch 4, Train Loss: 2.4756097969140125, Validation Loss: 2.468148507214736, Validation Accuracy: 0.26627815757032386\n",
            "Epoch 5, Train Loss: 2.4736373651705277, Validation Loss: 2.466481680662178, Validation Accuracy: 0.2681250700437073\n",
            "Epoch 6, Train Loss: 2.472124269203345, Validation Loss: 2.466429543674199, Validation Accuracy: 0.2679816205312115\n",
            "Epoch 7, Train Loss: 2.4708480818571146, Validation Loss: 2.465883294150098, Validation Accuracy: 0.27069819567410064\n",
            "Epoch 8, Train Loss: 2.4700088234584983, Validation Loss: 2.4655031073458136, Validation Accuracy: 0.2700078448952146\n",
            "Epoch 9, Train Loss: 2.4691298227613427, Validation Loss: 2.4636749804352585, Validation Accuracy: 0.2685912809593186\n",
            "Epoch 10, Train Loss: 2.4685771482319216, Validation Loss: 2.4629407434208064, Validation Accuracy: 0.2688916283761067\n",
            "Epoch 11, Train Loss: 2.4682173842300705, Validation Loss: 2.464664359976061, Validation Accuracy: 0.26891404236243416\n",
            "Epoch 12, Train Loss: 2.4675634227750054, Validation Loss: 2.4639094737034855, Validation Accuracy: 0.27076992043034853\n",
            "Epoch 13, Train Loss: 2.4673875704975967, Validation Loss: 2.4630202108293613, Validation Accuracy: 0.2691068026448504\n",
            "Epoch 14, Train Loss: 2.4667001955523604, Validation Loss: 2.4644776798563797, Validation Accuracy: 0.26791886136949455\n",
            "Epoch 15, Train Loss: 2.4666027852701307, Validation Loss: 2.4611131442139587, Validation Accuracy: 0.26997646531435615\n",
            "Epoch 16, Train Loss: 2.466086490335273, Validation Loss: 2.461149749013056, Validation Accuracy: 0.2700257760842766\n",
            "Epoch 17, Train Loss: 2.46551224953332, Validation Loss: 2.4616997523095647, Validation Accuracy: 0.2685464529866637\n",
            "Epoch 18, Train Loss: 2.4655852406267913, Validation Loss: 2.460694065853898, Validation Accuracy: 0.2709940602936232\n",
            "Epoch 19, Train Loss: 2.465334618372366, Validation Loss: 2.460307739619702, Validation Accuracy: 0.2693533564944525\n",
            "Epoch 20, Train Loss: 2.4652009040190106, Validation Loss: 2.461686493193744, Validation Accuracy: 0.2680130001120699\n",
            "\n",
            "Training models for sequence length: 50\n",
            "\n",
            "Training Transformer model...\n",
            "Epoch 1, Train Loss: 2.5128709207837883, Validation Loss: 2.477766037758965, Validation Accuracy: 0.26756920318278604\n",
            "Epoch 2, Train Loss: 2.4835912550188364, Validation Loss: 2.4707322531891127, Validation Accuracy: 0.26925025215734616\n",
            "Epoch 3, Train Loss: 2.478121971048902, Validation Loss: 2.46977524611069, Validation Accuracy: 0.2701871567858344\n",
            "Epoch 4, Train Loss: 2.475177505273078, Validation Loss: 2.4686968011534374, Validation Accuracy: 0.2667981620531211\n",
            "Epoch 5, Train Loss: 2.4732361167014365, Validation Loss: 2.4661953546839768, Validation Accuracy: 0.26906645746946095\n",
            "Epoch 6, Train Loss: 2.4716466452084407, Validation Loss: 2.4657850951789326, Validation Accuracy: 0.27033508909559567\n",
            "Epoch 7, Train Loss: 2.4708482640336418, Validation Loss: 2.4649765959430487, Validation Accuracy: 0.26985542978818783\n",
            "Epoch 8, Train Loss: 2.469855371010583, Validation Loss: 2.4650559040496236, Validation Accuracy: 0.2655788411969069\n",
            "Epoch 9, Train Loss: 2.469059393463546, Validation Loss: 2.4650548849850775, Validation Accuracy: 0.27004370727333854\n",
            "Epoch 10, Train Loss: 2.468256292546807, Validation Loss: 2.4629673392631326, Validation Accuracy: 0.2644940042586574\n",
            "Epoch 11, Train Loss: 2.467792551797286, Validation Loss: 2.4632597284424964, Validation Accuracy: 0.26940715006163846\n",
            "Epoch 12, Train Loss: 2.4672915398588993, Validation Loss: 2.46258389899833, Validation Accuracy: 0.2697343942620195\n",
            "Epoch 13, Train Loss: 2.467046474996548, Validation Loss: 2.461089156150925, Validation Accuracy: 0.27020508797489634\n",
            "Epoch 14, Train Loss: 2.4666214490600034, Validation Loss: 2.4623923302327357, Validation Accuracy: 0.2696895662893646\n",
            "Epoch 15, Train Loss: 2.4661808165335897, Validation Loss: 2.4611708020517327, Validation Accuracy: 0.2708326795920655\n",
            "Epoch 16, Train Loss: 2.466084174316484, Validation Loss: 2.4626256485780527, Validation Accuracy: 0.2704247450409055\n",
            "Epoch 17, Train Loss: 2.4656635430698586, Validation Loss: 2.4607084218255744, Validation Accuracy: 0.2699988793006836\n",
            "Epoch 18, Train Loss: 2.465439052883152, Validation Loss: 2.4624357541516186, Validation Accuracy: 0.26963128992491314\n",
            "Epoch 19, Train Loss: 2.4651993016268663, Validation Loss: 2.460739530655056, Validation Accuracy: 0.27046957301356045\n",
            "Epoch 20, Train Loss: 2.465210493829139, Validation Loss: 2.460428448021579, Validation Accuracy: 0.2681295528409728\n",
            "\n",
            "Results for sequence length: 20\n",
            "\n",
            "Transformer Model:\n",
            "Training Loss: 2.464954891251369\n",
            "Validation Loss: 2.459702318759693\n",
            "Validation Accuracy: 0.2703843998655161\n",
            "Execution Time: 1216.8855650424957 seconds\n",
            "\n",
            "Results for sequence length: 30\n",
            "\n",
            "Transformer Model:\n",
            "Training Loss: 2.4652009040190106\n",
            "Validation Loss: 2.461686493193744\n",
            "Validation Accuracy: 0.2680130001120699\n",
            "Execution Time: 1232.8566801548004 seconds\n",
            "\n",
            "Results for sequence length: 50\n",
            "\n",
            "Transformer Model:\n",
            "Training Loss: 2.465210493829139\n",
            "Validation Loss: 2.460428448021579\n",
            "Validation Accuracy: 0.2681295528409728\n",
            "Execution Time: 1243.4182229042053 seconds\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "V100",
      "machine_shape": "hm",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}