Neural_Machine_Translation_(ru2en)_Assignment_3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Neural Machine Translation (ru2en) - Assignment 3.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oh_4KjlAvztn",
        "outputId": "38f6eb98-1204-4f1e-95b7-eb852f1a1387"
      },
      "source": [
        "# the Russian lemmatizer requires the pymorphy2 library\r\n",
        "!pip install pymorphy2==0.8"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pymorphy2==0.8\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a3/33/fff9675c68b5f6c63ec8c6e6ff57827dda28a1fa5b2c2d727dffff92dd47/pymorphy2-0.8-py2.py3-none-any.whl (46kB)\n",
            "\r\u001b[K     |███████                         | 10kB 13.9MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 20kB 16.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 30kB 10.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 40kB 8.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 51kB 2.7MB/s \n",
            "\u001b[?25hCollecting dawg-python>=0.7\n",
            "  Downloading https://files.pythonhosted.org/packages/6a/84/ff1ce2071d4c650ec85745766c0047ccc3b5036f1d03559fd46bb38b5eeb/DAWG_Python-0.7.2-py2.py3-none-any.whl\n",
            "Requirement already satisfied: docopt>=0.6 in /usr/local/lib/python3.7/dist-packages (from pymorphy2==0.8) (0.6.2)\n",
            "Collecting pymorphy2-dicts<3.0,>=2.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/02/51/2465fd4f72328ab50877b54777764d928da8cb15b74e2680fc1bd8cb3173/pymorphy2_dicts-2.4.393442.3710985-py2.py3-none-any.whl (7.1MB)\n",
            "\u001b[K     |████████████████████████████████| 7.1MB 7.7MB/s \n",
            "\u001b[?25hInstalling collected packages: dawg-python, pymorphy2-dicts, pymorphy2\n",
            "Successfully installed dawg-python-0.7.2 pymorphy2-0.8 pymorphy2-dicts-2.4.393442.3710985\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "df1_gFwwu5k0"
      },
      "source": [
        "import io\r\n",
        "import math\r\n",
        "import time\r\n",
        "\r\n",
        "import torch\r\n",
        "import torch.nn as nn\r\n",
        "import torch.nn.functional as F\r\n",
        "import torch.optim as optim\r\n",
        "import torchtext\r\n",
        "from torchtext.legacy.data import Field, Dataset, Example, BucketIterator\r\n",
        "\r\n",
        "import spacy\r\n",
        "from spacy.lang.ru import Russian\r\n",
        "\r\n",
        "from tqdm import tqdm\r\n",
        "\r\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aNeCltinu6dR"
      },
      "source": [
        "nlp_ru = Russian()\r\n",
        "nlp_en = spacy.load(\"en_core_web_sm\", disable = [\"parser\", \"tagger\", \"ner\"])\r\n",
        "\r\n",
        "def tokenize_ru(text):\r\n",
        "  return [tok.text for tok in nlp_ru.tokenizer(text)]\r\n",
        "\r\n",
        "def tokenize_en(text):\r\n",
        "  return [tok.text for tok in nlp_en.tokenizer(text)]\r\n",
        "\r\n",
        "\r\n",
        "SRC = Field(tokenize = tokenize_ru, include_lengths = True, lower = True)\r\n",
        "TRG = Field(tokenize = tokenize_en, init_token = '<sos>', eos_token = '<eos>',\r\n",
        "                     include_lengths = True, lower = True)\r\n",
        "\r\n",
        "fields = [('rus', SRC), ('eng', TRG)]\r\n"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k6aNfXIbu6Z5",
        "outputId": "13a547f3-216a-4261-82f0-d94674a3c938"
      },
      "source": [
        "# Get the dataset\r\n",
        "torchtext.utils.download_from_url('https://github.com/bsbor/data/releases/download/test3/1mcorpus.zip', '1mcorpus.zip')\r\n",
        "torchtext.utils.extract_archive('1mcorpus.zip')\r\n",
        "\r\n",
        "ru_lines = io.open(\"corpus.en_ru.1m.ru\", encoding='UTF-8').read().splitlines()\r\n",
        "en_lines = io.open(\"corpus.en_ru.1m.en\", encoding='UTF-8').read().splitlines()\r\n"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1mcorpus.zip: 100%|██████████| 129M/129M [00:06<00:00, 20.6MB/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Ypf8SfqkLUe"
      },
      "source": [
        "dataset_size = 200000\r\n",
        "temp_ru_lines = ru_lines[:dataset_size]\r\n",
        "temp_en_lines = en_lines[:dataset_size]\r\n",
        "sentences = list(zip(temp_ru_lines, temp_en_lines))\r\n",
        "\r\n",
        "data = [ Example.fromlist(item, fields) for item in sentences ]\r\n",
        "\r\n",
        "data = Dataset(data, fields=fields)\r\n",
        "SRC.build_vocab(data)\r\n",
        "TRG.build_vocab(data)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h5e3NeVkMBtm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f28bb70e-49b0-4aba-91d2-fdf35afb4faa"
      },
      "source": [
        "sentences[0]"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('Такое развитие характера Гарри может разочаровать читателей, полюбивших его былую мстительность, но с другой стороны это преображение укрепляет позицию тех, кто не видит глубже сюжета и изображения героев.',\n",
              " \"This new development in Harry's character may be a disappointment to those readers who enjoyed his old vindictive ways, but it also reinforces the position of pro-Potter people who do not see beneath the surface appearance of the characters and plots.\")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9aIumt45u6VT"
      },
      "source": [
        "class Attention(nn.Module):\r\n",
        "  def __init__(self, hidden_size):\r\n",
        "    super(Attention, self).__init__()        \r\n",
        "    self.hidden_size = hidden_size\r\n",
        "      \r\n",
        "  def forward(self, hidden, encoder_outputs, mask):\r\n",
        "    # dot score\r\n",
        "    attn_scores = torch.sum(hidden * encoder_outputs, dim=2)\r\n",
        "    \r\n",
        "    # Transpose max_length and batch_size dimensions\r\n",
        "    attn_scores = attn_scores.t()\r\n",
        "    \r\n",
        "    # Apply mask so network does not attend <pad> tokens        \r\n",
        "    attn_scores = attn_scores.masked_fill(mask == 0, -1e5)\r\n",
        "    \r\n",
        "    # Return softmax over attention scores      \r\n",
        "    return F.softmax(attn_scores, dim=1).unsqueeze(1)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QC0JJOeQu6R2"
      },
      "source": [
        "class Encoder(nn.Module):\r\n",
        "  def __init__(self, hidden_size, embedding_size, num_layers=2, dropout=0.3):\r\n",
        "    \r\n",
        "    super(Encoder, self).__init__()\r\n",
        "    \r\n",
        "    # Basic network params\r\n",
        "    self.hidden_size = hidden_size\r\n",
        "    self.embedding_size = embedding_size\r\n",
        "    self.num_layers = num_layers\r\n",
        "    self.dropout = dropout\r\n",
        "    \r\n",
        "    # Embedding layer that will be shared with Decoder\r\n",
        "    self.embedding = nn.Embedding(len(SRC.vocab), embedding_size)\r\n",
        "    # GRU layer\r\n",
        "    self.gru = nn.GRU(embedding_size, hidden_size,\r\n",
        "                      num_layers=num_layers,\r\n",
        "                      dropout=dropout)\r\n",
        "      \r\n",
        "  def forward(self, input_sequence):\r\n",
        "    # Convert input_sequence to word embeddings\r\n",
        "    embedded = self.embedding(input_sequence)\r\n",
        "            \r\n",
        "    outputs, hidden = self.gru(embedded)\r\n",
        "    \r\n",
        "    # The ouput of a GRU has shape -> (seq_len, batch, hidden_size)\r\n",
        "    return outputs, hidden\r\n",
        "\r\n",
        "class Decoder(nn.Module):\r\n",
        "  def __init__(self, embedding_size, hidden_size, output_size, n_layers=2, dropout=0.3): \r\n",
        "    super(Decoder, self).__init__()\r\n",
        "    \r\n",
        "    # Basic network params\r\n",
        "    self.hidden_size = hidden_size\r\n",
        "    self.output_size = output_size\r\n",
        "    self.n_layers = n_layers\r\n",
        "    self.dropout = dropout\r\n",
        "    self.embedding = nn.Embedding(output_size, embedding_size)\r\n",
        "            \r\n",
        "    self.gru = nn.GRU(embedding_size, hidden_size, n_layers, \r\n",
        "                      dropout=dropout)\r\n",
        "    \r\n",
        "    self.concat = nn.Linear(hidden_size * 2, hidden_size)\r\n",
        "    self.out = nn.Linear(hidden_size, output_size)\r\n",
        "    self.attn = Attention(hidden_size)\r\n",
        "      \r\n",
        "  def forward(self, current_token, hidden_state, encoder_outputs, mask):\r\n",
        "    # convert current_token to word_embedding\r\n",
        "    embedded = self.embedding(current_token)\r\n",
        "    \r\n",
        "    # Pass through GRU\r\n",
        "    gru_output, hidden_state = self.gru(embedded, hidden_state)\r\n",
        "    \r\n",
        "    # Calculate attention weights\r\n",
        "    attention_weights = self.attn(gru_output, encoder_outputs, mask)\r\n",
        "    \r\n",
        "    # Calculate context vector (weigthed average)\r\n",
        "    context = attention_weights.bmm(encoder_outputs.transpose(0, 1))\r\n",
        "    \r\n",
        "    # Concatenate  context vector and GRU output\r\n",
        "    gru_output = gru_output.squeeze(0)\r\n",
        "    context = context.squeeze(1)\r\n",
        "    concat_input = torch.cat((gru_output, context), 1)\r\n",
        "    concat_output = torch.tanh(self.concat(concat_input))\r\n",
        "    \r\n",
        "    # Pass concat_output to final output layer\r\n",
        "    output = self.out(concat_output)\r\n",
        "    \r\n",
        "    # Return output and final hidden state\r\n",
        "    return output, hidden_state"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pgwnG8D_vQVm"
      },
      "source": [
        "class seq2seq(nn.Module):\r\n",
        "  def __init__(self, embedding_size, hidden_size, vocab_size, device, pad_idx, eos_idx, sos_idx):\r\n",
        "    super(seq2seq, self).__init__()\r\n",
        "    \r\n",
        "    # Embedding layer shared by encoder and decoder\r\n",
        "    self.embedding = nn.Embedding(vocab_size, embedding_size)\r\n",
        "    \r\n",
        "    # Encoder network\r\n",
        "    self.encoder = Encoder(hidden_size, embedding_size, num_layers=2, dropout=0.3)\r\n",
        "    \r\n",
        "    # Decoder network        \r\n",
        "    self.decoder = Decoder(embedding_size, hidden_size, vocab_size, n_layers=2, dropout=0.3)\r\n",
        "    \r\n",
        "    # Indices of special tokens and hardware device \r\n",
        "    self.pad_idx = pad_idx\r\n",
        "    self.eos_idx = eos_idx\r\n",
        "    self.sos_idx = sos_idx\r\n",
        "    self.device = device\r\n",
        "      \r\n",
        "  def create_mask(self, input_sequence):\r\n",
        "    return (input_sequence != self.pad_idx).permute(1, 0)\r\n",
        "      \r\n",
        "      \r\n",
        "  def forward(self, input_sequence, output_sequence):\r\n",
        "    \r\n",
        "    # Unpack input_sequence tuple\r\n",
        "    input_tokens = input_sequence[0]\r\n",
        "  \r\n",
        "    # Unpack output_tokens, or create an empty tensor for text generation\r\n",
        "    if output_sequence is None:\r\n",
        "      inference = True\r\n",
        "      output_tokens = torch.zeros((100, input_tokens.shape[1])).long().fill_(self.sos_idx).to(self.device)\r\n",
        "    else:\r\n",
        "      inference = False\r\n",
        "      output_tokens = output_sequence[0]\r\n",
        "    \r\n",
        "    vocab_size = self.decoder.output_size\r\n",
        "    batch_size = len(input_sequence[1])\r\n",
        "    max_seq_len = len(output_tokens)\r\n",
        "    \r\n",
        "    # tensor to store decoder outputs\r\n",
        "    outputs = torch.zeros(max_seq_len, batch_size, vocab_size).to(self.device)        \r\n",
        "    \r\n",
        "    # pass input sequence to the encoder\r\n",
        "    encoder_outputs, hidden = self.encoder(input_tokens)\r\n",
        "    \r\n",
        "    # first input to the decoder is the <sos> tokens\r\n",
        "    output = output_tokens[0,:]\r\n",
        "    \r\n",
        "    # create mask\r\n",
        "    mask = self.create_mask(input_tokens)\r\n",
        "    \r\n",
        "    \r\n",
        "    # Step through the length of the output sequence one token at a time\r\n",
        "    for t in range(1, max_seq_len):\r\n",
        "      output = output.unsqueeze(0)\r\n",
        "      \r\n",
        "      output, hidden = self.decoder(output, hidden, encoder_outputs, mask)\r\n",
        "      outputs[t] = output\r\n",
        "      \r\n",
        "      if inference:\r\n",
        "        output = output.max(1)[1]\r\n",
        "      else:\r\n",
        "        output = output_tokens[t]\r\n",
        "      \r\n",
        "      # If we're in inference mode, keep generating until we produce an\r\n",
        "      # <eos> token\r\n",
        "      if inference and output.item() == self.eos_idx:\r\n",
        "        return outputs[:t]\r\n",
        "        \r\n",
        "    return outputs"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LGXYcj8Nvw-4"
      },
      "source": [
        "\r\n",
        "train_data, val_data = data.split(split_ratio=0.8)\r\n",
        "train_iterator, valid_iterator = BucketIterator.splits(\r\n",
        "    (train_data, val_data), \r\n",
        "    batch_size = 64, \r\n",
        "    sort_within_batch = True, \r\n",
        "    sort_key = lambda x:len(x.rus),\r\n",
        "    device = device\r\n",
        ")\r\n",
        "\r\n",
        "# extract special tokens\r\n",
        "pad_idx = TRG.vocab.stoi['<pad>']\r\n",
        "eos_idx = TRG.vocab.stoi['<eos>']\r\n",
        "sos_idx = TRG.vocab.stoi['<sos>']\r\n",
        "\r\n",
        "# Size of embedding_dim should match the dim of pre-trained word embeddings!\r\n",
        "embedding_dim = 100\r\n",
        "hidden_dim = 256\r\n",
        "vocab_size = len(TRG.vocab)\r\n",
        "\r\n",
        "model = seq2seq(embedding_dim, hidden_dim, vocab_size, \r\n",
        "                device, pad_idx, eos_idx, sos_idx).to(device)\r\n",
        "\r\n",
        "optimizer = optim.Adam(model.parameters())\r\n",
        "\r\n",
        "# cross entropy loss with softmax\r\n",
        "criterion = nn.CrossEntropyLoss(ignore_index = pad_idx)\r\n",
        "\r\n",
        "def train(model, iterator, criterion, optimizer):\r\n",
        "  # Put the model in training mode!\r\n",
        "  model.train()\r\n",
        "  \r\n",
        "  epoch_loss = 0\r\n",
        "  i = 0\r\n",
        "  it_size = len(iterator)\r\n",
        "  with tqdm(total=it_size) as progress_bar:\r\n",
        "    for (idx, batch) in enumerate(iterator):\r\n",
        "        #if (idx % (round(it_size/500)) == 0):\r\n",
        "        #    print(\"\\tCompleted: {} / {} batches\".format(idx, it_size))\r\n",
        "\r\n",
        "        input_sequence = batch.rus\r\n",
        "        output_sequence = batch.eng\r\n",
        "\r\n",
        "        target_tokens = output_sequence[0]\r\n",
        "\r\n",
        "        # zero out the gradient for the current batch\r\n",
        "        optimizer.zero_grad()\r\n",
        "\r\n",
        "        # Run the batch through our model\r\n",
        "        output = model(input_sequence, output_sequence)\r\n",
        "\r\n",
        "        # Throw it through our loss function\r\n",
        "        output = output[1:].view(-1, output.shape[-1])\r\n",
        "        target_tokens = target_tokens[1:].view(-1)\r\n",
        "\r\n",
        "        loss = criterion(output, target_tokens)\r\n",
        "\r\n",
        "        # Perform back-prop and calculate the gradient of our loss function\r\n",
        "        loss.backward()\r\n",
        "\r\n",
        "        # Update model parameters\r\n",
        "        optimizer.step()\r\n",
        "\r\n",
        "        epoch_loss += loss.item()\r\n",
        "        i+= 1\r\n",
        "        progress_bar.update(1) # update progress\r\n",
        "        \r\n",
        "  return epoch_loss / len(iterator)\r\n",
        "\r\n",
        "def evaluate(model, iterator, criterion):\r\n",
        "  # Put the model in training mode!\r\n",
        "  model.eval()\r\n",
        "  \r\n",
        "  epoch_loss = 0\r\n",
        "  \r\n",
        "  for (idx, batch) in enumerate(iterator):\r\n",
        "    input_sequence = batch.rus\r\n",
        "    output_sequence = batch.eng\r\n",
        "\r\n",
        "    target_tokens = output_sequence[0]\r\n",
        "\r\n",
        "    # Run the batch through our model\r\n",
        "    output = model(input_sequence, output_sequence)\r\n",
        "\r\n",
        "    # Throw it through our loss function\r\n",
        "    output = output[1:].view(-1, output.shape[-1])\r\n",
        "    target_tokens = target_tokens[1:].view(-1)\r\n",
        "\r\n",
        "    loss = criterion(output, target_tokens)\r\n",
        "\r\n",
        "    epoch_loss += loss.item()\r\n",
        "      \r\n",
        "  return epoch_loss / len(iterator)\r\n",
        "\r\n",
        "def epoch_time(start_time, end_time):\r\n",
        "  elapsed_time = end_time - start_time\r\n",
        "  elapsed_mins = int(elapsed_time / 60)\r\n",
        "  elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\r\n",
        "  return elapsed_mins, elapsed_secs\r\n",
        "\r\n"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zADLVnePYvUJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eb1c0f0f-c518-4bae-9ffe-f7bb984b97d5"
      },
      "source": [
        "# %%script false\r\n",
        "# Train\r\n",
        "N_EPOCHS = 5\r\n",
        "\r\n",
        "best_valid_loss = float('inf')\r\n",
        "\r\n",
        "# start model training\r\n",
        "print('Epoch 1 Training started....')\r\n",
        "for epoch in range(N_EPOCHS):\r\n",
        "  start_time = time.time()\r\n",
        "  \r\n",
        "  train_loss = train(model, train_iterator, criterion, optimizer)\r\n",
        "  valid_loss = evaluate(model, valid_iterator, criterion)\r\n",
        "  \r\n",
        "  end_time = time.time()\r\n",
        "  \r\n",
        "  epoch_mins, epoch_secs = epoch_time(start_time, end_time)\r\n",
        "  \r\n",
        "  # compare validation loss\r\n",
        "  if valid_loss < best_valid_loss:\r\n",
        "    best_valid_loss = valid_loss\r\n",
        "    torch.save(model.state_dict(), 'best_model.pt')\r\n",
        "  \r\n",
        "  print(f'\\nEpoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\r\n",
        "  print(f'  > Train Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\r\n",
        "  print(f'  > Val. Loss: {valid_loss:.3f} |  Val. PPL: {math.exp(valid_loss):7.3f}')\r\n",
        "  print('')\r\n"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/625 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1 Training started....\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 625/625 [09:57<00:00,  1.05it/s]\n",
            "  0%|          | 0/625 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 01 | Time: 10m 45s\n",
            "  > Train Loss: 6.857 | Train PPL: 950.624\n",
            "  > Val. Loss: 6.377 |  Val. PPL: 588.427\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 625/625 [09:48<00:00,  1.06it/s]\n",
            "  0%|          | 0/625 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 02 | Time: 10m 37s\n",
            "  > Train Loss: 5.962 | Train PPL: 388.454\n",
            "  > Val. Loss: 5.803 |  Val. PPL: 331.285\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 625/625 [09:51<00:00,  1.06it/s]\n",
            "  0%|          | 0/625 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 03 | Time: 10m 39s\n",
            "  > Train Loss: 5.485 | Train PPL: 241.073\n",
            "  > Val. Loss: 5.590 |  Val. PPL: 267.815\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 625/625 [09:51<00:00,  1.06it/s]\n",
            "  0%|          | 0/625 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 04 | Time: 10m 39s\n",
            "  > Train Loss: 5.136 | Train PPL: 170.019\n",
            "  > Val. Loss: 5.465 |  Val. PPL: 236.373\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 625/625 [09:54<00:00,  1.05it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Epoch: 05 | Time: 10m 42s\n",
            "  > Train Loss: 4.842 | Train PPL: 126.746\n",
            "  > Val. Loss: 5.409 |  Val. PPL: 223.446\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q4wexpJcXM09"
      },
      "source": [
        "%%script false\r\n",
        "\r\n",
        "# saving & loading the model\r\n",
        "saved_model_path = \"best_model.pt\"\r\n",
        "model.load_state_dict(torch.load(saved_model_path))\r\n",
        "    "
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IRgEEF0VWV5D"
      },
      "source": [
        "def translate_sentence(model, sentence):\r\n",
        "    model.eval()\r\n",
        "    \r\n",
        "    # tokenization\r\n",
        "    tokenized = nlp_ru(sentence) \r\n",
        "    # convert tokens to lowercase\r\n",
        "    tokenized = [t.lower_ for t in tokenized]\r\n",
        "    # convert tokens to integers\r\n",
        "    int_tokenized = [SRC.vocab.stoi[t] for t in tokenized] \r\n",
        "    \r\n",
        "    # convert list to tensor\r\n",
        "    sentence_length = torch.LongTensor([len(int_tokenized)]).to(model.device) \r\n",
        "    tensor = torch.LongTensor(int_tokenized).unsqueeze(1).to(model.device) \r\n",
        "    \r\n",
        "    # get predictions\r\n",
        "    translation_tensor_logits = model((tensor, sentence_length), None) \r\n",
        "    \r\n",
        "    # get token index with highest score\r\n",
        "    translation_tensor = torch.argmax(translation_tensor_logits.squeeze(1), 1)\r\n",
        "    # convert indices (integers) to tokens\r\n",
        "    translation = [TRG.vocab.itos[t] for t in translation_tensor]\r\n",
        " \r\n",
        "    # Start at the first index.  We don't need to return the <sos> token...\r\n",
        "    translation = translation[1:]\r\n",
        "    return \" \".join(translation)\r\n"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WU2NV3vUPcnK",
        "outputId": "5003924e-059c-4858-c723-581886ab9553"
      },
      "source": [
        "translate_sentence(model, \"это холодно\")"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "it s cold\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JTP_CmiDPFHc"
      },
      "source": [
        "eval_lines = io.open('eval-ru-100.txt').read().splitlines()\r\n",
        "\r\n",
        "eval_en_out = [ translate_sentence(model, s) for s in eval_lines ]\r\n",
        "\r\n",
        "eval_output = list(zip(eval_lines, eval_en_out))"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xXrEA4Tzvx9L",
        "outputId": "f7bf1212-0219-4eaf-9ca1-6984ed581a76"
      },
      "source": [
        "eval_output[:4]"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('26. Вопрос о лесах необходимо вывести на более высокий уровень в рамках целей устойчивого развития, в том числе посредством включения в такие цели убедительных и четких целевых и рабочих показателей по лесам.',\n",
              "  'Forest need to increase with the develop goal the including with force and clear development mark of forests.'),\n",
              " ('В рамках экологической экспертизы определены пять вариантов строительства и эксплуатации замещающей электростанции, которая восстановит мощность энергораспределительной сети Управления по состоянию до стихийного бедствия.',\n",
              "  'in the end of the company , the company will be provided by the international and the international sector , which is also to be able to provide the new system of the new system .'),\n",
              " ('В ходе рассмотрения данного пункта повестки дня Рабочая группа будет кратко проинформирована Секретариатом о работе УНП ООН по содействию ратификации и осуществлению Протокола об огнестрельном оружии в рамках Глобальной программы по огнестрельному оружию.',\n",
              "  'during the taking of this item, work group will be able to establish on the work of international security council and the international level of the international system.'),\n",
              " ('В последние месяцы сирийское правительство позволило террористам использовать территорию своей страны в качестве базы, действуя с которой они устанавливают взрывные устройства на обочинах дорог, наносят ракетные удары по Израилю и обстреливают подразделения Армии обороны Израиля, дислоцированные на территории страны.',\n",
              "  'in the last year , the united states has been held in the world of the world , which had been held in the beginning of the 1990s , with the most important of the most important of the most important of the most important - scale and the soviet union , the government was held in the beginning of the ussr .')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FNOJidUKThPV"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}