3_CBOW_slow.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "3. CBOW_slow.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Northwind01/metaphors/blob/master/3_CBOW_slow.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KfHIu5UNNxyr",
        "colab_type": "text"
      },
      "source": [
        "# CBOW"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "82qCVhhqQ3Sx",
        "colab_type": "text"
      },
      "source": [
        "https://www.kdnuggets.com/2018/04/implementing-deep-learning-methods-feature-engineering-text-data-cbow.html\n",
        "\n",
        "https://medium.com/@TalPerry/getting-text-into-tensorflow-with-the-dataset-api-ffb832c8bec6"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HO3ljwUCPdif",
        "colab_type": "text"
      },
      "source": [
        "## 0. Set-up"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OCejqcScPiHT",
        "colab_type": "text"
      },
      "source": [
        "### Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "od3OWtF458Jh",
        "colab_type": "code",
        "outputId": "d1d8cf0f-60e7-4b1d-ca10-7db136347337",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%tensorflow_version 2.x\n",
        "import tensorflow as tf\n",
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "  raise SystemError('GPU device not found')\n",
        "print('Found GPU at: {}'.format(device_name))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 2.x selected.\n",
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G2ei2FEDN2B3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import _pickle as cPickle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B_VWuULGPrqY",
        "colab_type": "text"
      },
      "source": [
        "### Get Google drive access"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ad9qpjzlPpX4",
        "colab_type": "code",
        "outputId": "d3cf759e-280d-4347-fdc8-995c3e6e840e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uuntwCvRP5KS",
        "colab_type": "text"
      },
      "source": [
        "### Set the paths"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dXUBsn2rPwkW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "root_path = 'gdrive/My Drive/metaphors/'\n",
        "corpus_dir = root_path + 'data/Wikipedia/'\n",
        "pickle_dir = root_path + 'data/pickles/'\n",
        "CBOW_dir = root_path + 'models/CBOW/'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dk0ZO1TxShGw",
        "colab_type": "text"
      },
      "source": [
        "## 1. Build the corpus vocabulary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WQ0Li7zASmc4",
        "colab_type": "code",
        "outputId": "114bea91-2fa5-45ab-b0b3-4927fa27a530",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "from tensorflow.keras.preprocessing import text, sequence\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "from nltk.tokenize import sent_tokenize"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A708LIxLWYR5",
        "colab_type": "text"
      },
      "source": [
        "### Get the corpus"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z7JzJcFCTGTj",
        "colab_type": "code",
        "outputId": "e12f5fcd-290e-4554-a063-73c22f7608c2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "corpus_text_file = os.path.join(corpus_dir, 'wiki_en.txt')\n",
        "corpus_file = open(corpus_text_file,'r')\n",
        "corpus = corpus_file.read()\n",
        "len(corpus)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "327728373"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SoNLzPObhSlU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sents = sent_tokenize(corpus)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uv9cXy_arqmy",
        "colab_type": "code",
        "outputId": "eff92a8a-7d38-4e80-cff5-5ede666d1a13",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "sents[4][2]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "' '"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c-RujicJWbjU",
        "colab_type": "text"
      },
      "source": [
        "### Build corpus vocabulary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w3uQJBoSrfSC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "path = os.path.join(pickle_dir + 'CBOW/', 'tokenizer.pickle')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MzQpSb8Gr3Kk",
        "colab_type": "text"
      },
      "source": [
        "#### First time"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uXatSyMIV6vv",
        "colab_type": "code",
        "outputId": "da492923-1f4f-4c0b-866c-fb248b51f628",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%%time\n",
        "tokenizer = text.Tokenizer()\n",
        "tokenizer.fit_on_texts(sents)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 57.3 s, sys: 66.4 ms, total: 57.4 s\n",
            "Wall time: 57.3 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ohSGa4yItCkF",
        "colab_type": "code",
        "outputId": "160d5114-2197-4282-bd34-96735ab6c3ae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%%time\n",
        "word2id = tokenizer.word_index\n",
        "word2id['PAD'] = 0\n",
        "id2word = {v:k for k, v in word2id.items()}\n",
        "wids = [[word2id[w] for w in text.text_to_word_sequence(sent)] for sent in sents]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 38.6 s, sys: 312 ms, total: 38.9 s\n",
            "Wall time: 38.9 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-zCOUu3bV6-0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(path, \"wb\") as output_file:\n",
        "  cPickle.dump(tokenizer, output_file)\n",
        "  cPickle.dump(word2id, output_file)\n",
        "  cPickle.dump(id2word, output_file)\n",
        "  cPickle.dump(wids, output_file)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1uN8mhrisnyj",
        "colab_type": "text"
      },
      "source": [
        "#### Unpickle vocabulary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QqXCCv9cV7Kl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(path, \"rb\") as input_file:\n",
        "  tokenizer = cPickle.load(input_file)\n",
        "  word2id = cPickle.load(input_file)\n",
        "  id2word = cPickle.load(input_file)\n",
        "  wids = cPickle.load(input_file)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sk-RTu2kwIsc",
        "colab_type": "text"
      },
      "source": [
        "#### Define CBOW parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IWUZwEUdi6gr",
        "colab_type": "code",
        "outputId": "b543b481-b971-4e56-88d1-7f69fc1f520d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "vocab_size = len(word2id)\n",
        "embed_size = 100\n",
        "window_size = 2 # context window size\n",
        "\n",
        "print('Vocabulary size:', vocab_size)\n",
        "print('Vocabulary sample:', list(word2id.items())[:10])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Vocabulary size: 859144\n",
            "Vocabulary sample: [('the', 1), ('of', 2), ('and', 3), ('in', 4), ('to', 5), ('a', 6), ('is', 7), ('as', 8), ('was', 9), ('for', 10)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AS6fiLa2Sm89",
        "colab_type": "text"
      },
      "source": [
        "## 2. Build a CBOW (context, target) generator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hpe0lmCzSqSH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_context_word_pairs(corpus, window_size, vocab_size):\n",
        "  context_length = window_size*2\n",
        "  for words in corpus:\n",
        "    sentence_length = len(words)\n",
        "    for index, word in enumerate(words):\n",
        "      context_words = []\n",
        "      label_word   = []            \n",
        "      start = index - window_size\n",
        "      end = index + window_size + 1\n",
        "            \n",
        "      context_words.append([words[i] \n",
        "                            for i in range(start, end) \n",
        "                            if 0 <= i < sentence_length \n",
        "                            and i != index])\n",
        "      label_word.append(word)\n",
        "\n",
        "      x = sequence.pad_sequences(context_words, maxlen=context_length)\n",
        "      y = to_categorical(label_word, vocab_size)\n",
        "      yield (x, y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XOoOUG5CdjDZ",
        "colab_type": "code",
        "outputId": "d4c5a22f-8839-4e37-c800-858be2e56c6e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "# Just checking\n",
        "i = 0\n",
        "for x, y in generate_context_word_pairs(corpus=wids, window_size=window_size, vocab_size=vocab_size):\n",
        "    if 0 not in x[0]:\n",
        "        print('Context (X):', [id2word[w] for w in x[0]], '-> Target (Y):', id2word[np.argwhere(y[0])[0][0]])\n",
        "    \n",
        "        if i == 10:\n",
        "            break\n",
        "        i += 1"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Context (X): [\"'''anarchism'''\", 'is', 'political', 'and'] -> Target (Y): an\n",
            "Context (X): ['is', 'an', 'and', 'social'] -> Target (Y): political\n",
            "Context (X): ['an', 'political', 'social', 'philosophy'] -> Target (Y): and\n",
            "Context (X): ['political', 'and', 'philosophy', 'that'] -> Target (Y): social\n",
            "Context (X): ['and', 'social', 'that', 'rejects'] -> Target (Y): philosophy\n",
            "Context (X): ['social', 'philosophy', 'rejects', 'hierarchies'] -> Target (Y): that\n",
            "Context (X): ['philosophy', 'that', 'hierarchies', 'deemed'] -> Target (Y): rejects\n",
            "Context (X): ['that', 'rejects', 'deemed', 'unjust'] -> Target (Y): hierarchies\n",
            "Context (X): ['rejects', 'hierarchies', 'unjust', 'and'] -> Target (Y): deemed\n",
            "Context (X): ['hierarchies', 'deemed', 'and', 'advocates'] -> Target (Y): unjust\n",
            "Context (X): ['deemed', 'unjust', 'advocates', 'their'] -> Target (Y): and\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SaVie0eGSqxD",
        "colab_type": "text"
      },
      "source": [
        "## 3. Build the CBOW model architecture"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sFH2C0KqSuXT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow.keras.backend as K\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Embedding, Lambda"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6NrPtQEJe2Ik",
        "colab_type": "code",
        "outputId": "6ccd22a4-e490-4077-dbab-faefcc537a2f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        }
      },
      "source": [
        "cbow = Sequential()\n",
        "cbow.add(Embedding(input_dim=vocab_size, output_dim=embed_size, input_length=window_size*2))\n",
        "cbow.add(Lambda(lambda x: K.mean(x, axis=1), output_shape=(embed_size,)))\n",
        "cbow.add(Dense(vocab_size, activation='softmax'))\n",
        "cbow.compile(loss='categorical_crossentropy', optimizer='rmsprop')\n",
        "\n",
        "# View model summary\n",
        "print(cbow.summary())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding (Embedding)        (None, 4, 100)            85914400  \n",
            "_________________________________________________________________\n",
            "lambda (Lambda)              (None, 100)               0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 859144)            86773544  \n",
            "=================================================================\n",
            "Total params: 172,687,944\n",
            "Trainable params: 172,687,944\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d3WS8anPe2Ow",
        "colab_type": "code",
        "outputId": "6526802e-4eb2-45fd-80f8-8363161e5e3f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 386
        }
      },
      "source": [
        "# Visualize model structure\n",
        "from IPython.display import SVG\n",
        "from tensorflow.keras.utils import model_to_dot\n",
        "\n",
        "SVG(model_to_dot(cbow, show_shapes=True, show_layer_names=False, dpi=65,\n",
        "                 rankdir='TB').create(prog='dot', format='svg'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<IPython.core.display.SVG object>"
            ],
            "image/svg+xml": "<svg height=\"274pt\" viewBox=\"0.00 0.00 227.00 304.00\" width=\"205pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g class=\"graph\" id=\"graph0\" transform=\"scale(.9028 .9028) rotate(0) translate(4 300)\">\n<title>G</title>\n<polygon fill=\"#ffffff\" points=\"-4,4 -4,-300 223,-300 223,4 -4,4\" stroke=\"transparent\"/>\n<!-- 140091849097288 -->\n<g class=\"node\" id=\"node1\">\n<title>140091849097288</title>\n<polygon fill=\"none\" points=\"12.5,-249.5 12.5,-295.5 206.5,-295.5 206.5,-249.5 12.5,-249.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"52.5\" y=\"-268.8\">InputLayer</text>\n<polyline fill=\"none\" points=\"92.5,-249.5 92.5,-295.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"121.5\" y=\"-280.3\">input:</text>\n<polyline fill=\"none\" points=\"92.5,-272.5 150.5,-272.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"121.5\" y=\"-257.3\">output:</text>\n<polyline fill=\"none\" points=\"150.5,-249.5 150.5,-295.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"178.5\" y=\"-280.3\">[(?, 4)]</text>\n<polyline fill=\"none\" points=\"150.5,-272.5 206.5,-272.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"178.5\" y=\"-257.3\">[(?, 4)]</text>\n</g>\n<!-- 140091849099864 -->\n<g class=\"node\" id=\"node2\">\n<title>140091849099864</title>\n<polygon fill=\"none\" points=\"0,-166.5 0,-212.5 219,-212.5 219,-166.5 0,-166.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"42\" y=\"-185.8\">Embedding</text>\n<polyline fill=\"none\" points=\"84,-166.5 84,-212.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"113\" y=\"-197.3\">input:</text>\n<polyline fill=\"none\" points=\"84,-189.5 142,-189.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"113\" y=\"-174.3\">output:</text>\n<polyline fill=\"none\" points=\"142,-166.5 142,-212.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"180.5\" y=\"-197.3\">(?, 4)</text>\n<polyline fill=\"none\" points=\"142,-189.5 219,-189.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"180.5\" y=\"-174.3\">(?, 4, 100)</text>\n</g>\n<!-- 140091849097288&#45;&gt;140091849099864 -->\n<g class=\"edge\" id=\"edge1\">\n<title>140091849097288-&gt;140091849099864</title>\n<path d=\"M109.5,-249.3799C109.5,-241.1745 109.5,-231.7679 109.5,-222.8786\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"113.0001,-222.784 109.5,-212.784 106.0001,-222.784 113.0001,-222.784\" stroke=\"#000000\"/>\n</g>\n<!-- 140091849099024 -->\n<g class=\"node\" id=\"node3\">\n<title>140091849099024</title>\n<polygon fill=\"none\" points=\"10,-83.5 10,-129.5 209,-129.5 209,-83.5 10,-83.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"42\" y=\"-102.8\">Lambda</text>\n<polyline fill=\"none\" points=\"74,-83.5 74,-129.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"103\" y=\"-114.3\">input:</text>\n<polyline fill=\"none\" points=\"74,-106.5 132,-106.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"103\" y=\"-91.3\">output:</text>\n<polyline fill=\"none\" points=\"132,-83.5 132,-129.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"170.5\" y=\"-114.3\">(?, 4, 100)</text>\n<polyline fill=\"none\" points=\"132,-106.5 209,-106.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"170.5\" y=\"-91.3\">(?, 100)</text>\n</g>\n<!-- 140091849099864&#45;&gt;140091849099024 -->\n<g class=\"edge\" id=\"edge2\">\n<title>140091849099864-&gt;140091849099024</title>\n<path d=\"M109.5,-166.3799C109.5,-158.1745 109.5,-148.7679 109.5,-139.8786\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"113.0001,-139.784 109.5,-129.784 106.0001,-139.784 113.0001,-139.784\" stroke=\"#000000\"/>\n</g>\n<!-- 140088896198864 -->\n<g class=\"node\" id=\"node4\">\n<title>140088896198864</title>\n<polygon fill=\"none\" points=\"12.5,-.5 12.5,-46.5 206.5,-46.5 206.5,-.5 12.5,-.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"38.5\" y=\"-19.8\">Dense</text>\n<polyline fill=\"none\" points=\"64.5,-.5 64.5,-46.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"93.5\" y=\"-31.3\">input:</text>\n<polyline fill=\"none\" points=\"64.5,-23.5 122.5,-23.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"93.5\" y=\"-8.3\">output:</text>\n<polyline fill=\"none\" points=\"122.5,-.5 122.5,-46.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"164.5\" y=\"-31.3\">(?, 100)</text>\n<polyline fill=\"none\" points=\"122.5,-23.5 206.5,-23.5 \" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"164.5\" y=\"-8.3\">(?, 859144)</text>\n</g>\n<!-- 140091849099024&#45;&gt;140088896198864 -->\n<g class=\"edge\" id=\"edge3\">\n<title>140091849099024-&gt;140088896198864</title>\n<path d=\"M109.5,-83.3799C109.5,-75.1745 109.5,-65.7679 109.5,-56.8786\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"113.0001,-56.784 109.5,-46.784 106.0001,-56.784 113.0001,-56.784\" stroke=\"#000000\"/>\n</g>\n</g>\n</svg>"
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U9cPTmdwSu6g",
        "colab_type": "text"
      },
      "source": [
        "## 4. Train the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pnoGU5EKSx9I",
        "colab_type": "code",
        "outputId": "9c9706a8-2ba1-430b-f1d5-0e4dc719aec3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442
        }
      },
      "source": [
        "for epoch in range(1, 6):\n",
        "  loss = 0.\n",
        "  i = 0\n",
        "  for x, y in generate_context_word_pairs(corpus=wids, window_size=window_size, vocab_size=vocab_size):\n",
        "    i += 1\n",
        "    loss += cbow.train_on_batch(x, y)\n",
        "    if i % 10000 == 0:\n",
        "      print('Processed {} (context, word) pairs'.format(i))\n",
        "\n",
        "  print('Epoch:', epoch, '\\tLoss:', loss)\n",
        "  print()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Processed 10000 (context, word) pairs\n",
            "Processed 20000 (context, word) pairs\n",
            "Processed 30000 (context, word) pairs\n",
            "Processed 40000 (context, word) pairs\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-25-c5004fd3755a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m   \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgenerate_context_word_pairs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwindow_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwindow_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvocab_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvocab_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mi\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mloss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mcbow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_on_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m10000\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m       \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Processed {} (context, word) pairs'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_core/python/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight, reset_metrics)\u001b[0m\n\u001b[1;32m    971\u001b[0m       outputs = training_v2_utils.train_on_batch(\n\u001b[1;32m    972\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 973\u001b[0;31m           class_weight=class_weight, reset_metrics=reset_metrics)\n\u001b[0m\u001b[1;32m    974\u001b[0m       outputs = (outputs['total_loss'] + outputs['output_losses'] +\n\u001b[1;32m    975\u001b[0m                  outputs['metrics'])\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_core/python/keras/engine/training_v2_utils.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(model, x, y, sample_weight, class_weight, reset_metrics)\u001b[0m\n\u001b[1;32m    262\u001b[0m       \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m       \u001b[0msample_weights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 264\u001b[0;31m       output_loss_metrics=model._output_loss_metrics)\n\u001b[0m\u001b[1;32m    265\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    266\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mreset_metrics\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_core/python/keras/engine/training_eager.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(model, inputs, targets, sample_weights, output_loss_metrics)\u001b[0m\n\u001b[1;32m    309\u001b[0m           \u001b[0msample_weights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m           \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 311\u001b[0;31m           output_loss_metrics=output_loss_metrics))\n\u001b[0m\u001b[1;32m    312\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    313\u001b[0m     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_core/python/keras/engine/training_eager.py\u001b[0m in \u001b[0;36m_process_single_batch\u001b[0;34m(model, inputs, targets, output_loss_metrics, sample_weights, training)\u001b[0m\n\u001b[1;32m    270\u001b[0m                         loss_scale_optimizer.LossScaleOptimizer):\n\u001b[1;32m    271\u001b[0m             \u001b[0mgrads\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_unscaled_gradients\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 272\u001b[0;31m           \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_gradients\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrads\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainable_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    273\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    274\u001b[0m         logging.warning('The list of trainable weights is empty. Make sure that'\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_core/python/keras/optimizer_v2/optimizer_v2.py\u001b[0m in \u001b[0;36mapply_gradients\u001b[0;34m(self, grads_and_vars, name)\u001b[0m\n\u001b[1;32m    439\u001b[0m           \u001b[0mfunctools\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpartial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_distributed_apply\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mapply_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mapply_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    440\u001b[0m           \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrads_and_vars\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 441\u001b[0;31m           kwargs={\"name\": name})\n\u001b[0m\u001b[1;32m    442\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    443\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_distributed_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdistribution\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrads_and_vars\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mapply_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_core/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36mmerge_call\u001b[0;34m(self, merge_fn, args, kwargs)\u001b[0m\n\u001b[1;32m   1915\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1916\u001b[0m       \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1917\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_merge_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmerge_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1918\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1919\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_merge_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmerge_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_core/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36m_merge_call\u001b[0;34m(self, merge_fn, args, kwargs)\u001b[0m\n\u001b[1;32m   1922\u001b[0m         distribution_strategy_context._CrossReplicaThreadMode(self._strategy))  # pylint: disable=protected-access\n\u001b[1;32m   1923\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1924\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mmerge_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_strategy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1925\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1926\u001b[0m       \u001b[0m_pop_per_thread_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_core/python/keras/optimizer_v2/optimizer_v2.py\u001b[0m in \u001b[0;36m_distributed_apply\u001b[0;34m(self, distribution, grads_and_vars, name, apply_state)\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0massign_add\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    496\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 497\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0massign_add\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    498\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    499\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mget_updates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_core/python/ops/resource_variable_ops.py\u001b[0m in \u001b[0;36massign_add\u001b[0;34m(self, delta, use_locking, name, read_value)\u001b[0m\n\u001b[1;32m    779\u001b[0m       assign_add_op = gen_resource_variable_ops.assign_add_variable_op(\n\u001b[1;32m    780\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvert_to_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdelta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 781\u001b[0;31m           name=name)\n\u001b[0m\u001b[1;32m    782\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mread_value\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    783\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lazy_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0massign_add_op\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/tensorflow-2.0.0/python3.6/tensorflow_core/python/ops/gen_resource_variable_ops.py\u001b[0m in \u001b[0;36massign_add_variable_op\u001b[0;34m(resource, value, name)\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0m_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_context_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_thread_local_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m         \u001b[0;34m\"AssignAddVariableOp\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_post_execution_callbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m         value)\n\u001b[0m\u001b[1;32m     53\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_FallbackException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1TdnfUIKynjp",
        "colab_type": "code",
        "outputId": "01a8a251-46b8-43a8-e656-8c2dc7ccd6e5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        }
      },
      "source": [
        "tf.keras.models.save_model(\n",
        "    cbow,\n",
        "    CBOW_dir+'model',\n",
        "    overwrite=True,\n",
        "    include_optimizer=True,\n",
        "    save_format=None,\n",
        "    signatures=None,\n",
        "    options=None\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /tensorflow-2.0.0/python3.6/tensorflow_core/python/ops/resource_variable_ops.py:1781: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n",
            "INFO:tensorflow:Assets written to: gdrive/My Drive/metaphors/models/CBOW/model/assets\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NgyjV016Syn4",
        "colab_type": "text"
      },
      "source": [
        "## 5. Get word embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uedmmA86S1Cs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "weights = cbow.get_weights()[0]\n",
        "weights = weights[1:]\n",
        "print(weights.shape)\n",
        "\n",
        "pd.DataFrame(weights, index=list(id2word.values())[1:]).head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JRjsvIJ_uCmm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics.pairwise import euclidean_distances\n",
        "\n",
        "# compute pairwise distance matrix\n",
        "distance_matrix = euclidean_distances(weights)\n",
        "print(distance_matrix.shape)\n",
        "\n",
        "# view contextually similar words\n",
        "similar_words = {search_term: [id2word[idx] for idx in distance_matrix[word2id[search_term]-1].argsort()[1:6]+1] \n",
        "                   for search_term in ['god', 'jesus', 'noah', 'egypt', 'john', 'gospel', 'moses','famine']}\n",
        "\n",
        "similar_words"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}