LSTMLanguagemodel.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LSTMLanguagemodel.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "FGd7TsISWnBZ",
        "colab_type": "code",
        "outputId": "d76bffa1-b671-4102-8dd2-8af31e23eaa0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        }
      },
      "source": [
        "import time\n",
        "import numpy as np\n",
        "import tensorflow as tf"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:523: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:524: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:532: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eqtLuUevW2wZ",
        "colab_type": "code",
        "outputId": "f600c69f-d98a-4a17-e8a5-58d5bedc10cc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "!mkdir data\n",
        "!wget -q -O data/ptb.zip https://ibm.box.com/shared/static/z2yvmhbskc45xd2a9a4kkn6hg4g4kj5r.zip\n",
        "!unzip -o data/ptb.zip -d data\n",
        "!cp data/ptb/reader.py .\n",
        "import reader"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  data/ptb.zip\n",
            "   creating: data/ptb/\n",
            "  inflating: data/ptb/reader.py      \n",
            "   creating: data/__MACOSX/\n",
            "   creating: data/__MACOSX/ptb/\n",
            "  inflating: data/__MACOSX/ptb/._reader.py  \n",
            "  inflating: data/__MACOSX/._ptb     \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EBIwq-USXFLT",
        "colab_type": "code",
        "outputId": "c4822ade-8030-452b-edcc-d9672c476e84",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "!wget http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz \n",
        "!tar xzf simple-examples.tgz -C data/"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-05-06 08:28:17--  http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz\n",
            "Resolving www.fit.vutbr.cz (www.fit.vutbr.cz)... 147.229.9.23, 2001:67c:1220:809::93e5:917\n",
            "Connecting to www.fit.vutbr.cz (www.fit.vutbr.cz)|147.229.9.23|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 34869662 (33M) [application/x-gtar]\n",
            "Saving to: ‘simple-examples.tgz’\n",
            "\n",
            "simple-examples.tgz 100%[===================>]  33.25M  4.03MB/s    in 9.3s    \n",
            "\n",
            "2020-05-06 08:28:27 (3.58 MB/s) - ‘simple-examples.tgz’ saved [34869662/34869662]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L7FPdsuhXSxQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "init_scale = 0.1\n",
        "learning_rate = 1.0\n",
        "max_grad_norm = 5\n",
        "num_layers = 2\n",
        "num_steps = 20\n",
        "hidden_size_l1 = 256\n",
        "hidden_size_l2 = 128\n",
        "max_epoch_decay_lr = 4\n",
        "max_epoch = 15\n",
        "keep_prob = 1\n",
        "decay = 0.5\n",
        "batch_size = 60\n",
        "vocab_size = 10000\n",
        "embeding_vector_size = 200\n",
        "is_training = 1\n",
        "data_dir = \"data/simple-examples/data/\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fw49VCh0bIBK",
        "colab_type": "code",
        "outputId": "e8af3b7e-8d13-434d-8f5d-ac401d65a041",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 632
        }
      },
      "source": [
        "!pip install tensorflow==1.12.0"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorflow==1.12.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/22/cc/ca70b78087015d21c5f3f93694107f34ebccb3be9624385a911d4b52ecef/tensorflow-1.12.0-cp36-cp36m-manylinux1_x86_64.whl (83.1MB)\n",
            "\u001b[K     |████████████████████████████████| 83.1MB 35kB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0) (3.10.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0) (1.1.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0) (1.0.8)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0) (1.28.1)\n",
            "Requirement already satisfied: absl-py>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0) (0.9.0)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0) (0.3.3)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0) (1.1.0)\n",
            "\u001b[33mWARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProtocolError('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))': /simple/tensorboard/\u001b[0m\n",
            "Collecting tensorboard<1.13.0,>=1.12.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/07/53/8d32ce9471c18f8d99028b7cef2e5b39ea8765bd7ef250ca05b490880971/tensorboard-1.12.2-py3-none-any.whl (3.0MB)\n",
            "\u001b[K     |████████████████████████████████| 3.1MB 34.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0) (1.12.0)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0) (0.8.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0) (0.34.2)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.12.0) (1.18.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow==1.12.0) (46.1.3)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow==1.12.0) (2.10.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.10 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.13.0,>=1.12.0->tensorflow==1.12.0) (1.0.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.13.0,>=1.12.0->tensorflow==1.12.0) (3.2.1)\n",
            "Installing collected packages: tensorboard, tensorflow\n",
            "  Found existing installation: tensorboard 2.2.1\n",
            "    Uninstalling tensorboard-2.2.1:\n",
            "      Successfully uninstalled tensorboard-2.2.1\n",
            "  Found existing installation: tensorflow 2.2.0rc4\n",
            "    Uninstalling tensorflow-2.2.0rc4:\n",
            "      Successfully uninstalled tensorflow-2.2.0rc4\n",
            "Successfully installed tensorboard-1.12.2 tensorflow-1.12.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "tensorboard",
                  "tensorflow"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L5r2slamY1fO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "session = tf.InteractiveSession()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6LCjrz3gZoEi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "raw_data = reader.ptb_raw_data(data_dir)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F6lNn3UVapnG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_data, valid_data, test_data, vocab, word_to_id = raw_data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F7osbmOBcGTz",
        "colab_type": "code",
        "outputId": "64547ba7-7183-40c6-f5b8-9d7465bb0a06",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(train_data)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "929589"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AF4TeQTHcKw2",
        "colab_type": "code",
        "outputId": "6b31b589-03e5-4211-d638-4903d41f5d19",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "print(train_data[0:100])"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[9970, 9971, 9972, 9974, 9975, 9976, 9980, 9981, 9982, 9983, 9984, 9986, 9987, 9988, 9989, 9991, 9992, 9993, 9994, 9995, 9996, 9997, 9998, 9999, 2, 9256, 1, 3, 72, 393, 33, 2133, 0, 146, 19, 6, 9207, 276, 407, 3, 2, 23, 1, 13, 141, 4, 1, 5465, 0, 3081, 1596, 96, 2, 7682, 1, 3, 72, 393, 8, 337, 141, 4, 2477, 657, 2170, 955, 24, 521, 6, 9207, 276, 4, 39, 303, 438, 3684, 2, 6, 942, 4, 3150, 496, 263, 5, 138, 6092, 4241, 6036, 30, 988, 6, 241, 760, 4, 1015, 2786, 211, 6, 96, 4]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ve4H40b8cVUu",
        "colab_type": "code",
        "outputId": "56f71d8b-df74-451b-f1ac-fb9fdbea3d61",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "def id_to_word(id_list):\n",
        "    line = []\n",
        "    for w in id_list:\n",
        "        for word, wid in word_to_id.items():\n",
        "            if wid == w:\n",
        "                line.append(word)\n",
        "    return line            \n",
        "                \n",
        "\n",
        "print(id_to_word(train_data[0:100]))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['aer', 'banknote', 'berlitz', 'calloway', 'centrust', 'cluett', 'fromstein', 'gitano', 'guterman', 'hydro-quebec', 'ipo', 'kia', 'memotec', 'mlx', 'nahb', 'punts', 'rake', 'regatta', 'rubens', 'sim', 'snack-food', 'ssangyong', 'swapo', 'wachter', '<eos>', 'pierre', '<unk>', 'N', 'years', 'old', 'will', 'join', 'the', 'board', 'as', 'a', 'nonexecutive', 'director', 'nov.', 'N', '<eos>', 'mr.', '<unk>', 'is', 'chairman', 'of', '<unk>', 'n.v.', 'the', 'dutch', 'publishing', 'group', '<eos>', 'rudolph', '<unk>', 'N', 'years', 'old', 'and', 'former', 'chairman', 'of', 'consolidated', 'gold', 'fields', 'plc', 'was', 'named', 'a', 'nonexecutive', 'director', 'of', 'this', 'british', 'industrial', 'conglomerate', '<eos>', 'a', 'form', 'of', 'asbestos', 'once', 'used', 'to', 'make', 'kent', 'cigarette', 'filters', 'has', 'caused', 'a', 'high', 'percentage', 'of', 'cancer', 'deaths', 'among', 'a', 'group', 'of']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IqZQlQ6UchVm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "itera = reader.ptb_iterator(train_data, batch_size, num_steps)\n",
        "first_touple = itera.__next__()\n",
        "x = first_touple[0]\n",
        "y = first_touple[1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ugxcl66MdCt3",
        "colab_type": "code",
        "outputId": "dff4b79f-d0ab-4127-a18b-0df125112071",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "print(x)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[9970 9971 9972 ... 9993 9994 9995]\n",
            " [ 901   33 3361 ...  241   13 2420]\n",
            " [2654    6  334 ...  514    8  605]\n",
            " ...\n",
            " [7831   36 1678 ...    4 4558  157]\n",
            " [  59 2070 2433 ...  400    1 1173]\n",
            " [2097    3    2 ... 2043   23    1]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_u96CW7vdGL9",
        "colab_type": "code",
        "outputId": "576966d6-f958-4629-df44-9740b44c3b17",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x.shape"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(60, 20)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qyyfpdkZds38",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "_input_data = tf.placeholder(tf.int32 , [batch_size, num_steps])\n",
        "_targets = tf.placeholder(tf.int32 , [batch_size, num_steps])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PkNJllkXeIvF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "feed_dict = {_input_data:x, _targets:y}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pWCho2i4eZWQ",
        "colab_type": "code",
        "outputId": "9172a596-521f-49df-b660-bda3f5d33466",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "session.run(_input_data, feed_dict)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[9970, 9971, 9972, ..., 9993, 9994, 9995],\n",
              "       [ 901,   33, 3361, ...,  241,   13, 2420],\n",
              "       [2654,    6,  334, ...,  514,    8,  605],\n",
              "       ...,\n",
              "       [7831,   36, 1678, ...,    4, 4558,  157],\n",
              "       [  59, 2070, 2433, ...,  400,    1, 1173],\n",
              "       [2097,    3,    2, ..., 2043,   23,    1]], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Aw1WEIdekBJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "dbd5d2b4-ca55-4369-d0f6-7cc2c414d762"
      },
      "source": [
        "lstm_cell_l1 = tf.contrib.rnn.BasicLSTMCell(hidden_size_l1, forget_bias=0.0)\n",
        "lstm_cell_l2 = tf.contrib.rnn.BasicLSTMCell(hidden_size_l2, forget_bias=0.0)\n",
        "stacked_lstm = tf.contrib.rnn.MultiRNNCell([lstm_cell_l1, lstm_cell_l2])"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-18-ddf565d0e753>:1: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This class is deprecated, please use tf.nn.rnn_cell.LSTMCell, which supports all the feature this cell currently has. Please replace the existing code with tf.nn.rnn_cell.LSTMCell(name='basic_lstm_cell').\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CUpI9ez2ez-5",
        "colab_type": "code",
        "outputId": "f92b89df-5403-432f-9571-6996bf8287a6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "_initial_state = stacked_lstm.zero_state(batch_size, tf.float32)\n",
        "_initial_state"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(LSTMStateTuple(c=<tf.Tensor 'MultiRNNCellZeroState/BasicLSTMCellZeroState/zeros:0' shape=(60, 256) dtype=float32>, h=<tf.Tensor 'MultiRNNCellZeroState/BasicLSTMCellZeroState/zeros_1:0' shape=(60, 256) dtype=float32>),\n",
              " LSTMStateTuple(c=<tf.Tensor 'MultiRNNCellZeroState/BasicLSTMCellZeroState_1/zeros:0' shape=(60, 128) dtype=float32>, h=<tf.Tensor 'MultiRNNCellZeroState/BasicLSTMCellZeroState_1/zeros_1:0' shape=(60, 128) dtype=float32>))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IJMa3n6hfKEd",
        "colab_type": "code",
        "outputId": "dccc6458-6982-40cd-bf94-c65119c3a8a6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 459
        }
      },
      "source": [
        "session.run(_initial_state, feed_dict)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(LSTMStateTuple(c=array([[0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       ...,\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32), h=array([[0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       ...,\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32)),\n",
              " LSTMStateTuple(c=array([[0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       ...,\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32), h=array([[0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       ...,\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32)))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IPAz2230fRC1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embedding_vocab = tf.get_variable(\"embedding_vocab\", [vocab_size, embeding_vector_size])  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uBX_kqeef-Lm",
        "colab_type": "code",
        "outputId": "81bf9f46-fd0f-4783-d7b7-05d0df3a1867",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        }
      },
      "source": [
        "session.run(tf.global_variables_initializer())\n",
        "session.run(embedding_vocab)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-7.1576815e-03, -2.3444863e-02,  2.1636765e-02, ...,\n",
              "        -2.3458648e-02,  1.1177074e-02, -2.3858456e-02],\n",
              "       [-1.8274082e-02,  1.1337671e-02, -1.0738964e-02, ...,\n",
              "         5.1089767e-03,  3.0456204e-03,  6.9164876e-03],\n",
              "       [-2.3984347e-02,  5.0297678e-03, -2.2850571e-02, ...,\n",
              "        -1.9178528e-02,  1.2938876e-02,  2.2238284e-02],\n",
              "       ...,\n",
              "       [ 1.2332119e-02,  7.6370314e-05,  2.1143086e-02, ...,\n",
              "         1.7066214e-02, -3.7131514e-03,  1.5743989e-02],\n",
              "       [-9.2057111e-03, -2.0477977e-03,  1.2206256e-02, ...,\n",
              "        -1.7645882e-02,  1.1226237e-02, -1.5508312e-02],\n",
              "       [ 2.7251542e-03, -2.2815628e-02, -1.9703440e-02, ...,\n",
              "         4.0978212e-03,  2.2724506e-02, -4.0931590e-03]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2wmG7XdbgNzP",
        "colab_type": "code",
        "outputId": "2f27d270-4996-48c2-8e3e-864c9e44625a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "inputs = tf.nn.embedding_lookup(embedding_vocab, _input_data)  #shape=(30, 20, 200) \n",
        "inputs"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'embedding_lookup/Identity:0' shape=(60, 20, 200) dtype=float32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4iVEY3whgqYa",
        "colab_type": "code",
        "outputId": "c1bfd9f7-1a55-4084-f1f4-a16321dfa340",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        }
      },
      "source": [
        "session.run(inputs[0], feed_dict)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[ 0.00325009,  0.01081089,  0.0200482 , ...,  0.0022178 ,\n",
              "         0.01401596, -0.00581577],\n",
              "       [ 0.00442274,  0.01126799, -0.00109616, ..., -0.00493266,\n",
              "         0.00903059,  0.00655018],\n",
              "       [ 0.02124366,  0.02348461,  0.02135952, ...,  0.01759367,\n",
              "         0.01497137, -0.01829221],\n",
              "       ...,\n",
              "       [-0.01907893,  0.01818139,  0.02366877, ...,  0.02151349,\n",
              "        -0.0106791 , -0.00556704],\n",
              "       [ 0.00410514,  0.00279766,  0.00547559, ..., -0.01219951,\n",
              "        -0.02211702, -0.00371548],\n",
              "       [-0.02399711,  0.02415726, -0.00398123, ..., -0.01793151,\n",
              "         0.00649729, -0.00648275]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OSKNxo3dg22_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "outputs, new_state =  tf.nn.dynamic_rnn(stacked_lstm, inputs, initial_state=_initial_state)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E49F22xyg-a-",
        "colab_type": "code",
        "outputId": "60fe83f9-f6f6-48d8-ea3d-aa5269908ef6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        }
      },
      "source": [
        "session.run(tf.global_variables_initializer())\n",
        "session.run(outputs[0], feed_dict)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[ 1.00989251e-04,  6.92237154e-05,  1.73357417e-04, ...,\n",
              "         4.05761442e-04, -2.25753283e-05,  9.50943941e-05],\n",
              "       [ 3.96683870e-04,  4.82247618e-04, -1.18771946e-04, ...,\n",
              "         5.07148216e-04,  2.40567402e-04,  4.75475361e-04],\n",
              "       [ 7.95443717e-04,  3.01503838e-04, -2.29054000e-04, ...,\n",
              "         7.91834085e-04,  3.37273232e-04,  8.74314515e-04],\n",
              "       ...,\n",
              "       [-6.63069077e-04,  8.78425606e-04,  8.69163632e-05, ...,\n",
              "        -2.73895886e-04,  1.56508409e-04, -8.33585102e-04],\n",
              "       [-1.12435629e-03,  6.06384070e-04,  3.30563111e-04, ...,\n",
              "        -2.38942710e-04, -1.14292467e-04, -9.45392763e-04],\n",
              "       [-6.93089038e-04,  1.38926538e-04,  1.05895603e-03, ...,\n",
              "         3.86264612e-04,  2.89802731e-04, -6.68803346e-04]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HQZYo1JqhHML",
        "colab_type": "code",
        "outputId": "ba050164-58fb-4018-bcb2-83231f03cc66",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "output = tf.reshape(outputs, [-1, hidden_size_l2])\n",
        "output"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'Reshape:0' shape=(1200, 128) dtype=float32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7NNt-G2shR-0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "softmax_w = tf.get_variable(\"softmax_w\", [hidden_size_l2, vocab_size]) #[200x1000]\n",
        "softmax_b = tf.get_variable(\"softmax_b\", [vocab_size]) #[1x1000]\n",
        "logits = tf.matmul(output, softmax_w) + softmax_b\n",
        "prob = tf.nn.softmax(logits)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NU9iPHZjhdie",
        "colab_type": "code",
        "outputId": "4ee14ac3-9b63-4fa8-fe4f-5f36b84bfef4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        }
      },
      "source": [
        "session.run(tf.global_variables_initializer())\n",
        "output_words_prob = session.run(prob, feed_dict)\n",
        "print(\"shape of the output: \", output_words_prob.shape)\n",
        "print(\"The probability of observing words in t=0 to t=20\", output_words_prob[0:20])"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "shape of the output:  (1200, 10000)\n",
            "The probability of observing words in t=0 to t=20 [[1.01613296e-04 9.94720831e-05 1.00765646e-04 ... 1.00619422e-04\n",
            "  9.99091353e-05 9.86637606e-05]\n",
            " [1.01628037e-04 9.94681322e-05 1.00769597e-04 ... 1.00622696e-04\n",
            "  9.98977921e-05 9.86743762e-05]\n",
            " [1.01631325e-04 9.94678558e-05 1.00774392e-04 ... 1.00621925e-04\n",
            "  9.98932446e-05 9.86833547e-05]\n",
            " ...\n",
            " [1.01605583e-04 9.94582515e-05 1.00770769e-04 ... 1.00610567e-04\n",
            "  9.99066324e-05 9.86663072e-05]\n",
            " [1.01606274e-04 9.94520960e-05 1.00770529e-04 ... 1.00608457e-04\n",
            "  9.99003241e-05 9.86626983e-05]\n",
            " [1.01607540e-04 9.94602306e-05 1.00769787e-04 ... 1.00601661e-04\n",
            "  9.98939940e-05 9.86595041e-05]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mpPRf6oqhmRB",
        "colab_type": "code",
        "outputId": "2c9dff59-5131-435e-af67-2f90ab3e245b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "np.argmax(output_words_prob[0:20], axis=1)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 343,  343,  343,  343, 9540, 9540, 6745, 8953, 8953, 9995, 9995,\n",
              "       3928, 3928, 3928, 6755, 6755, 9923, 9923, 3928, 3928])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q_jTyTm1iBn4",
        "colab_type": "code",
        "outputId": "72f2ffa4-0d26-472a-f038-c02ada85dd5d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "y[0]"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([9971, 9972, 9974, 9975, 9976, 9980, 9981, 9982, 9983, 9984, 9986,\n",
              "       9987, 9988, 9989, 9991, 9992, 9993, 9994, 9995, 9996], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Li_S3WKBiDjr",
        "colab_type": "code",
        "outputId": "c4fa16ba-d4a7-48d3-f454-b1761f610995",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "targ = session.run(_targets, feed_dict) \n",
        "targ[0]"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([9971, 9972, 9974, 9975, 9976, 9980, 9981, 9982, 9983, 9984, 9986,\n",
              "       9987, 9988, 9989, 9991, 9992, 9993, 9994, 9995, 9996], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kFeYaqFYiMt-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example([logits], [tf.reshape(_targets, [-1])],[tf.ones([batch_size * num_steps])])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VvccjGadiUw1",
        "colab_type": "code",
        "outputId": "a724146a-a4fc-4dce-cfc2-94199dc263fe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "session.run(loss, feed_dict)[:10]"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([9.204273, 9.224012, 9.216199, 9.221151, 9.195596, 9.204564,\n",
              "       9.216711, 9.209148, 9.20055 , 9.223702], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rk3BPQ3DicLR",
        "colab_type": "code",
        "outputId": "d15ba4ce-0356-434e-a7f8-ca31f4ce50d6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "cost = tf.reduce_sum(loss) / batch_size\n",
        "session.run(tf.global_variables_initializer())\n",
        "session.run(cost, feed_dict)"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "184.19994"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vy31kgNjim8P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lr = tf.Variable(0.0, trainable=False)\n",
        "# Create the gradient descent optimizer with our learning rate\n",
        "optimizer = tf.train.GradientDescentOptimizer(lr)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Z6mXcNgivjj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "f8b78f5e-3f09-4c98-87a3-00184cf422ec"
      },
      "source": [
        "tvars = tf.trainable_variables()\n",
        "tvars"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<tf.Variable 'embedding_vocab:0' shape=(10000, 200) dtype=float32_ref>,\n",
              " <tf.Variable 'rnn/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0' shape=(456, 1024) dtype=float32_ref>,\n",
              " <tf.Variable 'rnn/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0' shape=(1024,) dtype=float32_ref>,\n",
              " <tf.Variable 'rnn/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0' shape=(384, 512) dtype=float32_ref>,\n",
              " <tf.Variable 'rnn/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0' shape=(512,) dtype=float32_ref>,\n",
              " <tf.Variable 'softmax_w:0' shape=(128, 10000) dtype=float32_ref>,\n",
              " <tf.Variable 'softmax_b:0' shape=(10000,) dtype=float32_ref>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m6j122bgi0Wz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "7f901d78-b5a3-4be4-8a66-cf88e8690b67"
      },
      "source": [
        "[v.name for v in tvars]"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['embedding_vocab:0',\n",
              " 'rnn/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0',\n",
              " 'rnn/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0',\n",
              " 'rnn/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0',\n",
              " 'rnn/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0',\n",
              " 'softmax_w:0',\n",
              " 'softmax_b:0']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YkFneN3ui6y-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6bdf9db7-d406-4b83-cd97-195762925952"
      },
      "source": [
        "var_x = tf.placeholder(tf.float32)\n",
        "var_y = tf.placeholder(tf.float32) \n",
        "func_test = 2.0 * var_x * var_x + 3.0 * var_x * var_y\n",
        "session.run(tf.global_variables_initializer())\n",
        "session.run(func_test, {var_x:1.0,var_y:2.0})"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "8.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ie6cDj6EjA8H",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0e68cded-ff07-4ff7-cc4b-03d9e9916247"
      },
      "source": [
        "var_grad = tf.gradients(func_test, [var_x])\n",
        "session.run(var_grad, {var_x:1.0,var_y:2.0})"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[10.0]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HI4BaVRVjKF6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "44cbd84b-be4b-48b5-979a-dc000be9cd1c"
      },
      "source": [
        "var_grad = tf.gradients(func_test, [var_y])\n",
        "session.run(var_grad, {var_x:1.0, var_y:2.0})"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[3.0]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c0CgD-W-jQh3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "031289b3-46a0-4de4-eebd-977eca739363"
      },
      "source": [
        "tf.gradients(cost, tvars)"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<tensorflow.python.framework.ops.IndexedSlices at 0x7fe2a7951a90>,\n",
              " <tf.Tensor 'gradients_4/rnn/while/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/MatMul/Enter_grad/b_acc_3:0' shape=(456, 1024) dtype=float32>,\n",
              " <tf.Tensor 'gradients_4/rnn/while/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/BiasAdd/Enter_grad/b_acc_3:0' shape=(1024,) dtype=float32>,\n",
              " <tf.Tensor 'gradients_4/rnn/while/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/MatMul/Enter_grad/b_acc_3:0' shape=(384, 512) dtype=float32>,\n",
              " <tf.Tensor 'gradients_4/rnn/while/rnn/multi_rnn_cell/cell_1/basic_lstm_cell/BiasAdd/Enter_grad/b_acc_3:0' shape=(512,) dtype=float32>,\n",
              " <tf.Tensor 'gradients_4/MatMul_grad/MatMul_1:0' shape=(128, 10000) dtype=float32>,\n",
              " <tf.Tensor 'gradients_4/add_grad/Reshape_1:0' shape=(10000,) dtype=float32>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IkvlMAu9jXfO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "941153e3-0a23-4e82-94f5-f41073012a03"
      },
      "source": [
        "grad_t_list = tf.gradients(cost, tvars)\n",
        "session.run(grad_t_list,feed_dict)"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[IndexedSlicesValue(values=array([[-2.7893816e-07, -1.2814845e-05, -1.3136873e-05, ...,\n",
              "        -7.3561187e-06, -4.3364307e-06, -1.0902732e-05],\n",
              "       [ 1.6519586e-07, -1.7336493e-05, -1.3297089e-05, ...,\n",
              "        -1.1260923e-05, -3.6407571e-06, -7.2892703e-06],\n",
              "       [ 1.3327696e-06,  2.7600042e-06, -4.5080787e-06, ...,\n",
              "        -5.6128933e-06, -5.8137848e-06, -5.4516358e-06],\n",
              "       ...,\n",
              "       [-5.8048096e-07,  5.8552064e-06,  5.5862770e-06, ...,\n",
              "         1.0751618e-06, -9.3951442e-08,  1.0573129e-05],\n",
              "       [-8.6072878e-06,  5.7813136e-06,  3.8972007e-06, ...,\n",
              "        -4.3460022e-06, -1.1080484e-06,  9.0330959e-06],\n",
              "       [-9.8562978e-06,  9.1688526e-06,  1.5621805e-06, ...,\n",
              "        -1.9780257e-06,  6.0257344e-06,  1.1340519e-05]], dtype=float32), indices=array([9970, 9971, 9972, ..., 2043,   23,    1], dtype=int32), dense_shape=array([10000,   200], dtype=int32)),\n",
              " array([[-5.60766473e-08,  3.01972598e-08, -4.73208672e-09, ...,\n",
              "          2.93886622e-08, -1.85395344e-08,  4.80393254e-08],\n",
              "        [ 1.71820957e-08, -1.95767704e-08,  4.07862988e-09, ...,\n",
              "         -3.94924271e-08, -1.83959266e-08,  1.72417245e-08],\n",
              "        [-3.70883413e-09, -1.79185484e-08,  1.61177134e-08, ...,\n",
              "         -1.01243192e-09,  4.17874890e-09, -3.96522957e-08],\n",
              "        ...,\n",
              "        [-8.94279495e-09, -2.39824782e-09,  5.79599790e-09, ...,\n",
              "         -1.44804275e-08, -4.19407398e-10, -3.08480930e-09],\n",
              "        [-3.28318195e-09, -2.86681612e-09, -2.78562151e-09, ...,\n",
              "          3.12178328e-09, -9.93178095e-10, -5.48603474e-09],\n",
              "        [ 6.75314027e-10, -4.57058569e-09,  4.62280791e-09, ...,\n",
              "         -4.37559677e-09,  7.90688126e-10, -2.88399011e-08]], dtype=float32),\n",
              " array([-1.5913596e-06,  1.0382903e-06,  1.3557336e-06, ...,\n",
              "        -3.5235050e-06, -4.0233942e-08, -1.8944552e-06], dtype=float32),\n",
              " array([[-3.1531850e-09,  3.9578683e-09, -5.2248579e-09, ...,\n",
              "         -1.1490903e-09,  7.6460305e-09,  7.3503434e-09],\n",
              "        [ 1.1700975e-09,  4.1059303e-09,  1.0339167e-09, ...,\n",
              "          2.7343035e-09, -1.1514814e-09, -1.2018908e-08],\n",
              "        [-3.1777687e-09,  3.0072356e-10, -4.7151389e-09, ...,\n",
              "          4.1989705e-09, -6.4952044e-09,  1.2984300e-08],\n",
              "        ...,\n",
              "        [ 2.6501059e-09, -2.4809688e-10, -1.3518880e-09, ...,\n",
              "          4.5523429e-10,  4.6023427e-10,  3.1385496e-09],\n",
              "        [-5.4524640e-10, -1.9285287e-09,  1.2471371e-10, ...,\n",
              "          1.5320135e-09,  2.4076481e-09, -1.1025443e-09],\n",
              "        [ 1.6644157e-09,  3.3786271e-10, -1.9236248e-09, ...,\n",
              "          3.0947300e-09, -1.4776135e-09,  1.2152175e-08]], dtype=float32),\n",
              " array([-4.88378873e-06, -4.13190310e-06,  2.69728730e-06,  1.57278805e-06,\n",
              "        -2.97681686e-06, -5.10778045e-06,  2.41158403e-07,  1.21385381e-06,\n",
              "        -9.88886427e-07, -2.27843839e-06, -4.68380767e-06, -2.68677718e-06,\n",
              "        -2.46014520e-06, -2.41076850e-06,  2.04575713e-06,  3.95849398e-08,\n",
              "        -4.52658952e-07,  2.27277246e-06,  4.12368848e-07,  4.83852864e-06,\n",
              "        -7.92494757e-06,  1.24531994e-06, -2.88416550e-06,  5.66277458e-06,\n",
              "         1.47988123e-06, -1.02949434e-06,  1.86922807e-06, -1.91365780e-06,\n",
              "         7.09533424e-07,  4.44163470e-06,  2.32406319e-06,  1.02701324e-06,\n",
              "         2.21399006e-07, -6.20714263e-06,  1.36593155e-06, -4.80390952e-07,\n",
              "         1.53454721e-06,  4.34707499e-06, -2.13167868e-06, -4.92736444e-06,\n",
              "         4.50687685e-06, -4.47616685e-06, -2.89162494e-06,  9.76832212e-07,\n",
              "        -1.68226541e-06,  1.13509884e-07,  4.76994728e-08,  1.50830920e-06,\n",
              "         4.04220100e-06,  3.90814705e-08, -2.37905147e-06,  1.56349643e-07,\n",
              "        -1.23645771e-06, -9.73865372e-06,  2.00330987e-08,  3.74465856e-07,\n",
              "         2.66228176e-06, -3.47049581e-06,  5.78261734e-07, -8.39276481e-07,\n",
              "        -4.45589376e-06,  1.83863256e-06, -6.54268945e-07,  6.96019606e-06,\n",
              "        -2.67118389e-06,  1.23611233e-06, -1.17471870e-06, -2.20969264e-06,\n",
              "         2.58412661e-06, -8.73472516e-07, -8.61359513e-06,  5.16431840e-08,\n",
              "        -3.32281547e-06,  2.89640457e-06,  9.35871128e-07,  7.05174534e-06,\n",
              "        -2.03580021e-06,  5.24662892e-06,  4.47693310e-06, -2.69713314e-06,\n",
              "        -2.76416472e-07,  2.71869385e-06, -2.69586644e-06,  1.12339217e-06,\n",
              "         9.65491154e-07, -2.05802007e-06, -5.44284148e-07, -2.19263529e-06,\n",
              "        -2.47574781e-06, -1.33308686e-06, -1.59611898e-06,  2.43460659e-06,\n",
              "        -1.20743744e-06, -3.55802945e-06, -1.28837792e-06, -6.79587720e-06,\n",
              "         4.17263800e-06,  2.25027725e-06,  1.39842689e-06, -3.66234190e-06,\n",
              "        -3.48062963e-06, -4.84117209e-08, -3.00496094e-06, -2.38488678e-06,\n",
              "        -6.40428289e-06, -5.12352517e-06,  2.02773003e-06,  3.17878417e-08,\n",
              "         1.87773048e-06, -5.39145219e-07,  2.34144773e-06, -3.92581023e-06,\n",
              "         5.41821919e-07,  6.68338544e-07,  3.22792516e-06,  1.46995126e-06,\n",
              "        -1.59985984e-06,  1.21002654e-06, -4.31857302e-07,  2.73906130e-06,\n",
              "        -3.20947083e-06,  1.22053834e-07,  4.44860098e-06, -9.04749641e-07,\n",
              "        -7.50588981e-07, -1.48517449e-06,  2.40128452e-06, -6.03200988e-06,\n",
              "         1.39242737e-02,  5.54974750e-03,  3.05335084e-03, -3.02047061e-04,\n",
              "         3.24388617e-03,  2.13072519e-03,  2.08082311e-02, -2.08542626e-02,\n",
              "         4.33728303e-04,  7.72338454e-03,  2.39868667e-02,  1.99148431e-02,\n",
              "        -4.12102370e-03, -1.22182192e-02,  1.72865614e-02, -1.44097246e-02,\n",
              "        -2.44710967e-02,  3.98896355e-03, -1.25209624e-02, -2.23097298e-03,\n",
              "        -4.78580315e-03, -1.45192482e-02, -1.13069732e-02, -1.64531805e-02,\n",
              "        -6.97623938e-03,  1.26333418e-03, -8.50428268e-03,  2.66466493e-04,\n",
              "        -1.51594291e-02, -1.37884710e-02, -2.74920743e-03,  9.04281437e-03,\n",
              "        -1.52048666e-03, -2.29096338e-02, -8.15972500e-03,  1.80550746e-03,\n",
              "        -1.82293577e-03,  2.32013594e-02, -3.24593647e-03,  2.08144505e-02,\n",
              "         3.81885562e-03, -2.66568419e-02, -1.12625565e-02,  3.48657300e-03,\n",
              "         4.61633375e-04, -8.20975192e-03, -6.41340762e-03, -9.33795981e-03,\n",
              "         1.38781508e-02,  1.86218563e-02, -1.51847713e-02,  1.37258158e-03,\n",
              "         1.25292584e-03, -1.91873237e-02,  5.13088144e-03, -6.12962153e-03,\n",
              "         5.84190944e-03,  1.16460193e-02,  9.68271587e-03,  3.86986020e-03,\n",
              "        -1.28079588e-02, -2.06657569e-03, -1.42742193e-03,  1.82190500e-02,\n",
              "         3.64407478e-03,  1.02758622e-02,  1.35673187e-03, -1.19023863e-02,\n",
              "         1.51308365e-02,  3.93285038e-04,  1.98362600e-02,  2.53575221e-02,\n",
              "         1.87595328e-03,  1.62954815e-02,  8.35802592e-03, -1.57758817e-02,\n",
              "        -1.36727793e-03, -4.92050126e-03,  1.70726515e-02, -1.28969271e-02,\n",
              "         1.79837625e-02,  2.06838883e-02,  3.80276632e-03, -1.05241723e-02,\n",
              "         1.85937509e-02, -3.36697139e-02, -5.94925415e-03, -1.21918833e-02,\n",
              "        -2.82976567e-03, -1.99882314e-04,  2.08421741e-02,  2.37090625e-02,\n",
              "         1.31583214e-02,  1.53059280e-02, -1.09626446e-02,  2.41414681e-02,\n",
              "        -2.38002613e-02,  9.27094929e-03,  1.20362630e-02,  8.34529474e-03,\n",
              "         1.77534111e-02, -1.12054003e-02, -1.45916240e-02, -3.39754834e-03,\n",
              "         6.24024635e-03, -1.36032151e-02, -2.45527569e-02, -5.45615191e-03,\n",
              "        -6.31176354e-03, -1.94209665e-02, -1.38772046e-02, -2.67241034e-03,\n",
              "        -1.60574785e-03,  1.41333099e-02,  1.58162159e-03,  8.07921751e-05,\n",
              "         1.74026992e-02,  1.14477184e-02,  9.50258691e-03, -1.70847699e-02,\n",
              "        -6.06383150e-03,  1.78761373e-03,  5.48334047e-03, -1.13184676e-02,\n",
              "        -1.11478064e-02,  1.21656582e-02, -3.54498182e-03,  1.31940674e-02,\n",
              "        -3.66292716e-06, -1.67709197e-06,  1.15489365e-06,  2.04090384e-06,\n",
              "        -1.97214399e-06, -1.94169706e-06, -1.54499412e-06,  1.51349195e-06,\n",
              "         1.90516789e-06, -2.84500561e-06, -8.89056366e-07, -3.36458947e-06,\n",
              "        -9.97944426e-07, -1.80570339e-06, -1.81552377e-06,  3.53210169e-07,\n",
              "        -2.42168062e-06,  6.28737837e-07, -4.57541944e-07,  2.37957647e-06,\n",
              "        -8.36917025e-06,  1.59378010e-06, -5.18979459e-06,  3.12828138e-06,\n",
              "         2.93772314e-06, -2.37422728e-06,  3.01512642e-07,  2.84448629e-06,\n",
              "        -8.92568096e-07,  2.78261064e-06, -3.07997908e-07, -3.48850699e-06,\n",
              "         1.78894788e-06, -5.76378216e-06, -1.19583365e-06,  3.61119010e-06,\n",
              "         1.38640780e-06,  7.27255929e-06, -7.13258032e-07, -2.42387796e-06,\n",
              "         4.81660391e-06, -6.05220657e-06, -1.20786240e-07, -9.63081220e-07,\n",
              "         6.28920418e-07,  2.50208899e-07,  1.46303012e-06, -2.06748246e-06,\n",
              "         3.27549719e-06,  3.18689104e-07, -3.28411375e-07, -9.97600523e-07,\n",
              "        -9.33672652e-07, -7.75691569e-06,  1.83845543e-08, -2.23615211e-06,\n",
              "         2.06670302e-06, -1.50715732e-06, -1.23903760e-06, -6.32621095e-07,\n",
              "        -3.87274940e-06,  1.30004466e-07, -1.56119836e-06,  2.95061045e-06,\n",
              "        -2.07922722e-06,  1.32001583e-06,  1.97937629e-06,  1.97699683e-06,\n",
              "         1.71079682e-06,  1.14813520e-06, -4.84415023e-06,  6.13900227e-07,\n",
              "        -1.26074929e-06,  1.64287928e-06, -9.00038913e-07,  3.33668163e-06,\n",
              "        -8.04801516e-07,  2.54634233e-06,  1.35993344e-06, -9.45049749e-07,\n",
              "        -2.26901398e-06,  3.30952184e-06, -1.23058055e-06,  1.23759025e-06,\n",
              "        -1.29110026e-06, -1.95543385e-06,  1.21082860e-06, -6.37032770e-07,\n",
              "        -2.75677689e-06,  1.71108638e-08, -1.22725510e-06,  5.48550452e-06,\n",
              "        -5.16780290e-07,  1.61851986e-07, -2.48144806e-06, -7.60292824e-06,\n",
              "         4.07707057e-06,  8.54864481e-07,  3.85257863e-07, -3.18585830e-06,\n",
              "        -2.36469376e-07, -1.39002009e-06, -2.07322273e-06, -3.60677495e-06,\n",
              "        -4.13536736e-06, -2.92734171e-06,  1.72655393e-06, -8.13934207e-07,\n",
              "         5.36086645e-07,  1.67940016e-06, -6.56056500e-07, -1.22003371e-06,\n",
              "         3.07312439e-06, -4.23844909e-07,  5.54079975e-07,  1.85878866e-06,\n",
              "        -8.07406934e-07, -8.08691084e-07, -7.44424426e-07,  2.25764484e-06,\n",
              "        -1.95123062e-06, -4.70720352e-07,  3.32219270e-06, -2.13086150e-06,\n",
              "        -1.34969036e-06, -1.95391681e-06, -7.53713209e-07, -6.07828497e-06,\n",
              "        -4.88081605e-06, -4.13471980e-06,  2.69602901e-06,  1.56886767e-06,\n",
              "        -2.98207669e-06, -5.10695054e-06,  2.48035235e-07,  1.21070343e-06,\n",
              "        -9.89848218e-07, -2.28240106e-06, -4.68119788e-06, -2.68705026e-06,\n",
              "        -2.46373384e-06, -2.40817781e-06,  2.03848572e-06,  4.54288198e-08,\n",
              "        -4.57164049e-07,  2.27584928e-06,  4.16878066e-07,  4.84208385e-06,\n",
              "        -7.92735682e-06,  1.25396025e-06, -2.89140667e-06,  5.66579047e-06,\n",
              "         1.47904996e-06, -1.02820059e-06,  1.86925911e-06, -1.90934611e-06,\n",
              "         7.12811982e-07,  4.44179477e-06,  2.32633147e-06,  1.02868739e-06,\n",
              "         2.19701064e-07, -6.21427762e-06,  1.36217898e-06, -4.80901122e-07,\n",
              "         1.53778296e-06,  4.35151696e-06, -2.12600298e-06, -4.91887249e-06,\n",
              "         4.50140260e-06, -4.46854210e-06, -2.88772208e-06,  9.74723889e-07,\n",
              "        -1.68251108e-06,  1.14085310e-07,  5.35013669e-08,  1.51363417e-06,\n",
              "         4.04000411e-06,  4.35830998e-08, -2.37468930e-06,  1.57842408e-07,\n",
              "        -1.23476138e-06, -9.73542501e-06,  2.07632525e-08,  3.75153348e-07,\n",
              "         2.66025768e-06, -3.46976731e-06,  5.77859623e-07, -8.39574454e-07,\n",
              "        -4.45489468e-06,  1.84165026e-06, -6.52981441e-07,  6.95018298e-06,\n",
              "        -2.66763186e-06,  1.23637790e-06, -1.18329149e-06, -2.20795437e-06,\n",
              "         2.58487898e-06, -8.70961571e-07, -8.61560966e-06,  4.86135150e-08,\n",
              "        -3.32118043e-06,  2.89433206e-06,  9.34584705e-07,  7.05653747e-06,\n",
              "        -2.03681157e-06,  5.24489951e-06,  4.48043011e-06, -2.70098872e-06,\n",
              "        -2.69784266e-07,  2.71467275e-06, -2.69793259e-06,  1.12840758e-06,\n",
              "         9.65819254e-07, -2.05789570e-06, -5.44561431e-07, -2.18705964e-06,\n",
              "        -2.47609819e-06, -1.33124081e-06, -1.59643776e-06,  2.43209706e-06,\n",
              "        -1.20729135e-06, -3.55698990e-06, -1.28926649e-06, -6.79971663e-06,\n",
              "         4.17126694e-06,  2.24649852e-06,  1.40196630e-06, -3.66347876e-06,\n",
              "        -3.47397531e-06, -5.14192777e-08, -3.00153397e-06, -2.38521898e-06,\n",
              "        -6.40659573e-06, -5.12215956e-06,  2.03400464e-06,  3.40062201e-08,\n",
              "         1.87842181e-06, -5.44717807e-07,  2.33790388e-06, -3.92147331e-06,\n",
              "         5.45191767e-07,  6.69800784e-07,  3.22032815e-06,  1.47051833e-06,\n",
              "        -1.60124841e-06,  1.21032929e-06, -4.30917112e-07,  2.73895967e-06,\n",
              "        -3.21418929e-06,  1.22858694e-07,  4.45136084e-06, -9.09784944e-07,\n",
              "        -7.51503421e-07, -1.48715139e-06,  2.40184841e-06, -6.03161698e-06],\n",
              "       dtype=float32),\n",
              " array([[ 1.4947556e-04,  2.3939194e-04,  5.2482792e-05, ...,\n",
              "         -2.1421333e-07, -2.1313690e-07, -2.1203239e-07],\n",
              "        [ 2.5827577e-04,  2.4813495e-04,  2.1275255e-04, ...,\n",
              "         -3.7740159e-07, -3.7550285e-07, -3.7354494e-07],\n",
              "        [ 1.3274519e-04,  8.9440102e-05,  9.4746589e-05, ...,\n",
              "         -2.6668624e-07, -2.6535332e-07, -2.6397137e-07],\n",
              "        ...,\n",
              "        [ 9.5092211e-05,  1.9221485e-04,  9.5722055e-05, ...,\n",
              "         -3.1345070e-07, -3.1189174e-07, -3.1021872e-07],\n",
              "        [-1.3263794e-04, -8.1470949e-05, -2.1943815e-04, ...,\n",
              "          3.5827429e-07,  3.5647301e-07,  3.5463933e-07],\n",
              "        [ 9.4148636e-05,  3.9820693e-04,  1.9381891e-04, ...,\n",
              "         -3.9441599e-07, -3.9249034e-07, -3.9041552e-07]], dtype=float32),\n",
              " array([-0.7813496 , -1.0980005 , -0.98132986, ...,  0.00200399,\n",
              "         0.00199393,  0.00198356], dtype=float32)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HH1jPnY1jdEO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "4ce4d7c7-2d05-44e0-da91-6f170685d207"
      },
      "source": [
        "grads, _ = tf.clip_by_global_norm(grad_t_list, max_grad_norm)\n",
        "grads"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<tensorflow.python.framework.ops.IndexedSlices at 0x7fe2a7918080>,\n",
              " <tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_1:0' shape=(456, 1024) dtype=float32>,\n",
              " <tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_2:0' shape=(1024,) dtype=float32>,\n",
              " <tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_3:0' shape=(384, 512) dtype=float32>,\n",
              " <tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_4:0' shape=(512,) dtype=float32>,\n",
              " <tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_5:0' shape=(128, 10000) dtype=float32>,\n",
              " <tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_6:0' shape=(10000,) dtype=float32>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Awl6-W-jjjhd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "d5e2b0a4-2091-47ef-cf26-8145ac25f411"
      },
      "source": [
        "session.run(grads, feed_dict)"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[IndexedSlicesValue(values=array([[-2.7893816e-07, -1.2814845e-05, -1.3136873e-05, ...,\n",
              "        -7.3561187e-06, -4.3364307e-06, -1.0902732e-05],\n",
              "       [ 1.6519586e-07, -1.7336493e-05, -1.3297089e-05, ...,\n",
              "        -1.1260923e-05, -3.6407571e-06, -7.2892703e-06],\n",
              "       [ 1.3327696e-06,  2.7600042e-06, -4.5080787e-06, ...,\n",
              "        -5.6128933e-06, -5.8137848e-06, -5.4516358e-06],\n",
              "       ...,\n",
              "       [-5.8048096e-07,  5.8552064e-06,  5.5862770e-06, ...,\n",
              "         1.0751618e-06, -9.3951442e-08,  1.0573129e-05],\n",
              "       [-8.6072878e-06,  5.7813136e-06,  3.8972007e-06, ...,\n",
              "        -4.3460022e-06, -1.1080484e-06,  9.0330959e-06],\n",
              "       [-9.8562978e-06,  9.1688526e-06,  1.5621805e-06, ...,\n",
              "        -1.9780257e-06,  6.0257344e-06,  1.1340519e-05]], dtype=float32), indices=array([9970, 9971, 9972, ..., 2043,   23,    1], dtype=int32), dense_shape=array([10000,   200], dtype=int32)),\n",
              " array([[-5.60766473e-08,  3.01972598e-08, -4.73208672e-09, ...,\n",
              "          2.93886622e-08, -1.85395344e-08,  4.80393254e-08],\n",
              "        [ 1.71820957e-08, -1.95767704e-08,  4.07862988e-09, ...,\n",
              "         -3.94924271e-08, -1.83959266e-08,  1.72417245e-08],\n",
              "        [-3.70883413e-09, -1.79185484e-08,  1.61177134e-08, ...,\n",
              "         -1.01243192e-09,  4.17874890e-09, -3.96522957e-08],\n",
              "        ...,\n",
              "        [-8.94279495e-09, -2.39824782e-09,  5.79599790e-09, ...,\n",
              "         -1.44804275e-08, -4.19407398e-10, -3.08480930e-09],\n",
              "        [-3.28318195e-09, -2.86681612e-09, -2.78562151e-09, ...,\n",
              "          3.12178328e-09, -9.93178095e-10, -5.48603474e-09],\n",
              "        [ 6.75314027e-10, -4.57058569e-09,  4.62280791e-09, ...,\n",
              "         -4.37559677e-09,  7.90688126e-10, -2.88399011e-08]], dtype=float32),\n",
              " array([-1.5913596e-06,  1.0382903e-06,  1.3557336e-06, ...,\n",
              "        -3.5235050e-06, -4.0233942e-08, -1.8944552e-06], dtype=float32),\n",
              " array([[-3.1531850e-09,  3.9578683e-09, -5.2248579e-09, ...,\n",
              "         -1.1490903e-09,  7.6460305e-09,  7.3503434e-09],\n",
              "        [ 1.1700975e-09,  4.1059303e-09,  1.0339167e-09, ...,\n",
              "          2.7343035e-09, -1.1514814e-09, -1.2018908e-08],\n",
              "        [-3.1777687e-09,  3.0072356e-10, -4.7151389e-09, ...,\n",
              "          4.1989705e-09, -6.4952044e-09,  1.2984300e-08],\n",
              "        ...,\n",
              "        [ 2.6501059e-09, -2.4809688e-10, -1.3518880e-09, ...,\n",
              "          4.5523429e-10,  4.6023427e-10,  3.1385496e-09],\n",
              "        [-5.4524640e-10, -1.9285287e-09,  1.2471371e-10, ...,\n",
              "          1.5320135e-09,  2.4076481e-09, -1.1025443e-09],\n",
              "        [ 1.6644157e-09,  3.3786271e-10, -1.9236248e-09, ...,\n",
              "          3.0947300e-09, -1.4776135e-09,  1.2152175e-08]], dtype=float32),\n",
              " array([-4.88378873e-06, -4.13190310e-06,  2.69728730e-06,  1.57278805e-06,\n",
              "        -2.97681686e-06, -5.10778045e-06,  2.41158403e-07,  1.21385381e-06,\n",
              "        -9.88886427e-07, -2.27843839e-06, -4.68380767e-06, -2.68677718e-06,\n",
              "        -2.46014520e-06, -2.41076850e-06,  2.04575713e-06,  3.95849398e-08,\n",
              "        -4.52658952e-07,  2.27277246e-06,  4.12368848e-07,  4.83852864e-06,\n",
              "        -7.92494757e-06,  1.24531994e-06, -2.88416550e-06,  5.66277458e-06,\n",
              "         1.47988123e-06, -1.02949434e-06,  1.86922807e-06, -1.91365780e-06,\n",
              "         7.09533424e-07,  4.44163470e-06,  2.32406319e-06,  1.02701324e-06,\n",
              "         2.21399006e-07, -6.20714263e-06,  1.36593155e-06, -4.80390952e-07,\n",
              "         1.53454721e-06,  4.34707499e-06, -2.13167868e-06, -4.92736444e-06,\n",
              "         4.50687685e-06, -4.47616685e-06, -2.89162494e-06,  9.76832212e-07,\n",
              "        -1.68226541e-06,  1.13509884e-07,  4.76994728e-08,  1.50830920e-06,\n",
              "         4.04220100e-06,  3.90814705e-08, -2.37905147e-06,  1.56349643e-07,\n",
              "        -1.23645771e-06, -9.73865372e-06,  2.00330987e-08,  3.74465856e-07,\n",
              "         2.66228176e-06, -3.47049581e-06,  5.78261734e-07, -8.39276481e-07,\n",
              "        -4.45589376e-06,  1.83863256e-06, -6.54268945e-07,  6.96019606e-06,\n",
              "        -2.67118389e-06,  1.23611233e-06, -1.17471870e-06, -2.20969264e-06,\n",
              "         2.58412661e-06, -8.73472516e-07, -8.61359513e-06,  5.16431840e-08,\n",
              "        -3.32281547e-06,  2.89640457e-06,  9.35871128e-07,  7.05174534e-06,\n",
              "        -2.03580021e-06,  5.24662892e-06,  4.47693310e-06, -2.69713314e-06,\n",
              "        -2.76416472e-07,  2.71869385e-06, -2.69586644e-06,  1.12339217e-06,\n",
              "         9.65491154e-07, -2.05802007e-06, -5.44284148e-07, -2.19263529e-06,\n",
              "        -2.47574781e-06, -1.33308686e-06, -1.59611898e-06,  2.43460659e-06,\n",
              "        -1.20743744e-06, -3.55802945e-06, -1.28837792e-06, -6.79587720e-06,\n",
              "         4.17263800e-06,  2.25027725e-06,  1.39842689e-06, -3.66234190e-06,\n",
              "        -3.48062963e-06, -4.84117209e-08, -3.00496094e-06, -2.38488678e-06,\n",
              "        -6.40428289e-06, -5.12352517e-06,  2.02773003e-06,  3.17878417e-08,\n",
              "         1.87773048e-06, -5.39145219e-07,  2.34144773e-06, -3.92581023e-06,\n",
              "         5.41821919e-07,  6.68338544e-07,  3.22792516e-06,  1.46995126e-06,\n",
              "        -1.59985984e-06,  1.21002654e-06, -4.31857302e-07,  2.73906130e-06,\n",
              "        -3.20947083e-06,  1.22053834e-07,  4.44860098e-06, -9.04749641e-07,\n",
              "        -7.50588981e-07, -1.48517449e-06,  2.40128452e-06, -6.03200988e-06,\n",
              "         1.39242737e-02,  5.54974750e-03,  3.05335084e-03, -3.02047061e-04,\n",
              "         3.24388617e-03,  2.13072519e-03,  2.08082311e-02, -2.08542626e-02,\n",
              "         4.33728303e-04,  7.72338454e-03,  2.39868667e-02,  1.99148431e-02,\n",
              "        -4.12102370e-03, -1.22182192e-02,  1.72865614e-02, -1.44097246e-02,\n",
              "        -2.44710967e-02,  3.98896355e-03, -1.25209624e-02, -2.23097298e-03,\n",
              "        -4.78580315e-03, -1.45192482e-02, -1.13069732e-02, -1.64531805e-02,\n",
              "        -6.97623938e-03,  1.26333418e-03, -8.50428268e-03,  2.66466493e-04,\n",
              "        -1.51594291e-02, -1.37884710e-02, -2.74920743e-03,  9.04281437e-03,\n",
              "        -1.52048666e-03, -2.29096338e-02, -8.15972500e-03,  1.80550746e-03,\n",
              "        -1.82293577e-03,  2.32013594e-02, -3.24593647e-03,  2.08144505e-02,\n",
              "         3.81885562e-03, -2.66568419e-02, -1.12625565e-02,  3.48657300e-03,\n",
              "         4.61633375e-04, -8.20975192e-03, -6.41340762e-03, -9.33795981e-03,\n",
              "         1.38781508e-02,  1.86218563e-02, -1.51847713e-02,  1.37258158e-03,\n",
              "         1.25292584e-03, -1.91873237e-02,  5.13088144e-03, -6.12962153e-03,\n",
              "         5.84190944e-03,  1.16460193e-02,  9.68271587e-03,  3.86986020e-03,\n",
              "        -1.28079588e-02, -2.06657569e-03, -1.42742193e-03,  1.82190500e-02,\n",
              "         3.64407478e-03,  1.02758622e-02,  1.35673187e-03, -1.19023863e-02,\n",
              "         1.51308365e-02,  3.93285038e-04,  1.98362600e-02,  2.53575221e-02,\n",
              "         1.87595328e-03,  1.62954815e-02,  8.35802592e-03, -1.57758817e-02,\n",
              "        -1.36727793e-03, -4.92050126e-03,  1.70726515e-02, -1.28969271e-02,\n",
              "         1.79837625e-02,  2.06838883e-02,  3.80276632e-03, -1.05241723e-02,\n",
              "         1.85937509e-02, -3.36697139e-02, -5.94925415e-03, -1.21918833e-02,\n",
              "        -2.82976567e-03, -1.99882314e-04,  2.08421741e-02,  2.37090625e-02,\n",
              "         1.31583214e-02,  1.53059280e-02, -1.09626446e-02,  2.41414681e-02,\n",
              "        -2.38002613e-02,  9.27094929e-03,  1.20362630e-02,  8.34529474e-03,\n",
              "         1.77534111e-02, -1.12054003e-02, -1.45916240e-02, -3.39754834e-03,\n",
              "         6.24024635e-03, -1.36032151e-02, -2.45527569e-02, -5.45615191e-03,\n",
              "        -6.31176354e-03, -1.94209665e-02, -1.38772046e-02, -2.67241034e-03,\n",
              "        -1.60574785e-03,  1.41333099e-02,  1.58162159e-03,  8.07921751e-05,\n",
              "         1.74026992e-02,  1.14477184e-02,  9.50258691e-03, -1.70847699e-02,\n",
              "        -6.06383150e-03,  1.78761373e-03,  5.48334047e-03, -1.13184676e-02,\n",
              "        -1.11478064e-02,  1.21656582e-02, -3.54498182e-03,  1.31940674e-02,\n",
              "        -3.66292716e-06, -1.67709197e-06,  1.15489365e-06,  2.04090384e-06,\n",
              "        -1.97214399e-06, -1.94169706e-06, -1.54499412e-06,  1.51349195e-06,\n",
              "         1.90516789e-06, -2.84500561e-06, -8.89056366e-07, -3.36458947e-06,\n",
              "        -9.97944426e-07, -1.80570339e-06, -1.81552377e-06,  3.53210169e-07,\n",
              "        -2.42168062e-06,  6.28737837e-07, -4.57541944e-07,  2.37957647e-06,\n",
              "        -8.36917025e-06,  1.59378010e-06, -5.18979459e-06,  3.12828138e-06,\n",
              "         2.93772314e-06, -2.37422728e-06,  3.01512642e-07,  2.84448629e-06,\n",
              "        -8.92568096e-07,  2.78261064e-06, -3.07997908e-07, -3.48850699e-06,\n",
              "         1.78894788e-06, -5.76378216e-06, -1.19583365e-06,  3.61119010e-06,\n",
              "         1.38640780e-06,  7.27255929e-06, -7.13258032e-07, -2.42387796e-06,\n",
              "         4.81660391e-06, -6.05220657e-06, -1.20786240e-07, -9.63081220e-07,\n",
              "         6.28920418e-07,  2.50208899e-07,  1.46303012e-06, -2.06748246e-06,\n",
              "         3.27549719e-06,  3.18689104e-07, -3.28411375e-07, -9.97600523e-07,\n",
              "        -9.33672652e-07, -7.75691569e-06,  1.83845543e-08, -2.23615211e-06,\n",
              "         2.06670302e-06, -1.50715732e-06, -1.23903760e-06, -6.32621095e-07,\n",
              "        -3.87274940e-06,  1.30004466e-07, -1.56119836e-06,  2.95061045e-06,\n",
              "        -2.07922722e-06,  1.32001583e-06,  1.97937629e-06,  1.97699683e-06,\n",
              "         1.71079682e-06,  1.14813520e-06, -4.84415023e-06,  6.13900227e-07,\n",
              "        -1.26074929e-06,  1.64287928e-06, -9.00038913e-07,  3.33668163e-06,\n",
              "        -8.04801516e-07,  2.54634233e-06,  1.35993344e-06, -9.45049749e-07,\n",
              "        -2.26901398e-06,  3.30952184e-06, -1.23058055e-06,  1.23759025e-06,\n",
              "        -1.29110026e-06, -1.95543385e-06,  1.21082860e-06, -6.37032770e-07,\n",
              "        -2.75677689e-06,  1.71108638e-08, -1.22725510e-06,  5.48550452e-06,\n",
              "        -5.16780290e-07,  1.61851986e-07, -2.48144806e-06, -7.60292824e-06,\n",
              "         4.07707057e-06,  8.54864481e-07,  3.85257863e-07, -3.18585830e-06,\n",
              "        -2.36469376e-07, -1.39002009e-06, -2.07322273e-06, -3.60677495e-06,\n",
              "        -4.13536736e-06, -2.92734171e-06,  1.72655393e-06, -8.13934207e-07,\n",
              "         5.36086645e-07,  1.67940016e-06, -6.56056500e-07, -1.22003371e-06,\n",
              "         3.07312439e-06, -4.23844909e-07,  5.54079975e-07,  1.85878866e-06,\n",
              "        -8.07406934e-07, -8.08691084e-07, -7.44424426e-07,  2.25764484e-06,\n",
              "        -1.95123062e-06, -4.70720352e-07,  3.32219270e-06, -2.13086150e-06,\n",
              "        -1.34969036e-06, -1.95391681e-06, -7.53713209e-07, -6.07828497e-06,\n",
              "        -4.88081605e-06, -4.13471980e-06,  2.69602901e-06,  1.56886767e-06,\n",
              "        -2.98207669e-06, -5.10695054e-06,  2.48035235e-07,  1.21070343e-06,\n",
              "        -9.89848218e-07, -2.28240106e-06, -4.68119788e-06, -2.68705026e-06,\n",
              "        -2.46373384e-06, -2.40817781e-06,  2.03848572e-06,  4.54288198e-08,\n",
              "        -4.57164049e-07,  2.27584928e-06,  4.16878066e-07,  4.84208385e-06,\n",
              "        -7.92735682e-06,  1.25396025e-06, -2.89140667e-06,  5.66579047e-06,\n",
              "         1.47904996e-06, -1.02820059e-06,  1.86925911e-06, -1.90934611e-06,\n",
              "         7.12811982e-07,  4.44179477e-06,  2.32633147e-06,  1.02868739e-06,\n",
              "         2.19701064e-07, -6.21427762e-06,  1.36217898e-06, -4.80901122e-07,\n",
              "         1.53778296e-06,  4.35151696e-06, -2.12600298e-06, -4.91887249e-06,\n",
              "         4.50140260e-06, -4.46854210e-06, -2.88772208e-06,  9.74723889e-07,\n",
              "        -1.68251108e-06,  1.14085310e-07,  5.35013669e-08,  1.51363417e-06,\n",
              "         4.04000411e-06,  4.35830998e-08, -2.37468930e-06,  1.57842408e-07,\n",
              "        -1.23476138e-06, -9.73542501e-06,  2.07632525e-08,  3.75153348e-07,\n",
              "         2.66025768e-06, -3.46976731e-06,  5.77859623e-07, -8.39574454e-07,\n",
              "        -4.45489468e-06,  1.84165026e-06, -6.52981441e-07,  6.95018298e-06,\n",
              "        -2.66763186e-06,  1.23637790e-06, -1.18329149e-06, -2.20795437e-06,\n",
              "         2.58487898e-06, -8.70961571e-07, -8.61560966e-06,  4.86135150e-08,\n",
              "        -3.32118043e-06,  2.89433206e-06,  9.34584705e-07,  7.05653747e-06,\n",
              "        -2.03681157e-06,  5.24489951e-06,  4.48043011e-06, -2.70098872e-06,\n",
              "        -2.69784266e-07,  2.71467275e-06, -2.69793259e-06,  1.12840758e-06,\n",
              "         9.65819254e-07, -2.05789570e-06, -5.44561431e-07, -2.18705964e-06,\n",
              "        -2.47609819e-06, -1.33124081e-06, -1.59643776e-06,  2.43209706e-06,\n",
              "        -1.20729135e-06, -3.55698990e-06, -1.28926649e-06, -6.79971663e-06,\n",
              "         4.17126694e-06,  2.24649852e-06,  1.40196630e-06, -3.66347876e-06,\n",
              "        -3.47397531e-06, -5.14192777e-08, -3.00153397e-06, -2.38521898e-06,\n",
              "        -6.40659573e-06, -5.12215956e-06,  2.03400464e-06,  3.40062201e-08,\n",
              "         1.87842181e-06, -5.44717807e-07,  2.33790388e-06, -3.92147331e-06,\n",
              "         5.45191767e-07,  6.69800784e-07,  3.22032815e-06,  1.47051833e-06,\n",
              "        -1.60124841e-06,  1.21032929e-06, -4.30917112e-07,  2.73895967e-06,\n",
              "        -3.21418929e-06,  1.22858694e-07,  4.45136084e-06, -9.09784944e-07,\n",
              "        -7.51503421e-07, -1.48715139e-06,  2.40184841e-06, -6.03161698e-06],\n",
              "       dtype=float32),\n",
              " array([[ 1.4947556e-04,  2.3939194e-04,  5.2482792e-05, ...,\n",
              "         -2.1421333e-07, -2.1313690e-07, -2.1203239e-07],\n",
              "        [ 2.5827577e-04,  2.4813495e-04,  2.1275255e-04, ...,\n",
              "         -3.7740159e-07, -3.7550285e-07, -3.7354494e-07],\n",
              "        [ 1.3274519e-04,  8.9440102e-05,  9.4746589e-05, ...,\n",
              "         -2.6668624e-07, -2.6535332e-07, -2.6397137e-07],\n",
              "        ...,\n",
              "        [ 9.5092211e-05,  1.9221485e-04,  9.5722055e-05, ...,\n",
              "         -3.1345070e-07, -3.1189174e-07, -3.1021872e-07],\n",
              "        [-1.3263794e-04, -8.1470949e-05, -2.1943815e-04, ...,\n",
              "          3.5827429e-07,  3.5647301e-07,  3.5463933e-07],\n",
              "        [ 9.4148636e-05,  3.9820693e-04,  1.9381891e-04, ...,\n",
              "         -3.9441599e-07, -3.9249034e-07, -3.9041552e-07]], dtype=float32),\n",
              " array([-0.7813496 , -1.0980005 , -0.98132986, ...,  0.00200399,\n",
              "         0.00199393,  0.00198356], dtype=float32)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y6ADca39jxus",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_op = optimizer.apply_gradients(zip(grads, tvars))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ktq68fBGj2rW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "session.run(tf.global_variables_initializer())\n",
        "session.run(train_op, feed_dict)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UzGECgZVj7zd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b09571ce-35ab-48f4-d970-8daf2b36dd3d"
      },
      "source": [
        "hidden_size_l1"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "256"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6TbZFmTikARt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PTBModel(object):\n",
        "\n",
        "    def __init__(self, action_type):\n",
        "        ######################################\n",
        "        # Setting parameters for ease of use #\n",
        "        ######################################\n",
        "        self.batch_size = batch_size\n",
        "        self.num_steps = num_steps\n",
        "        self.hidden_size_l1 = hidden_size_l1\n",
        "        self.hidden_size_l2 = hidden_size_l2\n",
        "        self.vocab_size = vocab_size\n",
        "        self.embeding_vector_size = embeding_vector_size\n",
        "        ###############################################################################\n",
        "        # Creating placeholders for our input data and expected outputs (target data) #\n",
        "        ###############################################################################\n",
        "        self._input_data = tf.placeholder(tf.int32, [batch_size, num_steps]) #[30#20]\n",
        "        self._targets = tf.placeholder(tf.int32, [batch_size, num_steps]) #[30#20]\n",
        "\n",
        "        ##########################################################################\n",
        "        # Creating the LSTM cell structure and connect it with the RNN structure #\n",
        "        ##########################################################################\n",
        "        # Create the LSTM unit. \n",
        "        # This creates only the structure for the LSTM and has to be associated with a RNN unit still.\n",
        "        # The argument n_hidden(size=200) of BasicLSTMCell is size of hidden layer, that is, the number of hidden units of the LSTM (inside A).\n",
        "        # Size is the same as the size of our hidden layer, and no bias is added to the Forget Gate. \n",
        "        # LSTM cell processes one word at a time and computes probabilities of the possible continuations of the sentence.\n",
        "        lstm_cell_l1 = tf.contrib.rnn.BasicLSTMCell(self.hidden_size_l1, forget_bias=0.0)\n",
        "        lstm_cell_l2 = tf.contrib.rnn.BasicLSTMCell(self.hidden_size_l2, forget_bias=0.0)\n",
        "        \n",
        "        # Unless you changed keep_prob, this won't actually execute -- this is a dropout wrapper for our LSTM unit\n",
        "        # This is an optimization of the LSTM output, but is not needed at all\n",
        "        if action_type == \"is_training\" and keep_prob < 1:\n",
        "            lstm_cell_l1 = tf.contrib.rnn.DropoutWrapper(lstm_cell_l1, output_keep_prob=keep_prob)\n",
        "            lstm_cell_l2 = tf.contrib.rnn.DropoutWrapper(lstm_cell_l2, output_keep_prob=keep_prob)\n",
        "        \n",
        "        # By taking in the LSTM cells as parameters, the MultiRNNCell function junctions the LSTM units to the RNN units.\n",
        "        # RNN cell composed sequentially of multiple simple cells.\n",
        "        stacked_lstm = tf.contrib.rnn.MultiRNNCell([lstm_cell_l1, lstm_cell_l2])\n",
        "\n",
        "        # Define the initial state, i.e., the model state for the very first data point\n",
        "        # It initialize the state of the LSTM memory. The memory state of the network is initialized with a vector of zeros and gets updated after reading each word.\n",
        "        self._initial_state = stacked_lstm.zero_state(batch_size, tf.float32)\n",
        "\n",
        "        ####################################################################\n",
        "        # Creating the word embeddings and pointing them to the input data #\n",
        "        ####################################################################\n",
        "        with tf.device(\"/cpu:0\"):\n",
        "            # Create the embeddings for our input data. Size is hidden size.\n",
        "            embedding = tf.get_variable(\"embedding\", [vocab_size, self.embeding_vector_size])  #[10000x200]\n",
        "            # Define where to get the data for our embeddings from\n",
        "            inputs = tf.nn.embedding_lookup(embedding, self._input_data)\n",
        "\n",
        "        # Unless you changed keep_prob, this won't actually execute -- this is a dropout addition for our inputs\n",
        "        # This is an optimization of the input processing and is not needed at all\n",
        "        if action_type == \"is_training\" and keep_prob < 1:\n",
        "            inputs = tf.nn.dropout(inputs, keep_prob)\n",
        "\n",
        "        ############################################\n",
        "        # Creating the input structure for our RNN #\n",
        "        ############################################\n",
        "        # Input structure is 20x[30x200]\n",
        "        # Considering each word is represended by a 200 dimentional vector, and we have 30 batchs, we create 30 word-vectors of size [30xx2000]\n",
        "        # inputs = [tf.squeeze(input_, [1]) for input_ in tf.split(1, num_steps, inputs)]\n",
        "        # The input structure is fed from the embeddings, which are filled in by the input data\n",
        "        # Feeding a batch of b sentences to a RNN:\n",
        "        # In step 1,  first word of each of the b sentences (in a batch) is input in parallel.  \n",
        "        # In step 2,  second word of each of the b sentences is input in parallel. \n",
        "        # The parallelism is only for efficiency.  \n",
        "        # Each sentence in a batch is handled in parallel, but the network sees one word of a sentence at a time and does the computations accordingly. \n",
        "        # All the computations involving the words of all sentences in a batch at a given time step are done in parallel. \n",
        "\n",
        "        ####################################################################################################\n",
        "        # Instantiating our RNN model and retrieving the structure for returning the outputs and the state #\n",
        "        ####################################################################################################\n",
        "        \n",
        "        outputs, state = tf.nn.dynamic_rnn(stacked_lstm, inputs, initial_state=self._initial_state)\n",
        "        #########################################################################\n",
        "        # Creating a logistic unit to return the probability of the output word #\n",
        "        #########################################################################\n",
        "        output = tf.reshape(outputs, [-1, self.hidden_size_l2])\n",
        "        softmax_w = tf.get_variable(\"softmax_w\", [self.hidden_size_l2, vocab_size]) #[200x1000]\n",
        "        softmax_b = tf.get_variable(\"softmax_b\", [vocab_size]) #[1x1000]\n",
        "        logits = tf.matmul(output, softmax_w) + softmax_b\n",
        "        logits = tf.reshape(logits, [self.batch_size, self.num_steps, vocab_size])\n",
        "        prob = tf.nn.softmax(logits)\n",
        "        out_words = tf.argmax(prob, axis=2)\n",
        "        self._output_words = out_words\n",
        "        #########################################################################\n",
        "        # Defining the loss and cost functions for the model's learning to work #\n",
        "        #########################################################################\n",
        "            \n",
        "\n",
        "        # Use the contrib sequence loss and average over the batches\n",
        "        loss = tf.contrib.seq2seq.sequence_loss(\n",
        "            logits,\n",
        "            self.targets,\n",
        "            tf.ones([batch_size, num_steps], dtype=tf.float32),\n",
        "            average_across_timesteps=False,\n",
        "            average_across_batch=True)\n",
        "    \n",
        "#         loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example([logits], [tf.reshape(self._targets, [-1])],\n",
        "#                                                       [tf.ones([batch_size * num_steps])])\n",
        "        self._cost = tf.reduce_sum(loss)\n",
        "\n",
        "        # Store the final state\n",
        "        self._final_state = state\n",
        "\n",
        "        #Everything after this point is relevant only for training\n",
        "        if action_type != \"is_training\":\n",
        "            return\n",
        "\n",
        "        #################################################\n",
        "        # Creating the Training Operation for our Model #\n",
        "        #################################################\n",
        "        # Create a variable for the learning rate\n",
        "        self._lr = tf.Variable(0.0, trainable=False)\n",
        "        # Get all TensorFlow variables marked as \"trainable\" (i.e. all of them except _lr, which we just created)\n",
        "        tvars = tf.trainable_variables()\n",
        "        # Define the gradient clipping threshold\n",
        "        grads, _ = tf.clip_by_global_norm(tf.gradients(self._cost, tvars), max_grad_norm)\n",
        "        # Create the gradient descent optimizer with our learning rate\n",
        "        optimizer = tf.train.GradientDescentOptimizer(self.lr)\n",
        "        # Create the training TensorFlow Operation through our optimizer\n",
        "        self._train_op = optimizer.apply_gradients(zip(grads, tvars))\n",
        "\n",
        "    # Helper functions for our LSTM RNN class\n",
        "\n",
        "    # Assign the learning rate for this model\n",
        "    def assign_lr(self, session, lr_value):\n",
        "        session.run(tf.assign(self.lr, lr_value))\n",
        "\n",
        "    # Returns the input data for this model at a point in time\n",
        "    @property\n",
        "    def input_data(self):\n",
        "        return self._input_data\n",
        "\n",
        "\n",
        "    \n",
        "    # Returns the targets for this model at a point in time\n",
        "    @property\n",
        "    def targets(self):\n",
        "        return self._targets\n",
        "\n",
        "    # Returns the initial state for this model\n",
        "    @property\n",
        "    def initial_state(self):\n",
        "        return self._initial_state\n",
        "\n",
        "    # Returns the defined Cost\n",
        "    @property\n",
        "    def cost(self):\n",
        "        return self._cost\n",
        "\n",
        "    # Returns the final state for this model\n",
        "    @property\n",
        "    def final_state(self):\n",
        "        return self._final_state\n",
        "    \n",
        "    # Returns the final output words for this model\n",
        "    @property\n",
        "    def final_output_words(self):\n",
        "        return self._output_words\n",
        "    \n",
        "    # Returns the current learning rate for this model\n",
        "    @property\n",
        "    def lr(self):\n",
        "        return self._lr\n",
        "\n",
        "    # Returns the training operation defined for this model\n",
        "    @property\n",
        "    def train_op(self):\n",
        "        return self._train_op"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZrtbXUMNkK-u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def run_one_epoch(session, m, data, eval_op, verbose=False):\n",
        "\n",
        "    #Define the epoch size based on the length of the data, batch size and the number of steps\n",
        "    epoch_size = ((len(data) // m.batch_size) - 1) // m.num_steps\n",
        "    start_time = time.time()\n",
        "    costs = 0.0\n",
        "    iters = 0\n",
        "\n",
        "    state = session.run(m.initial_state)\n",
        "    \n",
        "    #For each step and data point\n",
        "    for step, (x, y) in enumerate(reader.ptb_iterator(data, m.batch_size, m.num_steps)):\n",
        "        \n",
        "        #Evaluate and return cost, state by running cost, final_state and the function passed as parameter\n",
        "        cost, state, out_words, _ = session.run([m.cost, m.final_state, m.final_output_words, eval_op],\n",
        "                                     {m.input_data: x,\n",
        "                                      m.targets: y,\n",
        "                                      m.initial_state: state})\n",
        "\n",
        "        #Add returned cost to costs (which keeps track of the total costs for this epoch)\n",
        "        costs += cost\n",
        "        \n",
        "        #Add number of steps to iteration counter\n",
        "        iters += m.num_steps\n",
        "\n",
        "        if verbose and step % (epoch_size // 10) == 10:\n",
        "            print(\"Itr %d of %d, perplexity: %.3f speed: %.0f wps\" % (step , epoch_size, np.exp(costs / iters), iters * m.batch_size / (time.time() - start_time)))\n",
        "\n",
        "    # Returns the Perplexity rating for us to keep track of how the model is evolving\n",
        "    return np.exp(costs / iters)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ip6Cr96hkQg1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "raw_data = reader.ptb_raw_data(data_dir)\n",
        "train_data, valid_data, test_data, _, _ = raw_data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q1ZgUhmYkT84",
        "colab_type": "code",
        "outputId": "275791e8-eb28-47c1-c294-4b4d894c2a1d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 561
        }
      },
      "source": [
        "with tf.Graph().as_default(), tf.Session() as session:\n",
        "    initializer = tf.random_uniform_initializer(-init_scale, init_scale)\n",
        "    \n",
        "    # Instantiates the model for training\n",
        "    # tf.variable_scope add a prefix to the variables created with tf.get_variable\n",
        "    with tf.variable_scope(\"model\", reuse=None, initializer=initializer):\n",
        "        m = PTBModel(\"is_training\")\n",
        "        \n",
        "    # Reuses the trained parameters for the validation and testing models\n",
        "    # They are different instances but use the same variables for weights and biases, they just don't change when data is input\n",
        "    with tf.variable_scope(\"model\", reuse=True, initializer=initializer):\n",
        "        mvalid = PTBModel(\"is_validating\")\n",
        "        mtest = PTBModel(\"is_testing\")\n",
        "\n",
        "    #Initialize all variables\n",
        "    tf.global_variables_initializer().run()\n",
        "\n",
        "    for i in range(max_epoch):\n",
        "        # Define the decay for this epoch\n",
        "        lr_decay = decay ** max(i - max_epoch_decay_lr, 0.0)\n",
        "        \n",
        "        # Set the decayed learning rate as the learning rate for this epoch\n",
        "        m.assign_lr(session, learning_rate * lr_decay)\n",
        "\n",
        "        print(\"Epoch %d : Learning rate: %.3f\" % (i + 1, session.run(m.lr)))\n",
        "        \n",
        "        # Run the loop for this epoch in the training model\n",
        "        train_perplexity = run_one_epoch(session, m, train_data, m.train_op, verbose=True)\n",
        "        print(\"Epoch %d : Train Perplexity: %.3f\" % (i + 1, train_perplexity))\n",
        "        \n",
        "        # Run the loop for this epoch in the validation model\n",
        "        valid_perplexity = run_one_epoch(session, mvalid, valid_data, tf.no_op())\n",
        "        print(\"Epoch %d : Valid Perplexity: %.3f\" % (i + 1, valid_perplexity))\n",
        "    \n",
        "    # Run the loop in the testing model to see how effective was our training\n",
        "    test_perplexity = run_one_epoch(session, mtest, test_data, tf.no_op())\n",
        "    \n",
        "    print(\"Test Perplexity: %.3f\" % test_perplexity)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1 : Learning rate: 1.000\n",
            "Itr 10 of 774, perplexity: 3650.990 speed: 1645 wps\n",
            "Itr 87 of 774, perplexity: 1248.877 speed: 1682 wps\n",
            "Itr 164 of 774, perplexity: 966.082 speed: 1682 wps\n",
            "Itr 241 of 774, perplexity: 807.103 speed: 1681 wps\n",
            "Itr 318 of 774, perplexity: 714.737 speed: 1683 wps\n",
            "Itr 395 of 774, perplexity: 640.670 speed: 1685 wps\n",
            "Itr 472 of 774, perplexity: 582.377 speed: 1690 wps\n",
            "Itr 549 of 774, perplexity: 529.034 speed: 1690 wps\n",
            "Itr 626 of 774, perplexity: 487.013 speed: 1685 wps\n",
            "Itr 703 of 774, perplexity: 453.350 speed: 1682 wps\n",
            "Epoch 1 : Train Perplexity: 429.255\n",
            "Epoch 1 : Valid Perplexity: 266.269\n",
            "Epoch 2 : Learning rate: 1.000\n",
            "Itr 10 of 774, perplexity: 278.182 speed: 1664 wps\n",
            "Itr 87 of 774, perplexity: 239.337 speed: 1677 wps\n",
            "Itr 164 of 774, perplexity: 229.089 speed: 1688 wps\n",
            "Itr 241 of 774, perplexity: 218.988 speed: 1692 wps\n",
            "Itr 318 of 774, perplexity: 216.391 speed: 1695 wps\n",
            "Itr 395 of 774, perplexity: 210.586 speed: 1698 wps\n",
            "Itr 472 of 774, perplexity: 206.363 speed: 1698 wps\n",
            "Itr 549 of 774, perplexity: 199.621 speed: 1699 wps\n",
            "Itr 626 of 774, perplexity: 194.165 speed: 1698 wps\n",
            "Itr 703 of 774, perplexity: 190.007 speed: 1695 wps\n",
            "Epoch 2 : Train Perplexity: 187.201\n",
            "Epoch 2 : Valid Perplexity: 173.701\n",
            "Epoch 3 : Learning rate: 1.000\n",
            "Itr 10 of 774, perplexity: 182.841 speed: 1680 wps\n",
            "Itr 87 of 774, perplexity: 159.949 speed: 1693 wps\n",
            "Itr 164 of 774, perplexity: 156.312 speed: 1696 wps\n",
            "Itr 241 of 774, perplexity: 151.588 speed: 1699 wps\n",
            "Itr 318 of 774, perplexity: 151.668 speed: 1699 wps\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bEMytNiykYa2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}