8_rnn_names.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "8_rnn_names.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/wallykop/iad-deep-learning/blob/master/8_rnn_names.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "a9GueUvbshgh"
      },
      "source": [
        "# Генерация имени с помощью рекуррентной нейронной сети\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:42.696201Z",
          "start_time": "2018-08-13T20:26:38.104103Z"
        },
        "colab_type": "code",
        "id": "Y0lb0Mi1shgj",
        "outputId": "f565185d-3229-4828-908c-b56939b99684",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "print(tf.__version__)\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import os\n",
        "import sys\n",
        "sys.path.append(\"..\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.15.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "McTYOtRQshgn"
      },
      "source": [
        "# Загружаем данные\n",
        "\n",
        "Набор данных содержит ~8 тыс. имен людей из разных культур, все в латинской транскрипции.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "E5rmKo4muer5",
        "outputId": "04999114-341f-4213-cc02-c818dc587336",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 261
        }
      },
      "source": [
        "from google.colab import files\n",
        "files.upload()\n",
        "files.download()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-2026b107-3573-4f48-8d13-6eab00023eb5\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-2026b107-3573-4f48-8d13-6eab00023eb5\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving names.txt to names (1).txt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-6-ea3b0ab26250>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mgoogle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolab\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mfiles\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m: download() missing 1 required positional argument: 'filename'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "srd-kDMl8_xm",
        "colab_type": "code",
        "outputId": "0cf38f94-370b-4e8d-8391-5e87bd693876",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "! ls"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "'names (1).txt'   names.txt   sample_data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:42.701832Z",
          "start_time": "2018-08-13T20:26:42.697766Z"
        },
        "colab_type": "code",
        "id": "KHvai_G0shgo",
        "colab": {}
      },
      "source": [
        "start_token = \" \"  # so that the network knows that we're generating a first token\n",
        "\n",
        "# this is the token for padding,\n",
        "# we will add fake pad token at the end of names \n",
        "# to make them of equal size for further batching\n",
        "pad_token = \"#\"\n",
        "\n",
        "with open(\"names.txt\") as f:\n",
        "    names = f.read()[:-1].split('\\n')\n",
        "    names = [start_token + name for name in names]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:42.707885Z",
          "start_time": "2018-08-13T20:26:42.703302Z"
        },
        "colab_type": "code",
        "id": "GQAH7UOushgs",
        "outputId": "d997c56d-fb6b-4eb5-9114-0939b69e0a17",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 179
        }
      },
      "source": [
        "print('number of samples:', len(names))\n",
        "for x in names[::1000]:\n",
        "    print(x)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "number of samples: 7944\n",
            " Abagael\n",
            " Claresta\n",
            " Glory\n",
            " Liliane\n",
            " Prissie\n",
            " Geeta\n",
            " Giovanne\n",
            " Piggy\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:42.857411Z",
          "start_time": "2018-08-13T20:26:42.709371Z"
        },
        "colab_type": "code",
        "id": "Q1vZ7fufshgv",
        "outputId": "a1488b8b-b6fa-4eef-9ed7-74398a79ff85",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        }
      },
      "source": [
        "MAX_LENGTH = max(map(len, names))\n",
        "print(\"max length:\", MAX_LENGTH)\n",
        "\n",
        "plt.title('Sequence length distribution')\n",
        "plt.hist(list(map(len, names)), bins=25);"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "max length: 16\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAab0lEQVR4nO3dfZRddX3v8feH8FBAHoIZAySBQQwo\nsDTgFLAK4qVAeLgEvbcY6oWgaKAFq1fW9QK9LVSkK7VSKksMDZAGKiSmPJRUQIhUpbQGmWAMCQ8y\nQCATJslgeLDgiga+94/9G90Mc2bO05yT5Pd5rXXW7PP77f3b33Mm+cye395ntiICMzPLwzbtLsDM\nzFrHoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvm3VJIWk97Rhv8dI6m1g+8skfTst7yPpvySN\naVJt10r6i2bUOcTYR0l6slnjWfM59DMg6SOS/lPSK5I2SPoPSb/f7rq2JqP5wyUino+Id0TEGyPU\ncLakB6sY77yIuLwZtQ1+3RHx7xFxYDPGttGxbbsLsNElaVfgu8CfAAuB7YGjgI3trMvaQ9KYkX54\n2NbNR/pbvwMAImJ+RLwREb+KiPsiYvnACpI+I+lxSS9JulfSvqW+4yQ9kX5L+KakH0n6bOr77RRE\net6Zjvy2Tc93k3SDpD5JayR9dWCKYuCoVNLX036flXRiaaw9JP2jpBdS/7+U+k6RtEzSy+k3mPdX\n80ZI2iHt73lJ69I0x46p7xhJvZIulLQ+1fzp0rbvlPSvkl6V9HB6LQ+mvgfSaj9L0zCfLG035HhD\n1LZfem9/KWkxMG6Y9/VsSc+kdZ+V9ClJ7wOuBT6Uang5rTtP0mxJd0t6DfhYavvqoP1fIulFSask\nfarU/sOB73f5+1bpdQ+eLpL0vjTGy5JWSjq11DdP0jWS7kqv5SFJ+4/0fbTGOPS3fj8H3pB0o6QT\nJY0td0qaBlwCfALoAP4dmJ/6xgG3A/+PIoSeBj5cw77nAZuA9wCHAscDny31HwE8mcb+GnCDJKW+\nfwJ2Ag4G3gVclWo6FJgLnAu8E/gHYJGkHaqoZxbFD8EpqaYJwF+W+vcEdkvt5wDXlN6va4DX0joz\n0gOAiDg6LX4gTcN8p4rxBrsFWJrei8vL45dJ2hm4GjgxInYB/gBYFhGPA+cBP0417F7a7I+BK4Bd\ngKGmf/ZM+52Q9jtH0ohTNMO87oFatwP+FbiP4nv4eeDmQWNPB/4KGAv0pDptNEWEH1v5A3gfRQD3\nUoTwImB86rsHOKe07jbA68C+wFnAklKf0hifTc8vA75d6u8EgmLacDzFFNKOpf4zgB+k5bOBnlLf\nTmnbPYG9gDeBsUO8ltnA5YPangQ+WuG1B0XAiyK09y/1fQh4Ni0fA/wK2LbUvx44EhgD/AY4sNT3\nVeDBwfspPa843hA17pO+LzuX2m4ZeG8Hva87Ay8D/6P83pbe0wcHtc0Dbhqi7aulOgfveyHwF2n5\nhwPf76H2UeF196blo4C1wDal/vnAZaU6ri/1nQQ80e7/L1v7w0f6GYiIxyPi7IiYCBwC7A38fere\nF/hG+vX7ZWADRUBOSOutLo0T5ecj2BfYDugrjf0PFEd8A9aWxn49Lb4DmARsiIiXKox74cCYadxJ\nqdbhdFD8YFla2u57qX3ALyJiU+n566meDorALb/2at6HSuMNtjfwUkS8Vmp7bqgB0zqfpDiq70tT\nI+8doY6Rah1q3yO9n9XYG1gdEW8OGntC6fna0nKl98eayKGfmYh4guII65DUtBo4NyJ2Lz12jIj/\nBPooAhWANPUyqTTcaxRBOmDP0vJqiiP9caVxd42Ig6soczWwh6TdK/RdMajenSJi/ghjvkhx5H1w\nabvdIqKakOmnOBqeWGqbVGHdevQBY9PUzYB9Kq0cEfdGxHEUvxE9AVw30FVpkxH2P9S+X0jLw32P\nR/ICMElSOWf2AdbUMIY1mUN/Kyfpvelk4sT0fBLFNMuStMq1wMWSDk79u0n6o9R3F3CwpE+kk4h/\nxlv/0y8DjlZxHfluwMUDHRHRRzGXe6WkXSVtI2l/SR8dqea07T3AtySNlbSdpIH54+uA8yQdocLO\nkk6WtMsIY76Ztr1K0rvSa50g6YQq6nmD4tzGZZJ2SkfWZw1abR3w7pHGqjD+c0A38FeStpf0EeC/\nD7WupPGSpqWQ3gj8F8VU2EANEyVtX0cZA/s+CjgF+OfUvgz4RHrd76E4N1E23Ot+iOLo/cvpe3hM\nel0L6qjPmsShv/X7JcUJ04fS1RtLgBXAhQARcQfwN8ACSa+mvhNT34vAH1GcAP0FMBn4j4GBI2Ix\n8B1gOcVJyO8O2vdZFJeIPga8BNxKcXRajTMp5tGfoJgL/2LaZzfwOeCbacweinnmavzftP6S9Fq/\nD1R7TfkFFCdl11KcZJ7PWy97vQy4MU0dnV7lmGV/TPF92gBcCtxUYb1tgC9RHEVvAD5KcTkuwL8B\nK4G1kl6sYd9rKd7LF4CbgfPSb4RQnED/NUW435j6yy6jwuuOiF9ThPyJFL9pfQs4qzS2tYGKaVqz\n6kj6IcUJxuvbXUs7SfobYM+IGPIqG7PNlY/0zaqQpsnen6aUDqeY5rij3XWZ1cqfyDWrzi4UUzp7\nU0x1XAnc2daKzOrg6R0zs4x4esfMLCOb/fTOuHHjorOzs91lmJltMZYuXfpiRHQM1bfZh35nZyfd\n3d3tLsPMbIshachPdIOnd8zMsuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w4\n9M3MMrLZfyLXNi+dF91V0/qrZp08SpWYWT18pG9mlpERQ1/SJEk/kPSYpJWSvpDa95C0WNJT6evY\n1C5JV0vqkbRc0mGlsWak9Z+S5DsOmZm1WDVH+puACyPiIOBI4HxJBwEXAfdHxGTg/vQcivthTk6P\nmcBsKH5IUNz78wjgcODSgR8UZmbWGiOGfkT0RcQjafmXwOPABGAaxY2SSV9PS8vTgJuisATYXdJe\nwAnA4ojYEBEvAYuBqU19NWZmNqya5vQldQKHAg8B4yOiL3WtBcan5QnA6tJmvamtUvtQ+5kpqVtS\nd39/fy0lmpnZMKoOfUnvAG4DvhgRr5b7orjnYtPuuxgRcyKiKyK6OjqGvA+AmZnVoarQl7QdReDf\nHBG3p+Z1adqG9HV9al8DTCptPjG1VWo3M7MWqebqHQE3AI9HxN+VuhYBA1fgzADuLLWfla7iORJ4\nJU0D3QscL2lsOoF7fGozM7MWqebDWR8GzgQelbQstV0CzAIWSjoHeA44PfXdDZwE9ACvA58GiIgN\nki4HHk7rfSUiNjTlVZiZWVVGDP2IeBBQhe5jh1g/gPMrjDUXmFtLgWZm1jz+RK6ZWUYc+mZmGXHo\nm5llxKFvZpYRh76ZWUYc+mZmGfFNVLYyvsmJmQ3HR/pmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhlx\n6JuZZcShb2aWEYe+mVlGHPpmZhmp5naJcyWtl7Si1PYdScvSY9XAHbUkdUr6Vanv2tI2H5T0qKQe\nSVen2zCamVkLVfNnGOYB3wRuGmiIiE8OLEu6EniltP7TETFliHFmA58DHqK4peJU4J7aSzYzs3qN\neKQfEQ8AQ97LNh2tnw7MH24MSXsBu0bEknQ7xZuA02ov18zMGtHonP5RwLqIeKrUtp+kn0r6kaSj\nUtsEoLe0Tm9qG5KkmZK6JXX39/c3WKKZmQ1oNPTP4K1H+X3APhFxKPAl4BZJu9Y6aETMiYiuiOjq\n6OhosEQzMxtQ959WlrQt8AnggwNtEbER2JiWl0p6GjgAWANMLG0+MbWZmVkLNXKk/4fAExHx22kb\nSR2SxqTldwOTgWciog94VdKR6TzAWcCdDezbzMzqUM0lm/OBHwMHSuqVdE7qms7bT+AeDSxPl3De\nCpwXEQMngf8UuB7oAZ7GV+6YmbXciNM7EXFGhfazh2i7DbitwvrdwCE11mdmZk3kT+SamWXEoW9m\nlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceib\nmWXEoW9mlhGHvplZRhz6ZmYZqebOWXMlrZe0otR2maQ1kpalx0mlvosl9Uh6UtIJpfapqa1H0kXN\nfylmZjaSao705wFTh2i/KiKmpMfdAJIOoriN4sFpm29JGpPum3sNcCJwEHBGWtfMzFqomtslPiCp\ns8rxpgELImIj8KykHuDw1NcTEc8ASFqQ1n2s5orNzKxujczpXyBpeZr+GZvaJgCrS+v0prZK7UOS\nNFNSt6Tu/v7+Bko0M7OyekN/NrA/MAXoA65sWkVARMyJiK6I6Oro6Gjm0GZmWRtxemcoEbFuYFnS\ndcB309M1wKTSqhNTG8O0m5lZi9R1pC9pr9LTjwMDV/YsAqZL2kHSfsBk4CfAw8BkSftJ2p7iZO+i\n+ss2M7N6jHikL2k+cAwwTlIvcClwjKQpQACrgHMBImKlpIUUJ2g3AedHxBtpnAuAe4ExwNyIWNn0\nV2NmZsOq5uqdM4ZovmGY9a8Arhii/W7g7pqqMzOzpqprTt9stHRedFfN26yadfIoVGK2dfKfYTAz\ny4hD38wsIw59M7OMOPTNzDLi0Dczy4hD38wsIw59M7OMOPTNzDLi0Dczy4hD38wsIw59M7OMOPTN\nzDLi0Dczy4hD38wsIw59M7OMjBj6kuZKWi9pRantbyU9IWm5pDsk7Z7aOyX9StKy9Li2tM0HJT0q\nqUfS1ZI0Oi/JzMwqqeZIfx4wdVDbYuCQiHg/8HPg4lLf0xExJT3OK7XPBj5Hcd/cyUOMaWZmo2zE\n0I+IB4ANg9rui4hN6ekSYOJwY6Qbqe8aEUsiIoCbgNPqK9nMzOrVjDn9zwD3lJ7vJ+mnkn4k6ajU\nNgHoLa3Tm9qGJGmmpG5J3f39/U0o0czMoMHQl/TnwCbg5tTUB+wTEYcCXwJukbRrreNGxJyI6IqI\nro6OjkZKNDOzkrpvjC7pbOAU4Ng0ZUNEbAQ2puWlkp4GDgDW8NYpoImpzczMWqiuI31JU4EvA6dG\nxOul9g5JY9LyuylO2D4TEX3Aq5KOTFftnAXc2XD1ZmZWkxGP9CXNB44BxknqBS6luFpnB2BxuvJy\nSbpS52jgK5J+A7wJnBcRAyeB/5TiSqAdKc4BlM8DmJlZC4wY+hFxxhDNN1RY9zbgtgp93cAhNVVn\nZmZN5U/kmpllxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYR\nh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llpKrQlzRX0npJK0pte0haLOmp9HVs\napekqyX1SFou6bDSNjPS+k9JmtH8l2NmZsOp9kh/HjB1UNtFwP0RMRm4Pz0HOJHihuiTgZnAbCh+\nSFDcX/cI4HDg0oEfFGZm1hpVhX5EPABsGNQ8DbgxLd8InFZqvykKS4DdJe0FnAAsjogNEfESsJi3\n/yAxM7NR1Mic/viI6EvLa4HxaXkCsLq0Xm9qq9T+NpJmSuqW1N3f399AiWZmVtaUE7kREUA0Y6w0\n3pyI6IqIro6OjmYNa2aWvUZCf12atiF9XZ/a1wCTSutNTG2V2s3MrEUaCf1FwMAVODOAO0vtZ6Wr\neI4EXknTQPcCx0sam07gHp/azMysRbatZiVJ84FjgHGSeimuwpkFLJR0DvAccHpa/W7gJKAHeB34\nNEBEbJB0OfBwWu8rETH45LCZmY2iqkI/Is6o0HXsEOsGcH6FceYCc6uuzszMmsqfyDUzy0hVR/rW\nHJ0X3VXT+qtmnTxKlZhZrnykb2aWEYe+mVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhnxdfqW\nHX9ewnLmI30zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMlJ36Es6UNKy0uNVSV+UdJmkNaX2\nk0rbXCypR9KTkk5ozkswM7Nq1X2dfkQ8CUwBkDSG4ibnd1DcHvGqiPh6eX1JBwHTgYOBvYHvSzog\nIt6otwYzM6tNs6Z3jgWejojnhllnGrAgIjZGxLMU99A9vEn7NzOzKjQr9KcD80vPL5C0XNJcSWNT\n2wRgdWmd3tT2NpJmSuqW1N3f39+kEs3MrOHQl7Q9cCrwz6lpNrA/xdRPH3BlrWNGxJyI6IqIro6O\njkZLNDOzpBlH+icCj0TEOoCIWBcRb0TEm8B1/G4KZw0wqbTdxNRmZmYt0ozQP4PS1I6kvUp9HwdW\npOVFwHRJO0jaD5gM/KQJ+zczsyo19Fc2Je0MHAecW2r+mqQpQACrBvoiYqWkhcBjwCbgfF+5Y2bW\nWg2FfkS8BrxzUNuZw6x/BXBFI/s0M7P6+RO5ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXE\noW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRppxY/RV\nkh6VtExSd2rbQ9JiSU+lr2NTuyRdLalH0nJJhzW6fzMzq16zjvQ/FhFTIqIrPb8IuD8iJgP3p+dQ\n3ER9cnrMBGY3af9mZlaF0ZremQbcmJZvBE4rtd8UhSXA7oNupG5mZqOoGaEfwH2SlkqamdrGR0Rf\nWl4LjE/LE4DVpW17U9tbSJopqVtSd39/fxNKNDMzaPDG6MlHImKNpHcBiyU9Ue6MiJAUtQwYEXOA\nOQBdXV01bWtmZpU1fKQfEWvS1/XAHcDhwLqBaZv0dX1afQ0wqbT5xNRmZmYt0FDoS9pZ0i4Dy8Dx\nwApgETAjrTYDuDMtLwLOSlfxHAm8UpoGMjOzUdbo9M544A5JA2PdEhHfk/QwsFDSOcBzwOlp/buB\nk4Ae4HXg0w3u38zMatBQ6EfEM8AHhmj/BXDsEO0BnN/IPs3MrH7+RK6ZWUYc+mZmGXHom5llxKFv\nZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUaa8Vc2zayk86K7alp/1ayTR6kSs7fzkb6ZWUYc\n+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGak79CVNkvQDSY9JWinpC6n9MklrJC1Lj5NK21wsqUfS\nk5JOaMYLMDOz6jVynf4m4MKIeCTdJ3eppMWp76qI+Hp5ZUkHAdOBg4G9ge9LOiAi3mighqby9dVm\ntrWr+0g/Ivoi4pG0/EvgcWDCMJtMAxZExMaIeJbiPrmH17t/MzOrXVPm9CV1AocCD6WmCyQtlzRX\n0tjUNgFYXdqsl+F/SJiZWZM1HPqS3gHcBnwxIl4FZgP7A1OAPuDKOsacKalbUnd/f3+jJZqZWdJQ\n6EvajiLwb46I2wEiYl1EvBERbwLX8bspnDXApNLmE1Pb20TEnIjoioiujo6ORko0M7OSRq7eEXAD\n8HhE/F2pfa/Sah8HVqTlRcB0STtI2g+YDPyk3v2bmVntGrl658PAmcCjkpaltkuAMyRNAQJYBZwL\nEBErJS0EHqO48uf8zenKHTOzHNQd+hHxIKAhuu4eZpsrgCvq3aeZmTXGn8g1M8uIQ9/MLCMOfTOz\njDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy0sgncs2sDWq97wP43g/2Oz7SNzPLiEPfzCwjDn0z\ns4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMtLyD2dJmgp8AxgDXB8Rs1pdg5kNr9YPgPnDX1uO\nloa+pDHANcBxQC/wsKRFEfHYaOyvnk8umpltzVp9pH840BMRzwBIWgBMo7hZupllYrR/k/CfqqhM\nEdG6nUn/E5gaEZ9Nz88EjoiICwatNxOYmZ4eCDzZsiKrNw54sd1F1Mm1t4drb70ttW5orPZ9I6Jj\nqI7N8g+uRcQcYE676xiOpO6I6Gp3HfVw7e3h2ltvS60bRq/2Vl+9swaYVHo+MbWZmVkLtDr0HwYm\nS9pP0vbAdGBRi2swM8tWS6d3ImKTpAuAeyku2ZwbEStbWUMTbdbTTyNw7e3h2ltvS60bRqn2lp7I\nNTOz9vIncs3MMuLQNzPLiEO/TpLGSPqppO+2u5ZaSNpd0q2SnpD0uKQPtbumakj635JWSlohab6k\n32t3TZVImitpvaQVpbY9JC2W9FT6OradNVZSofa/Tf9elku6Q9Lu7ayxkqFqL/VdKCkkjWtHbSOp\nVLukz6f3fqWkrzVjXw79+n0BeLzdRdThG8D3IuK9wAfYAl6DpAnAnwFdEXEIxUUA09tb1bDmAVMH\ntV0E3B8Rk4H70/PN0TzeXvti4JCIeD/wc+DiVhdVpXm8vXYkTQKOB55vdUE1mMeg2iV9jOIvFnwg\nIg4Gvt6MHTn06yBpInAycH27a6mFpN2Ao4EbACLi1xHxcnurqtq2wI6StgV2Al5ocz0VRcQDwIZB\nzdOAG9PyjcBpLS2qSkPVHhH3RcSm9HQJxedrNjsV3neAq4AvA5vtVSsVav8TYFZEbEzrrG/Gvhz6\n9fl7in9Eb7a7kBrtB/QD/5impq6XtHO7ixpJRKyhOMp5HugDXomI+9pbVc3GR0RfWl4LjG9nMQ34\nDHBPu4uolqRpwJqI+Fm7a6nDAcBRkh6S9CNJv9+MQR36NZJ0CrA+Ipa2u5Y6bAscBsyOiEOB19h8\npxl+K81/T6P4obU3sLOk/9XequoXxXXSm+1RZyWS/hzYBNzc7lqqIWkn4BLgL9tdS522BfYAjgT+\nD7BQkhod1KFfuw8Dp0paBSwA/pukb7e3pKr1Ar0R8VB6fivFD4HN3R8Cz0ZEf0T8Brgd+IM211Sr\ndZL2Akhfm/KreqtIOhs4BfhUbDkf7tmf4kDhZ+n/60TgEUl7trWq6vUCt0fhJxQzCw2fiHbo1ygi\nLo6IiRHRSXEy8d8iYos46oyItcBqSQempmPZMv6s9fPAkZJ2Skc6x7IFnIAeZBEwIy3PAO5sYy01\nSTc++jJwakS83u56qhURj0bEuyKiM/1/7QUOS/8PtgT/AnwMQNIBwPY04S+GOvTz83ngZknLgSnA\nX7e5nhGl30xuBR4BHqX4d7vZfrxe0nzgx8CBknolnQPMAo6T9BTFby6b5R3jKtT+TWAXYLGkZZKu\nbWuRFVSofYtQofa5wLvTZZwLgBnN+C3Lf4bBzCwjPtI3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uI\nQ9/MLCMOfTOzjPx/p/4cUF1Gcl0AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "IvnJHPAJshgx"
      },
      "source": [
        "# Обрабатываем текст\n",
        "\n",
        "Сначала нужно собрать словарь всех уникальных токенов, т. е. уникальных символов. Затем мы можем кодировать входные данные в виде последовательности кодов символов."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:42.864592Z",
          "start_time": "2018-08-13T20:26:42.858725Z"
        },
        "colab_type": "code",
        "id": "ii1qX1YLshgy",
        "outputId": "a316aefe-53a6-481e-b291-ba563a7d4a38",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "tokens = np.unique(list(''.join(names))+ ['#']) ### YOUR CODE HERE: all unique characters go here, padding included!\n",
        "\n",
        "tokens = list(tokens)\n",
        "n_tokens = len(tokens)\n",
        "print ('n_tokens:', n_tokens)\n",
        "\n",
        "assert 50 < n_tokens < 60"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "n_tokens: 56\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "jh6aK0VZshg1"
      },
      "source": [
        "### Переводим все от символов к идентификаторам\n",
        "\n",
        "В TensorFlow работа со строками непростая, поэтому упростим ее. Мы будем подавать в нашу рекуррентную нейронную сеть идентификаторы символов из нашего словаря.\n",
        "\n",
        "Для создания такого словаря определим `token_to_id`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:42.870330Z",
          "start_time": "2018-08-13T20:26:42.866135Z"
        },
        "colab_type": "code",
        "id": "LqoijSz0shg2",
        "colab": {}
      },
      "source": [
        "token_to_id = {char:i for i, char in enumerate(tokens)}### YOUR CODE HERE: create a dictionary of {symbol -> its  index in tokens}\n",
        "\n",
        "assert len(tokens) == len(token_to_id), \"dictionaries must have same size\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_JKNWimj_bNU",
        "colab_type": "code",
        "outputId": "63f8a152-eac7-449e-f8a0-7b2a24365857",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "token_to_id"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{' ': 0,\n",
              " '#': 1,\n",
              " \"'\": 2,\n",
              " '-': 3,\n",
              " 'A': 4,\n",
              " 'B': 5,\n",
              " 'C': 6,\n",
              " 'D': 7,\n",
              " 'E': 8,\n",
              " 'F': 9,\n",
              " 'G': 10,\n",
              " 'H': 11,\n",
              " 'I': 12,\n",
              " 'J': 13,\n",
              " 'K': 14,\n",
              " 'L': 15,\n",
              " 'M': 16,\n",
              " 'N': 17,\n",
              " 'O': 18,\n",
              " 'P': 19,\n",
              " 'Q': 20,\n",
              " 'R': 21,\n",
              " 'S': 22,\n",
              " 'T': 23,\n",
              " 'U': 24,\n",
              " 'V': 25,\n",
              " 'W': 26,\n",
              " 'X': 27,\n",
              " 'Y': 28,\n",
              " 'Z': 29,\n",
              " 'a': 30,\n",
              " 'b': 31,\n",
              " 'c': 32,\n",
              " 'd': 33,\n",
              " 'e': 34,\n",
              " 'f': 35,\n",
              " 'g': 36,\n",
              " 'h': 37,\n",
              " 'i': 38,\n",
              " 'j': 39,\n",
              " 'k': 40,\n",
              " 'l': 41,\n",
              " 'm': 42,\n",
              " 'n': 43,\n",
              " 'o': 44,\n",
              " 'p': 45,\n",
              " 'q': 46,\n",
              " 'r': 47,\n",
              " 's': 48,\n",
              " 't': 49,\n",
              " 'u': 50,\n",
              " 'v': 51,\n",
              " 'w': 52,\n",
              " 'x': 53,\n",
              " 'y': 54,\n",
              " 'z': 55}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:42.875943Z",
          "start_time": "2018-08-13T20:26:42.871834Z"
        },
        "colab_type": "code",
        "id": "LHfmIPm9shg4",
        "colab": {}
      },
      "source": [
        "def to_matrix(names, max_len=None, pad=token_to_id[pad_token], dtype=np.int32):\n",
        "    \"\"\"Casts a list of names into rnn-digestable padded matrix\"\"\"\n",
        "    \n",
        "    max_len = max_len or max(map(len, names))\n",
        "    names_ix = np.zeros([len(names), max_len], dtype) + pad\n",
        "\n",
        "    for i in range(len(names)):\n",
        "        name_ix = list(map(token_to_id.get, names[i]))\n",
        "        names_ix[i, :len(name_ix)] = name_ix\n",
        "\n",
        "    return names_ix"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:42.883107Z",
          "start_time": "2018-08-13T20:26:42.877186Z"
        },
        "colab_type": "code",
        "id": "Qip2yO3Bshg5",
        "outputId": "a1472f6a-515f-461f-d63a-2b0259038018",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 161
        }
      },
      "source": [
        "# Example: cast 4 random names to padded matrices (so that we can easily batch them)\n",
        "print('\\n'.join(names[::2000]))\n",
        "print(to_matrix(names[::2000])) # после пробела начинается имя"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Abagael\n",
            " Glory\n",
            " Prissie\n",
            " Giovanne\n",
            "[[ 0  4 31 30 36 30 34 41  1]\n",
            " [ 0 10 41 44 47 54  1  1  1]\n",
            " [ 0 19 47 38 48 48 38 34  1]\n",
            " [ 0 10 38 44 51 30 43 43 34]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "f1iTFbSfshg7"
      },
      "source": [
        "# Строим рекуррентную нейронную сеть\n",
        "\n",
        "Мы можем переписать рекуррентную нейронную сеть как последовательное применение полносвязного слоя ко входу $x_t$ и предыдущему состоянию РНС $h_t$.\n",
        "\n",
        "<img src=\"https://github.com/grafft/hse-tasks/blob/master/ida-18/sem8/rnn.png?raw=1\" width=600>\n",
        "\n",
        "Поскольку мы обучаем языковую модель, у нас должен быть:\n",
        "* Трансформационный слой, преобразующий идентификатор символа $x_t$ в вектор.\n",
        "* Выходной слой, который предсказывает вероятность следующего символа на основе $h_{t+1}$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:44.039419Z",
          "start_time": "2018-08-13T20:26:42.884581Z"
        },
        "colab_type": "code",
        "id": "D5qQZFrLshg8",
        "colab": {}
      },
      "source": [
        "# remember to reset your session if you change your graph!\n",
        "\n",
        "from keras import backend as K\n",
        "\n",
        "def reset_tf_session():\n",
        "    curr_session = tf.get_default_session()\n",
        "    # close current session\n",
        "    if curr_session is not None:\n",
        "        curr_session.close()\n",
        "    # reset graph\n",
        "    K.clear_session()\n",
        "    # create new session\n",
        "    config = tf.ConfigProto()\n",
        "    config.gpu_options.allow_growth = True\n",
        "    s = tf.InteractiveSession(config=config)\n",
        "    K.set_session(s)\n",
        "    return s\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "h0GLev1jtEm9",
        "outputId": "f7c3f98f-271b-4771-bbdb-a55faa55c87b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145
        }
      },
      "source": [
        "s = reset_tf_session()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:107: The name tf.reset_default_graph is deprecated. Please use tf.compat.v1.reset_default_graph instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:111: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:44.044903Z",
          "start_time": "2018-08-13T20:26:44.041084Z"
        },
        "colab_type": "code",
        "id": "Sw3iZq4pshg-",
        "colab": {}
      },
      "source": [
        "import keras\n",
        "from keras.layers import concatenate, Dense, Embedding\n",
        "\n",
        "rnn_num_units = 64  # size of hidden state\n",
        "embedding_size = 16  # for characters\n",
        "\n",
        "# Let's create layers for our recurrent network\n",
        "# Note: we create layers but we don't \"apply\" them yet (this is a \"functional API\" of Keras)\n",
        "# Note: set the correct activation (from keras.activations) to Dense layers!\n",
        "\n",
        "# an embedding layer that converts character ids into embeddings\n",
        "embed_x = Embedding(n_tokens, embedding_size)\n",
        "\n",
        "# a dense layer that maps input and previous state to new hidden state, [x_t,h_t]->h_t+1\n",
        "get_h_next = Dense(rnn_num_units, activation='tanh')### YOUR CODE HERE\n",
        "\n",
        "# a dense layer that maps current hidden state to probabilities of characters [h_t+1]->P(x_t+1|h_t+1)\n",
        "get_probas = Dense(n_tokens, activation = 'softmax')### YOUR CODE HERE "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Qd0x8-MIshhA"
      },
      "source": [
        "Генерируем символа имен буква за буквой, начиная с `start_token`:\n",
        "\n",
        "<img src=\"https://github.com/grafft/hse-tasks/blob/master/ida-18/sem8/char-nn.png?raw=1\" width=600>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:44.053212Z",
          "start_time": "2018-08-13T20:26:44.048389Z"
        },
        "colab_type": "code",
        "id": "zh8t4CJdshhB",
        "colab": {}
      },
      "source": [
        "def rnn_one_step(x_t, h_t):\n",
        "    \"\"\"\n",
        "    Recurrent neural network step that produces \n",
        "    probabilities for next token x_t+1 and next state h_t+1\n",
        "    given current input x_t and previous state h_t.\n",
        "    We'll call this method repeatedly to produce the whole sequence.\n",
        "    \n",
        "    You're supposed to \"apply\" above layers to produce new tensors.\n",
        "    Follow inline instructions to complete the function.\n",
        "    \"\"\"\n",
        "    # convert character id into embedding\n",
        "    x_t_emb = embed_x(tf.reshape(x_t, [-1, 1]))[:, 0]\n",
        "    \n",
        "    # concatenate x_t embedding and previous h_t state\n",
        "    x_and_h = concatenate([x_t_emb, h_t])### YOUR CODE HERE\n",
        "    \n",
        "    # compute next state given x_and_h\n",
        "    h_next = get_h_next(x_and_h)### YOUR CODE HERE\n",
        "    \n",
        "    # get probabilities for language model P(x_next|h_next)\n",
        "    output_probas = get_probas(h_next)### YOUR CODE HERE\n",
        "    \n",
        "    return output_probas, h_next"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Qb3-tXUWshhC"
      },
      "source": [
        "# Цикл РНС\n",
        "\n",
        "Будем применять `rnn_one_step` в цикле по символами имен, чтобы получить прогнозы.\n",
        "\n",
        "Предположим, что все имена имеют максимальную длину-16, поэтому мы можем просто перебирать их в цикле for."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:44.342948Z",
          "start_time": "2018-08-13T20:26:44.056136Z"
        },
        "colab_type": "code",
        "id": "cGf1ExBmshhC",
        "outputId": "0acb1637-f139-4690-8fcd-afa5e46951c1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        }
      },
      "source": [
        "input_sequence = tf.placeholder(tf.int32, (None, MAX_LENGTH))  # batch of token ids\n",
        "batch_size = tf.shape(input_sequence)[0]\n",
        "\n",
        "predicted_probas = []\n",
        "h_prev = tf.zeros([batch_size, rnn_num_units])  # initial hidden state\n",
        "\n",
        "for t in range(MAX_LENGTH):\n",
        "    x_t = input_sequence[:, t]  # column t\n",
        "    probas_next, h_next = rnn_one_step(x_t, h_prev)\n",
        "    \n",
        "    h_prev = h_next\n",
        "    predicted_probas.append(probas_next)\n",
        "    \n",
        "# combine predicted_probas into [batch, time, n_tokens] tensor\n",
        "predicted_probas = tf.transpose(tf.stack(predicted_probas), [1, 0, 2])\n",
        "\n",
        "# next to last token prediction is not needed\n",
        "predicted_probas = predicted_probas[:, :-1, :]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "r8rOiHLdshhD"
      },
      "source": [
        "# РНС: функция потерь и градиенты\n",
        "\n",
        "Посторим матрицу предсказаний для $P (x_{next}|h)$ и соответствующих правильных ответов.\n",
        "\n",
        "Мы будем растягивать эту матрицу в вид [None, n_tokens].\n",
        "\n",
        "РНС может быть обучена путем минимизации кроссэнтропии между предсказанными вероятностями и этими ответами."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:44.354310Z",
          "start_time": "2018-08-13T20:26:44.344648Z"
        },
        "colab_type": "code",
        "id": "EBVuLyvFshhE",
        "colab": {}
      },
      "source": [
        "# flatten predictions to [batch*time, n_tokens]\n",
        "predictions_matrix = tf.reshape(predicted_probas, [-1, n_tokens])\n",
        "\n",
        "# flatten answers (next tokens) and one-hot encode them\n",
        "answers_matrix = tf.one_hot(tf.reshape(input_sequence[:, 1:], [-1]), n_tokens)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "M3VVEd_s3fD3",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:45.076642Z",
          "start_time": "2018-08-13T20:26:44.355594Z"
        },
        "colab_type": "code",
        "id": "56X_-hphshhF",
        "outputId": "13619575-4d26-4eb3-80b1-2538ac9a7486",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 91
        }
      },
      "source": [
        "# Define the loss as categorical cross-entropy (e.g. from keras.losses).\n",
        "# Mind that predictions are probabilities and NOT logits!\n",
        "# Remember to apply tf.reduce_mean to get a scalar loss!\n",
        "loss = tf.reduce_mean(tf.keras.losses.categorical_crossentropy(answers_matrix, predictions_matrix))### YOUR CODE HERE\n",
        "\n",
        "optimize = tf.train.AdamOptimizer().minimize(loss)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RNS3xq1eshhG"
      },
      "source": [
        "# РНС: обучение"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:55.322187Z",
          "start_time": "2018-08-13T20:26:45.078296Z"
        },
        "colab_type": "code",
        "id": "-dnq5IQHshhH",
        "outputId": "2c0292a5-f704-4d07-aae6-0f4c21fb0870",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        }
      },
      "source": [
        "from IPython.display import clear_output\n",
        "from random import sample\n",
        "\n",
        "s.run(tf.global_variables_initializer())\n",
        "\n",
        "batch_size = 32\n",
        "history = []\n",
        "\n",
        "for i in range(1000):\n",
        "    batch = to_matrix(sample(names, batch_size), max_len=MAX_LENGTH)\n",
        "    loss_i, _ = s.run([loss, optimize], {input_sequence: batch})\n",
        "    \n",
        "    history.append(loss_i)\n",
        "    \n",
        "    if (i + 1) % 100 == 0:\n",
        "        clear_output(True)\n",
        "        plt.plot(history, label='loss')\n",
        "        plt.legend()\n",
        "        plt.show()\n",
        "\n",
        "assert np.mean(history[:10]) > np.mean(history[-10:]), \"RNN didn't converge\""
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxTVdrA8d/TJG2Blr0UZCsIyCaL\nFgSRRUDB3XFn3hFxw1FGRWacEX3d0Bl1fF3GZWQYxX0URxlXBAFxEGQrCLKvshRZSoGWrft5/8hN\nmqRJm9K0JTfP9/Ppx5t7T5JzE3zuyXPPIsYYlFJKRb+42q6AUkqpyNCArpRSNqEBXSmlbEIDulJK\n2YQGdKWUsglnbb1x06ZNTVpaWm29vVJKRaXly5cfMMakBDtWawE9LS2NjIyM2np7pZSKSiKyI9Qx\nTbkopZRNaEBXSimb0ICulFI2UWs5dKWUioTCwkIyMzPJy8ur7apEVGJiIq1atcLlcoX9HA3oSqmo\nlpmZSXJyMmlpaYhIbVcnIowxZGdnk5mZSbt27cJ+nqZclFJRLS8vjyZNmtgmmAOICE2aNKn0r46w\nA7qIOETkRxH5MsixBBGZJiJbRGSJiKRVqhZKKVUFdgrmHidzTpVpod8LrA9x7FbgkDGmA/AC8Eyl\naxKmTfuO8OSX68grLK6ut1BKqagUVkAXkVbAJcDrIYpcAbxtbX8MDJNqumRmHjrO6wt+ZsWOQ9Xx\n8kopVWlJSUm1XQUg/Bb6i8AfgZIQx1sCuwCMMUVADtAksJCIjBWRDBHJyMrKOonqQt92TXDGCQu3\nHjip5yullF1VGNBF5FJgvzFmeVXfzBgzxRiTboxJT0kJOhVBhZISnPRs3ZCFW7KrWh2llIooYwz3\n338/3bt358wzz2TatGkA7Nmzh0GDBtGrVy+6d+/O999/T3FxMWPGjPGWfeGFF6r8/uF0WxwAXC4i\nFwOJQH0Rec8Y8xufMruB1kCmiDiBBkC1RdwBpzfhlXlbyM0rpH5i+H00lVL29vgXa1n3S25EX7Pr\nafV59LJuYZWdPn06K1euZNWqVRw4cIA+ffowaNAg/vWvfzFixAgeeughiouLOX78OCtXrmT37t2s\nWbMGgMOHD1e5rhW20I0xE40xrYwxacANwLcBwRzgc+Ama/saq0y1LVba//SmlBhYuu1gdb2FUkpV\n2oIFCxg1ahQOh4PU1FQGDx7MsmXL6NOnD2+++SaPPfYYq1evJjk5mfbt27Nt2zbuvvtuZs6cSf36\n9av8/ic9sEhEJgEZxpjPgTeAd0VkC3AQd+CvNr3bNMQRJ6zKPMzwrqnV+VZKqSgSbku6pg0aNIj5\n8+fz1VdfMWbMGCZMmMDo0aNZtWoVs2bNYvLkyXz00UdMnTq1Su9TqYFFxpjvjDGXWtuPWMEcY0ye\nMeZaY0wHY0xfY8y2KtWqAokuB6en1GP9niPV+TZKKVUpAwcOZNq0aRQXF5OVlcX8+fPp27cvO3bs\nIDU1ldtvv53bbruNFStWcODAAUpKSrj66qt58sknWbFiRZXfP2qH/rdoUIf9R+w1d4NSKrr96le/\nYtGiRfTs2RMR4a9//SvNmzfn7bff5tlnn8XlcpGUlMQ777zD7t27ufnmmykpcXcefOqpp6r8/lKN\nqe5ypaenm6oscHH/v1fx/eYDLH5wWARrpZSKNuvXr6dLly61XY1qEezcRGS5MSY9WPmonculWf0E\nso7mU1xSOxckpZQ61URvQE9OpLjEcPBYQW1XRSmlTglRHNATADSPrpSitlLH1elkzil6A3p9T0DP\nr+WaKKVqU2JiItnZ2bYK6p750BMTEyv1vKjt5dIs2X2iWbka0JWKZa1atSIzM5OTnR/qVOVZsagy\nojagp2jKRSkFuFyuSq3qY2dRm3JJdDloUMelKRellLJEbUAHaFwvnsPHC2u7GkopdUqI6oCenOgk\nN08DulJKQZQH9PqJLnJPaEBXSimI9oBex0luXlFtV0MppU4J0R3QE13kaAtdKaWAKA/oiS4HeQXF\ntV0NpZQ6JUR/QC/SgK6UUhD1AT2OwmKjMy4qpRRRH9AdAOQVaitdKaWiO6A73dXXgK6UUtEe0D0t\n9KKSWq6JUkrVPnsEdG2hK6VUtAd0TbkopZRHVAf0BG8LXVMuSikV1QE90ekO6PnaQldKqYoDuogk\nishSEVklImtF5PEgZcaISJaIrLT+bque6vrzplx0cJFSSoW1YlE+MNQYc1REXMACEfnaGLM4oNw0\nY8zvIl/F0BI15aKUUl4VBnTjXnn1qPXQZf2dEkMztZeLUkqVCiuHLiIOEVkJ7AdmG2OWBCl2tYj8\nJCIfi0jrEK8zVkQyRCQjEgu6lvZy0Ra6UkqFFdCNMcXGmF5AK6CviHQPKPIFkGaM6QHMBt4O8TpT\njDHpxpj0lJSUqtQbKL0pqi10pZSqZC8XY8xhYB4wMmB/tjHGs1rz68DZkale+UpHimpAV0qpcHq5\npIhIQ2u7DnABsCGgTAufh5cD6yNZyVASnJpyUUopj3B6ubQA3hYRB+4LwEfGmC9FZBKQYYz5HLhH\nRC4HioCDwJjqqrCvuDgh3hmn/dCVUorwern8BPQOsv8Rn+2JwMTIVi08ic44zaErpRRRPlIUrFWL\nNOWilFI2Ceh6U1QppewQ0DXlopRSYIuArikXpZQCOwR0p0Nb6EophQ0CeoIrjnxdgk4ppaI/oLtT\nLtpCV0qpqA/o8c44Cou1ha6UUlEf0F1xQmHxKTGbr1JK1aqoD+hORxxF2kJXSqnoD+guRxwF2kJX\nSik7BHShqERb6EopZYOAHkehdltUSqnoD+hOh1BYoikXpZSK+oAerzdFlVIKsEFAd8bFUWKgWFvp\nSqkYF/0B3SEAOrhIKRXzoj6gxzvcp6ABXSkV66I+oHta6EXaF10pFeOiPqC7tIWulFKALQK6lUPX\nm6JKqRgX9QE9wekAIF+n0FVKxbioD+iJLndAP6EBXSkV4yoM6CKSKCJLRWSViKwVkceDlEkQkWki\nskVElohIWnVUNpi68VZAL9CArpSKbeG00POBocaYnkAvYKSI9AsocytwyBjTAXgBeCay1QytTry2\n0JVSCsII6MbtqPXQZf0F3oG8Anjb2v4YGCYiErFalqOOlXI5ri10pVSMCyuHLiIOEVkJ7AdmG2OW\nBBRpCewCMMYUATlAkyCvM1ZEMkQkIysrq2o1t3ha6LquqFIq1oUV0I0xxcaYXkAroK+IdD+ZNzPG\nTDHGpBtj0lNSUk7mJcrw5NC1ha6UinWV6uVijDkMzANGBhzaDbQGEBEn0ADIjkQFK5Lo1JuiSikF\n4fVySRGRhtZ2HeACYENAsc+Bm6zta4BvjTE1MtIn3qkjRZVSCsAZRpkWwNsi4sB9AfjIGPOliEwC\nMowxnwNvAO+KyBbgIHBDtdU4gA79V0optwoDujHmJ6B3kP2P+GznAddGtmrh8Qz914WilVKxLupH\niooILodoC10pFfOiPqCDLhStlFJgp4CuLXSlVIyzTUDXHLpSKtbZIqDHO4QibaErpWKcLQK6y6kp\nF6WUskdAd8RRqCkXpVSMs01AL9AWulIqxtkioMdrP3SllLJHQNdui0opZaeAXqQ5dKVUbLNHQHdq\nDl0ppWwR0DWHrpRSNgnomkNXSilbBXTNoSulYpttAnqBzraolIpxtgjo8U7NoSullC0CuubQlVLK\nVgFdc+hKqdhmm4Cu/dCVUrHOFgHd0w/dGG2lK6Vily0CussRhzFQXKIBXSkVu+wR0J3u09A8ulIq\nltkjoDvcp6F5dKVULKswoItIaxGZJyLrRGStiNwbpMwQEckRkZXW3yPVU93g4h0CoF0XlVIxzRlG\nmSLg98aYFSKSDCwXkdnGmHUB5b43xlwa+SpWzNNC14CulIplFbbQjTF7jDErrO0jwHqgZXVXrDKc\nnoCuc6IrpWJYpXLoIpIG9AaWBDncX0RWicjXItItxPPHikiGiGRkZWVVurKhuKyUi+bQlVKxLOyA\nLiJJwCfAeGNMbsDhFUBbY0xP4GXg02CvYYyZYoxJN8akp6SknGydy4jXlItSSoUX0EXEhTuYv2+M\nmR543BiTa4w5am3PAFwi0jSiNS2H5tCVUiq8Xi4CvAGsN8Y8H6JMc6scItLXet3sSFa0PKX90DWg\nK6ViVzi9XAYANwKrRWSlte9BoA2AMWYycA1wp4gUASeAG0wNjsP35tD1pqhSKoZVGNCNMQsAqaDM\nK8ArkapUZWkOXSmlbDZSVAO6UiqWaUBXSimbsEVAj3e6M0L5uq6oUiqG2SKg10tw3wo4ll9cyzVR\nSqnaY4uAnpzoAuBIXmEt10QppWqPLQJ6vXgHInA0v6i2q6KUUrXGFgFdREhKcHIkTwO6Uip22SKg\nA9RPdJGrKRelVAyzTUBPTtQWulIqttksoGsLXSkVu2wU0F3aQldKxTQbBXRNuSilYpvNArqmXJRS\nscs2Ab1egpNjBTpSVCkVu2wT0BMccRQUlVCD07ArpdQpxT4B3eUAdKFopVTssk1A9yxyUaAzLiql\nYpRtAnqCy30qOoWuUipW2SagawtdKRXrbBPQtYWulIp19gnoTuumqAZ0pVSMsk1A96Rc8ou0L7pS\nKjbZJqB7Ui7aQldKxaoKA7qItBaReSKyTkTWisi9QcqIiLwkIltE5CcROat6qhta3Xj3uqI6J7pS\nKlaF00IvAn5vjOkK9APGiUjXgDIXAR2tv7HAaxGtZRg6piYBsO6X3Jp+a6WUOiVUGNCNMXuMMSus\n7SPAeqBlQLErgHeM22KgoYi0iHhty1E/0UXz+olszz5ek2+rlFKnjErl0EUkDegNLAk41BLY5fM4\nk7JBHxEZKyIZIpKRlZVVuZqGIbVBIvty8yL+ukopFQ3CDugikgR8Aow3xpxUXsMYM8UYk26MSU9J\nSTmZlyhXanICa3bnUFyiE3QppWJPWAFdRFy4g/n7xpjpQYrsBlr7PG5l7atRvds04tDxQjbtO1LT\nb62UUrUunF4uArwBrDfGPB+i2OfAaKu3Sz8gxxizJ4L1DEsn68Zooc64qJSKQc4wygwAbgRWi8hK\na9+DQBsAY8xkYAZwMbAFOA7cHPmqVsxlDS7SgK6UikUVBnRjzAJAKihjgHGRqtTJcjrc1Swo0hy6\nUir22GakKJQO/y8q0Ra6Uir22Cqga8pFKRXLbBXQPSmXt37YweHjBbVcG6WUqlm2CuielMv8TVn8\n4d+rvPuXbT/Ilv3alVEpZW/h9HKJGp6UC8CBo+4WenGJ4drJi3A5hM1/vri2qqaUUtXOVi10T8oF\nQKzNnQfdc7sUFmvPF6WUvdkqoMf7tNA9of1EgS54oZSKDbZNuazYeZhPf9xNvNNW1yyllArJVgHd\nN+UCMH7ayhAllVLKfmzVfPVtoSulVKyxVQRMdDm4Z2iHsMuXlBjcsxYopVT0s1VAB5hw4RmsfXwE\n15zdqtxyxwuKaP/gDF75dksN1UwppaqX7QI6QL0EJ3VcjjL7dx8+4d0+fNy9mPT7S3bWWL2UUqo6\n2TKgQ2k/dF8Dnv6WvEJ3N0ZTTjmllIpGtg3oocxY7V53o8Rapi5OI7pSyiZsG9BDhelnZ23kaH6R\nd91R3zSMUkpFM/sG9BAt7z05eXR/dBZLfs727vvtu8v59MfgS6B++uNuHv50TbXUUSmlIsm2Ab0i\nf/pktXd75tq9jJ+2kiN5hRzLL/IrN37aSt5dvKOmq6eUUpUWEwG9YV1XWOXOfOwb+j01t5prU9b/\nvL6Y95foRUMpVTUxEdATnHEM75LKM1efySU9WpRb9kheUbnHq8PCLdk89B9N6yilqsa2AX1wpxTv\n9tNX9eD1m9K5vk8bXv31WWG/hu8o0ukrMr3bm/YdYf2e3MhUVCmlIsRWk3P5Or9zM9ZNGkHd+LKn\neMfg9vzjv9vKff5nK3fzzNcbvI8nfLSKTqnJdG/ZgAtfmA/A9qcviWyllVKqCmzbQgeCBnOAiRd1\noWXDOuU+994PV/JLTp7fvty8wkrXYeaaPcxet6/Sz1NKqcqybQu9IsmJoU99w97g6ZT8opJKv89v\n31sBhG7NewY4KaVUVVXYQheRqSKyX0SC3rUTkSEikiMiK62/RyJfzcgLDOjjh3cEoEm9eEa++H3Q\n59z85jLm+LS2iysIxuEMWiosqfxFoiqMMcxdv6/Cuiulok84KZe3gJEVlPneGNPL+ptU9WpVv9sG\ntvduz7hnIOOHd2J0/7ZkHyso/3nvZHi3c04U8pcZ61mx81DQsgOe/ta7XVJiWLXrcJkyVQ2sRcUl\nbNl/NOzys9ft49a3M/jH/K1Vel+l1KmnwoBujJkPHKyButSoEd2ae7e7nlYfgK9+2lOp19iefYwp\n87cxZupSDh4r4K73l3MoxAWh/YMzuOLVhWRs9/8oi4IE9MXbssk6ku+3r7jEcCRIDv/ZWRsZ/vx/\n2Zl9PKw6Zx11v+6ug+GVV0pFj0jdFO0vIqtE5GsR6RaqkIiMFZEMEcnIysqK0FtHzmkV3CgN5Glx\n10twctYTs5mxei+9n5jNF6t+CfmcwDRMcbF/QDfGcMOUxVwz+Qe//Y9+voYzH/uGD5buZMv+I979\nS352XyA8gboinsnIajjTo5SqAZEI6CuAtsaYnsDLwKehChpjphhj0o0x6SkpKaGK1Zopo88Ouv+5\na3sG3f/95gOAe34YX3d/8CMvzN4U9Dm/HM7jnL/M4YOlO/nqpz1lcuieG687rBZ3SYmhqLiEj5a5\n+8FPnL6aC6xukwBx1pQ14a685JnhpkRXalLKdqoc0I0xucaYo9b2DMAlIk2rXLNa0Lx+Ypl96W0b\ncfXZrUh0xXnLjDk3DYDvN4f+lfG3uZuD7n9m5gb25eYzcfpqxv1rBTnH/dMogSNVx7y1jA4Pfe03\nb7snFi/YfIBdh9wt/nBT8d4WusZzpWynyt0WRaQ5sM8YY0SkL+6LRHYFTzslPHFl99LoSPAZGk9Y\nC2KI1baddkc/GtaN560ftlNYXPWouDMgl718x0G/7fmbsqy6lX3ub95Y4t0Ot8XteZ1PVmTSokEi\nF3RN5YpXFzJz/EA6N69fydorpU4lFQZ0EfkAGAI0FZFM4FHABWCMmQxcA9wpIkXACeAGEyUrL9/Y\nr22Zfee0a+zNS0NpvPcEwkSXg+SEyHXff+LLdd7tvMJib791gKtfW+TdrmghjsLi8JLivq/zyrwt\n7M11p4uW/XwwIgH95wPHMMbQPiWpyq+llKqcCiOTMWZUBcdfAV6JWI1q2bQ7+rNmdw6XvrwAwJtq\n8Uh0OoiLi9wqR9t9eqd0fnhm2M8LHJCUX1hCcYlh7S859GjVMOhztmUd5c8z1vvtO3zc3SunTohR\ntZV1/v99B+i0CErVBlsP/T9Z3Vs2YPVjF9KmcV0evLgLAB2auVucCQEBvl/7xjVSp8BRqoGPb3sn\ng9MfnMHlryzkR59+8f/471bSHviKvMJirnh1IQcDulXGO93nk3ui8tMaKKVOLRrQQ0hOdDH/j+eT\nnuYO2G+O6cMbN6WT6HIAcHnP0wCIdzpYN2kEn44bwMCO1XcvOHAAkie3H8yv/v4DJwqKMcbw7KyN\ngDsVEmxqYGec+59AjhXQS0oMt7y1jJlrgvfJX7Itm66PzCxzM1cpVfs0oIepSVICw7qkeh8P69IM\ncM+1XjfeSa/WDXEEScUkOKvnI173S/nT9/584Bijpy71DlzalnUsaDlP7t0T7O94bznfbtjvl8v3\n2LA3l+unLOZ4QTH/WrqzKtVXSlUDDehV5GmxQ/Abl84I5tt9+fZwCebil7739pOH0DNF5lkt/bwi\n9399Z4YMHH3qO8fNMzM3hDX75KFjBToBmVI1RAP6ScovdLdsfVvgwUK3byjz9HN//rrSgUrtm9Yr\n85xhnZvRNCkhIvX0OBpiJaY86zzyCovLDE4a9Ow8pi74mV//czFpD3xV5rk9HvuGidNXs3hbdtA5\naQ4czaf3E7N5Zd4Wv/0lJYYfth4oU76ouIRX523hREHodJJSKjQN6CepUb14ADo2K+2eF6wfe55P\nrnvCBZ0AGN41lQu7pvL66HT+/dv+3D/iDL/nTBmdzuz7BkW0voG9Wzw8rezpK3YzxOqh4mvSl+v4\nYWvoYQUfLN3JDVMW8+bCn/32FxaXsNcaQfv87E1+F4vJ87fy638uYYH1C+LA0Xy2HzjGJysyeXbW\nRl6Z5z8o691F20l74KtyfxEczS+iwOdG8fIdB+nw4Az2H8kL+ZxTQc6JQn4+EDwdplRlxex86FU1\nvEsz3hzTh0E+S905rMvjbwefTokxTJm/jfYpSd7ZEK/r05rr+rQG3EHbIyXZ3Rrv264x/7wxHUec\nhL2wdVX59nrZEeYEX8GsC1iSb/G2bOZtKB1Je6ygmCSr//6PO91z4Kzfkxs0dZRXWMK7i3eQ4Izj\nuvTWvLHAfbHYn5tP/cTgn0v3R2dxTrvGTLujPwAvf7uFohLDql05XNDV/ctoa5b7ezi9hvvIT1+R\nyfxNWbx4Q+8yx66bvIiN+45oN08VERrQT5KIcH7nZn77hnZuxqy1+7j6rJZ0TE1mUMcUOjVP4q73\nVtC7TfC+4QAN6riDVN14Bw2sQB6stV8d9uVGpgXbvH6iXyv8xjeW+h0/fLyA5TsOMbhTireLZKhf\nDS5HHA9/6p5+/7r01hRXME7Nk6P3DAgzxvDdRvfFpF5C6T2OYc/9F6j+PvJH8grJOpLvHVw14aNV\nAEED+sZ97onW8ouKSXA6yhw/Gfty86gT7wh58VP2pSmXCLouvTU/PnwBHVOTATivY1OaJSfy8Z3n\n8tAlXUM+b3CnFC7omsrtPnO0Ayx7aDg/PXYhn/9uAB+O7Rd2PcJZCPu9W88BIjeny7o9ueW28J/4\nch03TV3K3+ZsJmNH8PnjPeIdpRez9Xty2XXQPV/N4eMFvPbd1jKjYgPnsP/cZ7bL/MISjDEs93lP\n3/sBm/YdIe2Br7yt92837GNb1lFe+87df78ozBG4vm58YylDrYtHuAKnS66Kc/4ylx6PfUOHB2dw\nIMxZOJU9aECPIBHx5tYrI9Hl4J+j0xnQwb8fe0pyAvUTXfRo1ZB+7Zt493dunhz0dUb1bcP8+8/n\nkh4tvPs6NEvizJYN/MrVi3fQtkndoK/x8KXuC09gDv+3g08v9xy+25jFFa8uDHl8rdXN8oU5mypc\n1MPlKP1n6Ttr5dNfb+CZmRsY9Nd5bMs6yqy1e3lxzqYy8+FkHiqdonjdnlw+W/kLV7/mPx1x2gNf\nce5Tc/ls5W6gdC78W97KYOhz/+WZme4Fwn0HcK3ZncN+6xfNRxm7GPrcd0HnqF9pTaucFzBWoLwZ\nMT5ativksZNVVGL4fnOW370FZW8a0KOIpwvkB7f3Y/JvSqf6fd3Kx3dunkybgEA9Z8JgBnXyv1DU\nr+OiVaPgc7+f064x25++hI6pyX4Lad8yIK3C+uWUM9o03LlmAL7duN+7/Y1PN8oNe93piT05edz+\nTgZ3vLucF+ds9gbf01PK9hh6dtZGxk9bGfR9fsnJ8w6sKiou4XhB2Z5AvvW+9OUF3rTNO4u2sy3r\nGD9l5gCQfTSfEwXF5BeVBvHA0bcF5XwGL327JeSxqrhv2ioufbnskoqrdh3maH7wnk+1rbjEhD0d\n9KmooKik1j5bDehR5Ot7B/LyqN40qhfPyO6lKy4N75rKZ+MG8BufycaaJpX+UhjdP83vdZITnYgI\nr49O551b+vrdgPVtuX/j00pvVj+xSl0p9+WG/9Pfc9M0kO//JFt9BkottXLnSScxaZrLSu+89cN2\nuj4yq8xxTxD2LExyxKqDJzZ7ulie/eQcrnx1oTf3D2UvcPlFJRQUlYRc1SqUZdsPlpmyweO7jft5\nZ9H2cp+/aZ//EoWeaSDuer/s4DGAP368ivtCXAQB5m3cz7FqClj5RcWc/uAM7vlwpbeXVLS5aepS\nuj9a9t9STdCAHkU6piZzmTXlAOCXWukZMFL1m/sGM/f3gwFIrZ/IiG6lo1xHdnc/b3jXVAZ1SmHm\nvYOYOX4g25++hGSfG2nxAaNcZ9xznne7VaM6XNbzNP7nnDYV1vv8M2pmMZNVmTl8seqXSqUYPOmd\n3BD99D1TJPuuD3s0v8g75mDz/qPe99u474j3ZizA4ROFbNpXurpUfmEJd763nN5PzA67fvty87h2\n8iLOemK2X1pmze4cso/mM+bNZTzy2VrvBSeclq1nVHDgcogeH2Vk8p8fdwc9ti3rKDe/uczvwlWR\nHdnhd8v0jJf4YtUv9HtqbtjPi4R/Z+xi8baKZ/4ONmbD16IwXqO6aECPYq/++qyQPTYa14v36543\n8aIuDDkjhdn3DWL8sI5+ZZs3SAw6dW7gKNdmPguATL/zXF4e1Zs//+rMMs8LnLDsjyM7V3wyEXL3\nBz+GXFwkmIqC/4tBVp7ybX09M3MDU3364O/3ubm5YU8uF/qsLpVfVMzcDe500v4wexd55sMH+GzV\nbgqKSli+4xCXvryAX/+ztMtnoXUe327YX+Y1Ar1qDfQKFpOW/hx6+eC3f9juff3tAUH6m7V7uW7y\nIq58daHfwi9LtmUz+Nnv+Hi5e8Wt4wVFvDx3s9/9hQ17c70Xl/LSUoGO5Rd55xR6fvYm77iG8mza\ndyTkyOX7P/6JG6Ys5mh+UcjFa/bn5tH54Zm8/cP2Ct+rNtJGGtBjRFrTerx1c186piaHPf1vsK6T\n484/nZHdmvsFd48x56YxvEsqYwf599ZpklT+jeKfn7qY7U9f4p3wrCY9F2KpQI9/L8/kl4B1YMG/\n3/3TX28I+tzANNN5z8zzbvf9y1y2Bwwo2rA3l+U7DvL87E089vlavvzpF+7/+Cfv8YVbsun0v197\nUxEbfVr/OScKef37bdz6dka555NXWMxbVjAyGI7mF/HinE3sz83DGMN1/1gU8rmPfr6WJ79ydzX1\nvXFdUFTC2HeXs3T7QVbuOsyNbyzlm7V7KSou8QZyz4XpiS/X8dzsTXy7Yb/3/sTIF7/nmsmLrPqF\nH9CH/N939Jz0DcYYXpq7ucyYhitfXchVf3ffqM8+ms9tby/jwhfm88mKTIwx3pvzxwuK6PjQDO/z\n7v7XCm58Yyl/nbmBRT6D6mq1+dIAABAySURBVLZmHWWb9Z19GHATu7jEcPYTs5m+ItO7L3BG1Jqg\n/dBVuZrUi+eGvq29j+8fUba1HSfu7o8TL+5MgtPBD1tKW0qPXdaVxnWDB/QWDRKpl+D0XjheGtXb\nr8thMFed1ZJDxwqYt7HmFhk/1yfdUhnBesD4ChyZe8lLC/x6AL0VohUYbBK48noYAUz6Yh2DOjVl\nzJvLvPvyCkv4dsN+XpyzmRfnbOaBi/y/27zCYu9cRYE5c9903GafRcs9xr673O9x9rF8dmQf44Ol\n7kA4Y/Ue7np/BQv+dL5fuXCnfTiWX+Tt6nnEp24b9uby9eq9jB/e0dvbCOCWt5axyrqBvXLXYTIP\nneBvczdzY7+2XJveym/1Mc+/rb9/t5W/f7eV1PoJ9GzV0O8Gve/MpTknCjleUET2sQIe/Wytd//x\ngmK/uZ6MMdU+vkQDuirX8ocvqLDMoonD+HHnYe/AmBY+vWN+fU5bnI44xg/vSHGJ4WWf3hw/PDC0\nzM/+3/Rrw3uLdzKqb2vv//weyQlObjuvPYePuwO6CLSon8gvp+jNs4ouToEq6s5ZFVMX/uyXGvLw\nvXH73uIdfsc8C65MuqIb01f459TjHXHkFRaHvSjLwi3u1IvHl1Y30YVb/NMkgdNC5xwv9A628/j0\nx91+PZd6PPaNd/vPX63n+80HynRl9QRzgPeXlM4U+u7iHRX21d+Xm+8XzMH/Bn3Px0vf33fR93Of\nnsvDl3ald+tGLN95iIc/XcOqRy/0DiSsDhrQVZWl1k/063XTrmk9Jv/mbDqlJnlbcuOHd/LLS/7r\ntnMQkTJrpXoCfGBOP94Rx+rHRwClUwcbA05H2azhqL5t+OAUmN73UDXNGR/Yv70qfG9uhupq94hP\nq9Nj3Z5cXqrEvYpQDvt8RnmFxWVa6KOnLiH7WAH/uWsAM9fu5YY+rUOmuMDdgwvwu6lbXo8dgK/X\n7K10vXNOFAadsM43ZZRXWMJD//G/eZx1JL9aA7rm0FW1GNm9eZl1RXu1bkibxnX55M5zObdD8MVA\nPG1UEbhlQDvvft/A36FZEi0b1uGeoR1wOsr+hB3ZvTlf3zuQb61ePoEC+6u385nx8iNrLphhAdM6\nVMacCcHfN1K+DrH4SGU0CRgA9+DFnf2Cq69gKZ49OXn8/butVa7HXp+bw/dNW1kmD74qM4fMQyfo\n8+c5PPzpGp6dtdHvOYGOB0nZhOqxUxuqq7unhwZ0VWM8q0Cd3bZRyDLNrInKGtRx8chlXVlhpXx8\n55qPd8ax4E/nM+HCM3ju2tKpiIdY3SPjBLq0qE/7lCTvxGd/uLCTt9zZbRux9S8Xex97Zszs3DyZ\nvtbAqjt8RsYO7dyMNo3rsn7SSO716SH0xe/OY8ufL/Kr/6fjBniXKwzlL0F6BlXGrLX7Ki5UgdMa\n+g8sa90o+MhhqN5U0GafPvLhtJSnzN9W7vHvKri34vn3VVuCrRoWSRrQ1SnlriEd+OvVPbish7vH\ni6cFHthI9Nxc6t2mEYsmDiXjf4fz1FVncmO/tn7TJKRbF4/Le7b07nv40q444oS7hriD9hnNk/nL\nr87kzZv7eMsczXe3Vgd3SmHqmD7M/+P51Il3cHkvd71cDuHMVg1wOuK8Lf7HLutKr9buSdhaNw4+\nEhfg6rNb8sZN6dwR0BuoPPP+MMT72lV1Za/TePbaHn77EuMjMzEYwM1hjCr2WLCl4q6GJ6vbaWW7\n4tYN4zyresEtT25eISt2HmJPTtmeU5GgAV2dUuKdcVzXp7W3a2W8lSMPlaIBaNGgDk2TEmjRoA5P\nXNndr0vd89f1Ys6EwbRuXId7h3Vk3h+GeAdPpVpdL4tLDL8+pw0tGpQG4bQm7iB9RS//rpSeFt74\n4aUt/rm/H8JHd/T3G5E7+77QaZcEp4NhXVK9k7gFuqp3yzL72jWtxzVntwr5mqEEBrXR/dvy4g29\ny9yjCJzvx3cgWuBnUBHfC2ptuX/EGbx4fa8y+0WkTOPA14dj+9EnLfQvyFBaNgx9Afc16Yt1XPX3\nH0L2YKoqDejqlJbocjD7vkG8FGTq2XDUiXfQoVkSIsJ9F3Tyy5df36c1tw9sx20Dy7aU26cksX7S\nSK46yz+IJie62PDESG/r3qNvu8Z+/fsTXQ62P31JuVP1XtazBbee167M/rvOP50x56aV2Z99tOIp\nA/56TQ/O69CUT+7sz91DO/DJnef6HU/1GT/wyZ39uaLXabx1cx+aJiX4dUW82ef+xchuzanIGT4X\np5Ry0hovjerNlQEXiD5pjVj20HC/xWKqatz5HYLOey+4R1GH0qCOi4ZWN9vLAsZFNPf57C45s4Xf\nsd8N7eD32HcaCt/nefL/yScxTUU4KgzoIjJVRPaLSNCxvuL2kohsEZGfRKTiuVuVqoSOqcnUiWBK\nwCPR5eChS7rSOMQMmaHeM9HlOOn+xP+5qzTAJjgd3tktfdWv4+J/L+nC0oeG+e33bVl6bmp6pkH2\nuOasVrx32zmc3bYxv7/wDL9+0IDffD9nt23M327ozZAz3DeAfUfNHssv4nfnd6BjsySa1S8N0E2T\nEnj88m5M/o3//+aeSeEa14unV6uG3NCntd/xQZ1SmHRFNy7veRoXBwTDBy7qTEpyArMnDGZgx9C/\nxCor6AA6cf/auaBratljuAN6SnICcyYM4m8+LfwWDRKZOX6g9/E9AaOtkxKcfumchX8aSiOru+Wk\nK7qVOa96tRXQgbeAkeUcvwjoaP2NBV6rerWUso9nr+nBtLH9eP+2c+jdpuzPed9Vr8AdVJyOOFIC\nJkO7fVB7b2CYOX4Qax4fwXkdm7LpydIbs8GC2GfjBvi9dkUu73kaAzum8IcRZzB7wmDObtuYxy/v\nRlKCk3//tj83nZvGiG7N+eD2fsyZMJj/u7anN03xwvW9iIsTnr66BxdaQfPas1vxzi19vSmpYV1S\n+eJ3pfMCtWxYekP23YALlMdLo3rz2bgBXH1W6LTT7PsGcc+wjjx9VWkOPHBtgDgRHHHCP0ens27S\nCDY+OdJvXV/P59OhmXtEtedXw6KJw2hYN56rerdExH8SuzHnpjGsSzPOP6O0Z1TdBId3XEbX0+qH\nPK9Iq/AyYYyZLyJp5RS5AnjHuCcuWCwiDUWkhTGm6n2rlLKBa9Nbl3v8kUu78vgXa2nZsA4fLtvl\nDQSeXwGeOXUSXY6ggSHe6R64FThIx6NjauVSGc9d19PvPgTATeemcZNPGkhE6H+6O1feoVkSxSWG\nM1s29O4DePyKbvRr36TMTVJHnPuGcoIzjvyikpC/kHq0auCdntgzLcTV+UV8siKTOi4HJwqLOa2B\ne2BZcoKTjqnJTLjA/77EgA7++Xzfy13deHf4e+aaHox7fwUTL+5cpuX81T0DKfEZ/fb89b143mq5\nfzi2Hx2bJdHEuvA+d11PvlrtDnsuRxzjhnbg4U/XBJ2ltLp6DkWi3d8S8B3Sl2ntKxPQRWQs7lY8\nbdpUPEufUrGgQ7Mkb6B++mr/3icvXt+LM1s1CPY0P+OHd/K7UesrPsjgq2Aev7wbby78uUwwD4cj\nTvyCObhvVt8S5B6Bx5d3n8eibdllZvX0+HBsPw4cKaBRvdJfFS2tefzHDEjjT9akb9lH80lwBU+P\nBU6pHBckVdYnrTFLHxoe9Pmh6gZlb/4Gprdu7NeWG31SXDWhRkeKGmOmAFMA0tPTo3cGe6VqyJVB\nerxUlmc07fAu5Q+WCmyFV7eOqclBe/q8dXMf8otKqBvvpE0T/xDVtkk95kwYTJpPyqNJOfP0B44k\nHhdw87ImfTpuAD/uPMSqXYe5vk/5v9pOViQC+m7At3atrH1KqVPEkgeH+S1kciobckb5F56KBm4F\neuLK7nQ/rX7Q+xeRNv2uc0P2de/VumHExhKEEomA/jnwOxH5EDgHyNH8uVKnltQg0x3HippMe5xV\nAxeN8lQY0EXkA2AI0FREMoFHAReAMWYyMAO4GNgCHAdurq7KKqWUCi2cXi6jKjhugHERq5FSSqmT\noiNFlVLKJjSgK6WUTWhAV0opm9CArpRSNqEBXSmlbEIDulJK2YSYwGXXa+qNRbKAHRUWDK4pUH1L\nnZya9Jxjg55zbKjKObc1xqQEO1BrAb0qRCTDGJNe2/WoSXrOsUHPOTZU1zlrykUppWxCA7pSStlE\ntAb0KbVdgVqg5xwb9JxjQ7Wcc1Tm0JVSSpUVrS10pZRSATSgK6WUTURdQBeRkSKyUUS2iMgDtV2f\nSBGR1iIyT0TWichaEbnX2t9YRGaLyGbrv42s/SIiL1mfw08iclb573BqEhGHiPwoIl9aj9uJyBLr\nvKaJSLy1P8F6vMU6nlab9a4KayH1j0Vkg4isF5H+dv6eReQ+69/0GhH5QEQS7fg9i8hUEdkvImt8\n9lX6exWRm6zym0XkpsrUIaoCuog4gFeBi4CuwCgR6Vq7tYqYIuD3xpiuQD9gnHVuDwBzjTEdgbnW\nY3B/Bh2tv7HAazVf5Yi4F1jv8/gZ4AVjTAfgEHCrtf9W4JC1/wWrXLT6GzDTGNMZ6In7/G35PYtI\nS+AeIN0Y0x1wADdgz+/5LWBkwL5Kfa8i0hj3IkLnAH2BRz0XgbAYY6LmD+gPzPJ5PBGYWNv1qqZz\n/Qy4ANgItLD2tQA2Wtv/AEb5lPeWi5Y/3OvPzgWGAl8Cgnv0nDPw+wZmAf2tbadVTmr7HE7inBsA\nPwfW3a7fM9AS2AU0tr63L4ERdv2egTRgzcl+r8Ao4B8++/3KVfQXVS10Sv9xeGRa+2zF+pnZG1gC\npJrSNVr3AqnWth0+ixeBPwIl1uMmwGFjTJH12PecvOdrHc+xykebdkAW8KaVanpdROph0+/ZGLMb\n+D9gJ7AH9/e2HPt/zx6V/V6r9H1HW0C3PRFJAj4Bxhtjcn2PGfcl2xb9TEXkUmC/MWZ5bdelhjmB\ns4DXjDG9gWOU/gwHbPc9NwKuwH0hOw2oR9m0REyoie812gL6bqC1z+NW1j5bEBEX7mD+vjFmurV7\nn4i0sI63APZb+6P9sxgAXC4i24EPcadd/gY0FBHPWre+5+Q9X+t4AyC7JiscIZlApjFmifX4Y9wB\n3q7f83DgZ2NMljGmEJiO+7u3+/fsUdnvtUrfd7QF9GVAR+sOeTzumyuf13KdIkJEBHgDWG+Med7n\n0OeA5073Tbhz6579o6275f2AHJ+fdqc8Y8xEY0wrY0wa7u/xW2PM/wDzgGusYoHn6/kcrrHKR10r\n1hizF9glImdYu4YB67Dp94w71dJPROpa/8Y952vr79lHZb/XWcCFItLI+nVzobUvPLV9E+Ekbjpc\nDGwCtgIP1XZ9Inhe5+H+OfYTsNL6uxh3/nAusBmYAzS2ygvuHj9bgdW4exHU+nmc5LkPAb60ttsD\nS4EtwL+BBGt/ovV4i3W8fW3Xuwrn2wvIsL7rT4FGdv6egceBDcAa4F0gwY7fM/AB7vsEhbh/id16\nMt8rcIt1/luAmytTBx36r5RSNhFtKRellFIhaEBXSimb0ICulFI2oQFdKaVsQgO6UkrZhAZ0pZSy\nCQ3oSillE/8P2JPaixdvMoIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "ikShs99SshhH"
      },
      "source": [
        "# РНС: генерация\n",
        "\n",
        "После того, как мы немного обучили нашу сеть, давайте перейдем к фактической генерации имен. Все, что нам нужно-это функция `rnn_one_step'."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:55.341196Z",
          "start_time": "2018-08-13T20:26:55.323787Z"
        },
        "colab_type": "code",
        "id": "nqis4zI4shhI",
        "colab": {}
      },
      "source": [
        "x_t = tf.placeholder(tf.int32, (1,))\n",
        "h_t = tf.Variable(np.zeros([1, rnn_num_units], np.float32))  # we will update hidden state in this variable\n",
        "\n",
        "# For sampling we need to define `rnn_one_step` tensors only once in our graph.\n",
        "# We reuse all parameters thanks to functional API usage.\n",
        "# Then we can feed appropriate tensor values using feed_dict in a loop.\n",
        "# Note how different it is from training stage, where we had to unroll the whole sequence for backprop.\n",
        "next_probs, next_h = rnn_one_step(x_t, h_t)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:55.346422Z",
          "start_time": "2018-08-13T20:26:55.342659Z"
        },
        "colab_type": "code",
        "id": "fXEVUeTsshhJ",
        "colab": {}
      },
      "source": [
        "def generate_sample(seed_phrase=start_token, max_length=MAX_LENGTH):\n",
        "    '''\n",
        "    This function generates text given a `seed_phrase` as a seed.\n",
        "    Remember to include start_token in seed phrase!\n",
        "    Parameter `max_length` is used to set the number of characters in prediction.\n",
        "    '''\n",
        "    x_sequence = [token_to_id[token] for token in seed_phrase]\n",
        "    s.run(tf.assign(h_t, h_t.initial_value))\n",
        "    \n",
        "    # feed the seed phrase, if any\n",
        "    for ix in x_sequence[:-1]:\n",
        "         s.run(tf.assign(h_t, next_h), {x_t: [ix]})\n",
        "    \n",
        "    # start generating\n",
        "    for _ in range(max_length-len(seed_phrase)):\n",
        "        x_probs,_ = s.run([next_probs, tf.assign(h_t, next_h)], {x_t: [x_sequence[-1]]})\n",
        "        x_sequence.append(np.random.choice(n_tokens, p=x_probs[0]))\n",
        "        \n",
        "    return ''.join([tokens[ix] for ix in x_sequence if tokens[ix] != pad_token])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:26:58.458115Z",
          "start_time": "2018-08-13T20:26:55.347900Z"
        },
        "colab_type": "code",
        "id": "gzyjLrtyshhJ",
        "outputId": "9ec483cc-1169-474c-a457-bb2a1bae1088",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        }
      },
      "source": [
        "# without prefix\n",
        "for _ in range(10):\n",
        "    print(generate_sample())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Ro ey\n",
            " Kerrete\n",
            " Nodden\n",
            " Maody\n",
            " Rotce\n",
            " Hiacne\n",
            " Mancil\n",
            " Ararmlan\n",
            " Budra\n",
            " Colien\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:27:01.986726Z",
          "start_time": "2018-08-13T20:26:58.459810Z"
        },
        "colab_type": "code",
        "id": "U7o-Z7fhshhL",
        "outputId": "b4cbef80-80e1-4100-a110-6754ccd45774",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        }
      },
      "source": [
        "# with prefix conditioning\n",
        "for _ in range(10):\n",
        "    print(generate_sample(' Valentin'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Valentin\n",
            " Valentin\n",
            " Valentine\n",
            " Valentins\n",
            " Valentina\n",
            " Valentin\n",
            " Valentinr\n",
            " Valentina\n",
            " Valentin\n",
            " Valentin\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "collapsed": true,
        "id": "psNKnoLlshhL"
      },
      "source": [
        "# Динамические РНС\n",
        "\n",
        "Помимо Keras, есть также TensorFlow API для рекуррентных нейронных сетей. В его основе лежит функция символьного цикла (ака [tf.scan](https://www.tensorflow.org/api_docs/python/tf/scan)).\n",
        "\n",
        "РНС цикл, который мы реализовали для обучения, может быть заменен на однуTensorFlow инструкцию: [tf.nn.dynamic_rnn](https://www.tensorflow.org/api_docs/python/tf/nn/dynamic_rnn). Этот интерфейс учитывает динамическую длину последовательности и идет с некоторыми предобученными архитектурами.\n",
        "\n",
        "[tf.nn.rnn_cell.BasicRNNCell](https://www.tensorflow.org/api_docs/python/tf/contrib/rnn/BasicRNNCell)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:27:12.975354Z",
          "start_time": "2018-08-13T20:27:12.737529Z"
        },
        "colab_type": "code",
        "id": "qtV0qgBRshhM",
        "outputId": "011bae71-0c7a-4296-df4a-8e55eb54e9f9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 361
        }
      },
      "source": [
        "class CustomRNN(tf.nn.rnn_cell.BasicRNNCell):\n",
        "    def call(self, input, state):\n",
        "        # from docs:\n",
        "        # Returns:\n",
        "        # Output: A 2-D tensor with shape [batch_size, self.output_size].\n",
        "        # New state: Either a single 2-D tensor, or a tuple of tensors matching the arity and shapes of state.\n",
        "        return rnn_one_step(input[:, 0], state)\n",
        "    \n",
        "    @property\n",
        "    def output_size(self):\n",
        "        return n_tokens\n",
        "    \n",
        "cell = CustomRNN(rnn_num_units)\n",
        "\n",
        "input_sequence = tf.placeholder(tf.float32, (None, None))\n",
        "    \n",
        "predicted_probas, last_state = tf.nn.dynamic_rnn(cell, input_sequence[:, :, None], dtype=tf.float32)\n",
        "\n",
        "print('LSTM outputs for each step [batch,time,n_tokens]:')\n",
        "print(predicted_probas.eval({input_sequence: to_matrix(names[:10], max_len=50)}).shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-47-932903461488>:13: BasicRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This class is equivalent as tf.keras.layers.SimpleRNNCell, and will be replaced by that in Tensorflow 2.0.\n",
            "WARNING:tensorflow:From <ipython-input-47-932903461488>:17: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:456: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `layer.add_weight` method instead.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py:460: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.sparse_tensor_to_dense is deprecated. Please use tf.sparse.to_dense instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.sparse_tensor_dense_matmul is deprecated. Please use tf.sparse.sparse_dense_matmul instead.\n",
            "\n",
            "LSTM outputs for each step [batch,time,n_tokens]:\n",
            "(10, 50, 56)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "N6f-mPfVshhM"
      },
      "source": [
        "Обратите внимание, что мы никогда не использовали MAX_LENGTH в коде выше: TF будет перебирать столько временных шагов, сколько вы ему дали.\n",
        "\n",
        "Можно также использовать любую предварительно реализованную ячейку РНС:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:27:12.981697Z",
          "start_time": "2018-08-13T20:27:12.977590Z"
        },
        "colab_type": "code",
        "id": "iTfENkYMshhN",
        "outputId": "a0fa1e68-969c-49f1-b21c-6815e5c34515",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 647
        }
      },
      "source": [
        "for obj in dir(tf.nn.rnn_cell) + dir(tf.contrib.rnn):\n",
        "    if obj.endswith('Cell'):\n",
        "        print(obj)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "BasicLSTMCell\n",
            "BasicRNNCell\n",
            "GRUCell\n",
            "LSTMCell\n",
            "MultiRNNCell\n",
            "RNNCell\n",
            "BasicLSTMCell\n",
            "BasicRNNCell\n",
            "BidirectionalGridLSTMCell\n",
            "Conv1DLSTMCell\n",
            "Conv2DLSTMCell\n",
            "Conv3DLSTMCell\n",
            "ConvLSTMCell\n",
            "CoupledInputForgetGateLSTMCell\n",
            "FusedRNNCell\n",
            "GLSTMCell\n",
            "GRUBlockCell\n",
            "GRUCell\n",
            "GridLSTMCell\n",
            "IndRNNCell\n",
            "IndyGRUCell\n",
            "IndyLSTMCell\n",
            "IntersectionRNNCell\n",
            "LSTMBlockCell\n",
            "LSTMBlockFusedCell\n",
            "LSTMCell\n",
            "LayerNormBasicLSTMCell\n",
            "LayerRNNCell\n",
            "MultiRNNCell\n",
            "NASCell\n",
            "PhasedLSTMCell\n",
            "RNNCell\n",
            "SRUCell\n",
            "TimeFreqLSTMCell\n",
            "UGRNNCell\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2018-08-13T20:27:13.168207Z",
          "start_time": "2018-08-13T20:27:12.986884Z"
        },
        "colab_type": "code",
        "id": "HATgwfAVshhO",
        "outputId": "8974b7f3-9491-4c90-dee0-9208edf8dcd5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 520
        }
      },
      "source": [
        "input_sequence = tf.placeholder(tf.int32, (None, None))\n",
        "\n",
        "inputs_embedded = embed_x(input_sequence)\n",
        "\n",
        "# standard cell returns hidden state as output!\n",
        "cell = tf.nn.rnn_cell.LSTMCell(rnn_num_units)\n",
        "\n",
        "state_sequence, last_state = tf.nn.dynamic_rnn(cell, inputs_embedded, dtype=tf.float32)\n",
        "\n",
        "s.run(tf.global_variables_initializer())\n",
        "\n",
        "print('LSTM hidden state for each step [batch,time,rnn_num_units]:')\n",
        "print(state_sequence.eval({input_sequence: to_matrix(names[:10], max_len=50)}).shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-50-62766a2145fb>:6: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36mBuildLoop\u001b[0;34m(self, pred, body, loop_vars, shape_invariants, return_same_structure)\u001b[0m\n\u001b[1;32m   2244\u001b[0m         original_body_result, exit_vars = self._BuildLoop(\n\u001b[0;32m-> 2245\u001b[0;31m             pred, body, original_loop_vars, loop_vars, shape_invariants)\n\u001b[0m\u001b[1;32m   2246\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36m_BuildLoop\u001b[0;34m(self, pred, body, original_loop_vars, loop_vars, shape_invariants)\u001b[0m\n\u001b[1;32m   2169\u001b[0m     \u001b[0mpre_summaries\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_collection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGraphKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_SUMMARY_COLLECTION\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2170\u001b[0;31m     \u001b[0mbody_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbody\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mpacked_vars_for_body\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2171\u001b[0m     \u001b[0mpost_summaries\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_collection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGraphKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_SUMMARY_COLLECTION\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(i, lv)\u001b[0m\n\u001b[1;32m   2704\u001b[0m             math_ops.logical_and(i < maximum_iterations, orig_cond(*lv)))\n\u001b[0;32m-> 2705\u001b[0;31m         \u001b[0mbody\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlv\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morig_body\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mlv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2706\u001b[0m       \u001b[0mtry_to_pack\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn.py\u001b[0m in \u001b[0;36m_time_step\u001b[0;34m(time, output_ta_t, state)\u001b[0m\n\u001b[1;32m    883\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 884\u001b[0;31m       \u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_state\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_cell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    885\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    869\u001b[0m       \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 870\u001b[0;31m     \u001b[0mcall_cell\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_t\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    871\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope, *args, **kwargs)\u001b[0m\n\u001b[1;32m    385\u001b[0m     return base_layer.Layer.__call__(\n\u001b[0;32m--> 386\u001b[0;31m         self, inputs, state, scope=scope, *args, **kwargs)\n\u001b[0m\u001b[1;32m    387\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/base.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    547\u001b[0m       \u001b[0;31m# Actually call layer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 548\u001b[0;31m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLayer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    549\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    801\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mbuild_graph\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mbase_layer_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mneeds_keras_history\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 802\u001b[0;31m       \u001b[0mbase_layer_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_keras_history\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    803\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/base_layer_utils.py\u001b[0m in \u001b[0;36mcreate_keras_history\u001b[0;34m(tensors)\u001b[0m\n\u001b[1;32m    183\u001b[0m   \"\"\"\n\u001b[0;32m--> 184\u001b[0;31m   \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreated_layers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_create_keras_history_helper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    185\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mcreated_layers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/base_layer_utils.py\u001b[0m in \u001b[0;36m_create_keras_history_helper\u001b[0;34m(tensors, processed_ops, created_layers)\u001b[0m\n\u001b[1;32m    230\u001b[0m       processed_ops, created_layers = _create_keras_history_helper(\n\u001b[0;32m--> 231\u001b[0;31m           layer_inputs, processed_ops, created_layers)\n\u001b[0m\u001b[1;32m    232\u001b[0m       \u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/base_layer_utils.py\u001b[0m in \u001b[0;36m_create_keras_history_helper\u001b[0;34m(tensors, processed_ops, created_layers)\u001b[0m\n\u001b[1;32m    230\u001b[0m       processed_ops, created_layers = _create_keras_history_helper(\n\u001b[0;32m--> 231\u001b[0;31m           layer_inputs, processed_ops, created_layers)\n\u001b[0m\u001b[1;32m    232\u001b[0m       \u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/base_layer_utils.py\u001b[0m in \u001b[0;36m_create_keras_history_helper\u001b[0;34m(tensors, processed_ops, created_layers)\u001b[0m\n\u001b[1;32m    230\u001b[0m       processed_ops, created_layers = _create_keras_history_helper(\n\u001b[0;32m--> 231\u001b[0;31m           layer_inputs, processed_ops, created_layers)\n\u001b[0m\u001b[1;32m    232\u001b[0m       \u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/base_layer_utils.py\u001b[0m in \u001b[0;36m_create_keras_history_helper\u001b[0;34m(tensors, processed_ops, created_layers)\u001b[0m\n\u001b[1;32m    228\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 229\u001b[0;31m               \u001b[0mconstants\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop_input\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    230\u001b[0m       processed_ops, created_layers = _create_keras_history_helper(\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3440\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3441\u001b[0;31m     \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3442\u001b[0m     \u001b[0mfeed_arrays\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/backend.py\u001b[0m in \u001b[0;36mget_session\u001b[0;34m(op_input_list)\u001b[0m\n\u001b[1;32m    485\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 486\u001b[0;31m       \u001b[0m_initialize_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    487\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/backend.py\u001b[0m in \u001b[0;36m_initialize_variables\u001b[0;34m(session)\u001b[0m\n\u001b[1;32m    902\u001b[0m     is_initialized = session.run(\n\u001b[0;32m--> 903\u001b[0;31m         [variables_module.is_variable_initialized(v) for v in candidate_vars])\n\u001b[0m\u001b[1;32m    904\u001b[0m     \u001b[0muninitialized_vars\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    955\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 956\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    957\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1179\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1180\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1181\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1358\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1359\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1360\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1364\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1365\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1366\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1347\u001b[0m       \u001b[0;31m# Ensure any changes to the graph are reflected in the runtime.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1348\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1349\u001b[0m       return self._call_tf_sessionrun(options, feed_dict, fetch_list,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_extend_graph\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1386\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1387\u001b[0;31m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session_run_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1388\u001b[0m       \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExtendSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/util/lock_util.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    123\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__enter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 124\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_group_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    125\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/util/lock_util.py\u001b[0m in \u001b[0;36macquire\u001b[0;34m(self, group_id)\u001b[0m\n\u001b[1;32m     91\u001b[0m     \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_another_group_active\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ready\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_group_member_counts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgroup_id\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.6/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    294\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    296\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-50-62766a2145fb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mcell\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrnn_cell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLSTMCell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnn_num_units\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mstate_sequence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlast_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdynamic_rnn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcell\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs_embedded\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_variables_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/util/deprecation.py\u001b[0m in \u001b[0;36mnew_func\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    322\u001b[0m               \u001b[0;34m'in a future version'\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'after %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mdate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m               instructions)\n\u001b[0;32m--> 324\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    325\u001b[0m     return tf_decorator.make_decorator(\n\u001b[1;32m    326\u001b[0m         \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'deprecated'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn.py\u001b[0m in \u001b[0;36mdynamic_rnn\u001b[0;34m(cell, inputs, sequence_length, initial_state, dtype, parallel_iterations, swap_memory, time_major, scope)\u001b[0m\n\u001b[1;32m    705\u001b[0m         \u001b[0mswap_memory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswap_memory\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    706\u001b[0m         \u001b[0msequence_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msequence_length\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 707\u001b[0;31m         dtype=dtype)\n\u001b[0m\u001b[1;32m    708\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    709\u001b[0m     \u001b[0;31m# Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth].\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/rnn.py\u001b[0m in \u001b[0;36m_dynamic_rnn_loop\u001b[0;34m(cell, inputs, initial_state, parallel_iterations, swap_memory, sequence_length, dtype)\u001b[0m\n\u001b[1;32m    914\u001b[0m       \u001b[0mparallel_iterations\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparallel_iterations\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    915\u001b[0m       \u001b[0mmaximum_iterations\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtime_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 916\u001b[0;31m       swap_memory=swap_memory)\n\u001b[0m\u001b[1;32m    917\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    918\u001b[0m   \u001b[0;31m# Unpack final output if not using output tuples.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36mwhile_loop\u001b[0;34m(cond, body, loop_vars, shape_invariants, parallel_iterations, back_prop, swap_memory, name, maximum_iterations, return_same_structure)\u001b[0m\n\u001b[1;32m   2751\u001b[0m       \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_to_collection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGraphKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mWHILE_CONTEXT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloop_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2752\u001b[0m     result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants,\n\u001b[0;32m-> 2753\u001b[0;31m                                     return_same_structure)\n\u001b[0m\u001b[1;32m   2754\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mmaximum_iterations\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2755\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36mBuildLoop\u001b[0;34m(self, pred, body, loop_vars, shape_invariants, return_same_structure)\u001b[0m\n\u001b[1;32m   2243\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_default_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mutation_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2244\u001b[0m         original_body_result, exit_vars = self._BuildLoop(\n\u001b[0;32m-> 2245\u001b[0;31m             pred, body, original_loop_vars, loop_vars, shape_invariants)\n\u001b[0m\u001b[1;32m   2246\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2247\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/util/lock_util.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    126\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__exit__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype_arg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue_arg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraceback_arg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m       \u001b[0;32mdel\u001b[0m \u001b[0mtype_arg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue_arg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraceback_arg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_group_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/util/lock_util.py\u001b[0m in \u001b[0;36mrelease\u001b[0;34m(self, group_id)\u001b[0m\n\u001b[1;32m     98\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_group_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ready\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    101\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_group_member_counts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgroup_id\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m-=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_group_member_counts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgroup_id\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "pIfsSdgq5VD2",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}