Kopylov_Sentiment_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Kopylov_Sentiment.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/wallykop/IAD_SMMO_2019/blob/master/Kopylov_Sentiment_Analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6y4HDrnM_gFs",
        "colab_type": "text"
      },
      "source": [
        "## ИАД KAGGLE тексты\n",
        "Решение для [соревнования](https://www.kaggle.com/c/iad-deep-learning-sentiment) на курсе СММО ИАД 2019\n",
        "\n",
        "[Папка](https://drive.google.com/drive/folders/1kHOgfrSSR1LBxIABHZ8Z03nUI7Jo0ExW?usp=sharing) с файлами для загрузки:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TX1NfhPk3mJP",
        "colab_type": "code",
        "outputId": "8f4a7a20-0b01-4534-cebb-087b332d0a44",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 131
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dTsTU8kl3raG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import string\n",
        "import re"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8fCVsoof34cD",
        "colab_type": "code",
        "outputId": "04fb92db-3310-46ab-84fa-328f38ae23d6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83
        }
      },
      "source": [
        "from keras.layers import Dense, Input, GRU, Embedding, Dropout, Bidirectional, BatchNormalization, LSTM, Activation, CuDNNGRU, CuDNNLSTM\n",
        "from keras.layers import Bidirectional, GlobalMaxPool1D, Conv1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D, Add, Flatten, BatchNormalization, GlobalAveragePooling1D\n",
        "from keras.models import Model, Sequential\n",
        "from keras.optimizers import Adam\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, LearningRateScheduler\n",
        "from tqdm import tqdm_notebook\n",
        "\n",
        "from keras.preprocessing import text, sequence\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "\n",
        "from keras.models import Model, load_model\n",
        "\n",
        "from keras import initializers, regularizers, constraints, optimizers, layers, callbacks\n",
        "from keras.callbacks import Callback\n",
        "\n",
        "# Сохраним модель\n",
        "import cv2\n",
        "import keras\n",
        "from keras.models import save_model, load_model\n",
        "\n",
        "def load_from_file(model_filename, last_epoch):\n",
        "        model = load_model(model_filename)\n",
        "        epoch = last_epoch\n",
        "        return model\n",
        "\n",
        "class ModelSaveCallback(keras.callbacks.Callback):\n",
        "\n",
        "    def __init__(self, file_name):\n",
        "        super(ModelSaveCallback, self).__init__()\n",
        "        self.file_name = file_name\n",
        "\n",
        "\n",
        "    def on_epoch_end(self, epoch, logs=None):\n",
        "        epoch += 1\n",
        "        model.save(self.file_name.format(epoch))\n",
        "        print(model.save(self.file_name.format(epoch)))\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vCo9kVzuCmsw",
        "colab_type": "code",
        "outputId": "74280ee4-8de7-4406-cb5e-fb776ec683ba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        }
      },
      "source": [
        "x_train = pd.read_csv('/content/drive/My Drive/Sentiment_analysis_Kopylov/x_train.txt', sep = '\\r\\t', header = None, names = ['Review'])\n",
        "x_test = pd.read_csv('/content/drive/My Drive/Sentiment_analysis_Kopylov/x_test.txt', sep = '\\r\\t', header = None, names = ['Review'])\n",
        "y_train = pd.read_csv('/content/drive/My Drive/Sentiment_analysis_Kopylov/y_train.csv')\n",
        "y_train = y_train.Probability.values"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
            "  \n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sIhM8muTE6Et",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embed_size = 300 \n",
        "max_features = 100000 \n",
        "maxlen = 150\n",
        "\n",
        "tokenizer = Tokenizer(num_words=max_features, filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n', lower=True,)\n",
        "list_sentences_train = x_train.Review.values\n",
        "list_sentences_test = x_test.Review.values\n",
        "tokenizer.fit_on_texts(list(np.concatenate((list_sentences_train, list_sentences_test))))\n",
        "list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)\n",
        "list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)\n",
        "x_train_pad = pad_sequences(list_tokenized_train, maxlen=maxlen)\n",
        "x_test_pad = pad_sequences(list_tokenized_test, maxlen=maxlen)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PtKNLxy1IMst",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras import backend as K\n",
        "from keras.engine.topology import Layer\n",
        "from keras import initializers, regularizers, constraints, optimizers, layers\n",
        "\n",
        "class Attention(Layer):\n",
        "    def __init__(self, step_dim,\n",
        "                 W_regularizer=None, b_regularizer=None,\n",
        "                 W_constraint=None, b_constraint=None,\n",
        "                 bias=True, **kwargs):\n",
        "        self.supports_masking = True\n",
        "        self.init = initializers.get('glorot_uniform')\n",
        "\n",
        "        self.W_regularizer = regularizers.get(W_regularizer)\n",
        "        self.b_regularizer = regularizers.get(b_regularizer)\n",
        "\n",
        "        self.W_constraint = constraints.get(W_constraint)\n",
        "        self.b_constraint = constraints.get(b_constraint)\n",
        "\n",
        "        self.bias = bias\n",
        "        self.step_dim = step_dim\n",
        "        self.features_dim = 0\n",
        "        super(Attention, self).__init__(**kwargs)\n",
        "\n",
        "    def build(self, input_shape):\n",
        "        assert len(input_shape) == 3\n",
        "\n",
        "        self.W = self.add_weight((input_shape[-1],),\n",
        "                                 initializer=self.init,\n",
        "                                 name='{}_W'.format(self.name),\n",
        "                                 regularizer=self.W_regularizer,\n",
        "                                 constraint=self.W_constraint)\n",
        "        self.features_dim = input_shape[-1]\n",
        "\n",
        "        if self.bias:\n",
        "            self.b = self.add_weight((input_shape[1],),\n",
        "                                     initializer='zero',\n",
        "                                     name='{}_b'.format(self.name),\n",
        "                                     regularizer=self.b_regularizer,\n",
        "                                     constraint=self.b_constraint)\n",
        "        else:\n",
        "            self.b = None\n",
        "\n",
        "        self.built = True\n",
        "\n",
        "    def compute_mask(self, input, input_mask=None):\n",
        "        return None\n",
        "\n",
        "    def call(self, x, mask=None):\n",
        "        features_dim = self.features_dim\n",
        "        step_dim = self.step_dim\n",
        "\n",
        "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n",
        "                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
        "\n",
        "        if self.bias:\n",
        "            eij += self.b\n",
        "\n",
        "        eij = K.tanh(eij)\n",
        "\n",
        "        a = K.exp(eij)\n",
        "\n",
        "        if mask is not None:\n",
        "            a *= K.cast(mask, K.floatx())\n",
        "\n",
        "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
        "\n",
        "        a = K.expand_dims(a)\n",
        "        weighted_input = x * a\n",
        "        return K.sum(weighted_input, axis=1)\n",
        "\n",
        "    def compute_output_shape(self, input_shape):\n",
        "        return input_shape[0],  self.features_dim"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PsaXVKVy7VTB",
        "colab_type": "code",
        "outputId": "8fae4436-32c9-4759-9096-58f9672f7c16",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "from keras.models import Model\n",
        "from keras.optimizers import Adam\n",
        "from keras.callbacks import LearningRateScheduler\n",
        "\n",
        "sequence_input = Input(shape=(maxlen,), dtype='int32')\n",
        "embedding_layer = Embedding(max_features,\n",
        "                            embed_size,\n",
        "                            input_length=maxlen)\n",
        "x = embedding_layer(sequence_input)\n",
        "\n",
        "x = BatchNormalization()(x)\n",
        "x = SpatialDropout1D(0.2)(x)\n",
        "x = Bidirectional(CuDNNLSTM(128, return_sequences=True, kernel_initializer='lecun_uniform', bias_initializer='lecun_uniform'))(x)\n",
        "x = Bidirectional(CuDNNLSTM(128, return_sequences=True, kernel_initializer='lecun_uniform', bias_initializer='lecun_uniform'))(x)\n",
        "x = BatchNormalization()(x)\n",
        "\n",
        "att = Attention(maxlen)(x)\n",
        "att = BatchNormalization()(att)\n",
        "att = Dropout(0.3)(att)\n",
        "avg_pool1 = GlobalAveragePooling1D()(x)\n",
        "max_pool1 = GlobalMaxPooling1D()(x)\n",
        "\n",
        "x = concatenate([att,avg_pool1, max_pool1])\n",
        "\n",
        "x = BatchNormalization()(x)\n",
        "x = Dropout(0.3)(x)\n",
        "x = Dense(512, kernel_initializer='lecun_uniform', bias_initializer='lecun_uniform')(x)\n",
        "\n",
        "x = BatchNormalization()(x)\n",
        "x = Dropout(0.2)(x)\n",
        "x = Dense(128, kernel_initializer='lecun_uniform', bias_initializer='lecun_uniform')(x)\n",
        "\n",
        "x = BatchNormalization()(x)\n",
        "x = Dropout(0.1)(x)\n",
        "\n",
        "preds = Dense(1, activation='sigmoid', kernel_initializer='lecun_uniform', bias_initializer='lecun_uniform')(x)\n",
        "\n",
        "model = Model(sequence_input, preds)\n",
        "\n",
        "\n",
        "\n",
        "def time_decay(epoch, initial_lrate):\n",
        "    decay_rate = 0.05\n",
        "    new_lrate = initial_lrate/(1+decay_rate*(epoch))\n",
        "    return new_lrate\n",
        "\n",
        "def schedule(ind):\n",
        "    a = [0.0001, 0.0001, 0.00005, 0.00005, 0.00001, 0.00001, 0.000005, 0.000005, 0.000001, 0.000001]\n",
        "    return a[ind]\n",
        "\n",
        "lr = LearningRateScheduler(schedule)\n",
        "\n",
        "lrate = LearningRateScheduler(time_decay,verbose=1)\n",
        "\n",
        "reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor = 0.5, patience = 2, min_lr = 0.0001)\n",
        "\n",
        "early_stop = EarlyStopping(monitor='val_loss',\n",
        "                           patience=5,\n",
        "                           verbose=1,\n",
        "                           min_delta=1e-4)\n",
        "\n",
        "print(model.summary())\n",
        "model.compile(loss = \"binary_crossentropy\", optimizer = Adam(lr = 0.001, decay = 0), metrics = [\"accuracy\"])\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:541: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:148: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3733: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            (None, 150)          0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_1 (Embedding)         (None, 150, 300)     30000000    input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_1 (BatchNor (None, 150, 300)     1200        embedding_1[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "spatial_dropout1d_1 (SpatialDro (None, 150, 300)     0           batch_normalization_1[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_1 (Bidirectional) (None, 150, 256)     440320      spatial_dropout1d_1[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_2 (Bidirectional) (None, 150, 256)     395264      bidirectional_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_2 (BatchNor (None, 150, 256)     1024        bidirectional_2[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "attention_1 (Attention)         (None, 256)          406         batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_3 (BatchNor (None, 256)          1024        attention_1[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dropout_1 (Dropout)             (None, 256)          0           batch_normalization_3[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling1d_1 (Glo (None, 256)          0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "global_max_pooling1d_1 (GlobalM (None, 256)          0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 768)          0           dropout_1[0][0]                  \n",
            "                                                                 global_average_pooling1d_1[0][0] \n",
            "                                                                 global_max_pooling1d_1[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_4 (BatchNor (None, 768)          3072        concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dropout_2 (Dropout)             (None, 768)          0           batch_normalization_4[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 512)          393728      dropout_2[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_5 (BatchNor (None, 512)          2048        dense_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dropout_3 (Dropout)             (None, 512)          0           batch_normalization_5[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 128)          65664       dropout_3[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_6 (BatchNor (None, 128)          512         dense_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dropout_4 (Dropout)             (None, 128)          0           batch_normalization_6[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 1)            129         dropout_4[0][0]                  \n",
            "==================================================================================================\n",
            "Total params: 31,304,391\n",
            "Trainable params: 31,299,951\n",
            "Non-trainable params: 4,440\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:793: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3657: The name tf.log is deprecated. Please use tf.math.log instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/nn_impl.py:183: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0i9fmJpWQtOP",
        "colab_type": "code",
        "outputId": "890e5267-f298-44a8-df97-840169e363c0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 452
        }
      },
      "source": [
        "model_filename = 'weights_{0:02d}.hdf5'\n",
        "history = model.fit(x_train_pad, \n",
        "                    y_train, \n",
        "                    batch_size = 512, \n",
        "                    epochs = 1, \n",
        "                    validation_split = 0.2, \n",
        "                    verbose = 1, callbacks = [ModelSaveCallback(model_filename), \n",
        "                                              lrate, \n",
        "                                              #lr,\n",
        "                                              reduce_lr,\n",
        "                                              early_stop])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1033: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1020: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3005: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
            "\n",
            "Train on 2880000 samples, validate on 720000 samples\n",
            "Epoch 1/1\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:190: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:197: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:207: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:216: The name tf.is_variable_initialized is deprecated. Please use tf.compat.v1.is_variable_initialized instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:223: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.\n",
            "\n",
            "\n",
            "Epoch 00001: LearningRateScheduler setting learning rate to 0.0010000000474974513.\n",
            "2880000/2880000 [==============================] - 1021s 354us/step - loss: 0.1482 - acc: 0.9442 - val_loss: 0.1253 - val_acc: 0.9540\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7FXzA9RNEZk-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "submission = model.predict(x_test_pad)\n",
        "predictions = submission\n",
        "answer = pd.DataFrame(predictions)\n",
        "answer = pd.DataFrame({\n",
        "    'Id': range(len(answer)),\n",
        "    'Probability':answer[0]})\n",
        "answer.Id = answer.Id + 1\n",
        "answer.to_csv('predictions.csv', index = False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4P3bRLCs47TT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}