Sentiment_analysis_LSTM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Sentiment analysis LSTM.ipynb",
      "version": "0.3.2",
      "views": {},
      "default_view": {},
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "vPNHCs-KULq5",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install -U -q PyDrive\n",
        "\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "# 1. Authenticate and create the PyDrive client.\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "kgB6mIj0UQp3",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 147
        },
        "outputId": "bce0b24a-36b8-4f9d-9280-69009eb02829",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528692701373,
          "user_tz": -480,
          "elapsed": 1213,
          "user": {
            "displayName": "Sauradeep Chakraborty",
            "photoUrl": "//lh6.googleusercontent.com/-QvGcv_ohGQE/AAAAAAAAAAI/AAAAAAAADh8/JnoW4qEexCc/s50-c-k-no/photo.jpg",
            "userId": "104205094747188221458"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "file_list = drive.ListFile({'q': \"'13uceITtEknqn4WiEC9fvpttTd9mn8mSe' in parents and trashed=false\"}).GetList()\n",
        "for file1 in file_list:\n",
        "  print('title: %s, id: %s' % (file1['title'], file1['id']))"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "title: train.csv, id: 106i86-VEOKrKWdT6xRHmX2tVfIe413rq\n",
            "title: text8.zip, id: 1by4oOo87N0vOTPbx3JZzWu-L3MJK5XDm\n",
            "title: test_data.csv, id: 1prpU3Z6pHHRpB5T4XvYBPkGzcz3FMU8a\n",
            "title: sampleTrain.csv, id: 1NnSvlLJTxQJnGGKHgbT8DYN7jWqQaaQl\n",
            "title: Untitled1.ipynb, id: 1Rr56M01SOnsC-LjVlXnl6dlTyyVTld00\n",
            "title: sampleSubmission.csv, id: 1oq7i_PoqMDgnbrEaXDAZ8fE85R1ZBpYH\n",
            "title: LSTM.ipynb, id: 1VGS5i3uLTKmB-CdWzsF5kYjSXrGt8lxN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "8sbaKIriUe4f",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "train_downloaded = drive.CreateFile({'id': '106i86-VEOKrKWdT6xRHmX2tVfIe413rq'})\n",
        "train_downloaded.GetContentFile('train.csv')\n",
        "test_downloaded = drive.CreateFile({'id': '1prpU3Z6pHHRpB5T4XvYBPkGzcz3FMU8a'})\n",
        "test_downloaded.GetContentFile('test.csv')\n",
        "sample_downloaded = drive.CreateFile({'id': '1NnSvlLJTxQJnGGKHgbT8DYN7jWqQaaQl'})\n",
        "sample_downloaded.GetContentFile('sample.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "VVJnbioWUkKR",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 110
        },
        "outputId": "0bd61d57-72c8-4817-e05a-85c4ed04500d",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528692710723,
          "user_tz": -480,
          "elapsed": 1484,
          "user": {
            "displayName": "Sauradeep Chakraborty",
            "photoUrl": "//lh6.googleusercontent.com/-QvGcv_ohGQE/AAAAAAAAAAI/AAAAAAAADh8/JnoW4qEexCc/s50-c-k-no/photo.jpg",
            "userId": "104205094747188221458"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "import warnings; warnings.simplefilter('ignore')\n",
        "import pickle\n",
        "import sys, os, re, csv, codecs, numpy as np, pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn import metrics\n",
        "from sklearn.model_selection import train_test_split\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation\n",
        "from keras.layers import Bidirectional, GlobalMaxPool1D\n",
        "from keras.models import Model\n",
        "from keras import initializers, regularizers, constraints, optimizers, layers\n",
        "import numpy as np \n",
        "import pandas as pd \n",
        "import time\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Embedding, LSTM\n",
        "from sklearn.model_selection import train_test_split\n",
        "from keras.utils.np_utils import to_categorical\n",
        "import re\n",
        "import pandas as pd \n",
        "import re\n",
        "import numpy as np\n",
        "#from gensim.models import Doc2Vec, doc2vec\n",
        "from nltk.corpus import stopwords\n",
        "#from textblob import TextBlob, Word\n",
        "import nltk\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "from keras import backend\n",
        "backend.tensorflow_backend._get_available_gpus()\n",
        "import nltk\n",
        "nltk.download('wordnet')\n",
        "nltk.download('stopwords')\n"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /content/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /content/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "metadata": {
        "id": "lPdSSHPwUrSn",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 1224
        },
        "outputId": "ddee792b-e51f-4634-b24f-3af11a94c795",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528692815307,
          "user_tz": -480,
          "elapsed": 104417,
          "user": {
            "displayName": "Sauradeep Chakraborty",
            "photoUrl": "//lh6.googleusercontent.com/-QvGcv_ohGQE/AAAAAAAAAAI/AAAAAAAADh8/JnoW4qEexCc/s50-c-k-no/photo.jpg",
            "userId": "104205094747188221458"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "tick = time.time()\n",
        "df = pd.read_csv(\"train.csv\", dtype={\"label\": object, \"text\": object})\n",
        "y = df.label \n",
        "text = df.text\n",
        "\n",
        "print(df)\n",
        "#Convert all text to lower case\n",
        "text = text.apply(lambda x: \" \".join(x.lower() for x in str(x).split()))\n",
        "\n",
        "#Remove all punctuation\n",
        "for i in range(0, len(text)):\n",
        "    text[i] = re.sub(r'[^\\w\\s]', \"\", text[i])\n",
        "    \n",
        "#for index,row in df.iterrows():\n",
        "#    if df.iloc[index]['text'] == 'My husband and I had not purchased a home before and we definitely needed some hand holding. They were patient and professional. We got our dream home and the entire experience was awesome! Thank you so much ladies for a job well done!':\n",
        "#        df.drop(index, inplace=True)\n",
        "\n",
        "#Remove stop words (removes important words! needs modification)\n",
        "stop = stopwords.words(\"english\")\n",
        "stop = stop[:143]\n",
        "stop.remove(\"not\")\n",
        "stop.remove(\"against\")\n",
        "stop.remove(\"no\")\n",
        "#stop.append(\"My husband and I had not purchased a home before and we definitely needed some hand holding. They were patient and professional. We got our dream home and the entire experience was awesome! Thank you so much ladies for a job well done! \")\n",
        "text = text.apply(lambda x: \" \".join(x for x in str(x).split() if x not in stop))\n",
        "\n",
        "#Lemmatize (not working)\n",
        "#text = text.apply(lambda x: \" \".join(Word(x).lemmatize() for x in x.split()))\n",
        "\n",
        "#Stemming\n",
        "\n",
        "#text.drop(\"My husband and I had not purchased a home before and we definitely needed some hand holding. They were patient and professional. We got our dream home and the entire experience was awesome! Thank you so much ladies for a job well done!\")\n",
        "#seperating words into lists\n",
        "for i, line in enumerate(text):\n",
        "    text[i] = line.split()\n",
        "#print(text)\n",
        "#df.text = text\n",
        "#text = list(text)\n",
        "print(time.time() - tick)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "       label                                               text\n",
            "0          1  Great mobile app with nice reward program. Mak...\n",
            "1          2  Really fast and polite. Definitely recommend. ...\n",
            "2          2  This place is always amazing, friendly staff a...\n",
            "3          1  We did a Wine 101 class on a Friday night. Coo...\n",
            "4          1  I am rounding up because I think this place ma...\n",
            "5          0  My romantic tryst with the Planet Hollywood Po...\n",
            "6          2  Just got color and a haircut. Finally after a ...\n",
            "7          0  Never will go there again.  Place was nearly e...\n",
            "8          2  I've come here for 2 years now, when it was NP...\n",
            "9          0  Am 07.08.2014 war ich mit 5 Kollegen mittags i...\n",
            "10         2  5 stars (for take-out); not recommended for di...\n",
            "11         1  Nachdem wir jetzt das dritte Mal vor der Türe ...\n",
            "12         2  Hands down the best polish boy sandwich in Cle...\n",
            "13         2  Oh Fuzzy's where to begin with you? Are any of...\n",
            "14         2  I've tried many a cupcakes (Magnolia, Prarie G...\n",
            "15         1  If you put an Irish pub into a Walmart, this i...\n",
            "16         2  Amazing Pho!!!   I was almost drinking the bro...\n",
            "17         1  ¿La Eiffel Tower en Las Vegas?¿posta?¿es eso p...\n",
            "18         2  I love this shop.  The best retro dress in veg...\n",
            "19         2  Definitely a sexy show! And surprisingly funny...\n",
            "20         0  Something must have gone horribly wrong since ...\n",
            "21         2  I was so happy to find Silk and found Nicole t...\n",
            "22         2  As good as it gets for a Qdoba. Seriously, it'...\n",
            "23         2  Another one of my favorite spots, this place i...\n",
            "24         2  Count on Charleston's for top-notch food, serv...\n",
            "25         2  Amazing place! Went for a massage, very relaxi...\n",
            "26         2  Man Oh Man was this place the FUCKING bomb. Th...\n",
            "27         1  Great food, big portions and excellent custome...\n",
            "28         2  My son's iphone 6 screen was busted. They fixe...\n",
            "29         1  Yes, the space is cozy and comfy but the food ...\n",
            "...      ...                                                ...\n",
            "399971     0  I came at 12pm on a tuesday and theyre suppose...\n",
            "399972     2  The pizza at this place was delicious! Our dau...\n",
            "399973     2  I love that place! Waiting in row for half an ...\n",
            "399974     2  Walk by this place 5 days a week since Novembe...\n",
            "399975     0  I don't usually write 1 stars but Moira does d...\n",
            "399976     1  Mediocre food. Overpriced buffet. Only come he...\n",
            "399977     1  Food is fantastic though on the pricier side f...\n",
            "399978     0  The service was ok- a C or 2.5 , the food was ...\n",
            "399979     0  I was in Las Vegas over the past weekend to at...\n",
            "399980     2  Whatever you need done on your subaru these ar...\n",
            "399981     2  We had a fantastic experience. The room was , ...\n",
            "399982     2  Well, first of all this is an official Verizon...\n",
            "399983     1  Got the vanilla spiced oatmeal for $8. It had ...\n",
            "399984     2  This place was delightful. The food was elegan...\n",
            "399985     2  Have stayed here several times.  Management an...\n",
            "399986     2  Great Chinese food. Great place to take friend...\n",
            "399987     2  I never have been to 18\\/8 before but I will f...\n",
            "399988     1  This is a perfectly adequate gyros place. It's...\n",
            "399989     2  I've used MBV for at least nine years to fix m...\n",
            "399990     0  Don't bother coming back (again). I hadn't bee...\n",
            "399991     2  I couldn't be happier with 5th and Wine - I wa...\n",
            "399992     2  Kasadee is a fantastic hair stylist and the be...\n",
            "399993     1  Decided to grab some food here and being that ...\n",
            "399994     1  The hot dogs (12-18 inches) are great!  Frozen...\n",
            "399995     2  I have never used a moving company prior to Fa...\n",
            "399996     2  Went here for my first time and got Kelsey to ...\n",
            "399997     2  This was my first experience with True Food Ki...\n",
            "399998     2  Must be one of the most talked-about second-ha...\n",
            "399999     2  Undeniably the best bang for your buck in the ...\n",
            "400000     0  Don't order the hamburgers here.  We both orde...\n",
            "\n",
            "[400001 rows x 2 columns]\n",
            "103.59741258621216\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "YlLdyXcOU5LO",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 1168
        },
        "outputId": "71840437-0da9-435c-9366-7c23a7dbaed1",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528692816329,
          "user_tz": -480,
          "elapsed": 796,
          "user": {
            "displayName": "Sauradeep Chakraborty",
            "photoUrl": "//lh6.googleusercontent.com/-QvGcv_ohGQE/AAAAAAAAAAI/AAAAAAAADh8/JnoW4qEexCc/s50-c-k-no/photo.jpg",
            "userId": "104205094747188221458"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "X = text\n",
        "\n",
        "Y = df['label']\n",
        "X"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0         [great, mobile, app, nice, reward, program, ma...\n",
              "1         [really, fast, polite, definitely, recommend, ...\n",
              "2         [place, always, amazing, friendly, staff, grea...\n",
              "3         [wine, 101, class, friday, night, cool, spot, ...\n",
              "4         [rounding, think, place, may, potential, coupl...\n",
              "5         [romantic, tryst, planet, hollywood, poker, ro...\n",
              "6         [got, color, haircut, finally, months, looking...\n",
              "7         [never, go, place, nearly, empty, 430, pm, sat...\n",
              "8         [ive, come, 2, years, nps, bought, staff, alwa...\n",
              "9         [07082014, war, ich, mit, 5, kollegen, mittags...\n",
              "10        [5, stars, takeout, not, recommended, dinein, ...\n",
              "11        [nachdem, wir, jetzt, das, dritte, mal, vor, d...\n",
              "12        [hands, best, polish, boy, sandwich, cleveland...\n",
              "13        [oh, fuzzys, begin, familiar, toby, keith, son...\n",
              "14        [ive, tried, many, cupcakes, magnolia, prarie,...\n",
              "15        [put, irish, pub, walmart, youd, get, dont, ge...\n",
              "16        [amazing, pho, almost, drinking, broth, straw,...\n",
              "17        [la, eiffel, tower, en, las, vegaspostaes, eso...\n",
              "18        [love, shop, best, retro, dress, vegas, girls,...\n",
              "19        [definitely, sexy, show, surprisingly, funny, ...\n",
              "20        [something, must, gone, horribly, wrong, since...\n",
              "21        [happy, find, silk, found, nicole, professiona...\n",
              "22        [good, gets, qdoba, seriously, qdoba, good, ta...\n",
              "23        [another, one, favorite, spots, place, awesome...\n",
              "24        [count, charlestons, topnotch, food, service, ...\n",
              "25        [amazing, place, went, massage, relaxing, staf...\n",
              "26        [man, oh, man, place, fucking, bomb, pastor, a...\n",
              "27        [great, food, big, portions, excellent, custom...\n",
              "28        [sons, iphone, 6, screen, busted, fixed, less,...\n",
              "29        [yes, space, cozy, comfy, food, le, club, chas...\n",
              "                                ...                        \n",
              "399971    [came, 12pm, tuesday, theyre, supposed, open, ...\n",
              "399972    [pizza, place, delicious, daughters, loved, bi...\n",
              "399973    [love, place, waiting, row, half, hour, defini...\n",
              "399974    [walk, place, 5, days, week, since, november, ...\n",
              "399975    [dont, usually, write, 1, stars, moira, deserv...\n",
              "399976    [mediocre, food, overpriced, buffet, come, sus...\n",
              "399977    [food, fantastic, though, pricier, side, brunc...\n",
              "399978    [service, ok, c, 25, food, quick, arrive, wait...\n",
              "399979    [las, vegas, past, weekend, attend, mountain, ...\n",
              "399980    [whatever, need, done, subaru, guys, reliable,...\n",
              "399981    [fantastic, experience, room, indeed, whole, h...\n",
              "399982    [well, first, official, verizon, wireless, sto...\n",
              "399983    [got, vanilla, spiced, oatmeal, 8, like, 4, sl...\n",
              "399984    [place, delightful, food, elegant, tasteful, a...\n",
              "399985    [stayed, several, times, management, staff, po...\n",
              "399986    [great, chinese, food, great, place, take, fri...\n",
              "399987    [never, 188, forever, continue, coming, back, ...\n",
              "399988    [perfectly, adequate, gyros, place, small, cle...\n",
              "399989    [ive, used, mbv, least, nine, years, fix, 2000...\n",
              "399990    [dont, bother, coming, back, hadnt, 10, years,...\n",
              "399991    [couldnt, happier, 5th, wine, first, introduce...\n",
              "399992    [kasadee, fantastic, hair, stylist, best, ive,...\n",
              "399993    [decided, grab, food, fish, chips, thought, no...\n",
              "399994    [hot, dogs, 1218, inches, great, frozen, hot, ...\n",
              "399995    [never, used, moving, company, prior, family, ...\n",
              "399996    [went, first, time, got, kelsey, dye, cut, hai...\n",
              "399997    [first, experience, true, food, kitchen, trip,...\n",
              "399998    [must, one, talkedabout, secondhand, bike, pla...\n",
              "399999    [undeniably, best, bang, buck, city, 10, bucks...\n",
              "400000    [dont, order, hamburgers, ordered, well, done,...\n",
              "Name: text, Length: 400001, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "metadata": {
        "id": "yyqfJ4RCXJTb",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=42)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "tB8gRYsFXLhZ",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "max_features = 20000\n",
        "#tokenizer = Tokenizer(num_words=max_features)\n",
        "tokenizer = Tokenizer(num_words=max_features, filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n', lower=True,split=' ')\n",
        "tokenizer.fit_on_texts(list(X_train))\n",
        "list_tokenized_train = tokenizer.texts_to_sequences(X_train)\n",
        "list_tokenized_test = tokenizer.texts_to_sequences(X_test)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "j6bt3WNUXNlY",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "outputId": "fbc5bc14-b8db-40c5-e8ce-f04de6e89048",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528692865870,
          "user_tz": -480,
          "elapsed": 18855,
          "user": {
            "displayName": "Sauradeep Chakraborty",
            "photoUrl": "//lh6.googleusercontent.com/-QvGcv_ohGQE/AAAAAAAAAAI/AAAAAAAADh8/JnoW4qEexCc/s50-c-k-no/photo.jpg",
            "userId": "104205094747188221458"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "X = tokenizer.texts_to_sequences(text)\n",
        "X = pad_sequences(X)\n",
        "\n",
        "\n",
        "Y = pd.get_dummies(y.values)\n",
        "X_train, X_test, Y_train, Y_test = train_test_split(X,Y, test_size = 0.2, random_state = 42)\n",
        "print(X_train.shape,Y_train.shape)\n",
        "print(X_test.shape,Y_test.shape)\n",
        "print(y.values)\n",
        "print(Y_train.values)\n"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(320000, 571) (320000, 3)\n",
            "(80001, 571) (80001, 3)\n",
            "['1' '2' '2' ... '2' '2' '0']\n",
            "[[0 0 1]\n",
            " [1 0 0]\n",
            " [0 0 1]\n",
            " ...\n",
            " [0 0 1]\n",
            " [0 0 1]\n",
            " [0 0 1]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "VEwsICECXWDf",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "maxlen = 100\n",
        "X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)\n",
        "X_te = pad_sequences(list_tokenized_test, maxlen=maxlen)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UrRqxUQlXmXf",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "outputId": "78777960-6e70-4dfa-bb31-25e74f68df96",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528692871362,
          "user_tz": -480,
          "elapsed": 912,
          "user": {
            "displayName": "Sauradeep Chakraborty",
            "photoUrl": "//lh6.googleusercontent.com/-QvGcv_ohGQE/AAAAAAAAAAI/AAAAAAAADh8/JnoW4qEexCc/s50-c-k-no/photo.jpg",
            "userId": "104205094747188221458"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "Y_train, Y_test = train_test_split(Y, test_size = 0.2, random_state = 42)\n",
        "print(X_train.shape,Y_train.shape)\n",
        "print(X_test.shape,Y_test.shape)\n",
        "print(y.values)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(320000, 571) (320000, 3)\n",
            "(80001, 571) (80001, 3)\n",
            "['1' '2' '2' ... '2' '2' '0']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "2i_VKEzbX0c1",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 537
        },
        "outputId": "fbb67d2b-a04c-4749-d898-1c399010276d",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528692873105,
          "user_tz": -480,
          "elapsed": 1374,
          "user": {
            "displayName": "Sauradeep Chakraborty",
            "photoUrl": "//lh6.googleusercontent.com/-QvGcv_ohGQE/AAAAAAAAAAI/AAAAAAAADh8/JnoW4qEexCc/s50-c-k-no/photo.jpg",
            "userId": "104205094747188221458"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "inp = Input(shape=(maxlen, ))\n",
        "embed_size = 128\n",
        "x = Embedding(max_features, embed_size)(inp)\n",
        "x = LSTM(200, return_sequences=True,name='lstm_layer')(x)\n",
        "x = GlobalMaxPool1D()(x)\n",
        "x = Dropout(0.1)(x)\n",
        "x = Dense(120, activation=\"relu\")(x)\n",
        "x = Dropout(0.1)(x)\n",
        "x = Dense(60, activation=\"relu\")(x)\n",
        "x = Dropout(0.1)(x)\n",
        "x = Dense(4, activation=\"softmax\")(x)\n",
        "model = Model(inputs=inp, outputs=x)\n",
        "model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "print(model.summary())\n"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_3 (InputLayer)         (None, 100)               0         \n",
            "_________________________________________________________________\n",
            "embedding_3 (Embedding)      (None, 100, 128)          2560000   \n",
            "_________________________________________________________________\n",
            "lstm_layer (LSTM)            (None, 100, 200)          263200    \n",
            "_________________________________________________________________\n",
            "global_max_pooling1d_2 (Glob (None, 200)               0         \n",
            "_________________________________________________________________\n",
            "dropout_6 (Dropout)          (None, 200)               0         \n",
            "_________________________________________________________________\n",
            "dense_6 (Dense)              (None, 120)               24120     \n",
            "_________________________________________________________________\n",
            "dropout_7 (Dropout)          (None, 120)               0         \n",
            "_________________________________________________________________\n",
            "dense_7 (Dense)              (None, 60)                7260      \n",
            "_________________________________________________________________\n",
            "dropout_8 (Dropout)          (None, 60)                0         \n",
            "_________________________________________________________________\n",
            "dense_8 (Dense)              (None, 4)                 244       \n",
            "=================================================================\n",
            "Total params: 2,854,824\n",
            "Trainable params: 2,854,824\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "dDYKbyEOX23I",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 186
        },
        "outputId": "f53958d2-ec5e-44d6-8c4c-28ef04676292"
      },
      "cell_type": "code",
      "source": [
        "print(\"start fitting...\")\n",
        "model.fit(X_t, y_train , epochs=3, batch_size=32, validation_split=0.1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "start fitting...\n",
            "Train on 288000 samples, validate on 32000 samples\n",
            "Epoch 1/3\n",
            " 19232/288000 [=>............................] - ETA: 35:47 - loss: 0.6607 - acc: 0.7312"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "287968/288000 [============================>.] - ETA: 0s - loss: 0.3219 - acc: 0.8754"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r288000/288000 [==============================] - 2321s 8ms/step - loss: 0.3219 - acc: 0.8755 - val_loss: 0.2610 - val_acc: 0.8999\n",
            "Epoch 2/3\n",
            " 10144/288000 [>.............................] - ETA: 36:29 - loss: 0.2165 - acc: 0.9198"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "287968/288000 [============================>.] - ETA: 0s - loss: 0.2258 - acc: 0.9154"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r288000/288000 [==============================] - 2305s 8ms/step - loss: 0.2258 - acc: 0.9154 - val_loss: 0.2435 - val_acc: 0.9070\n",
            "Epoch 3/3\n",
            " 10144/288000 [>.............................] - ETA: 35:49 - loss: 0.1757 - acc: 0.9356"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 28480/288000 [=>............................] - ETA: 47:32 - loss: 0.1757 - acc: 0.9357"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "9RAYLnkpX53p",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "outputId": "7dab3f52-4736-4a2e-fd3e-fb01ff4715f8",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528692491727,
          "user_tz": -480,
          "elapsed": 169869,
          "user": {
            "displayName": "Sauradeep Chakraborty",
            "photoUrl": "//lh6.googleusercontent.com/-QvGcv_ohGQE/AAAAAAAAAAI/AAAAAAAADh8/JnoW4qEexCc/s50-c-k-no/photo.jpg",
            "userId": "104205094747188221458"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "scores = model.evaluate(X_te, y_test)\n",
        "print(\"\\n%s: %.4f%%\" % (model.metrics_names[1], scores[1]*100))\n",
        "y_pred = model.predict(X_te, batch_size=1024)\n",
        "y_classes = y_pred.argmax(axis=-1)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "64672/80001 [=======================>......] - ETA: 30s"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "80001/80001 [==============================] - 160s 2ms/step\n",
            "\n",
            "acc: 90.7276%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "CwnaxZiowMyn",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "z=np.array(y_test)\n",
        "d=y_test.astype(int)\n",
        "d\n",
        "z=np.array(d)\n",
        "z[0]\n",
        "Y_TEST=z"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "t1Q7yj45x0jT",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 296
        },
        "outputId": "029b20fd-31aa-44c7-830f-dd3a1d1cb363",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528692638590,
          "user_tz": -480,
          "elapsed": 1468,
          "user": {
            "displayName": "Sauradeep Chakraborty",
            "photoUrl": "//lh6.googleusercontent.com/-QvGcv_ohGQE/AAAAAAAAAAI/AAAAAAAADh8/JnoW4qEexCc/s50-c-k-no/photo.jpg",
            "userId": "104205094747188221458"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "\n",
        "accScore = metrics.accuracy_score(Y_TEST,y_classes)\n",
        "\n",
        "lbl = [0,1,2]\n",
        "precision = metrics.precision_score(Y_TEST,y_classes,average=None,labels=lbl)\n",
        "recall = metrics.recall_score(Y_TEST,y_classes,average=None,labels=lbl)\n",
        "f1Score = metrics.f1_score(Y_TEST,y_classes,average=None,labels=lbl)\n",
        "\n",
        "print(\"\\nOverall Acurracy: \",accScore,\"\\n\")\n",
        "\n",
        "for i in range(len(lbl)):\n",
        "    print(\"Precision of %s class: %f\" %(lbl[i],precision[i]))\n",
        "    print(\"Recall of %s class: %f\" %(lbl[i],recall[i]))\n",
        "    print(\"F1-Score of %s class: %f\" %(lbl[i],f1Score[i]),\"\\n\")\n"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Overall Acurracy:  0.907276159048012 \n",
            "\n",
            "Precision of 0 class: 0.890900\n",
            "Recall of 0 class: 0.906200\n",
            "F1-Score of 0 class: 0.898485 \n",
            "\n",
            "Precision of 1 class: 0.771680\n",
            "Recall of 1 class: 0.737959\n",
            "F1-Score of 1 class: 0.754443 \n",
            "\n",
            "Precision of 2 class: 0.947654\n",
            "Recall of 2 class: 0.953694\n",
            "F1-Score of 2 class: 0.950664 \n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "2mGlszdzx3Pb",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 305
        },
        "outputId": "80fdb3ff-a8a5-48e0-800c-9a67f3bcb8c4",
        "executionInfo": {
          "status": "error",
          "timestamp": 1528549497961,
          "user_tz": -480,
          "elapsed": 1315,
          "user": {
            "displayName": "Sauradeep Chakraborty",
            "photoUrl": "//lh6.googleusercontent.com/-QvGcv_ohGQE/AAAAAAAAAAI/AAAAAAAADh8/JnoW4qEexCc/s50-c-k-no/photo.jpg",
            "userId": "104205094747188221458"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "import utils"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-69d9c258b31d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mutils\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'utils'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "DJtPOD5zpkMO",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "from keras.models import Sequential, load_model\n",
        "from keras.layers import Dense, Dropout, Activation\n",
        "from keras.layers import Embedding, Flatten\n",
        "from keras.layers import Conv1D, GlobalMaxPooling1D\n",
        "from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\n",
        "kernel_size=3\n",
        "embedding_matrix = np.random.randn(20000, 128) * 0.01"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "zOgR_M4PBJlg",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "inp = Input(shape=(100, ))\n",
        "embed_size = 128\n",
        "x = Embedding(20000,128,weights=[embedding_matrix])(inp)#input_length=maxlen)(inp)\n",
        "x=(Dropout(0.4))(x)\n",
        "x=(Conv1D(600, kernel_size, padding='valid', activation='relu', strides=1))(x)\n",
        "x=(Conv1D(300, kernel_size, padding='valid', activation='relu', strides=1))(x)\n",
        "x=(Conv1D(150, kernel_size, padding='valid', activation='relu', strides=1))(x)\n",
        "x=(Conv1D(75, kernel_size, padding='valid', activation='relu', strides=1))(x)\n",
        "x=(Flatten())(x)\n",
        "x=(Dense(600))(x)\n",
        "x=(Dropout(0.5))(x)\n",
        "x=(Activation('relu'))(x)\n",
        "x=(Dense(1))(x)\n",
        "x=(Activation('softmax'))(x)\n",
        "model=Model(inputs=inp, outputs=x)\n",
        "\n",
        "model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "k-kfwxG2CTqM",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 590
        },
        "outputId": "92a27b88-2057-47a8-ac82-46c8c4e94d2c",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528557389455,
          "user_tz": -480,
          "elapsed": 719,
          "user": {
            "displayName": "Sauradeep Chakraborty",
            "photoUrl": "//lh6.googleusercontent.com/-QvGcv_ohGQE/AAAAAAAAAAI/AAAAAAAADh8/JnoW4qEexCc/s50-c-k-no/photo.jpg",
            "userId": "104205094747188221458"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_6 (InputLayer)         (None, 100)               0         \n",
            "_________________________________________________________________\n",
            "embedding_5 (Embedding)      (None, 100, 128)          2560000   \n",
            "_________________________________________________________________\n",
            "dropout_9 (Dropout)          (None, 100, 128)          0         \n",
            "_________________________________________________________________\n",
            "conv1d_17 (Conv1D)           (None, 98, 600)           231000    \n",
            "_________________________________________________________________\n",
            "conv1d_18 (Conv1D)           (None, 96, 300)           540300    \n",
            "_________________________________________________________________\n",
            "conv1d_19 (Conv1D)           (None, 94, 150)           135150    \n",
            "_________________________________________________________________\n",
            "conv1d_20 (Conv1D)           (None, 92, 75)            33825     \n",
            "_________________________________________________________________\n",
            "flatten_5 (Flatten)          (None, 6900)              0         \n",
            "_________________________________________________________________\n",
            "dense_9 (Dense)              (None, 600)               4140600   \n",
            "_________________________________________________________________\n",
            "dropout_10 (Dropout)         (None, 600)               0         \n",
            "_________________________________________________________________\n",
            "activation_9 (Activation)    (None, 600)               0         \n",
            "_________________________________________________________________\n",
            "dense_10 (Dense)             (None, 1)                 601       \n",
            "_________________________________________________________________\n",
            "activation_10 (Activation)   (None, 1)                 0         \n",
            "=================================================================\n",
            "Total params: 7,641,476\n",
            "Trainable params: 7,641,476\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "bpSKbkAdCuZh",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 992
        },
        "outputId": "4718bbcf-a8ce-4ea9-dc8b-921906b89a49",
        "executionInfo": {
          "status": "error",
          "timestamp": 1528691497975,
          "user_tz": -480,
          "elapsed": 1765,
          "user": {
            "displayName": "Sauradeep Chakraborty",
            "photoUrl": "//lh6.googleusercontent.com/-QvGcv_ohGQE/AAAAAAAAAAI/AAAAAAAADh8/JnoW4qEexCc/s50-c-k-no/photo.jpg",
            "userId": "104205094747188221458"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "res = pd.read_csv('test_data.csv')\n",
        "res_text = res.text\n",
        "tick = time.time()\n",
        "#df = pd.read_csv(\"train.csv\", dtype={\"label\": object, \"text\": object})\n",
        "#y = df.label \n",
        "#text = df.text\n",
        "\n",
        "#print(df)\n",
        "#Convert all text to lower case\n",
        "res_text = res_text.apply(lambda x: \" \".join(x.lower() for x in str(x).split()))\n",
        "\n",
        "#Remove all punctuation\n",
        "for i in range(0, len(text)):\n",
        "    res_text[i] = re.sub(r'[^\\w\\s]', \"\", res_text[i])\n",
        "    \n",
        "#for index,row in df.iterrows():\n",
        "#    if df.iloc[index]['text'] == 'My husband and I had not purchased a home before and we definitely needed some hand holding. They were patient and professional. We got our dream home and the entire experience was awesome! Thank you so much ladies for a job well done!':\n",
        "#        df.drop(index, inplace=True)\n",
        "\n",
        "#Remove stop words (removes important words! needs modification)\n",
        "stop = stopwords.words(\"english\")\n",
        "stop = stop[:143]\n",
        "stop.remove(\"not\")\n",
        "stop.remove(\"against\")\n",
        "stop.remove(\"no\")\n",
        "#stop.append(\"My husband and I had not purchased a home before and we definitely needed some hand holding. They were patient and professional. We got our dream home and the entire experience was awesome! Thank you so much ladies for a job well done! \")\n",
        "res_text = res_text.apply(lambda x: \" \".join(x for x in str(x).split() if x not in stop))\n",
        "\n",
        "#Lemmatize (not working)\n",
        "#text = text.apply(lambda x: \" \".join(Word(x).lemmatize() for x in x.split()))\n",
        "\n",
        "#Stemming\n",
        "\n",
        "#text.drop(\"My husband and I had not purchased a home before and we definitely needed some hand holding. They were patient and professional. We got our dream home and the entire experience was awesome! Thank you so much ladies for a job well done!\")\n",
        "#seperating words into lists\n",
        "for i, line in enumerate(res_text):\n",
        "    res_text[i] = line.split()\n",
        "#print(text)\n",
        "#df.text = text\n",
        "#text = list(text)\n",
        "print(time.time() - tick)\n",
        "y_pred = model.predict(res_text, batch_size=1024)\n"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-15-b6973d4aa7d2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'test_data.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mres_text\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtick\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m#df = pd.read_csv(\"train.csv\", dtype={\"label\": object, \"text\": object})\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m#y = df.label\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, skip_footer, doublequote, delim_whitespace, as_recarray, compact_ints, use_unsigned, low_memory, buffer_lines, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    707\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[1;32m    708\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 709\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    710\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    711\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    447\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 449\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    450\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    451\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    816\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    817\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 818\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    819\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    820\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1047\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1048\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1049\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1050\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1051\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1693\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'allow_leading_cols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex_col\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1694\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1695\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1696\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1697\u001b[0m         \u001b[0;31m# XXX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: File b'test_data.csv' does not exist"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "k8ISExXnEIUI",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}