Document Classiication.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "from collections import Counter\n",
    "import csv\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "directory = \"/home/praveen/Documents/Data/Text_full/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f_names = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filenames = os.listdir(directory)\n",
    "    \n",
    "for file in filenames:\n",
    "    f_name = os.path.join(directory, file)\n",
    "    f_names.append(f_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_corpus = {}\n",
    "\n",
    "for f in filenames:\n",
    "    f_name = os.path.join(directory, f)\n",
    "    fi = open(f_name)\n",
    "    data = fi.read().lower()\n",
    "    data_corpus[f] = ' '.join(re.findall(r\"(?i)\\b([a-z]{3,})+\\b\", data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'united states america before the securities and exchange commission washington administrative proceedings rulings release april administrative proceeding file the matter texstar oil corporation and bonamour inc stay order today the parties filed joint motion stay this proceeding based upon their agreement principle settlement that will resolve the proceeding all major terms for good cause shown ordered that the joint motion granted and the proceeding stayed subject compliance with rule practice requiring that this office notified promptly any the requirements the rule are not met jason patil administrative law judge'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_corpus['ap-3815.pdf.txt']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = pd.DataFrame.from_dict(data_corpus,orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = data.reset_index()\n",
    "data.columns = ['file_name','text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ap-4081.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ap-3948.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id744cff.pdf.txt</td>\n",
       "      <td>initial decision release administrative procee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ap-3748.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33-10168.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>34-68988.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>33-10094.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ap-3777.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>34-75165.pdf.txt</td>\n",
       "      <td>securities and exchange commission release fil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>id302cff.pdf.txt</td>\n",
       "      <td>initial decision release administrative procee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ap-3776.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>34-53041.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>monyquiz.pdf.txt</td>\n",
       "      <td>test your money marts facts saving and investi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>34-77739.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>33-10228.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>34-77423.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>34-77250.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>34-77013.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>34-70082.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>34-77288.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ap-4044.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>sgtd.pdf.txt</td>\n",
       "      <td>louisiana state government telephone directory...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>id1047cff.pdf.txt</td>\n",
       "      <td>initial decision release administrative procee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>33-10225.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>ap-3492.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>34-72247.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>34-72505.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>ia-4455.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>ap-4000.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>34-69604-pdp.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3646</th>\n",
       "      <td>ap-4171.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3647</th>\n",
       "      <td>ap-3618.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3648</th>\n",
       "      <td>34-78184.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3649</th>\n",
       "      <td>34-79042-o.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3650</th>\n",
       "      <td>comp23658.pdf.txt</td>\n",
       "      <td>case jal document entered flsd docket page uni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3651</th>\n",
       "      <td>ap-4303.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3652</th>\n",
       "      <td>34-77563.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3653</th>\n",
       "      <td>34-77648.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3654</th>\n",
       "      <td>34-76722.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3655</th>\n",
       "      <td>id328bpm.pdf.txt</td>\n",
       "      <td>initial decision release administrative procee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3656</th>\n",
       "      <td>33-10158.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3657</th>\n",
       "      <td>34-78495.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3658</th>\n",
       "      <td>34-78560.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3659</th>\n",
       "      <td>33-9354.pdf.txt</td>\n",
       "      <td>securities and exchange commission cfr parts a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3660</th>\n",
       "      <td>34-78148.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3661</th>\n",
       "      <td>34-77870.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3662</th>\n",
       "      <td>id940jeg.pdf.txt</td>\n",
       "      <td>initial decision release administrative procee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3663</th>\n",
       "      <td>3-11498-order.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3664</th>\n",
       "      <td>34-78365.pdf.txt</td>\n",
       "      <td>securities and exchange commission washington ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3665</th>\n",
       "      <td>34-78708.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3666</th>\n",
       "      <td>comp-pr2016-170.pdf.txt</td>\n",
       "      <td>case document filed page page gary leung cal b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3667</th>\n",
       "      <td>34-59930.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3668</th>\n",
       "      <td>ap-3577.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3669</th>\n",
       "      <td>34-76743.pdf.txt</td>\n",
       "      <td>securities and exchange commission cfr part re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3670</th>\n",
       "      <td>34-77300.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3671</th>\n",
       "      <td>34-67824.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3672</th>\n",
       "      <td>ap-3589.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3673</th>\n",
       "      <td>34-59870.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3674</th>\n",
       "      <td>33-10002.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3675</th>\n",
       "      <td>ap-4145.pdf.txt</td>\n",
       "      <td>united states america before the securities an...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3676 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    file_name  \\\n",
       "0             ap-4081.pdf.txt   \n",
       "1             ap-3948.pdf.txt   \n",
       "2            id744cff.pdf.txt   \n",
       "3             ap-3748.pdf.txt   \n",
       "4            33-10168.pdf.txt   \n",
       "5            34-68988.pdf.txt   \n",
       "6            33-10094.pdf.txt   \n",
       "7             ap-3777.pdf.txt   \n",
       "8            34-75165.pdf.txt   \n",
       "9            id302cff.pdf.txt   \n",
       "10            ap-3776.pdf.txt   \n",
       "11           34-53041.pdf.txt   \n",
       "12           monyquiz.pdf.txt   \n",
       "13           34-77739.pdf.txt   \n",
       "14           33-10228.pdf.txt   \n",
       "15           34-77423.pdf.txt   \n",
       "16           34-77250.pdf.txt   \n",
       "17           34-77013.pdf.txt   \n",
       "18           34-70082.pdf.txt   \n",
       "19           34-77288.pdf.txt   \n",
       "20            ap-4044.pdf.txt   \n",
       "21               sgtd.pdf.txt   \n",
       "22          id1047cff.pdf.txt   \n",
       "23           33-10225.pdf.txt   \n",
       "24            ap-3492.pdf.txt   \n",
       "25           34-72247.pdf.txt   \n",
       "26           34-72505.pdf.txt   \n",
       "27            ia-4455.pdf.txt   \n",
       "28            ap-4000.pdf.txt   \n",
       "29       34-69604-pdp.pdf.txt   \n",
       "...                       ...   \n",
       "3646          ap-4171.pdf.txt   \n",
       "3647          ap-3618.pdf.txt   \n",
       "3648         34-78184.pdf.txt   \n",
       "3649       34-79042-o.pdf.txt   \n",
       "3650        comp23658.pdf.txt   \n",
       "3651          ap-4303.pdf.txt   \n",
       "3652         34-77563.pdf.txt   \n",
       "3653         34-77648.pdf.txt   \n",
       "3654         34-76722.pdf.txt   \n",
       "3655         id328bpm.pdf.txt   \n",
       "3656         33-10158.pdf.txt   \n",
       "3657         34-78495.pdf.txt   \n",
       "3658         34-78560.pdf.txt   \n",
       "3659          33-9354.pdf.txt   \n",
       "3660         34-78148.pdf.txt   \n",
       "3661         34-77870.pdf.txt   \n",
       "3662         id940jeg.pdf.txt   \n",
       "3663    3-11498-order.pdf.txt   \n",
       "3664         34-78365.pdf.txt   \n",
       "3665         34-78708.pdf.txt   \n",
       "3666  comp-pr2016-170.pdf.txt   \n",
       "3667         34-59930.pdf.txt   \n",
       "3668          ap-3577.pdf.txt   \n",
       "3669         34-76743.pdf.txt   \n",
       "3670         34-77300.pdf.txt   \n",
       "3671         34-67824.pdf.txt   \n",
       "3672          ap-3589.pdf.txt   \n",
       "3673         34-59870.pdf.txt   \n",
       "3674         33-10002.pdf.txt   \n",
       "3675          ap-4145.pdf.txt   \n",
       "\n",
       "                                                   text  \n",
       "0     united states america before the securities an...  \n",
       "1     united states america before the securities an...  \n",
       "2     initial decision release administrative procee...  \n",
       "3     united states america before the securities an...  \n",
       "4     united states america before the securities an...  \n",
       "5     united states america before the securities an...  \n",
       "6     united states america before the securities an...  \n",
       "7     united states america before the securities an...  \n",
       "8     securities and exchange commission release fil...  \n",
       "9     initial decision release administrative procee...  \n",
       "10    united states america before the securities an...  \n",
       "11    united states america before the securities an...  \n",
       "12    test your money marts facts saving and investi...  \n",
       "13    united states america before the securities an...  \n",
       "14    united states america before the securities an...  \n",
       "15    united states america before the securities an...  \n",
       "16    united states america before the securities an...  \n",
       "17    united states america before the securities an...  \n",
       "18    united states america before the securities an...  \n",
       "19    united states america before the securities an...  \n",
       "20    united states america before the securities an...  \n",
       "21    louisiana state government telephone directory...  \n",
       "22    initial decision release administrative procee...  \n",
       "23    united states america before the securities an...  \n",
       "24    united states america before the securities an...  \n",
       "25    united states america before the securities an...  \n",
       "26    united states america before the securities an...  \n",
       "27    united states america before the securities an...  \n",
       "28    united states america before the securities an...  \n",
       "29    united states america before the securities an...  \n",
       "...                                                 ...  \n",
       "3646  united states america before the securities an...  \n",
       "3647  united states america before the securities an...  \n",
       "3648  united states america before the securities an...  \n",
       "3649  united states america before the securities an...  \n",
       "3650  case jal document entered flsd docket page uni...  \n",
       "3651  united states america before the securities an...  \n",
       "3652  united states america before the securities an...  \n",
       "3653  united states america before the securities an...  \n",
       "3654  united states america before the securities an...  \n",
       "3655  initial decision release administrative procee...  \n",
       "3656  united states america before the securities an...  \n",
       "3657  united states america before the securities an...  \n",
       "3658  united states america before the securities an...  \n",
       "3659  securities and exchange commission cfr parts a...  \n",
       "3660  united states america before the securities an...  \n",
       "3661  united states america before the securities an...  \n",
       "3662  initial decision release administrative procee...  \n",
       "3663  united states america before the securities an...  \n",
       "3664  securities and exchange commission washington ...  \n",
       "3665  united states america before the securities an...  \n",
       "3666  case document filed page page gary leung cal b...  \n",
       "3667  united states america before the securities an...  \n",
       "3668  united states america before the securities an...  \n",
       "3669  securities and exchange commission cfr part re...  \n",
       "3670  united states america before the securities an...  \n",
       "3671  united states america before the securities an...  \n",
       "3672  united states america before the securities an...  \n",
       "3673  united states america before the securities an...  \n",
       "3674  united states america before the securities an...  \n",
       "3675  united states america before the securities an...  \n",
       "\n",
       "[3676 rows x 2 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import graphlab as gl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = gl.SFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\"><table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">file_name</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">ap-4081.pdf.txt</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">united states america<br>before the securities ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">ap-3948.pdf.txt</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">united states america<br>before the securities ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">id744cff.pdf.txt</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">initial decision release<br>administrative procee ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">ap-3748.pdf.txt</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">united states america<br>before the securities ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">33-10168.pdf.txt</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">united states america<br>before the securities ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">34-68988.pdf.txt</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">united states america<br>before the securities ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">33-10094.pdf.txt</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">united states america<br>before the securities ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">ap-3777.pdf.txt</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">united states america<br>before the securities ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">34-75165.pdf.txt</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">securities and exchange<br>commission release file ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">id302cff.pdf.txt</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">initial decision release<br>administrative procee ...</td>\n",
       "    </tr>\n",
       "</table>\n",
       "[3676 rows x 2 columns]<br/>Note: Only the head of the SFrame is printed.<br/>You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.\n",
       "</div>"
      ],
      "text/plain": [
       "Columns:\n",
       "\tfile_name\tstr\n",
       "\ttext\tstr\n",
       "\n",
       "Rows: 3676\n",
       "\n",
       "Data:\n",
       "+------------------+-------------------------------+\n",
       "|    file_name     |              text             |\n",
       "+------------------+-------------------------------+\n",
       "| ap-4081.pdf.txt  | united states america befo... |\n",
       "| ap-3948.pdf.txt  | united states america befo... |\n",
       "| id744cff.pdf.txt | initial decision release a... |\n",
       "| ap-3748.pdf.txt  | united states america befo... |\n",
       "| 33-10168.pdf.txt | united states america befo... |\n",
       "| 34-68988.pdf.txt | united states america befo... |\n",
       "| 33-10094.pdf.txt | united states america befo... |\n",
       "| ap-3777.pdf.txt  | united states america befo... |\n",
       "| 34-75165.pdf.txt | securities and exchange co... |\n",
       "| id302cff.pdf.txt | initial decision release a... |\n",
       "+------------------+-------------------------------+\n",
       "[3676 rows x 2 columns]\n",
       "Note: Only the head of the SFrame is printed.\n",
       "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns."
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data['tf_idf'] = gl.text_analytics.tf_idf(data['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "float() argument must be a string or a number",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-44-3d19c0717ba2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpreprocessing\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnormalize\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'tf_idf_norm'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnormalize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'tf_idf'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m/home/praveen/anaconda/lib/python2.7/site-packages/sklearn/preprocessing/data.pyc\u001b[0m in \u001b[0;36mnormalize\u001b[1;34m(X, norm, axis, copy, return_norm)\u001b[0m\n\u001b[0;32m   1342\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1343\u001b[0m     X = check_array(X, sparse_format, copy=copy, warn_on_dtype=True,\n\u001b[1;32m-> 1344\u001b[1;33m                     estimator='the normalize function', dtype=FLOAT_DTYPES)\n\u001b[0m\u001b[0;32m   1345\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0maxis\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1346\u001b[0m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mT\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/praveen/anaconda/lib/python2.7/site-packages/sklearn/utils/validation.pyc\u001b[0m in \u001b[0;36mcheck_array\u001b[1;34m(array, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)\u001b[0m\n\u001b[0;32m    380\u001b[0m                                       force_all_finite)\n\u001b[0;32m    381\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 382\u001b[1;33m         \u001b[0marray\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    383\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    384\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mensure_2d\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: float() argument must be a string or a number"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import normalize\n",
    "data['tf_idf_norm'] = normalize(data['tf_idf'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>Batch size is larger than the input dataset. Switching to an exact Kmeans method.</pre>"
      ],
      "text/plain": [
       "Batch size is larger than the input dataset. Switching to an exact Kmeans method."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Choosing initial cluster centers with Kmeans++.</pre>"
      ],
      "text/plain": [
       "Choosing initial cluster centers with Kmeans++."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+---------------+-----------+</pre>"
      ],
      "text/plain": [
       "+---------------+-----------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Center number | Row index |</pre>"
      ],
      "text/plain": [
       "| Center number | Row index |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+---------------+-----------+</pre>"
      ],
      "text/plain": [
       "+---------------+-----------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 0             | 161       |</pre>"
      ],
      "text/plain": [
       "| 0             | 161       |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 1             | 791       |</pre>"
      ],
      "text/plain": [
       "| 1             | 791       |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 2             | 352       |</pre>"
      ],
      "text/plain": [
       "| 2             | 352       |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 3             | 521       |</pre>"
      ],
      "text/plain": [
       "| 3             | 521       |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+---------------+-----------+</pre>"
      ],
      "text/plain": [
       "+---------------+-----------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Starting kmeans model training.</pre>"
      ],
      "text/plain": [
       "Starting kmeans model training."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>Assigning points to initial cluster centers.</pre>"
      ],
      "text/plain": [
       "Assigning points to initial cluster centers."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+-------------------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+-------------------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Iteration | Number of changed assignments |</pre>"
      ],
      "text/plain": [
       "| Iteration | Number of changed assignments |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+-------------------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+-------------------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 1         | 3                             |</pre>"
      ],
      "text/plain": [
       "| 1         | 3                             |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 2         | 0                             |</pre>"
      ],
      "text/plain": [
       "| 2         | 0                             |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+-----------+-------------------------------+</pre>"
      ],
      "text/plain": [
       "+-----------+-------------------------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = gl.clustering.kmeans.create(data,num_clusters=4,features=['tf_idf'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model.cluster_id.print_rows(num_rows=402, num_columns=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}