HN_DataTransform_XGBoost_Time.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "FILE = \"/Users/Shared/data/HN_posts_year_to_Sep_26_2016.csv\"\n",
    "\n",
    "data = pd.read_csv(FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>num_points</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12579008</td>\n",
       "      <td>9/26/2016 3:26</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12579005</td>\n",
       "      <td>9/26/2016 3:24</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12578997</td>\n",
       "      <td>9/26/2016 3:19</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12578989</td>\n",
       "      <td>9/26/2016 3:16</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12578979</td>\n",
       "      <td>9/26/2016 3:14</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12578975</td>\n",
       "      <td>9/26/2016 3:13</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>12578954</td>\n",
       "      <td>9/26/2016 3:06</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>12578942</td>\n",
       "      <td>9/26/2016 3:04</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>12578919</td>\n",
       "      <td>9/26/2016 2:57</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>12578918</td>\n",
       "      <td>9/26/2016 2:56</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>12578908</td>\n",
       "      <td>9/26/2016 2:53</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12578893</td>\n",
       "      <td>9/26/2016 2:46</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12578879</td>\n",
       "      <td>9/26/2016 2:40</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>12578866</td>\n",
       "      <td>9/26/2016 2:37</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>12578857</td>\n",
       "      <td>9/26/2016 2:34</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>12578834</td>\n",
       "      <td>9/26/2016 2:28</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>12578831</td>\n",
       "      <td>9/26/2016 2:28</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>12578822</td>\n",
       "      <td>9/26/2016 2:26</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>12578816</td>\n",
       "      <td>9/26/2016 2:23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>12578806</td>\n",
       "      <td>9/26/2016 2:21</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>12578796</td>\n",
       "      <td>9/26/2016 2:18</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>12578791</td>\n",
       "      <td>9/26/2016 2:17</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>12578786</td>\n",
       "      <td>9/26/2016 2:16</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>12578753</td>\n",
       "      <td>9/26/2016 2:07</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>12578738</td>\n",
       "      <td>9/26/2016 2:04</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>12578725</td>\n",
       "      <td>9/26/2016 2:01</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>12578705</td>\n",
       "      <td>9/26/2016 1:58</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>12578700</td>\n",
       "      <td>9/26/2016 1:57</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>12578694</td>\n",
       "      <td>9/26/2016 1:54</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>12578681</td>\n",
       "      <td>9/26/2016 1:49</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293089</th>\n",
       "      <td>10177071</td>\n",
       "      <td>9/6/2015 8:02</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293090</th>\n",
       "      <td>10177065</td>\n",
       "      <td>9/6/2015 7:58</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293091</th>\n",
       "      <td>10177062</td>\n",
       "      <td>9/6/2015 7:57</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293092</th>\n",
       "      <td>10177048</td>\n",
       "      <td>9/6/2015 7:50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293093</th>\n",
       "      <td>10177041</td>\n",
       "      <td>9/6/2015 7:45</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293094</th>\n",
       "      <td>10177034</td>\n",
       "      <td>9/6/2015 7:39</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293095</th>\n",
       "      <td>10177013</td>\n",
       "      <td>9/6/2015 7:28</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293096</th>\n",
       "      <td>10177011</td>\n",
       "      <td>9/6/2015 7:25</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293097</th>\n",
       "      <td>10177010</td>\n",
       "      <td>9/6/2015 7:24</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293098</th>\n",
       "      <td>10177004</td>\n",
       "      <td>9/6/2015 7:18</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293099</th>\n",
       "      <td>10176994</td>\n",
       "      <td>9/6/2015 7:09</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293100</th>\n",
       "      <td>10176983</td>\n",
       "      <td>9/6/2015 6:57</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293101</th>\n",
       "      <td>10176981</td>\n",
       "      <td>9/6/2015 6:55</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293102</th>\n",
       "      <td>10176980</td>\n",
       "      <td>9/6/2015 6:55</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293103</th>\n",
       "      <td>10176976</td>\n",
       "      <td>9/6/2015 6:49</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293104</th>\n",
       "      <td>10176974</td>\n",
       "      <td>9/6/2015 6:49</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293105</th>\n",
       "      <td>10176971</td>\n",
       "      <td>9/6/2015 6:47</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293106</th>\n",
       "      <td>10176962</td>\n",
       "      <td>9/6/2015 6:41</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293107</th>\n",
       "      <td>10176960</td>\n",
       "      <td>9/6/2015 6:41</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293108</th>\n",
       "      <td>10176959</td>\n",
       "      <td>9/6/2015 6:38</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293109</th>\n",
       "      <td>10176951</td>\n",
       "      <td>9/6/2015 6:33</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293110</th>\n",
       "      <td>10176942</td>\n",
       "      <td>9/6/2015 6:23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293111</th>\n",
       "      <td>10176938</td>\n",
       "      <td>9/6/2015 6:22</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293112</th>\n",
       "      <td>10176926</td>\n",
       "      <td>9/6/2015 6:10</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293113</th>\n",
       "      <td>10176923</td>\n",
       "      <td>9/6/2015 6:03</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293114</th>\n",
       "      <td>10176919</td>\n",
       "      <td>9/6/2015 6:02</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293115</th>\n",
       "      <td>10176917</td>\n",
       "      <td>9/6/2015 6:01</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293116</th>\n",
       "      <td>10176908</td>\n",
       "      <td>9/6/2015 5:56</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293117</th>\n",
       "      <td>10176907</td>\n",
       "      <td>9/6/2015 5:55</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293118</th>\n",
       "      <td>10176903</td>\n",
       "      <td>9/6/2015 5:50</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>293119 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              id      created_at  num_points\n",
       "0       12579008  9/26/2016 3:26           1\n",
       "1       12579005  9/26/2016 3:24           1\n",
       "2       12578997  9/26/2016 3:19           1\n",
       "3       12578989  9/26/2016 3:16           1\n",
       "4       12578979  9/26/2016 3:14           1\n",
       "5       12578975  9/26/2016 3:13           1\n",
       "6       12578954  9/26/2016 3:06           1\n",
       "7       12578942  9/26/2016 3:04           1\n",
       "8       12578919  9/26/2016 2:57           3\n",
       "9       12578918  9/26/2016 2:56           1\n",
       "10      12578908  9/26/2016 2:53           4\n",
       "11      12578893  9/26/2016 2:46           1\n",
       "12      12578879  9/26/2016 2:40           4\n",
       "13      12578866  9/26/2016 2:37           3\n",
       "14      12578857  9/26/2016 2:34           2\n",
       "15      12578834  9/26/2016 2:28           1\n",
       "16      12578831  9/26/2016 2:28           2\n",
       "17      12578822  9/26/2016 2:26           1\n",
       "18      12578816  9/26/2016 2:23           1\n",
       "19      12578806  9/26/2016 2:21           3\n",
       "20      12578796  9/26/2016 2:18           1\n",
       "21      12578791  9/26/2016 2:17           2\n",
       "22      12578786  9/26/2016 2:16           2\n",
       "23      12578753  9/26/2016 2:07           2\n",
       "24      12578738  9/26/2016 2:04           2\n",
       "25      12578725  9/26/2016 2:01           4\n",
       "26      12578705  9/26/2016 1:58           1\n",
       "27      12578700  9/26/2016 1:57           1\n",
       "28      12578694  9/26/2016 1:54           2\n",
       "29      12578681  9/26/2016 1:49           1\n",
       "...          ...             ...         ...\n",
       "293089  10177071   9/6/2015 8:02           3\n",
       "293090  10177065   9/6/2015 7:58           1\n",
       "293091  10177062   9/6/2015 7:57           4\n",
       "293092  10177048   9/6/2015 7:50           1\n",
       "293093  10177041   9/6/2015 7:45           7\n",
       "293094  10177034   9/6/2015 7:39           3\n",
       "293095  10177013   9/6/2015 7:28           1\n",
       "293096  10177011   9/6/2015 7:25          23\n",
       "293097  10177010   9/6/2015 7:24           1\n",
       "293098  10177004   9/6/2015 7:18          11\n",
       "293099  10176994   9/6/2015 7:09          27\n",
       "293100  10176983   9/6/2015 6:57           2\n",
       "293101  10176981   9/6/2015 6:55           1\n",
       "293102  10176980   9/6/2015 6:55           5\n",
       "293103  10176976   9/6/2015 6:49         144\n",
       "293104  10176974   9/6/2015 6:49          25\n",
       "293105  10176971   9/6/2015 6:47           2\n",
       "293106  10176962   9/6/2015 6:41           2\n",
       "293107  10176960   9/6/2015 6:41           1\n",
       "293108  10176959   9/6/2015 6:38           2\n",
       "293109  10176951   9/6/2015 6:33          20\n",
       "293110  10176942   9/6/2015 6:23           1\n",
       "293111  10176938   9/6/2015 6:22           2\n",
       "293112  10176926   9/6/2015 6:10           6\n",
       "293113  10176923   9/6/2015 6:03          34\n",
       "293114  10176919   9/6/2015 6:02          15\n",
       "293115  10176917   9/6/2015 6:01          14\n",
       "293116  10176908   9/6/2015 5:56          10\n",
       "293117  10176907   9/6/2015 5:55           2\n",
       "293118  10176903   9/6/2015 5:50           4\n",
       "\n",
       "[293119 rows x 3 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data =data[['id', 'created_at', 'num_points']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "seed = 7\n",
    "np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 87282 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "title = data[\"title\"]\n",
    "\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(title)\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "GOOD_THRESHOLD = 100\n",
    "MAX_SEQUENCE_LENGTH = 24\n",
    "\n",
    "train = data.sample(frac=0.8)\n",
    "test = data.drop(train.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepareData(df):    \n",
    "    good = df[df[\"num_points\"] >= GOOD_THRESHOLD]\n",
    "    bad = df[df[\"num_points\"] < GOOD_THRESHOLD]\n",
    "    bad = bad.sample(n=good.shape[0])\n",
    "    data = good.append(bad)\n",
    "    data = data.sample(frac=1).reset_index(drop=True)\n",
    "    \n",
    "    num_points = data[\"num_points\"].values\n",
    "\n",
    "    y_train = np.zeros((len(num_points), 2), dtype=int)\n",
    "    y_original = np.zeros((len(num_points)), dtype=int)\n",
    "    for i in range(0, len(num_points)):\n",
    "        y_train[i, 1] = int(num_points[i] >= GOOD_THRESHOLD)\n",
    "        y_train[i, 0] = int(num_points[i] < GOOD_THRESHOLD)\n",
    "        y_original[i] = int(num_points[i] >= GOOD_THRESHOLD)\n",
    "        \n",
    "    sequences = tokenizer.texts_to_sequences(data[\"title\"])\n",
    "    x_train = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "\n",
    "    return x_train, y_train, y_original\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 1 ..., 0 1 1]\n"
     ]
    }
   ],
   "source": [
    "x_full, y_full, y2_full = prepareData(data)\n",
    "x_train, y_train, y2_train = prepareData(train)\n",
    "x_test, y_test, y2_test = prepareData(test)\n",
    "\n",
    "print(y2_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "\n",
    "embeddings_index = {}\n",
    "f = open(os.path.join('/Users/Shared/data/glove.6B/', 'glove.6B.100d.txt'))\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "\n",
    "print('Found %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "EMBEDDING_DIM = 100\n",
    "\n",
    "embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "def validate(model, x_test, y_test):\n",
    "    test_truth = np.apply_along_axis(lambda x: np.argmax(x), 1, y_test)\n",
    "    test_pred = model.predict(x_test)\n",
    "    test_pred = np.apply_along_axis(lambda x: np.argmax(x), 1, test_pred)\n",
    "    precision = precision_score(test_truth, test_pred)\n",
    "    recall = recall_score(test_truth, test_pred)\n",
    "    print(precision)\n",
    "    print(recall)\n",
    "    return precision, recall\n",
    "\n",
    "def validate_2(truth, pred):\n",
    "    truth = np.apply_along_axis(lambda x: np.argmax(x), 1, truth)\n",
    "    pred = np.apply_along_axis(lambda x: np.argmax(x), 1, pred)\n",
    "    precision = precision_score(truth, pred)\n",
    "    recall = recall_score(truth, pred)\n",
    "    print(precision)\n",
    "    print(recall)\n",
    "    return precision, recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=500, n_jobs=-1, oob_score=False, random_state=1,\n",
       "            verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier(n_estimators=500, n_jobs=-1, criterion=\"entropy\", random_state=1)\n",
    "rf.fit(x_train, y2_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.520553872782\n",
      "0.496696944674\n"
     ]
    }
   ],
   "source": [
    "res = rf.predict(x_test)\n",
    "\n",
    "#validate_2(y_test, res)\n",
    "precision = precision_score(y2_test, res)\n",
    "recall = recall_score(y2_test, res)\n",
    "print(precision)\n",
    "print(recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18236, 24)\n",
      "(18236, 2)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.1, loss='deviance', max_depth=1,\n",
       "              max_features=None, max_leaf_nodes=None,\n",
       "              min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "              min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "              n_estimators=500, presort='auto', random_state=1,\n",
       "              subsample=1.0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "print(x_train.shape)\n",
    "print(y_train.shape)\n",
    "\n",
    "gbc = GradientBoostingClassifier(n_estimators=500, learning_rate=0.1, max_depth=1, random_state=1)\n",
    "gbc.fit(x_train, y2_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 ..., 1 0 0]\n",
      "0.527832609554\n",
      "0.552023121387\n"
     ]
    }
   ],
   "source": [
    "res = gbc.predict(x_test)\n",
    "print(res)\n",
    "\n",
    "precision = precision_score(y2_test, res)\n",
    "recall = recall_score(y2_test, res)\n",
    "print(precision)\n",
    "print(recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "\n",
    "params = {\"objective\": \"binary:logistic\",\n",
    "          \"eta\": 0.15,\n",
    "          \"max_depth\": 7,\n",
    "          \"min_child_weight\": 10,\n",
    "          \"silent\": 1,\n",
    "          \"subsample\": 0.7,\n",
    "          \"colsample_bytree\": 0.7,\n",
    "          \"seed\": 1}\n",
    "num_trees=500\n",
    "gbm = xgb.train(params, xgb.DMatrix(x_train, y2_train), num_trees)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.57553232  0.23762675  0.47189417 ...,  0.55435169  0.23429342\n",
      "  0.38367593]\n",
      "[1 0 0 ..., 1 0 0]\n",
      "0.541924824453\n",
      "0.541701073493\n"
     ]
    }
   ],
   "source": [
    "res = gbm.predict(xgb.DMatrix(x_test))\n",
    "\n",
    "print(res)\n",
    "res2 = np.zeros(len(res), dtype=int)\n",
    "for i in range(0, len(res)):\n",
    "    res2[i] = 1 if res[i] >= 0.5 else 0\n",
    "print(res2)\n",
    "\n",
    "precision = precision_score(y2_test, res2)\n",
    "recall = recall_score(y2_test, res2)\n",
    "print(precision)\n",
    "print(recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18236, 2)\n",
      "(18236, 2)\n",
      "(18236,)\n",
      "(18236, 5)\n"
     ]
    }
   ],
   "source": [
    "rf_x_group_train = rf.predict_proba(x_train)\n",
    "gbc_x_group_train = gbc.predict_proba(x_train)\n",
    "gbm_x_group_train = gbm.predict(xgb.DMatrix(x_train))\n",
    "\n",
    "print(rf_x_group_train.shape)\n",
    "print(gbc_x_group_train.shape)\n",
    "print(gbm_x_group_train.shape)\n",
    "\n",
    "x_group_train = np.zeros([rf_x_group_train.shape[0], 5])\n",
    "for i in range(0, rf_x_group_train.shape[0]):\n",
    "    x_group_train[i][0] = rf_x_group_train[i][0]\n",
    "    x_group_train[i][1] = rf_x_group_train[i][1]\n",
    "    \n",
    "    x_group_train[i][2] = gbc_x_group_train[i][0]\n",
    "    x_group_train[i][3] = gbc_x_group_train[i][1]\n",
    "    \n",
    "    x_group_train[i][4] = gbm_x_group_train[i]\n",
    "    \n",
    "print(x_group_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4844, 2)\n",
      "(4844, 2)\n",
      "(4844,)\n",
      "(4844, 5)\n"
     ]
    }
   ],
   "source": [
    "rf_x_group_test = rf.predict_proba(x_test)\n",
    "gbc_x_group_test = gbc.predict_proba(x_test)\n",
    "gbm_x_group_test = gbm.predict(xgb.DMatrix(x_test))\n",
    "\n",
    "print(rf_x_group_test.shape)\n",
    "print(gbc_x_group_test.shape)\n",
    "print(gbm_x_group_test.shape)\n",
    "\n",
    "x_group_test = np.zeros([rf_x_group_test.shape[0], 5])\n",
    "for i in range(0, rf_x_group_test.shape[0]):\n",
    "    x_group_test[i][0] = rf_x_group_test[i][0]\n",
    "    x_group_test[i][1] = rf_x_group_test[i][1]\n",
    "    \n",
    "    x_group_test[i][2] = gbc_x_group_test[i][0]\n",
    "    x_group_test[i][3] = gbc_x_group_test[i][1]\n",
    "    \n",
    "    x_group_test[i][4] = gbm_x_group_test[i]\n",
    "    \n",
    "print(x_group_test.shape)\n",
    "\n",
    "y_group_test = y_test\n",
    "    \n",
    "\n",
    "\n",
    "#print(zip(rf_x_group_train,gbc_x_group_train))\n",
    "#print(np.stack([rf_x_group_train,gbm_x_group_train]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.layers import Input, Convolution1D, MaxPooling1D, Dense, Flatten, Dropout, Embedding\n",
    "from keras.models import Model, Sequential\n",
    "from keras.regularizers import l2, activity_l2\n",
    "\n",
    "def create_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(10, input_shape=(5,)))\n",
    "    model.add(Dense(2, activation='softmax'))\n",
    "    model.compile(optimizer='rmsprop',\n",
    "          loss='categorical_crossentropy',\n",
    "          metrics=['accuracy', 'precision'])\n",
    "    return model\n",
    "    \n",
    "\n",
    "def create_baseline():\n",
    "    embedding_layer = Embedding(len(word_index) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)\n",
    "\n",
    "    sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences = embedding_layer(sequence_input)\n",
    "    #x = Convolution1D(16, 5, activation='relu')(embedded_sequences)\n",
    "    #x = Dropout(0.5)(x)\n",
    "    #x = MaxPooling1D()(x)\n",
    "    #x = Convolution1D(16, 5, activation='relu')(embedded_sequences)\n",
    "    #x = Dropout(0.5)(x)\n",
    "    #x = MaxPooling1D()(x)\n",
    "    #x = Dropout(0.5)(x)\n",
    "    x = Flatten()(embedded_sequences)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = Dense(64, init='uniform', activation='relu')(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    preds = Dense(2, activation='softmax')(x)\n",
    "    model = Model(sequence_input, preds)\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['acc', 'precision'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import EarlyStopping\n",
    "es = EarlyStopping('val_precision', patience=1, mode='max')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 0 ..., 1 0 1]\n",
      "TRAIN: [    0     1     2 ..., 18233 18234 18235] TEST: [    3    13    15 ..., 18225 18228 18232]\n",
      "Train on 14588 samples, validate on 3648 samples\n",
      "Epoch 1/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.3305 - acc: 0.9208 - precision: 0.9208 - val_loss: 0.0477 - val_acc: 0.9951 - val_precision: 0.9951\n",
      "Epoch 2/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0178 - acc: 0.9961 - precision: 0.9961 - val_loss: 0.0106 - val_acc: 0.9959 - val_precision: 0.9959\n",
      "Epoch 3/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0089 - acc: 0.9962 - precision: 0.9962 - val_loss: 0.0095 - val_acc: 0.9953 - val_precision: 0.9953\n",
      "Epoch 4/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0085 - acc: 0.9958 - precision: 0.9958 - val_loss: 0.0092 - val_acc: 0.9953 - val_precision: 0.9953\n",
      "Epoch 5/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0082 - acc: 0.9962 - precision: 0.9962 - val_loss: 0.0094 - val_acc: 0.9948 - val_precision: 0.9948\n",
      "Epoch 6/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0083 - acc: 0.9962 - precision: 0.9962 - val_loss: 0.0091 - val_acc: 0.9956 - val_precision: 0.9956\n",
      "Epoch 7/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0083 - acc: 0.9960 - precision: 0.9960 - val_loss: 0.0091 - val_acc: 0.9953 - val_precision: 0.9953\n",
      "Epoch 8/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0082 - acc: 0.9960 - precision: 0.9960 - val_loss: 0.0091 - val_acc: 0.9956 - val_precision: 0.9956\n",
      "Epoch 9/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0083 - acc: 0.9960 - precision: 0.9960 - val_loss: 0.0090 - val_acc: 0.9953 - val_precision: 0.9953\n",
      "Epoch 10/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0082 - acc: 0.9960 - precision: 0.9960 - val_loss: 0.0090 - val_acc: 0.9956 - val_precision: 0.9956\n",
      "0.996158068057\n",
      "0.995065789474\n",
      "0.530671859786\n",
      "0.450041288192\n",
      "TRAIN: [    0     1     3 ..., 18233 18234 18235] TEST: [    2     8    11 ..., 18213 18214 18224]\n",
      "Train on 14588 samples, validate on 3648 samples\n",
      "Epoch 1/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.1221 - acc: 0.9944 - precision: 0.9944 - val_loss: 0.0168 - val_acc: 0.9959 - val_precision: 0.9959\n",
      "Epoch 2/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0116 - acc: 0.9957 - precision: 0.9957 - val_loss: 0.0088 - val_acc: 0.9964 - val_precision: 0.9964\n",
      "Epoch 3/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0094 - acc: 0.9955 - precision: 0.9955 - val_loss: 0.0080 - val_acc: 0.9962 - val_precision: 0.9962\n",
      "Epoch 4/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0090 - acc: 0.9955 - precision: 0.9955 - val_loss: 0.0082 - val_acc: 0.9962 - val_precision: 0.9962\n",
      "Epoch 5/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0090 - acc: 0.9957 - precision: 0.9957 - val_loss: 0.0078 - val_acc: 0.9959 - val_precision: 0.9959\n",
      "Epoch 6/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0088 - acc: 0.9955 - precision: 0.9955 - val_loss: 0.0077 - val_acc: 0.9964 - val_precision: 0.9964\n",
      "Epoch 7/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0088 - acc: 0.9957 - precision: 0.9957 - val_loss: 0.0078 - val_acc: 0.9956 - val_precision: 0.9956\n",
      "Epoch 8/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0088 - acc: 0.9958 - precision: 0.9958 - val_loss: 0.0078 - val_acc: 0.9959 - val_precision: 0.9959\n",
      "Epoch 9/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0087 - acc: 0.9958 - precision: 0.9958 - val_loss: 0.0080 - val_acc: 0.9962 - val_precision: 0.9962\n",
      "Epoch 10/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0087 - acc: 0.9957 - precision: 0.9957 - val_loss: 0.0076 - val_acc: 0.9962 - val_precision: 0.9962\n",
      "0.996162280702\n",
      "0.996162280702\n",
      "0.532807104095\n",
      "0.445912469034\n",
      "TRAIN: [    0     1     2 ..., 18232 18234 18235] TEST: [    6    10    12 ..., 18227 18229 18233]\n",
      "Train on 14588 samples, validate on 3648 samples\n",
      "Epoch 1/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.1675 - acc: 0.9754 - precision: 0.9754 - val_loss: 0.0181 - val_acc: 0.9967 - val_precision: 0.9967\n",
      "Epoch 2/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0115 - acc: 0.9957 - precision: 0.9957 - val_loss: 0.0076 - val_acc: 0.9970 - val_precision: 0.9970\n",
      "Epoch 3/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0085 - acc: 0.9959 - precision: 0.9959 - val_loss: 0.0068 - val_acc: 0.9973 - val_precision: 0.9973\n",
      "Epoch 4/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0084 - acc: 0.9956 - precision: 0.9956 - val_loss: 0.0068 - val_acc: 0.9973 - val_precision: 0.9973\n",
      "Epoch 5/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0083 - acc: 0.9956 - precision: 0.9956 - val_loss: 0.0066 - val_acc: 0.9975 - val_precision: 0.9975\n",
      "Epoch 6/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0083 - acc: 0.9955 - precision: 0.9955 - val_loss: 0.0068 - val_acc: 0.9973 - val_precision: 0.9973\n",
      "Epoch 7/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0083 - acc: 0.9957 - precision: 0.9957 - val_loss: 0.0067 - val_acc: 0.9973 - val_precision: 0.9973\n",
      "Epoch 8/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0082 - acc: 0.9960 - precision: 0.9960 - val_loss: 0.0068 - val_acc: 0.9970 - val_precision: 0.9970\n",
      "Epoch 9/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0082 - acc: 0.9954 - precision: 0.9954 - val_loss: 0.0066 - val_acc: 0.9973 - val_precision: 0.9973\n",
      "Epoch 10/10\n",
      "14588/14588 [==============================] - 1s - loss: 0.0083 - acc: 0.9957 - precision: 0.9957 - val_loss: 0.0066 - val_acc: 0.9970 - val_precision: 0.9970\n",
      "0.996712328767\n",
      "0.99725877193\n",
      "0.522765598651\n",
      "0.511973575557\n",
      "TRAIN: [    0     1     2 ..., 18232 18233 18235] TEST: [    4     7    17 ..., 18230 18231 18234]\n",
      "Train on 14590 samples, validate on 3646 samples\n",
      "Epoch 1/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.3238 - acc: 0.8929 - precision: 0.8929 - val_loss: 0.0438 - val_acc: 0.9967 - val_precision: 0.9967\n",
      "Epoch 2/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0185 - acc: 0.9956 - precision: 0.9956 - val_loss: 0.0071 - val_acc: 0.9973 - val_precision: 0.9973\n",
      "Epoch 3/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0099 - acc: 0.9953 - precision: 0.9953 - val_loss: 0.0056 - val_acc: 0.9973 - val_precision: 0.9973\n",
      "Epoch 4/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0093 - acc: 0.9954 - precision: 0.9954 - val_loss: 0.0059 - val_acc: 0.9967 - val_precision: 0.9967\n",
      "Epoch 5/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0093 - acc: 0.9953 - precision: 0.9953 - val_loss: 0.0050 - val_acc: 0.9978 - val_precision: 0.9978\n",
      "Epoch 6/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0092 - acc: 0.9958 - precision: 0.9958 - val_loss: 0.0049 - val_acc: 0.9973 - val_precision: 0.9973\n",
      "Epoch 7/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0091 - acc: 0.9954 - precision: 0.9954 - val_loss: 0.0048 - val_acc: 0.9978 - val_precision: 0.9978\n",
      "Epoch 8/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0091 - acc: 0.9951 - precision: 0.9951 - val_loss: 0.0052 - val_acc: 0.9967 - val_precision: 0.9967\n",
      "Epoch 9/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0091 - acc: 0.9955 - precision: 0.9955 - val_loss: 0.0049 - val_acc: 0.9973 - val_precision: 0.9973\n",
      "Epoch 10/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0091 - acc: 0.9956 - precision: 0.9956 - val_loss: 0.0047 - val_acc: 0.9978 - val_precision: 0.9978\n",
      "0.998352553542\n",
      "0.997257268239\n",
      "0.532061473238\n",
      "0.414533443435\n",
      "TRAIN: [    2     3     4 ..., 18232 18233 18234] TEST: [    0     1     5 ..., 18221 18223 18235]\n",
      "Train on 14590 samples, validate on 3646 samples\n",
      "Epoch 1/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.1885 - acc: 0.9576 - precision: 0.9576 - val_loss: 0.0211 - val_acc: 0.9948 - val_precision: 0.9948\n",
      "Epoch 2/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0100 - acc: 0.9964 - precision: 0.9964 - val_loss: 0.0121 - val_acc: 0.9931 - val_precision: 0.9931\n",
      "Epoch 3/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0073 - acc: 0.9967 - precision: 0.9967 - val_loss: 0.0113 - val_acc: 0.9937 - val_precision: 0.9937\n",
      "Epoch 4/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0070 - acc: 0.9964 - precision: 0.9964 - val_loss: 0.0112 - val_acc: 0.9942 - val_precision: 0.9942\n",
      "Epoch 5/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0069 - acc: 0.9966 - precision: 0.9966 - val_loss: 0.0123 - val_acc: 0.9937 - val_precision: 0.9937\n",
      "Epoch 6/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0069 - acc: 0.9964 - precision: 0.9964 - val_loss: 0.0122 - val_acc: 0.9937 - val_precision: 0.9937\n",
      "Epoch 7/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0068 - acc: 0.9962 - precision: 0.9962 - val_loss: 0.0114 - val_acc: 0.9951 - val_precision: 0.9951\n",
      "Epoch 8/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0069 - acc: 0.9964 - precision: 0.9964 - val_loss: 0.0114 - val_acc: 0.9942 - val_precision: 0.9942\n",
      "Epoch 9/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0069 - acc: 0.9964 - precision: 0.9964 - val_loss: 0.0115 - val_acc: 0.9945 - val_precision: 0.9945\n",
      "Epoch 10/10\n",
      "14590/14590 [==============================] - 1s - loss: 0.0069 - acc: 0.9964 - precision: 0.9964 - val_loss: 0.0117 - val_acc: 0.9929 - val_precision: 0.9929\n",
      "0.994496422675\n",
      "0.991223258365\n",
      "0.528505392912\n",
      "0.424855491329\n",
      "Precision: 1.00\n",
      "Recall: 1.00\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "N = 5\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=N, shuffle=True, random_state=seed)\n",
    "kfold.get_n_splits(x_group_train, y_train)\n",
    "\n",
    "print(y2_train)\n",
    "\n",
    "precision = 0\n",
    "recall = 0\n",
    "for train_index, test_index in kfold.split(x_group_train, y2_train):\n",
    "    print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n",
    "    x_k_train, x_k_test = x_group_train[train_index], x_group_train[test_index]\n",
    "    y_k_train, y_k_test = y_train[train_index], y_train[test_index]\n",
    "    model = create_model()\n",
    "    model.fit(x_k_train, y_k_train, nb_epoch=10, batch_size=32, validation_data=(x_k_test, y_k_test), callbacks=[])\n",
    "    p, r = validate(model, x_k_test, y_k_test)\n",
    "    precision += p\n",
    "    recall += r\n",
    "    validate(model, x_group_test, y_group_test)\n",
    "    \n",
    "print(\"Precision: %.2f\" % (precision / N))\n",
    "print(\"Recall: %.2f\" % (recall / N))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "N = 5\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=N, shuffle=True, random_state=seed)\n",
    "kfold.get_n_splits(x_full, y_full)\n",
    "\n",
    "print(y2_full)\n",
    "\n",
    "precision = 0\n",
    "recall = 0\n",
    "for train_index, test_index in kfold.split(x_full, y2_full):\n",
    "    print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n",
    "    x_train, x_test = x_full[train_index], x_full[test_index]\n",
    "    y_train, y_test = y_full[train_index], y_full[test_index]\n",
    "    model = create_baseline()\n",
    "    model.fit(x_train, y_train, nb_epoch=100, batch_size=128, validation_data=(x_test, y_test), callbacks=[es])\n",
    "    p, r = validate(model, x_test, y_test)\n",
    "    precision += p\n",
    "    recall += r\n",
    "    \n",
    "print(\"Precision: %.2f\" % (precision / N))\n",
    "print(\"Recall: %.2f\" % (recall / N))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}